Note: The default ITS GitLab runner is a shared resource and is subject to slowdowns during heavy usage.
You can run your own GitLab runner that is dedicated just to your group if you need to avoid processing delays.

Commit e8262a69 authored by Qusai Al Shidi's avatar Qusai Al Shidi 💬
Browse files

FIX: Lazy load filecache to allow parallel processes.

parent 859e8081
Pipeline #70508 passed with stage
in 8 minutes and 18 seconds
......@@ -12,14 +12,15 @@ import datetime as dt
import ftplib
from functools import lru_cache
import gzip
import importlib
from operator import itemgetter
import os.path
import shutil
import sys
import urllib
import urllib.request
import warnings
from dateutil import rrule
from filecache import filecache
import numpy as np
from .tools import _nearest, carrington_rotation_number
......@@ -234,8 +235,7 @@ def get_omni_data(time_from, time_to, **kwargs):
return return_data
@lru_cache(maxsize=10)
@filecache
@lru_cache
def _download_static_page(url):
"""Downloads a cached webpage as a list of lines
......@@ -247,7 +247,12 @@ def _download_static_page(url):
list: A list of lines from the webpage. *Note*: Linebreaks are still
present.
"""
return list(urllib.request.urlopen(url))
# lazy load filecache to prevent parallel issues
from filecache import filecache
@filecache
def _download_static_page_lazy(arg):
return list(urllib.request.urlopen(arg))
return _download_static_page_lazy(url)
def _urls_omni_hires(time_from, time_to):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment