Note: The default ITS GitLab runner is a shared resource and is subject to slowdowns during heavy usage.
You can run your own GitLab runner that is dedicated just to your group if you need to avoid processing delays.

Commit e8262a69 authored by Qusai Al Shidi's avatar Qusai Al Shidi 💬
Browse files

FIX: Lazy load filecache to allow parallel processes.

parent 859e8081
Pipeline #70508 passed with stage
in 8 minutes and 18 seconds
...@@ -12,14 +12,15 @@ import datetime as dt ...@@ -12,14 +12,15 @@ import datetime as dt
import ftplib import ftplib
from functools import lru_cache from functools import lru_cache
import gzip import gzip
import importlib
from operator import itemgetter from operator import itemgetter
import os.path import os.path
import shutil import shutil
import sys
import urllib import urllib
import urllib.request import urllib.request
import warnings import warnings
from dateutil import rrule from dateutil import rrule
from filecache import filecache
import numpy as np import numpy as np
from .tools import _nearest, carrington_rotation_number from .tools import _nearest, carrington_rotation_number
...@@ -234,8 +235,7 @@ def get_omni_data(time_from, time_to, **kwargs): ...@@ -234,8 +235,7 @@ def get_omni_data(time_from, time_to, **kwargs):
return return_data return return_data
@lru_cache(maxsize=10) @lru_cache
@filecache
def _download_static_page(url): def _download_static_page(url):
"""Downloads a cached webpage as a list of lines """Downloads a cached webpage as a list of lines
...@@ -247,7 +247,12 @@ def _download_static_page(url): ...@@ -247,7 +247,12 @@ def _download_static_page(url):
list: A list of lines from the webpage. *Note*: Linebreaks are still list: A list of lines from the webpage. *Note*: Linebreaks are still
present. present.
""" """
return list(urllib.request.urlopen(url)) # lazy load filecache to prevent parallel issues
from filecache import filecache
@filecache
def _download_static_page_lazy(arg):
return list(urllib.request.urlopen(arg))
return _download_static_page_lazy(url)
def _urls_omni_hires(time_from, time_to): def _urls_omni_hires(time_from, time_to):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment