Note: The default ITS GitLab runner is a shared resource and is subject to slowdowns during heavy usage.
You can run your own GitLab runner that is dedicated just to your group if you need to avoid processing delays.

Commit 0d243de1 authored by Qusai Al Shidi's avatar Qusai Al Shidi 💬
Browse files

Not a working commit. Just pushing this to work on another computer

parent 67841097
...@@ -15,17 +15,19 @@ Clone into the directory you want to use it. ...@@ -15,17 +15,19 @@ Clone into the directory you want to use it.
*Note*: swmfpy also is part of the SWMF and gets cloned into `SWMF/share/Python`. However, if you would like to [develop](CONTRIBUTING.markdown) for swmfpy or have a local copy do the following: *Note*: swmfpy also is part of the SWMF and gets cloned into `SWMF/share/Python`. However, if you would like to [develop](CONTRIBUTING.markdown) for swmfpy or have a local copy do the following:
```bash ```bash
# Skip this if using it in SWMF directory.
git clone https://gitlab.umich.edu/swmf_sofware/swmfpy.git /path/to/my/dir git clone https://gitlab.umich.edu/swmf_sofware/swmfpy.git /path/to/my/dir
``` ```
Then go to its directory and run `setup.py` make sure to include `--user`. Then go to its directory and run [pip](https://pip.pypa.io/en/stable/) to install. Make sure to include `--user`.
```bash ```bash
cd /path/to/swmfpy cd SWMF/share/Python # or your clone directory
python3 -m pip install setuptools wheel twine --user # If you don't have these pip install --user .
python3 setup.py install --user
``` ```
*Note*: Depending on your system [pip](https://pip.pypa.io/en/stable/) may be ran in several ways: `pip`, `pip3`, or `python3 -m pip`
Then import it into your python project. Then import it into your python project.
```python ```python
......
...@@ -5,8 +5,11 @@ swmfpy. ...@@ -5,8 +5,11 @@ swmfpy.
import setuptools import setuptools
with open('README.markdown') as readme_fh: with open('README.markdown') as fh_readme:
LONG_DESCRIPTION = readme_fh.read() LONG_DESCRIPTION = fh_readme.read()
with open('requirements.txt') as fh_requirements:
REQUIREMENTS = list(fh_requirements)
setuptools.setup( setuptools.setup(
name='swmfpy', name='swmfpy',
...@@ -24,5 +27,6 @@ setuptools.setup( ...@@ -24,5 +27,6 @@ setuptools.setup(
'License :: OSI Approved :: MIT License', 'License :: OSI Approved :: MIT License',
'Operating System :: OS Independent', 'Operating System :: OS Independent',
], ],
install_requires=REQUIREMENTS,
python_requires='>=3.6' python_requires='>=3.6'
) )
...@@ -23,52 +23,7 @@ __email__ = 'qusai@umich.edu' ...@@ -23,52 +23,7 @@ __email__ = 'qusai@umich.edu'
import sys import sys
assert sys.version_info >= (3, 6), "swmfpy requires Python >=3.6. Sorry :(."
from . import paramin from . import paramin
from . import io from . import io
from . import web from . import web
assert sys.version_info >= (3, 6), "swmfpy requires Python >=3.6. Sorry :(."
# This is straight from the format guide on spdf with nicer names as second col
OMNI_COLS = (('ID for IMF spacecraft', 'id_imf'),
('ID for SW Plasma spacecraft', 'id_sw'),
('# of points in IMF averages', 'num_avg_imf'),
('# of points in Plasma averages', 'num_avg_sw'),
('Percent interp', 'interp'),
('Timeshift, sec', 'timeshift'),
('RMS, Timeshift', 'rms_timeshift'),
('RMS, Phase front normal', 'rms_phase'),
('Time btwn observations, sec', 'dt'),
('Field magnitude average, nT', 'b'),
('Bx, nT (GSE, GSM)', 'bx'),
('By, nT (GSE)', 'by_gse'),
('Bz, nT (GSE)', 'bz_gse'),
('By, nT (GSM)', 'by'),
('Bz, nT (GSM)', 'bz'),
('RMS SD B scalar, nT', 'rms_sd_b'),
('RMS SD field vector, nT', 'rms_sd_field'),
('Flow speed, km/s', 'v'),
('Vx Velocity, km/s, GSE', 'vx_gse'),
('Vy Velocity, km/s, GSE', 'vy_gse'),
('Vz Velocity, km/s, GSE', 'vz_gse'),
('Proton Density, n/cc', 'density'),
('Temperature, K', 'temperature'),
('Flow pressure, nPa', 'pressure'),
('Electric field, mV/m', 'e'),
('Plasma beta', 'beta'),
('Alfven mach number', 'alfven_mach'),
('X(s/c), GSE, Re', 'x_gse'),
('Y(s/c), GSE, Re', 'y_gse'),
('Z(s/c), GSE, Re', 'z_gse'),
('BSN location, Xgse, Re', 'bsn_x_gse'),
('BSN location, Ygse, Re', 'bsn_y_gse'),
('BSN location, Zgse, Re', 'bsn_z_gse'),
('AE-index, nT', 'ae'),
('AL-index, nT', 'al'),
('AU-index, nT', 'au'),
('SYM/D index, nT', 'sym_d'),
('SYM/H index, nT', 'sym_h'),
('ASY/D index, nT', 'asy_d'),
('ASY/H index, nT', 'asy_h'),
('PC(N) index', 'pc_n'),
('Magnetosonic mach number', 'mach'),
)
...@@ -45,7 +45,8 @@ def write_imf_input(imf_data, filename='IMF.dat', **kwargs): ...@@ -45,7 +45,8 @@ def write_imf_input(imf_data, filename='IMF.dat', **kwargs):
with open(filename, 'w') as file_imf: with open(filename, 'w') as file_imf:
# header # header
file_imf.write('\t'.join(columns_dat)+'\n\n') file_imf.write('Made with swmfpy ')
file_imf.write('(https://gitlab.umich.edu/swmf_software/swmfpy)\n\n')
# write commands # write commands
commands = kwargs.get('commands', None) commands = kwargs.get('commands', None)
...@@ -58,6 +59,7 @@ def write_imf_input(imf_data, filename='IMF.dat', **kwargs): ...@@ -58,6 +59,7 @@ def write_imf_input(imf_data, filename='IMF.dat', **kwargs):
file_imf.write(_make_line(command)+'\n') file_imf.write(_make_line(command)+'\n')
# write dat file # write dat file
file_imf.write('\n'+'\t'.join(columns_dat))
file_imf.write('#START\n') file_imf.write('#START\n')
lines = [] lines = []
for index, _time in enumerate(imf_data[columns_dict[0]]): for index, _time in enumerate(imf_data[columns_dict[0]]):
......
...@@ -5,6 +5,50 @@ ...@@ -5,6 +5,50 @@
__author__ = 'Qusai Al Shidi' __author__ = 'Qusai Al Shidi'
__email__ = 'qusai@umich.edu' __email__ = 'qusai@umich.edu'
import numpy as np
def get_v_gsm_from_omni(dict_omni):
"""Adds gsm values to the omni dictionary
This is helpful for the omni dict created with #swmfpy.web.get_omni_data()
Args:
dict_omni(dict): Dictionary with np.arrays in 'bz', 'by', 'bz_gse',
'by_gse', 'vx_gse', 'vy_gse'
Returns:
dict_omni with added 'vz' and 'vy' keys for gsm coordinates.
Raises:
KeyError: If above values don't exist.
Examples:
```python
import datetime as dt
from swmfpy.web import get_omni_data
from swmfpy.toos import get_v_gsm_from_omni
time_period = (dt.datetime(2012, 2, 1), dt.datetime(2012, 4, 5))
data = get_v_gsm_from_omni(get_omni_data(*time_period))
# You should now have the omni dict with gsm values
plot(data['times'], data['vz'])
```
"""
# Author: Qusai Al Shidi
# Email: qusai@umich.edu
vy_gse, vz_gse = dict_omni['vy_gse'], dict_omni['vz_gse']
by_gse, bz_gse = dict_omni['by_gse'], dict_omni['bz_gse']
by_gsm, bz_gsm = dict_omni['by'], dict_omni['bz']
velocity = np.sqrt(vx_gse**2 + vz_gse**2)
theta = np.arctan(bz_gsm, by_gsm)
dict_omni['vy'] = velocity*np.cos(theta)
dict_omni['vz'] = velocity*np.sin(theta)
return dict_omni
def _import_error_string(library): def _import_error_string(library):
return ( return (
......
...@@ -9,9 +9,118 @@ __author__ = 'Qusai Al Shidi' ...@@ -9,9 +9,118 @@ __author__ = 'Qusai Al Shidi'
__email__ = 'qusai@umich.edu' __email__ = 'qusai@umich.edu'
import datetime as dt import datetime as dt
import ftplib
from ftplib import FTP
import gzip
from operator import itemgetter
import shutil
import urllib import urllib
from .__init__ import OMNI_COLS from dateutil import rrule
from .tools import _import_error_string, _nearest import drms
import numpy as np
from sunpy.coordinates.sun import carrington_rotation_number
from .tools import _nearest
# Global defines
# This is straight from the format guide on spdf with nicer names as second col
# spdf_name nice_name type
OMNI_HIRES_COLS = (('ID for IMF spacecraft', 'id_imf'),
('ID for SW Plasma spacecraft', 'id_sw'),
('# of points in IMF averages', 'num_avg_imf'),
('# of points in Plasma averages', 'num_avg_sw'),
('Percent interp', 'interp'),
('Timeshift, sec', 'timeshift'),
('RMS, Timeshift', 'rms_timeshift'),
('RMS, Phase front normal', 'rms_phase'),
('Time btwn observations, sec', 'dt'),
('Field magnitude average, nT', 'b'),
('Bx, nT (GSE, GSM)', 'bx'),
('By, nT (GSE)', 'by_gse'),
('Bz, nT (GSE)', 'bz_gse'),
('By, nT (GSM)', 'by'),
('Bz, nT (GSM)', 'bz'),
('RMS SD B scalar, nT', 'rms_sd_b'),
('RMS SD field vector, nT', 'rms_sd_field'),
('Flow speed, km/s', 'v'),
('Vx Velocity, km/s, GSE', 'vx_gse'),
('Vy Velocity, km/s, GSE', 'vy_gse'),
('Vz Velocity, km/s, GSE', 'vz_gse'),
('Proton Density, n/cc', 'density'),
('Temperature, K', 'temperature'),
('Flow pressure, nPa', 'pressure'),
('Electric field, mV/m', 'e'),
('Plasma beta', 'beta'),
('Alfven mach number', 'alfven_mach'),
('X(s/c), GSE, Re', 'x_gse'),
('Y(s/c), GSE, Re', 'y_gse'),
('Z(s/c), GSE, Re', 'z_gse'),
('BSN location, Xgse, Re', 'bsn_x_gse'),
('BSN location, Ygse, Re', 'bsn_y_gse'),
('BSN location, Zgse, Re', 'bsn_z_gse'),
('AE-index, nT', 'ae'),
('AL-index, nT', 'al'),
('AU-index, nT', 'au'),
('SYM/D index, nT', 'sym_d'),
('SYM/H index, nT', 'sym_h'),
('ASY/D index, nT', 'asy_d'),
('ASY/H index, nT', 'asy_h'),
('PC(N) index', 'pc_n'),
('Magnetosonic mach number', 'mach'),
)
OMNI_LORES_COLS = (('Bartels rotation number', 'bartels'),
('ID for IMF spacecraft', 'id_imf'),
('ID for SW plasma spacecraft', 'id_sw'),
('# of points in the IMF averages', 'num_avg_imf'),
('# of points in the plasma averages', 'num_avg_sw'),
('Field Magnitude Average |B|', 'b_avg'),
('Magnitude of Average Field Vector', 'b'),
('Lat.Angle of Aver. Field Vector', 'b_angle_lat'),
('Long.Angle of Aver.Field Vector', 'b_angle_lon'),
('Bx GSE, GSM', 'bx'),
('By GSE', 'by_gse'),
('Bz GSE', 'bz_gse'),
('By GSM', 'by'),
('Bz GSM', 'bz'),
('sigma|B|', 'sigma_b_avg'),
('sigma B', 'sigma_b'),
('sigma Bx', 'sigma_bx'),
('sigma By', 'sigma_by'),
('sigma Bz', 'sigma_bz'),
('Proton temperature', 'temperature'),
('Proton Density', 'density'),
('Plasma (Flow) speed', 'v'),
('Plasma Flow Long. Angle', 'v_angle_lat'),
('Plasma Flow Lat. Angle', 'v_angle_lon'),
('Na/Np', 'na_np'),
('Flow Pressure', 'dyn_p'),
('sigma T', 'sigma_t'),
('sigma N', 'sigma_n'),
('sigma V', 'sigma_v'),
('sigma phi V', 'sigma_phi_v'),
('sigma theta V', 'sigma_theta_v'),
('sigma-Na/Np', 'sigma_na_np'),
('Electric field', 'e'),
('Plasma beta', 'beta'),
('Alfven mach number', 'alfven_mach'),
('Kp', 'kp'),
('R', 'sunspot'),
('DST Index', 'dst'),
('AE-index', 'ae'),
('Proton flux number/cmsq sec sr >1 Mev', 'p_flux_1'),
('Proton flux number/cmsq sec sr >2 Mev', 'p_flux_2'),
('Proton flux number/cmsq sec sr >4 Mev', 'p_flux_4'),
('Proton flux number/cmsq sec sr >10 Mev', 'p_flux_10'),
('Proton flux number/cmsq sec sr >30 Mev', 'p_flux_30'),
('Proton flux number/cmsq sec sr >60 Mev', 'p_flux_60'),
('Flag(***)', 'flag'),
('ap-index', 'ap'),
('f10.7_index', 'f10_7'),
('PC(N) index', 'pc_n'),
('AL-index', 'al'),
('AU-index', 'au'),
('Magnetosonic mach number', 'mach'),
)
def get_omni_data(time_from, time_to, **kwargs): def get_omni_data(time_from, time_to, **kwargs):
...@@ -31,6 +140,9 @@ def get_omni_data(time_from, time_to, **kwargs): ...@@ -31,6 +140,9 @@ def get_omni_data(time_from, time_to, **kwargs):
spdf specification. The alternative is spdf specification. The alternative is
nicer and shorter names. Defaults to nicer and shorter names. Defaults to
False. False.
resolution (str): (default: 'high') Here you can choose 'high' or
'low' resolution omni data. Some columns appear
in one but not the other.
Returns: Returns:
dict: This will be a list of *all* columns dict: This will be a list of *all* columns
...@@ -50,57 +162,86 @@ def get_omni_data(time_from, time_to, **kwargs): ...@@ -50,57 +162,86 @@ def get_omni_data(time_from, time_to, **kwargs):
# Author: Qusai Al Shidi # Author: Qusai Al Shidi
# Email: qusai@umich.edu # Email: qusai@umich.edu
from dateutil import rrule # Initialize
# Set the url
omni_url = 'https://spdf.gsfc.nasa.gov/pub/data/omni/'
if kwargs.get('high_res', True):
omni_url += 'high_res_omni/monthly_1min/'
# Initialize return dict
return_data = {} return_data = {}
return_data['times'] = [] return_data['times'] = []
col_names = [] name_type = int(not kwargs.get('original_colnames', False))
for name in OMNI_COLS: name_type = itemgetter(name_type)
if kwargs.get('original_colnames', False): resolution = kwargs.get('resolution', 'high')
col_names += [name[0]] # omni cols do not include time cols
return_data[name[0]] = [] omni = {
else: 'high': {
col_names += [name[1]] 'urls': _urls_omni_hires,
return_data[name[1]] = [] 'cols': list(map(name_type, OMNI_HIRES_COLS)),
'parsetime': lambda cols: (dt.datetime(int(cols[0]), 1, 1,
# Iterate monthly to save RAM int(cols[2]),
for date in rrule.rrule(rrule.MONTHLY, int(cols[3]))
dtstart=dt.datetime(time_from.year, + dt.timedelta(int(cols[1])-1)),
time_from.month, 'ntimecols': 4, # Minute
1), },
until=dt.datetime(time_to.year, 'low': {
time_to.month, 'urls': _urls_omni_lores,
25)): 'cols': list(map(name_type, OMNI_LORES_COLS)),
suffix = 'omni_min' 'parsetime': lambda cols: (dt.datetime(int(cols[0]), 1, 1,
suffix += str(date.year) + str(date.month).zfill(2) int(cols[2]))
suffix += '.asc' + dt.timedelta(int(cols[1])-1)),
omni_data = list(urllib.request.urlopen(omni_url+suffix)) 'ntimecols': 3, # Hourly
},
}
omni = omni[resolution] # To save line space
for col_name in omni['cols']:
return_data[col_name] = []
# Iterate by url to save RAM
for url in omni['urls'](time_from, time_to):
# Parse omni data # Parse omni data
for line in omni_data: for line in list(urllib.request.urlopen(url)):
cols = line.decode('ascii').split() cols = line.decode('ascii').split()
# Time uses day of year which must be parsed
time = dt.datetime.strptime(cols[0] + ' ' # year time = omni['parsetime'](cols)
+ cols[1] + ' ' # day of year if time_from <= time <= time_to:
+ cols[2] + ' ' # hour
+ cols[3], # minute
'%Y %j %H %M')
if time >= time_from and time <= time_to:
return_data['times'] += [time] return_data['times'] += [time]
# Assign the data from after the time columns (0:3) # Assign the data from after the time columns
for num, value in enumerate(cols[4:len(col_names)+4]): for col_name, value in zip(omni['cols'],
cols[omni['ntimecols']:]):
if _check_bad_omni_num(value): if _check_bad_omni_num(value):
return_data[col_names[num]] += [None] return_data[col_name] += [None]
else: else:
return_data[col_names[num]] += [float(value)] return_data[col_name] += [float(value)]
return return_data # dictionary with omni values where index is the row for col_name in omni['cols']:
return_data[col_name] = np.array(return_data[col_name],
dtype=float)
return return_data
def _urls_omni_hires(time_from, time_to):
"""Returns hires omni urls from time_from to time_to
"""
prefix = 'https://spdf.gsfc.nasa.gov/pub/data/omni/'
prefix += 'high_res_omni/monthly_1min/'
for date in rrule.rrule(rrule.MONTHLY,
dtstart=time_from,
until=dt.datetime(time_to.year,
time_to.month+1,
1)):
suffix = 'omni_min'
suffix += str(date.year) + str(date.month).zfill(2)
suffix += '.asc'
yield prefix+suffix
def _urls_omni_lores(time_from, time_to):
"""Returns lores omni urls from time_from to time_to
"""
prefix = 'https://spdf.gsfc.nasa.gov/pub/data/omni/'
prefix += 'low_res_omni/'
for year in range(time_from.year, time_to.year+1):
suffix = 'omni2_'
suffix += str(year)
suffix += '.dat'
yield prefix+suffix
def _check_bad_omni_num(value_string): def _check_bad_omni_num(value_string):
...@@ -166,12 +307,6 @@ def download_magnetogram_hmi(mag_time, hmi_map='hmi.B_720s', **kwargs): ...@@ -166,12 +307,6 @@ def download_magnetogram_hmi(mag_time, hmi_map='hmi.B_720s', **kwargs):
``` ```
""" """
# import drms dynamically
try:
import drms
except ImportError:
raise ImportError(_import_error_string('drms'))
get_urls = { get_urls = {
'hmi.B_720s': _get_urls_hmi_b720, 'hmi.B_720s': _get_urls_hmi_b720,
'hmi.b_synoptic_small': _get_urls_hmi_b_synoptic_small, 'hmi.b_synoptic_small': _get_urls_hmi_b_synoptic_small,
...@@ -219,11 +354,6 @@ def _get_urls_hmi_b_synoptic_small(client, mag_time): ...@@ -219,11 +354,6 @@ def _get_urls_hmi_b_synoptic_small(client, mag_time):
generator that yields (datetime.datetime, str): Time of magnetogram, generator that yields (datetime.datetime, str): Time of magnetogram,
suffix url of magnetogram suffix url of magnetogram
""" """
try:
from sunpy.coordinates.sun import carrington_rotation_number
except ImportError as error:
print(_import_error_string('sunpy'))
raise error
cr_number = int(round(carrington_rotation_number(mag_time))) cr_number = int(round(carrington_rotation_number(mag_time)))
query_string = f'hmi.b_synoptic_small[{int(round(cr_number))}]' query_string = f'hmi.b_synoptic_small[{int(round(cr_number))}]'
...@@ -246,7 +376,6 @@ def _get_urls_hmi_b720(client, mag_time): ...@@ -246,7 +376,6 @@ def _get_urls_hmi_b720(client, mag_time):
generator that yields (datetime.datetime, str): Time of magnetogram, generator that yields (datetime.datetime, str): Time of magnetogram,
suffix url of magnetogram suffix url of magnetogram
""" """
import drms
query_string = 'hmi.B_720s' query_string = 'hmi.B_720s'
query_string += f'[{mag_time.year}.' query_string += f'[{mag_time.year}.'
query_string += f'{str(mag_time.month).zfill(2)}.' query_string += f'{str(mag_time.month).zfill(2)}.'
...@@ -303,11 +432,6 @@ def download_magnetogram_adapt(time, map_type='fixed', **kwargs): ...@@ -303,11 +432,6 @@ def download_magnetogram_adapt(time, map_type='fixed', **kwargs):
# Author: Zhenguang Huang # Author: Zhenguang Huang
# Email: zghuang@umich.edu # Email: zghuang@umich.edu
import ftplib
from ftplib import FTP
import gzip
import shutil
if map_type == 'fixed': if map_type == 'fixed':
map_id = '0' map_id = '0'
elif map_type == 'central': elif map_type == 'central':
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment