web.py 8.56 KB
Newer Older
1
"""Tools to retrieve and send data on the web.
2

3
4
Here are a collection of tools to work with data on the internet. Thus,
this module mostly requires an internet connection.
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
"""
__author__ = 'Qusai Al Shidi'
__email__ = 'qusai@umich.edu'

import datetime as dt


def get_omni_data(time_from, time_to, **kwargs):
    """Retrieve omni solar wind data over http.

    This will download omni data from https://spdf.gsfc.nasa.gov/pub/data/omni
    and put it into a dictionary. If your data is large, then make a csv and
    use swmfpy.io.read_omni_data().

    Args:
        time_from (datetime.datetime): The start time of the solar wind
                                       data that you want to receive.
        time_to (datetime.datetime): The end time of the solar wind data
                                     you want to receive.

    Returns:
        dict: This will be a list of *all* columns
              available in the omni data set.

    Examples:
        ```python
        import datetime
32
        import swmfpy.web
33

34
35
36
37
        storm_start = datetime.datetime(year=2000, month=1, day=1)
        storm_end = datetime.datetime(year=2000, month=2, day=15)
        data = swmfpy.web.get_omni_data(time_from=storm_start,
                                        time_to=storm_end)
38
39
        ```
    """
40
41
42
    # Author: Qusai Al Shidi
    # Email: qusai@umich.edu

Qusai Al Shidi's avatar
Qusai Al Shidi committed
43
44
45
    import urllib.request
    from dateutil import rrule

46
    # This is straight from the format guide on spdf
47
    col_names = ('ID for IMF spacecraft',
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
                 'ID for SW Plasma spacecraft',
                 '# of points in IMF averages',
                 '# of points in Plasma averages',
                 'Percent interp',
                 'Timeshift, sec',
                 'RMS, Timeshift',
                 'RMS, Phase front normal',
                 'Time btwn observations, sec',
                 'Field magnitude average, nT',
                 'Bx, nT (GSE, GSM)',
                 'By, nT (GSE)',
                 'Bz, nT (GSE)',
                 'By, nT (GSM)',
                 'Bz, nT (GSM)',
                 'RMS SD B scalar, nT',
                 'RMS SD field vector, nT',
                 'Flow speed, km/s',
                 'Vx Velocity, km/s, GSE',
                 'Vy Velocity, km/s, GSE',
                 'Vz Velocity, km/s, GSE',
                 'Proton Density, n/cc',
                 'Temperature, K',
                 'Flow pressure, nPa',
                 'Electric field, mV/m',
                 'Plasma beta',
                 'Alfven mach number',
                 'X(s/c), GSE, Re',
                 'Y(s/c), GSE, Re',
                 'Z(s/c), GSE, Re',
                 'BSN location, Xgse, Re',
                 'BSN location, Ygse, Re',
79
80
81
82
83
84
85
86
87
88
                 'BSN location, Zgse, Re',
                 'AE-index, nT',
                 'AL-index, nT',
                 'AU-index, nT',
                 'SYM/D index, nT',
                 'SYM/H index, nT',
                 'ASY/D index, nT',
                 'ASY/H index, nT',
                 'PC(N) index',
                 'Magnetosonic mach number')
89
90
91
92
93
94
95

    # Set the url
    omni_url = 'https://spdf.gsfc.nasa.gov/pub/data/omni/'
    if kwargs.get('high_res', True):
        omni_url += 'high_res_omni/monthly_1min/'

    # Initialize return dict
96
    return_data = {}
97
    return_data['times'] = []
98
    for name in col_names:
99
        return_data[name] = []
100

101
    # Iterate monthly to save RAM
102
103
104
105
    for date in rrule.rrule(rrule.MONTHLY, dtstart=time_from, until=time_to):
        suffix = 'omni_min'
        suffix += str(date.year) + str(date.month).zfill(2)
        suffix += '.asc'
106
        omni_data = list(urllib.request.urlopen(omni_url+suffix))
107
108

        # Parse omni data
109
        for line in omni_data:
110
111
            cols = line.decode('ascii').split()
            # Time uses day of year which must be parsed
112
113
114
115
            time = dt.datetime.strptime(cols[0] + ' '  # year
                                        + cols[1] + ' '  # day of year
                                        + cols[2] + ' '  # hour
                                        + cols[3],  # minute
116
117
                                        '%Y %j %H %M')
            if time >= time_from and time <= time_to:
118
                return_data['times'].append(time)
119
120
                # Assign the data from after the time columns (0:3)
                for num, value in enumerate(cols[4:len(col_names)+4]):
121
122
123
124
                    if _check_bad_omni_num(value):
                        return_data[col_names[num]].append(None)
                    else:
                        return_data[col_names[num]].append(float(value))
125

126
    return return_data  # dictionary with omni values where index is the row
Qusai Al Shidi's avatar
Qusai Al Shidi committed
127
128


129
130
131
132
133
134
135
136
def _check_bad_omni_num(value_string):
    """Returns true if bad or false if not"""
    for char in value_string:
        if char != '9' and char != '.':
            return False
    return True


Qusai Al Shidi's avatar
Qusai Al Shidi committed
137
def download_magnetogram_adapt(time, map_type='fixed', **kwargs):
Qusai Al Shidi's avatar
Qusai Al Shidi committed
138
    """This routine downloads GONG ADAPT magnetograms.
Qusai Al Shidi's avatar
Qusai Al Shidi committed
139
140

    Downloads ADAPT magnetograms from ftp://gong2.nso.edu/adapt/maps/gong/
141
    to a local directory. It will download all maps with the regex file
142
    pattern: adapt4[0,1]3*yyyymmddhh
Qusai Al Shidi's avatar
Qusai Al Shidi committed
143
144
145
146
147
148
149

    Args:
        time (datetime.datetime): Time in which you want the magnetogram.
        map_type (str): (default: 'fixed')
                        Choose either 'fixed' or 'central' for
                        the map type you want.
        **kwargs:
Qusai Al Shidi's avatar
Qusai Al Shidi committed
150
            download_dir (str): (default is current dir) Relative directory
Qusai Al Shidi's avatar
Qusai Al Shidi committed
151
152
                                where you want the maps to be downloaded.

153
154
155
    Returns:
        str: First unzipped filename found.

Qusai Al Shidi's avatar
Qusai Al Shidi committed
156
    Raises:
157
158
        NotADirectoryError: If the adapt maps directory
                            is not found on the server.
Qusai Al Shidi's avatar
Qusai Al Shidi committed
159
160
        ValueError: If map_type is not recognized.
                    (i.e. not 'fixed' or 'central')
161
        FileNotFoundError: If maps were not found.
Qusai Al Shidi's avatar
Qusai Al Shidi committed
162
163
164
165
166
167

    Examples:
        ```python
        import datetime as dt

        # Use datetime objects for the time
168
        time_flare = dt.datetime(2018, 2, 12, hour=10)
Qusai Al Shidi's avatar
Qusai Al Shidi committed
169
170
171
172
        swmfpy.web.download_magnetogram_adapt(time=time_flare,
                                              map_type='central',
                                              download_dir='./mymaps/')
        ```
Qusai Al Shidi's avatar
Qusai Al Shidi committed
173
    """
Qusai Al Shidi's avatar
Qusai Al Shidi committed
174
175
176
177
178
179
180
181
182
183
184
185
186
    # Author: Zhenguang Huang
    # Email: zghuang@umich.edu

    import ftplib
    from ftplib import FTP
    import gzip
    import shutil

    if map_type == 'fixed':
        map_id = '0'
    elif map_type == 'central':
        map_id = '1'
    else:
187
        raise ValueError('Not recognized type of ADAPT map')
Qusai Al Shidi's avatar
Qusai Al Shidi committed
188
189
190
191
192
193
194
195
196
197
198
199
200

    # Go to the the ADAPT ftp server
    ftp = FTP('gong2.nso.edu')
    ftp.login()

    # Only ADAPT GONG is considered
    ftp.cwd('adapt/maps/gong')

    # Go to the specific year
    try:
        ftp.cwd(str(time.year))
    except ftplib.all_errors:
        ftp.quit()
201
        raise NotADirectoryError('Cannot go to the specific year directory')
Qusai Al Shidi's avatar
Qusai Al Shidi committed
202

203
204
205
206
    # ADAPT maps only contains the hours for even numbers
    if time.hour % 2 != 0:
        print('Warning: Hour must be an even number.',
              'The entered hour value is changed to',
Qusai Al Shidi's avatar
Qusai Al Shidi committed
207
              time.hour//2*2)
Qusai Al Shidi's avatar
Qusai Al Shidi committed
208
209
210
211
212
    # Only consider the public (4) Carrington Fixed (0) GONG (3) ADAPT maps
    file_pattern = 'adapt4' + map_id + '3*' \
        + str(time.year).zfill(4) \
        + str(time.month).zfill(2) \
        + str(time.day).zfill(2) \
Qusai Al Shidi's avatar
Qusai Al Shidi committed
213
        + str(time.hour//2*2).zfill(2) + '*'
214
    # adapt4[0,1]3*yyyymmddhh
Qusai Al Shidi's avatar
Qusai Al Shidi committed
215
216
217
218

    filenames = ftp.nlst(file_pattern)

    if len(filenames) < 1:
219
220
        raise FileNotFoundError('Could not find a file that matches'
                                + 'the pattern.')
Qusai Al Shidi's avatar
Qusai Al Shidi committed
221
222
223
224
225
226
227
228
229
230
231
232

    for filename in filenames:
        # open the file locally
        directory = kwargs.get('download_dir', './')
        if directory[-1] != '/':
            directory += '/'
        with open(directory + filename, 'wb') as fhandle:
            # try to download the magnetogram
            try:
                ftp.retrbinary('RETR ' + filename, fhandle.write)
            except ftplib.all_errors:
                ftp.quit()
233
                raise FileNotFoundError('Cannot download ', filename)
Qusai Al Shidi's avatar
Qusai Al Shidi committed
234
235
236
237
238
239
240
241
242
243

        # unzip the file
        if '.gz' in filename:
            filename_unzip = filename.replace('.gz', '')
            with gzip.open(directory + filename, 'rb') as s_file:
                with open(directory + filename_unzip, 'wb') as d_file:
                    shutil.copyfileobj(s_file, d_file, 65536)

    # close the connection
    ftp.quit()
244
245
246
247
248
249

    # return first file name if all goes well
    return_name = filenames[0]
    if '.gz' in return_name:
        return_name = return_name[:-3]
    return return_name