web.py 8.56 KB
Newer Older
1
"""Tools to retrieve and send data on the web.
2

3
4
Here are a collection of tools to work with data on the internet. Thus,
this module mostly requires an internet connection.
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
"""
__author__ = 'Qusai Al Shidi'
__email__ = 'qusai@umich.edu'

import datetime as dt


def get_omni_data(time_from, time_to, **kwargs):
    """Retrieve omni solar wind data over http.

    This will download omni data from https://spdf.gsfc.nasa.gov/pub/data/omni
    and put it into a dictionary. If your data is large, then make a csv and
    use swmfpy.io.read_omni_data().

    Args:
        time_from (datetime.datetime): The start time of the solar wind
                                       data that you want to receive.
        time_to (datetime.datetime): The end time of the solar wind data
                                     you want to receive.

    Returns:
        dict: This will be a list of *all* columns
              available in the omni data set.

    Examples:
        ```python
Qusai Al Shidi's avatar
Qusai Al Shidi committed
31

32
        import datetime
33
        import swmfpy.web
34

35
36
37
38
        storm_start = datetime.datetime(year=2000, month=1, day=1)
        storm_end = datetime.datetime(year=2000, month=2, day=15)
        data = swmfpy.web.get_omni_data(time_from=storm_start,
                                        time_to=storm_end)
39
40
        ```
    """
41
42
43
    # Author: Qusai Al Shidi
    # Email: qusai@umich.edu

Qusai Al Shidi's avatar
Qusai Al Shidi committed
44
45
46
    import urllib.request
    from dateutil import rrule

47
    # This is straight from the format guide on spdf
48
    col_names = ('ID for IMF spacecraft',
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
                 'ID for SW Plasma spacecraft',
                 '# of points in IMF averages',
                 '# of points in Plasma averages',
                 'Percent interp',
                 'Timeshift, sec',
                 'RMS, Timeshift',
                 'RMS, Phase front normal',
                 'Time btwn observations, sec',
                 'Field magnitude average, nT',
                 'Bx, nT (GSE, GSM)',
                 'By, nT (GSE)',
                 'Bz, nT (GSE)',
                 'By, nT (GSM)',
                 'Bz, nT (GSM)',
                 'RMS SD B scalar, nT',
                 'RMS SD field vector, nT',
                 'Flow speed, km/s',
                 'Vx Velocity, km/s, GSE',
                 'Vy Velocity, km/s, GSE',
                 'Vz Velocity, km/s, GSE',
                 'Proton Density, n/cc',
                 'Temperature, K',
                 'Flow pressure, nPa',
                 'Electric field, mV/m',
                 'Plasma beta',
                 'Alfven mach number',
                 'X(s/c), GSE, Re',
                 'Y(s/c), GSE, Re',
                 'Z(s/c), GSE, Re',
                 'BSN location, Xgse, Re',
                 'BSN location, Ygse, Re',
80
81
82
83
84
85
86
87
88
89
                 'BSN location, Zgse, Re',
                 'AE-index, nT',
                 'AL-index, nT',
                 'AU-index, nT',
                 'SYM/D index, nT',
                 'SYM/H index, nT',
                 'ASY/D index, nT',
                 'ASY/H index, nT',
                 'PC(N) index',
                 'Magnetosonic mach number')
90
91
92
93
94
95
96

    # Set the url
    omni_url = 'https://spdf.gsfc.nasa.gov/pub/data/omni/'
    if kwargs.get('high_res', True):
        omni_url += 'high_res_omni/monthly_1min/'

    # Initialize return dict
97
    return_data = {}
98
    return_data['times'] = []
99
    for name in col_names:
100
        return_data[name] = []
101

102
    # Iterate monthly to save RAM
103
104
105
106
    for date in rrule.rrule(rrule.MONTHLY, dtstart=time_from, until=time_to):
        suffix = 'omni_min'
        suffix += str(date.year) + str(date.month).zfill(2)
        suffix += '.asc'
107
        omni_data = list(urllib.request.urlopen(omni_url+suffix))
108
109

        # Parse omni data
110
        for line in omni_data:
111
112
            cols = line.decode('ascii').split()
            # Time uses day of year which must be parsed
113
114
115
116
            time = dt.datetime.strptime(cols[0] + ' '  # year
                                        + cols[1] + ' '  # day of year
                                        + cols[2] + ' '  # hour
                                        + cols[3],  # minute
117
118
                                        '%Y %j %H %M')
            if time >= time_from and time <= time_to:
119
                return_data['times'].append(time)
120
121
                # Assign the data from after the time columns (0:3)
                for num, value in enumerate(cols[4:len(col_names)+4]):
122
123
124
125
                    if _check_bad_omni_num(value):
                        return_data[col_names[num]].append(None)
                    else:
                        return_data[col_names[num]].append(float(value))
126

127
    return return_data  # dictionary with omni values where index is the row
Qusai Al Shidi's avatar
Qusai Al Shidi committed
128
129


130
131
132
133
134
135
136
137
def _check_bad_omni_num(value_string):
    """Returns true if bad or false if not"""
    for char in value_string:
        if char != '9' and char != '.':
            return False
    return True


Qusai Al Shidi's avatar
Qusai Al Shidi committed
138
def download_magnetogram_adapt(time, map_type='fixed', **kwargs):
Qusai Al Shidi's avatar
Qusai Al Shidi committed
139
    """This routine downloads GONG ADAPT magnetograms.
Qusai Al Shidi's avatar
Qusai Al Shidi committed
140
141

    Downloads ADAPT magnetograms from ftp://gong2.nso.edu/adapt/maps/gong/
142
    to a local directory. It will download all maps with the regex file
143
    pattern: adapt4[0,1]3*yyyymmddhh
Qusai Al Shidi's avatar
Qusai Al Shidi committed
144
145
146
147
148
149
150

    Args:
        time (datetime.datetime): Time in which you want the magnetogram.
        map_type (str): (default: 'fixed')
                        Choose either 'fixed' or 'central' for
                        the map type you want.
        **kwargs:
Qusai Al Shidi's avatar
Qusai Al Shidi committed
151
            download_dir (str): (default is current dir) Relative directory
Qusai Al Shidi's avatar
Qusai Al Shidi committed
152
153
                                where you want the maps to be downloaded.

154
155
156
    Returns:
        str: First unzipped filename found.

Qusai Al Shidi's avatar
Qusai Al Shidi committed
157
    Raises:
158
159
        NotADirectoryError: If the adapt maps directory
                            is not found on the server.
Qusai Al Shidi's avatar
Qusai Al Shidi committed
160
161
        ValueError: If map_type is not recognized.
                    (i.e. not 'fixed' or 'central')
162
        FileNotFoundError: If maps were not found.
Qusai Al Shidi's avatar
Qusai Al Shidi committed
163
164
165

    Examples:
        ```python
Qusai Al Shidi's avatar
Qusai Al Shidi committed
166

Qusai Al Shidi's avatar
Qusai Al Shidi committed
167
168
169
        import datetime as dt

        # Use datetime objects for the time
170
        time_flare = dt.datetime(2018, 2, 12, hour=10)
Qusai Al Shidi's avatar
Qusai Al Shidi committed
171
172
173
174
        swmfpy.web.download_magnetogram_adapt(time=time_flare,
                                              map_type='central',
                                              download_dir='./mymaps/')
        ```
Qusai Al Shidi's avatar
Qusai Al Shidi committed
175
    """
Qusai Al Shidi's avatar
Qusai Al Shidi committed
176
177
178
179
180
181
182
183
184
185
186
187
188
    # Author: Zhenguang Huang
    # Email: zghuang@umich.edu

    import ftplib
    from ftplib import FTP
    import gzip
    import shutil

    if map_type == 'fixed':
        map_id = '0'
    elif map_type == 'central':
        map_id = '1'
    else:
189
        raise ValueError('Not recognized type of ADAPT map')
Qusai Al Shidi's avatar
Qusai Al Shidi committed
190
191
192
193
194
195
196
197
198
199
200
201
202

    # Go to the the ADAPT ftp server
    ftp = FTP('gong2.nso.edu')
    ftp.login()

    # Only ADAPT GONG is considered
    ftp.cwd('adapt/maps/gong')

    # Go to the specific year
    try:
        ftp.cwd(str(time.year))
    except ftplib.all_errors:
        ftp.quit()
203
        raise NotADirectoryError('Cannot go to the specific year directory')
Qusai Al Shidi's avatar
Qusai Al Shidi committed
204

205
206
207
208
    # ADAPT maps only contains the hours for even numbers
    if time.hour % 2 != 0:
        print('Warning: Hour must be an even number.',
              'The entered hour value is changed to',
Qusai Al Shidi's avatar
Qusai Al Shidi committed
209
              time.hour//2*2)
Qusai Al Shidi's avatar
Qusai Al Shidi committed
210
211
212
213
214
    # Only consider the public (4) Carrington Fixed (0) GONG (3) ADAPT maps
    file_pattern = 'adapt4' + map_id + '3*' \
        + str(time.year).zfill(4) \
        + str(time.month).zfill(2) \
        + str(time.day).zfill(2) \
Qusai Al Shidi's avatar
Qusai Al Shidi committed
215
        + str(time.hour//2*2).zfill(2) + '*'
216
    # adapt4[0,1]3*yyyymmddhh
Qusai Al Shidi's avatar
Qusai Al Shidi committed
217
218
219
220

    filenames = ftp.nlst(file_pattern)

    if len(filenames) < 1:
221
222
        raise FileNotFoundError('Could not find a file that matches'
                                + 'the pattern.')
Qusai Al Shidi's avatar
Qusai Al Shidi committed
223
224
225
226
227
228
229
230
231
232
233
234

    for filename in filenames:
        # open the file locally
        directory = kwargs.get('download_dir', './')
        if directory[-1] != '/':
            directory += '/'
        with open(directory + filename, 'wb') as fhandle:
            # try to download the magnetogram
            try:
                ftp.retrbinary('RETR ' + filename, fhandle.write)
            except ftplib.all_errors:
                ftp.quit()
235
                raise FileNotFoundError('Cannot download ', filename)
Qusai Al Shidi's avatar
Qusai Al Shidi committed
236
237
238
239
240
241
242
243
244
245

        # unzip the file
        if '.gz' in filename:
            filename_unzip = filename.replace('.gz', '')
            with gzip.open(directory + filename, 'rb') as s_file:
                with open(directory + filename_unzip, 'wb') as d_file:
                    shutil.copyfileobj(s_file, d_file, 65536)

    # close the connection
    ftp.quit()
246
247
248
249
250
251

    # return first file name if all goes well
    return_name = filenames[0]
    if '.gz' in return_name:
        return_name = return_name[:-3]
    return return_name