Commit 41a6385e authored by Qusai Al Shidi's avatar Qusai Al Shidi 💬
Browse files

Added function combine data for ACE and WIND data that interpolates

parent ab0813ca
......@@ -4,6 +4,81 @@ Interpolate washed out data from OMNI
from datetime import datetime as d
import numpy as np
import pandas as pd
def combine_data(mfi_fname, swe_fname, plas_fname):
combine_data(mfi_fname, swe_fname, plas_fname)
Combines the three csv files from into a pandas DataFrame.
Recommended to be used with:
mfi_fname: AC_H0_MFI_***.csv
swe_fname: AC_H0_SWE_***.csv
plas_fname: WI_PM_3DP_***.csv
return pandas.DataFrame
Make sure to download the csv files with
the header seperated into a json file for safety.
bfile = open(mfi_fname, 'r')
except OSError:
print("Could not open the MFI data file.")
swfile = open(swe_fname, 'r')
except OSError:
print("Could not open the SWE data file.")
pfile = open(plas_fname, 'r')
except OSError:
print("Could not open the MFI data file.")
# Read the csv files and set the indexes to dates
ndata = pd.read_csv(pfile)
ndata = ndata.set_index(pd.to_datetime(ndata[ndata.columns[0]]))
ndata.index.names = ['date']
ndata = ndata.drop(ndata.columns[0], 1)
ndata = ndata.rename(columns={ndata.columns[0]: "density"})
bdata = pd.read_csv(bfile)
bdata = bdata.set_index(pd.to_datetime(bdata[bdata.columns[0]]))
bdata.index.names = ['date']
bdata = bdata.drop(bdata.columns[0], 1)
bdata = bdata.rename(columns={bdata.columns[0]: "bx",
bdata.columns[1]: "by",
bdata.columns[2]: "bz"})
swdata = pd.read_csv(swfile)
swdata = swdata.set_index(pd.to_datetime(swdata[swdata.columns[0]]))
swdata.index.names = ['date']
swdata = swdata.drop(swdata.columns[0], 1)
swdata = swdata.rename(columns={swdata.columns[0]: "temperature",
swdata.columns[1]: "vx",
swdata.columns[2]: "vy",
swdata.columns[3]: "vz"})
# Clean erroneous data found in SWEPAM data from ACE
for column in swdata.columns:
swdata = swdata[swdata[column].abs() < 1.e20]
# Merge the data
data = pd.merge(ndata, bdata, how='outer',
left_index=True, right_index=True)
data = pd.merge(data, swdata, how='outer',
left_index=True, right_index=True)
# Interpolate and fill
data = data.interpolate().ffill().bfill()
# Close and return pandas DataFrame
return data
def write_data(data, outfilename):
outfile = open(outfilename, 'w')
except OSError:
print("Could not open output file to write.")
outfile.write("CSV files downloaded from\n")
outfile.write("yr mn dy hr min sec msec bx by bz vx vy vz dens temp\n")
def convert(infile, outfile):
......@@ -18,11 +93,11 @@ def convert(infile, outfile):
date = d.strptime(line[:14], "%Y %j %H %M")
correctline = date.strftime("%Y %m %d %H %M %S") + ' 000' + line[14:]
# Close files
def clean(data):
Cleans the data for bad discontinuities.
......@@ -57,6 +132,7 @@ def clean(data):
data[prevlocation:nextlocation+1] = replacedata
return data
def read_data(filename):
Read the OMNI web data that was previously converted from
......@@ -70,4 +70,4 @@ def plotnt():
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment