Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 652e2b2

Browse files
jgieselerjgieseler
committed
refactor epd_load function for pos_timestamp with new data structure
1 parent e058cc1 commit 652e2b2

1 file changed

Lines changed: 34 additions & 27 deletions

File tree

‎solo_epd_loader/__init__.py‎

Lines changed: 34 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -559,7 +559,10 @@ def _autodownload_cdf(startdate, enddate, sensor, level, path):
559559

560560

561561
# TODO: remove old_ept_het_loading
562-
def epd_load(sensor, startdate, enddate=None, level='l2', viewing=None, path=None, autodownload=False, only_averages=False, old_step_loading=False, pos_timestamp='center', old_ept_het_loading=True):
562+
def epd_load(sensor, startdate, enddate=None, level='l2', viewing=None,
563+
path=None, autodownload=False, only_averages=False,
564+
old_step_loading=False, pos_timestamp='center',
565+
old_ept_het_loading=True):
563566
"""
564567
Load SolO/EPD data
565568
@@ -683,16 +686,10 @@ def epd_load(sensor, startdate, enddate=None, level='l2', viewing=None, path=Non
683686
path=path,
684687
autodownload=autodownload)
685688
# adjusting the position of the timestamp manually. original SolO/EPD data hast timestamp at 'start' of interval
686-
if pos_timestamp == 'center':
687-
# TODO: implement pos_timestamp for EPT L3 data also for df_rtn and df_hci?
688-
custom_warning("Note that for the Dataframes containing the flow direction and SC coordinates timestamp position will not be adjusted by 'pos_timestamp'!")
689-
if len(df) > 0:
690-
shift_index_start2center(df)
691-
# if len(df_rtn) > 0:
692-
# shift_index_start2center(df_rtn)
693-
# if len(df_hci) > 0:
694-
# shift_index_start2center(df_hci)
695-
# return data_dict, energies_dict, metadata_dict
689+
if pos_timestamp == 'center' and level.lower() != 'll' and len(data_dict[[*data_dict.keys()][0]]) > 0:
690+
for key in data_dict.keys(): # e.g. df, df_rtn, df_hci, ...
691+
if key not in ['df_rtn', 'df_hci']: # do not change timestamp for df_rtn and df_hci
692+
shift_index_start2center(data_dict[key])
696693
else:
697694
raise Exception("Level 3 data only available for EPT! No data read!")
698695
else:
@@ -701,15 +698,17 @@ def epd_load(sensor, startdate, enddate=None, level='l2', viewing=None, path=Non
701698
_read_step_cdf(level=level, startdate=startdate, enddate=enddate, path=path, autodownload=autodownload,
702699
only_averages=only_averages, old_loading=old_step_loading)
703700
# adjusting the position of the timestamp manually. original SolO/EPD data hast timestamp at 'start' of interval
704-
if pos_timestamp == 'center' and level.lower() != 'll' and old_step_loading is not True and len(datadf) > 0:
705-
shift_index_start2center(datadf) # TODO: implement pos_timestamp for STEP data
701+
if pos_timestamp == 'center' and level.lower() != 'll' and old_step_loading is not True and len(data_dict[[*data_dict.keys()][0]]) > 0:
702+
for key in data_dict.keys(): # e.g. df, df_rtn, df_hci, ...
703+
if key not in ['df_rtn', 'df_hci']: # do not change timestamp for df_rtn and df_hci
704+
shift_index_start2center(data_dict[key])
706705
# return data_dict, energies_dict, metadata_dict
707706
if sensor.lower() == 'ept' or sensor.lower() == 'het':
708707
if viewing is None:
709708
raise Exception("EPT and HET need a telescope 'viewing' direction! No data read!")
710-
df_epd_p = []
711-
df_epd_e = []
709+
data_dict = []
712710
energies_dict = []
711+
metadata_dict = []
713712
elif viewing == 'omni':
714713
data_dict = {}
715714
all_data_dict = {}
@@ -720,6 +719,12 @@ def epd_load(sensor, startdate, enddate=None, level='l2', viewing=None, path=Non
720719
# elif not old_ept_het_loading:
721720
t_data_dict, energies_dict, metadata_dict = \
722721
_read_epd_cdf(sensor=sensor, viewing=view, level=level, startdate=startdate, enddate=enddate, path=path, autodownload=autodownload)
722+
# adjusting the position of the timestamp manually. original SolO/EPD data hast timestamp at 'start' of interval
723+
# adjusting the position of the timestamp manually. original SolO/EPD data hast timestamp at 'start' of interval
724+
if pos_timestamp == 'center' and level.lower() != 'll' and len(t_data_dict[[*t_data_dict.keys()][0]]) > 0:
725+
for key in t_data_dict.keys(): # e.g. df_p, df_e, df_rtn, df_hci, ...
726+
if key not in ['df_rtn', 'df_hci']: # do not change timestamp for df_rtn and df_hci
727+
shift_index_start2center(t_data_dict[key])
723728
for key in t_data_dict.keys(): # e.g. df_p, df_e, df_rtn, df_hci, ...
724729
all_data_dict[f'{key}_{view}'] = t_data_dict[key] # e.g. all_data_dict['df_p_sun']
725730
# sum fluxes and uncertainties (TODO: make this bette for uncertainties!) from all four sectors and divide by 4
@@ -748,12 +753,14 @@ def epd_load(sensor, startdate, enddate=None, level='l2', viewing=None, path=Non
748753
# elif not old_ept_het_loading:
749754
data_dict, energies_dict, metadata_dict = \
750755
_read_epd_cdf(sensor=sensor, viewing=viewing, level=level, startdate=startdate, enddate=enddate, path=path, autodownload=autodownload)
751-
# adjusting the position of the timestamp manually. original SolO/EPD data hast timestamp at 'start' of interval
752-
if pos_timestamp == 'center' and level.lower() != 'll': # TODO: run a for-loop over all dataframes?
753-
if len(df_epd_p) > 0:
754-
shift_index_start2center(df_epd_p)
755-
if len(df_epd_e) > 0:
756-
shift_index_start2center(df_epd_e)
756+
# adjusting the position of the timestamp manually. original SolO/EPD data hast timestamp at 'start' of interval
757+
if pos_timestamp == 'center' and level.lower() != 'll' and len(data_dict[[*data_dict.keys()][0]]) > 0:
758+
for key in data_dict.keys(): # e.g. df_p, df_e, df_rtn, df_hci, ...
759+
if key not in ['df_rtn', 'df_hci']: # do not change timestamp for df_rtn and df_hci
760+
shift_index_start2center(data_dict[key])
761+
762+
if pos_timestamp == 'center' and level.lower() != 'll' and len(data_dict[[*data_dict.keys()][0]]) > 0:
763+
custom_warning("Note that for the DataFrames containing the flow direction and SC coordinates (df_rtn & df_hci) the timestamp positions will not be adjusted by 'pos_timestamp'!")
757764

758765
return data_dict, energies_dict, metadata_dict
759766

@@ -1533,7 +1540,7 @@ def _read_step_cdf(level, startdate, enddate=None, path=None, autodownload=False
15331540
col_list.append(t_df)
15341541
except TypeError:
15351542
print(' ')
1536-
print("WARNING: Gap in dataframe due to missing cdf file.")
1543+
custom_warning("WARNING: Gap in dataframe due to missing cdf file.")
15371544
break
15381545
try:
15391546
temp_df = pd.concat(col_list, axis=1, keys=param_list)
@@ -1619,7 +1626,7 @@ def _read_new_step_cdf(files, only_averages=False):
16191626
df_rtn = pd.DataFrame()
16201627
df_hci = pd.DataFrame()
16211628
for f in files:
1622-
print('Loading', f)
1629+
# print('Loading', f)
16231630
# data = TimeSeries(f, concatenate=True)
16241631
ignore_vars = []
16251632
if not all_columns:
@@ -2247,16 +2254,16 @@ def shift_index_start2center(df, delta_epoch_name=None):
22472254
Manual define name of the DELTA_EPOCH column, by default None
22482255
"""
22492256

2250-
# Make a copy of the index to work on
2251-
df['Time'] = df.index
2252-
22532257
if delta_epoch_name:
22542258
de = delta_epoch_name
22552259
else:
22562260
if df.columns.get_level_values(0).str.startswith('DELTA_EPOCH').sum() != 1:
2257-
custom_warning("DELTA_EPOCH column not available or not unique, aborting. Try using pos_timestamp='start' instead.")
2261+
custom_warning(f"DELTA_EPOCH column not available or not unique in DataFrame containing {df.columns[0]}, aborting. Try using pos_timestamp='start' instead.")
22582262
return
22592263
else:
2264+
# Make a copy of the index to work on
2265+
df['Time'] = df.index
2266+
22602267
# Obtain DELTA_EPOCH column name
22612268
de = df.columns.get_level_values(0)[df.columns.get_level_values(0).str.startswith('DELTA_EPOCH')][0]
22622269

0 commit comments

Comments
 (0)