diff --git a/main.py b/main.py index e6df9e0cd5a49164128543f1dc35b50950c5f1d0..6a35ca0e28185aba2e8a62dd6243bca9e2370e00 100644 --- a/main.py +++ b/main.py @@ -1,7 +1,8 @@ import os from glob import glob import re -#from aeri_tools.io.dmv.housekeeping import get_all_housekeeping +import netCDF4 +from aeri_tools.io.dmv.housekeeping import get_all_housekeeping import electronic_checks import global_checks @@ -19,9 +20,28 @@ levels = [ thermal_checks.CheckList() ] +def save_quality(frame, qc_path): + frame[['qc_percent','qc_notes','datetime']] + ncdf = netCDF4.Dataset(qc_path, 'w') + time = ncdf.createDimension('time', len(frame)) + base_time = ncdf.createVariable('base_time', 'i8', ()) + time_offset = ncdf.createVariable('time_offset', 'f8', ('time',)) + qc_percent = ncdf.createVariable('qc_percent', 'f4', ('time',)) + qc_notes = ncdf.createVariable('qc_notes', str, ('time',)) + for check_mask in frame.filter(like='check'): + ncdf.createVariable(check_mask, 'f4', ('time',))[:] = frame[check_mask].values + base_time[:] = frame.datetime[0].timestamp() + time_offset[:] = (frame.datetime - frame.datetime[0]).values / 1e9 + qc_percent[:] = frame['qc_percent'].values + qc_notes[:] = frame['qc_notes'].fillna('').values + ncdf.close() + + def read_frame(cxs_file, sum_file): - return get_all_housekeeping(cxs_file).combine_first( + hk = get_all_housekeeping(cxs_file).combine_first( get_all_housekeeping(sum_file)) + hk.index.name = 'datetime' + return hk.reset_index() def check_frame(frame, parameters): frame['qc_percent'] = 0 @@ -34,10 +54,12 @@ def check_frame(frame, parameters): def update_all(ftp_dir, parameters=None): cxs_files = glob(os.path.join(ftp_dir,'AE*','*B1.CXS')) for qc_file, cxs_file, sum_file in files_to_update(cxs_files): - hk = read_frame(cxs_file, sum_file) + print('Performing quality control for {}'.format(cxs_file)) + frame = read_frame(cxs_file, sum_file) if parameters is None: parameters = {} - check_frame(frame, parameters) + frame = check_frame(frame, parameters) + save_quality(frame, qc_file) def files_to_update(cxs_files): for cxs_file in cxs_files: @@ -52,7 +74,7 @@ def files_to_update(cxs_files): if max(os.path.getmtime(sum_file), os.path.getmtime(cxs_file)) > os.path.getmtime(qc_file): yield (qc_file, cxs_file, sum_file) else: - yield (qc_file, cxs_file, sum_file) + yield (possible_qc, cxs_file, sum_file) if __name__ == '__main__': import argparse