diff --git a/interpret_qc.py b/interpret_qc.py new file mode 100644 index 0000000000000000000000000000000000000000..11f6f2c4a43da5d8fc33f43c6081c347e0cc2778 --- /dev/null +++ b/interpret_qc.py @@ -0,0 +1,67 @@ +import netCDF4 +import pandas as pd +import numpy as np +import matplotlib.pyplot as plt +from aeri_tools.io.dmv.housekeeping import get_all_housekeeping +from main import files_to_update +from glob import glob +import os + +def save_plot(filename): + print('saving {}'.format(filename)) + plt.savefig(filename, transparent=True) + +def plot_variable_qc(frame, loc, filename): + qc_loc = 'qc_' + loc + if sum(frame[qc_loc]) > 0: + plt.figure() + plot_outliers(frame, frame[qc_loc] == 0, loc) + save_plot(filename) + +def plot_outliers(frame, qc_mask, loc): + if (~qc_mask).any(): + frame.ix[qc_mask & (frame['qc_percent'] == 0), loc].plot(style='b.') + frame.ix[qc_mask & (frame['qc_percent'] > 0), loc].plot(style='k.', alpha=.2) + frame.ix[~qc_mask, loc].plot(style='r.') + plt.xlabel('Time') + plt.title(loc) + +def make_plots_here(dirname, frame): + for qc_variable in ([ + 'ABBapexTemp', + 'ABBtopTemp', + 'ABBbottomTemp', + 'HBBapexTemp', + 'HBBtopTemp', + 'HBBbottomTemp', + 'calibrationAmbientTemp']): + plot_variable_qc(frame, qc_variable, os.path.join(dirname, qc_variable+'.png')) + +def get_qc_frame(qc_path): + nc = netCDF4.Dataset(qc_path) + frame = pd.DataFrame({k:v[:] for k,v in nc.variables.items()}) + frame.index = pd.to_datetime((frame['base_time'] + frame['time_offset']), utc=True) + nc.close() + return frame + +def generate_plots(qc_file, cxs_file, sum_file): + cxs = get_all_housekeeping(cxs_file) + frame = cxs.combine_first(get_all_housekeeping(sum_file)) + frame = frame.combine_first(get_qc_frame(qc_file)) + + make_plots_here(os.path.dirname(qc_file), frame) + +def generate_all_plots(ftp_dir): + for qc_file, cxs_file, sum_file in files_to_update(glob(os.path.join(ftp_dir,'AE*','*B1.CXS')), update_only=False): + if os.path.isfile(qc_file): + generate_plots(qc_file, cxs_file, sum_file) + +if __name__ == '__main__': + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('ftp') + + args = parser.parse_args() + + generate_all_plots(args.ftp) +