From e8599bb58c4ba890df55a6330d2a50e00f0cacf6 Mon Sep 17 00:00:00 2001 From: Coda Phillips <cphillips@sean.ssec.wisc.edu> Date: Thu, 25 Aug 2016 14:01:50 -0500 Subject: [PATCH] Update interpreting code --- interpret_qc.py | 93 +++++++++++++++-------------------------------- templates/qc.html | 8 ++-- 2 files changed, 34 insertions(+), 67 deletions(-) diff --git a/interpret_qc.py b/interpret_qc.py index a08811c..3a00ee5 100644 --- a/interpret_qc.py +++ b/interpret_qc.py @@ -26,70 +26,39 @@ def index(): @app.route('/qc/<path:qc_path>') def qc_day(qc_path): qc_path = '/'+qc_path - cxs_path = qc_path.replace('QC.nc','B1.CXS') sum_path = qc_path.replace('QC.nc','.SUM') - cxs = get_all_housekeeping(cxs_path) - frame = cxs.combine_first(get_all_housekeeping(sum_path)) - qc_frame = get_qc_frame(qc_path) - frame = frame.combine_first(qc_frame) - frame = frame.query('missingDataFlag == 0') + sum_hk = get_all_housekeeping(sum_path) - qc_frame_sum = (qc_frame > .95).sum(axis=0).to_string() + plots = {} - plots = [] - qc_variables = qc_frame.columns - - for qc_variable in qc_variables: - if qc_variable.startswith('qc_') and qc_variable not in ['qc_notes','qc_percent']: - qc_variable = qc_variable.replace('qc_','') - plot = plot_variable_qc(frame, qc_variable) - if plot is not None: - plots.append(plot) - - return flask.render_template('qc.html', qc_path=qc_path, plots=plots, qc_frame=qc_frame_sum) - -def save_plot(filename): - print('saving {}'.format(filename)) - plt.savefig(filename, transparent=True) - -def plot_variable_qc(frame, loc, filename=None): - qc_loc = 'qc_' + loc - if qc_loc not in frame.columns: - print('{qc_loc} not in frame'.format(qc_loc=qc_loc)) - return - if frame[qc_loc].sum() > 0: - fig = plt.figure(figsize=(10,5)) - plot_outliers(frame, frame[qc_loc], loc) - if filename is not None: - save_plot(filename) - else: - return mpld3.fig_to_html(fig) - -def plot_outliers(frame, qc_mask, loc): - if (~np.isnan(qc_mask) & qc_mask > .95).any(): - frame.ix[(np.isnan(qc_mask) | (qc_mask < .95)) & (frame['qc_percent'] < .95), loc].plot(style='b.') - frame.ix[(np.isnan(qc_mask) | (qc_mask < .95)) & (frame['qc_percent'] > .95), loc].plot(style='k.', alpha=.2) - frame.ix[~np.isnan(qc_mask) & (qc_mask > .95) & (frame['qc_percent'] > .95), loc].plot(style='r.') - plt.xlabel('Time') - plt.title(loc) - -def make_plots_here(dirname, frame): - for qc_variable in ([ - 'ABBapexTemp', - 'ABBtopTemp', - 'ABBbottomTemp', - 'HBBapexTemp', - 'HBBtopTemp', - 'HBBbottomTemp', - 'calibrationAmbientTemp']): - plot_variable_qc(frame, qc_variable, os.path.join(dirname, qc_variable+'.png')) - -def get_qc_frame(qc_path): nc = netCDF4.Dataset(qc_path) - frame = pd.DataFrame({k:v[:] for k,v in nc.variables.items()}) - frame.index = pd.to_datetime((frame['base_time'] + frame['time_offset']), utc=True) + qc_percent_var = nc.variables['qc_percent'] + if (qc_percent_var[:] > .95).any(): + dependent_checks = qc_percent_var.depends.split(',') + for check in dependent_checks: + if (nc.variables[check][:] > .95).any(): + plots[check] = list(plot_check(nc, check, sum_hk)) nc.close() + return flask.render_template('qc.html', qc_path=qc_path, plots=plots) + +def plot_check(nc, check, sum_hk): + check_var = nc.variables[check] + passes_qc = check_var[:] < .95 + for depend in check_var.depends.split(','): + if depend in sum_hk.columns: + fig = plt.figure() + sum_hk.reset_index()[depend].ix[(nc.variables['qc_percent'][:] < .95) & passes_qc].plot(style='b.') + sum_hk.reset_index()[depend].ix[(nc.variables['qc_percent'][:] > .95) & passes_qc].plot(style='k.',alpha=.2) + sum_hk.reset_index()[depend].ix[~passes_qc].plot(style='r.') + plt.xlabel('Record') + plt.title(depend) + yield mpld3.fig_to_html(fig) + elif depend in nc.variables.keys(): + # Recurse + yield from plot_check(nc, depend, sum_hk) + +def get_qc_frame(qc_path): return frame def generate_plots(qc_file, cxs_file, sum_file): @@ -108,13 +77,9 @@ if __name__ == '__main__': import argparse parser = argparse.ArgumentParser() parser.add_argument('ftp') - parser.add_argument('--serve', action='store_true') args = parser.parse_args() - if not args.serve: - generate_all_plots(os.path.abspath(args.ftp)) - else: - global_ftp_dir = os.path.abspath(args.ftp) - app.run('0.0.0.0', debug=True) + global_ftp_dir = os.path.abspath(args.ftp) + app.run('0.0.0.0', debug=True) diff --git a/templates/qc.html b/templates/qc.html index 45b241c..b86c70b 100644 --- a/templates/qc.html +++ b/templates/qc.html @@ -4,9 +4,11 @@ </head> <body> <h1>{{qc_path}}</h1> - <pre>{{qc_frame}}</pre> - {% for plot in plots %} - {{plot|safe}} + {% for check,plots in plots.items() %} + <h2>{{check}}</h2> + {% for plot in plots %} + {{plot|safe}} + {% endfor %} {% endfor %} </body </html> -- GitLab