From e8599bb58c4ba890df55a6330d2a50e00f0cacf6 Mon Sep 17 00:00:00 2001
From: Coda Phillips <cphillips@sean.ssec.wisc.edu>
Date: Thu, 25 Aug 2016 14:01:50 -0500
Subject: [PATCH] Update interpreting code
---
interpret_qc.py | 93 +++++++++++++++--------------------------------
templates/qc.html | 8 ++--
2 files changed, 34 insertions(+), 67 deletions(-)
diff --git a/interpret_qc.py b/interpret_qc.py
index a08811c..3a00ee5 100644
--- a/interpret_qc.py
+++ b/interpret_qc.py
@@ -26,70 +26,39 @@ def index():
@app.route('/qc/<path:qc_path>')
def qc_day(qc_path):
qc_path = '/'+qc_path
- cxs_path = qc_path.replace('QC.nc','B1.CXS')
sum_path = qc_path.replace('QC.nc','.SUM')
- cxs = get_all_housekeeping(cxs_path)
- frame = cxs.combine_first(get_all_housekeeping(sum_path))
- qc_frame = get_qc_frame(qc_path)
- frame = frame.combine_first(qc_frame)
- frame = frame.query('missingDataFlag == 0')
+ sum_hk = get_all_housekeeping(sum_path)
- qc_frame_sum = (qc_frame > .95).sum(axis=0).to_string()
+ plots = {}
- plots = []
- qc_variables = qc_frame.columns
-
- for qc_variable in qc_variables:
- if qc_variable.startswith('qc_') and qc_variable not in ['qc_notes','qc_percent']:
- qc_variable = qc_variable.replace('qc_','')
- plot = plot_variable_qc(frame, qc_variable)
- if plot is not None:
- plots.append(plot)
-
- return flask.render_template('qc.html', qc_path=qc_path, plots=plots, qc_frame=qc_frame_sum)
-
-def save_plot(filename):
- print('saving {}'.format(filename))
- plt.savefig(filename, transparent=True)
-
-def plot_variable_qc(frame, loc, filename=None):
- qc_loc = 'qc_' + loc
- if qc_loc not in frame.columns:
- print('{qc_loc} not in frame'.format(qc_loc=qc_loc))
- return
- if frame[qc_loc].sum() > 0:
- fig = plt.figure(figsize=(10,5))
- plot_outliers(frame, frame[qc_loc], loc)
- if filename is not None:
- save_plot(filename)
- else:
- return mpld3.fig_to_html(fig)
-
-def plot_outliers(frame, qc_mask, loc):
- if (~np.isnan(qc_mask) & qc_mask > .95).any():
- frame.ix[(np.isnan(qc_mask) | (qc_mask < .95)) & (frame['qc_percent'] < .95), loc].plot(style='b.')
- frame.ix[(np.isnan(qc_mask) | (qc_mask < .95)) & (frame['qc_percent'] > .95), loc].plot(style='k.', alpha=.2)
- frame.ix[~np.isnan(qc_mask) & (qc_mask > .95) & (frame['qc_percent'] > .95), loc].plot(style='r.')
- plt.xlabel('Time')
- plt.title(loc)
-
-def make_plots_here(dirname, frame):
- for qc_variable in ([
- 'ABBapexTemp',
- 'ABBtopTemp',
- 'ABBbottomTemp',
- 'HBBapexTemp',
- 'HBBtopTemp',
- 'HBBbottomTemp',
- 'calibrationAmbientTemp']):
- plot_variable_qc(frame, qc_variable, os.path.join(dirname, qc_variable+'.png'))
-
-def get_qc_frame(qc_path):
nc = netCDF4.Dataset(qc_path)
- frame = pd.DataFrame({k:v[:] for k,v in nc.variables.items()})
- frame.index = pd.to_datetime((frame['base_time'] + frame['time_offset']), utc=True)
+ qc_percent_var = nc.variables['qc_percent']
+ if (qc_percent_var[:] > .95).any():
+ dependent_checks = qc_percent_var.depends.split(',')
+ for check in dependent_checks:
+ if (nc.variables[check][:] > .95).any():
+ plots[check] = list(plot_check(nc, check, sum_hk))
nc.close()
+ return flask.render_template('qc.html', qc_path=qc_path, plots=plots)
+
+def plot_check(nc, check, sum_hk):
+ check_var = nc.variables[check]
+ passes_qc = check_var[:] < .95
+ for depend in check_var.depends.split(','):
+ if depend in sum_hk.columns:
+ fig = plt.figure()
+ sum_hk.reset_index()[depend].ix[(nc.variables['qc_percent'][:] < .95) & passes_qc].plot(style='b.')
+ sum_hk.reset_index()[depend].ix[(nc.variables['qc_percent'][:] > .95) & passes_qc].plot(style='k.',alpha=.2)
+ sum_hk.reset_index()[depend].ix[~passes_qc].plot(style='r.')
+ plt.xlabel('Record')
+ plt.title(depend)
+ yield mpld3.fig_to_html(fig)
+ elif depend in nc.variables.keys():
+ # Recurse
+ yield from plot_check(nc, depend, sum_hk)
+
+def get_qc_frame(qc_path):
return frame
def generate_plots(qc_file, cxs_file, sum_file):
@@ -108,13 +77,9 @@ if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('ftp')
- parser.add_argument('--serve', action='store_true')
args = parser.parse_args()
- if not args.serve:
- generate_all_plots(os.path.abspath(args.ftp))
- else:
- global_ftp_dir = os.path.abspath(args.ftp)
- app.run('0.0.0.0', debug=True)
+ global_ftp_dir = os.path.abspath(args.ftp)
+ app.run('0.0.0.0', debug=True)
diff --git a/templates/qc.html b/templates/qc.html
index 45b241c..b86c70b 100644
--- a/templates/qc.html
+++ b/templates/qc.html
@@ -4,9 +4,11 @@
</head>
<body>
<h1>{{qc_path}}</h1>
- <pre>{{qc_frame}}</pre>
- {% for plot in plots %}
- {{plot|safe}}
+ {% for check,plots in plots.items() %}
+ <h2>{{check}}</h2>
+ {% for plot in plots %}
+ {{plot|safe}}
+ {% endfor %}
{% endfor %}
</body
</html>
--
GitLab