Update interpreting code

interpret_qc.py

@@ -26,70 +26,39 @@ def index():
 @app.route('/qc/<path:qc_path>')
 def qc_day(qc_path):
     qc_path = '/'+qc_path
-    cxs_path = qc_path.replace('QC.nc','B1.CXS')
     sum_path = qc_path.replace('QC.nc','.SUM')
 
-    cxs = get_all_housekeeping(cxs_path)
-    frame = cxs.combine_first(get_all_housekeeping(sum_path))
-    qc_frame = get_qc_frame(qc_path)
-    frame = frame.combine_first(qc_frame)
-    frame = frame.query('missingDataFlag == 0')
+    sum_hk = get_all_housekeeping(sum_path)
 
-    qc_frame_sum = (qc_frame > .95).sum(axis=0).to_string()
+    plots = {}
 
-    plots = []
-    qc_variables = qc_frame.columns
-
-    for qc_variable in qc_variables:
-        if qc_variable.startswith('qc_') and qc_variable not in ['qc_notes','qc_percent']:
-            qc_variable = qc_variable.replace('qc_','')
-            plot = plot_variable_qc(frame, qc_variable)
-            if plot is not None:
-                plots.append(plot)
-
-    return flask.render_template('qc.html', qc_path=qc_path, plots=plots, qc_frame=qc_frame_sum)
-
-def save_plot(filename):
-    print('saving {}'.format(filename))
-    plt.savefig(filename, transparent=True)
-
-def plot_variable_qc(frame, loc, filename=None):
-    qc_loc = 'qc_' + loc
-    if qc_loc not in frame.columns:
-        print('{qc_loc} not in frame'.format(qc_loc=qc_loc))
-        return
-    if frame[qc_loc].sum() > 0:
-        fig = plt.figure(figsize=(10,5))
-        plot_outliers(frame, frame[qc_loc], loc)
-        if filename is not None:
-            save_plot(filename)
-        else:
-            return mpld3.fig_to_html(fig)
-
-def plot_outliers(frame, qc_mask, loc):
-    if (~np.isnan(qc_mask) & qc_mask > .95).any():
-        frame.ix[(np.isnan(qc_mask) | (qc_mask < .95)) & (frame['qc_percent'] < .95), loc].plot(style='b.')
-        frame.ix[(np.isnan(qc_mask) | (qc_mask < .95)) & (frame['qc_percent'] > .95), loc].plot(style='k.', alpha=.2)
-        frame.ix[~np.isnan(qc_mask) & (qc_mask > .95) & (frame['qc_percent'] > .95), loc].plot(style='r.')
-        plt.xlabel('Time')
-        plt.title(loc)
-
-def make_plots_here(dirname, frame):
-    for qc_variable in ([
-        'ABBapexTemp',
-        'ABBtopTemp',
-        'ABBbottomTemp',
-        'HBBapexTemp',
-        'HBBtopTemp',
-        'HBBbottomTemp',
-        'calibrationAmbientTemp']):
-        plot_variable_qc(frame, qc_variable, os.path.join(dirname, qc_variable+'.png'))
-
-def get_qc_frame(qc_path):
     nc = netCDF4.Dataset(qc_path)
-    frame = pd.DataFrame({k:v[:] for k,v in nc.variables.items()})
-    frame.index = pd.to_datetime((frame['base_time'] + frame['time_offset']), utc=True)
+    qc_percent_var = nc.variables['qc_percent']
+    if (qc_percent_var[:] > .95).any():
+        dependent_checks = qc_percent_var.depends.split(',')
+        for check in dependent_checks:
+            if (nc.variables[check][:] > .95).any():
+                plots[check] = list(plot_check(nc, check, sum_hk))
     nc.close()
+    return flask.render_template('qc.html', qc_path=qc_path, plots=plots)
+
+def plot_check(nc, check, sum_hk):
+    check_var = nc.variables[check]
+    passes_qc = check_var[:] < .95
+    for depend in check_var.depends.split(','):
+        if depend in sum_hk.columns:
+            fig = plt.figure()
+            sum_hk.reset_index()[depend].ix[(nc.variables['qc_percent'][:] < .95) & passes_qc].plot(style='b.')
+            sum_hk.reset_index()[depend].ix[(nc.variables['qc_percent'][:] > .95) & passes_qc].plot(style='k.',alpha=.2)
+            sum_hk.reset_index()[depend].ix[~passes_qc].plot(style='r.')
+            plt.xlabel('Record')
+            plt.title(depend)
+            yield mpld3.fig_to_html(fig)
+        elif depend in nc.variables.keys():
+            # Recurse
+            yield from plot_check(nc, depend, sum_hk)
+
+def get_qc_frame(qc_path):
     return frame
 
 def generate_plots(qc_file, cxs_file, sum_file):
@@ -108,13 +77,9 @@ if __name__ == '__main__':
     import argparse
     parser = argparse.ArgumentParser()
     parser.add_argument('ftp')
-    parser.add_argument('--serve', action='store_true')
 
     args = parser.parse_args()
 
-    if not args.serve:
-        generate_all_plots(os.path.abspath(args.ftp))
-    else:
-        global_ftp_dir = os.path.abspath(args.ftp)
-        app.run('0.0.0.0', debug=True)
+    global_ftp_dir = os.path.abspath(args.ftp)
+    app.run('0.0.0.0', debug=True)
     

templates/qc.html

@@ -4,9 +4,11 @@
     </head>
     <body>
     <h1>{{qc_path}}</h1>
-    <pre>{{qc_frame}}</pre>
-    {% for plot in plots %}
-        {{plot|safe}}
+    {% for check,plots in plots.items() %}
+        <h2>{{check}}</h2>
+        {% for plot in plots %}
+            {{plot|safe}}
+        {% endfor %}
     {% endfor %}
     </body
 </html>