interpret_qc.py

import netCDF4
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import mpld3
from aeri_tools.io.dmv.housekeeping import get_all_housekeeping
from main import files_to_update
from glob import glob
import os
import flask
from flask import render_template
from datetime import datetime

app = flask.Flask(__name__)

global_ftp_dir = None

@app.route('/')
def index():
    ae_dirs = glob(os.path.join(global_ftp_dir, 'AE*/*.qc'))
    ae_dirs = [a.lstrip('/') for a in ae_dirs]
    return flask.render_template('index.html', ae_dirs=ae_dirs)

@app.route('/qc/<path:qc_path>')
def qc_day(qc_path):
    qc_path = '/'+qc_path
    cxs_path = qc_path.replace('.qc','B1.CXS')
    sum_path = qc_path.replace('.qc','.SUM')

    cxs = get_all_housekeeping(cxs_path)
    frame = cxs.combine_first(get_all_housekeeping(sum_path))
    qc_frame = get_qc_frame(qc_path)
    frame = frame.combine_first(qc_frame)
    frame = frame.query('missingDataFlag == 0')

    qc_frame_sum = (qc_frame > .95).sum(axis=0).to_string()

    plots = []
    qc_variables = qc_frame.columns

    for qc_variable in qc_variables:
        if qc_variable.startswith('qc_') and qc_variable not in ['qc_notes','qc_percent']:
            qc_variable = qc_variable.replace('qc_','')
            plot = plot_variable_qc(frame, qc_variable)
            if plot is not None:
                plots.append(plot)

    return flask.render_template('qc.html', qc_path=qc_path, plots=plots, qc_frame=qc_frame_sum)

def save_plot(filename):
    print('saving {}'.format(filename))
    plt.savefig(filename, transparent=True)

def plot_variable_qc(frame, loc, filename=None):
    qc_loc = 'qc_' + loc
    if qc_loc not in frame.columns:
        print('{qc_loc} not in frame'.format(qc_loc=qc_loc))
        return
    if frame[qc_loc].sum() > 0:
        fig = plt.figure(figsize=(10,5))
        plot_outliers(frame, frame[qc_loc], loc)
        if filename is not None:
            save_plot(filename)
        else:
            return mpld3.fig_to_html(fig)

def plot_outliers(frame, qc_mask, loc):
    if (~np.isnan(qc_mask) & qc_mask > .95).any():
        frame.ix[(np.isnan(qc_mask) | (qc_mask < .95)) & (frame['qc_percent'] < .95), loc].plot(style='b.')
        frame.ix[(np.isnan(qc_mask) | (qc_mask < .95)) & (frame['qc_percent'] > .95), loc].plot(style='k.', alpha=.2)
        frame.ix[~np.isnan(qc_mask) & (qc_mask > .95) & (frame['qc_percent'] > .95), loc].plot(style='r.')
        plt.xlabel('Time')
        plt.title(loc)

def make_plots_here(dirname, frame):
    for qc_variable in ([
        'ABBapexTemp',
        'ABBtopTemp',
        'ABBbottomTemp',
        'HBBapexTemp',
        'HBBtopTemp',
        'HBBbottomTemp',
        'calibrationAmbientTemp']):
        plot_variable_qc(frame, qc_variable, os.path.join(dirname, qc_variable+'.png'))

def get_qc_frame(qc_path):
    nc = netCDF4.Dataset(qc_path)
    frame = pd.DataFrame({k:v[:] for k,v in nc.variables.items()})
    frame.index = pd.to_datetime((frame['base_time'] + frame['time_offset']), utc=True)
    nc.close()
    return frame

def generate_plots(qc_file, cxs_file, sum_file):
    cxs = get_all_housekeeping(cxs_file)
    frame = cxs.combine_first(get_all_housekeeping(sum_file))
    frame = frame.combine_first(get_qc_frame(qc_file))
    
    make_plots_here(os.path.dirname(qc_file), frame)

def generate_all_plots(ftp_dir):
    for qc_file, cxs_file, sum_file in files_to_update(glob(os.path.join(ftp_dir,'AE*','*B1.CXS')), update_only=False):
        if os.path.isfile(qc_file):
            generate_plots(qc_file, cxs_file, sum_file)

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('ftp')
    parser.add_argument('--serve', action='store_true')

    args = parser.parse_args()

    if not args.serve:
        generate_all_plots(os.path.abspath(args.ftp))
    else:
        global_ftp_dir = os.path.abspath(args.ftp)
        app.run(debug=True)