interpret_qc.py

import netCDF4
import pandas as pd
import numpy as np
import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot as plt
import mpld3
from aeri_tools.io.dmv.housekeeping import get_all_housekeeping
from main import files_to_update
from glob import glob
import os
import flask
from flask import render_template
from datetime import datetime

app = flask.Flask(__name__)

global_ftp_dir = None

@app.route('/')
def index():
    ae_dirs = glob(os.path.join(global_ftp_dir, 'AE*/*QC.nc'))
    ae_dirs = [a.lstrip('/') for a in ae_dirs]
    return flask.render_template('index.html', ae_dirs=ae_dirs)

@app.route('/qc/<path:qc_path>')
def qc_day(qc_path):
    qc_path = '/'+qc_path
    sum_path = qc_path.replace('QC.nc','.SUM')

    sum_hk = get_all_housekeeping(sum_path).reset_index()

    plots = {}

    nc = netCDF4.Dataset(qc_path)
    qc_percent_var = nc.variables['qc_percent']
    if (qc_percent_var[:] > .95).any():
        dependent_checks = qc_percent_var.depends.split(',')
        for check in dependent_checks:
            if (nc.variables[check][:] > .95).any():
                plots[check] = list(plot_check(nc, check, sum_hk))
    nc.close()
    return flask.render_template('qc.html', qc_path=qc_path, plots=plots)

def ignore_error(func, error):
    try:
        func()
    except error:
        pass

def plot_check(nc, check, sum_hk):
    check_var = nc.variables[check]
    passes_qc = check_var[:] < .95
    assert len(passes_qc) == len(sum_hk)
    for depend in check_var.depends.split(','):
        if depend in sum_hk.columns:
            fig = plt.figure()
            # Passing check and qc_percent
            ignore_error(lambda : sum_hk[depend].ix[(nc.variables['qc_percent'][:] < .95) & passes_qc].plot(style='b.'), TypeError)
            # Passing check but fails overall
            ignore_error(lambda : sum_hk[depend].ix[(nc.variables['qc_percent'][:] > .95) & passes_qc].plot(style='k.',alpha=.2), TypeError)
            # Failing check and implicitly failing overall
            ignore_error(lambda : sum_hk[depend].ix[~passes_qc].plot(style='r.'), TypeError)
            plt.xlabel('Record')
            plt.title(depend)
            yield mpld3.fig_to_html(fig)
            plt.close()
        elif depend in nc.variables.keys() and (nc.variables['depend'] > .95).any():
            # Recurse
            yield from plot_check(nc, depend, sum_hk)

def get_qc_frame(qc_path):
    return frame

def generate_plots(qc_file, cxs_file, sum_file):
    cxs = get_all_housekeeping(cxs_file)
    frame = cxs.combine_first(get_all_housekeeping(sum_file))
    frame = frame.combine_first(get_qc_frame(qc_file))
    
    make_plots_here(os.path.dirname(qc_file), frame)

def generate_all_plots(ftp_dir):
    for qc_file, cxs_file, sum_file in files_to_update(glob(os.path.join(ftp_dir,'AE*','*B1.CXS')), update_only=False):
        if os.path.isfile(qc_file):
            generate_plots(qc_file, cxs_file, sum_file)

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('ftp')

    args = parser.parse_args()

    global_ftp_dir = os.path.abspath(args.ftp)
    app.run('0.0.0.0', debug=True)