import netCDF4 import pandas as pd import numpy as np import matplotlib matplotlib.use('agg') import matplotlib.pyplot as plt import mpld3 from aeri_tools.io.dmv.housekeeping import get_all_housekeeping from main import files_to_update from glob import glob import os import flask from flask import render_template from datetime import datetime app = flask.Flask(__name__) global_ftp_dir = None @app.route('/') def index(): ae_dirs = glob(os.path.join(global_ftp_dir, 'AE*/*QC.nc')) ae_dirs = [a.lstrip('/') for a in ae_dirs] return flask.render_template('index.html', ae_dirs=ae_dirs) @app.route('/qc/<path:qc_path>') def qc_day(qc_path): qc_path = '/'+qc_path sum_path = qc_path.replace('QC.nc','.SUM') sum_hk = get_all_housekeeping(sum_path).reset_index() plots = {} nc = netCDF4.Dataset(qc_path) qc_percent_var = nc.variables['qc_percent'] if (qc_percent_var[:] > .95).any(): dependent_checks = qc_percent_var.depends.split(',') for check in dependent_checks: if (nc.variables[check][:] > .95).any(): plots[check] = list(plot_check(nc, check, sum_hk)) nc.close() return flask.render_template('qc.html', qc_path=qc_path, plots=plots) def ignore_error(func, error): try: func() except error: pass def plot_check(nc, check, sum_hk): check_var = nc.variables[check] passes_qc = check_var[:] < .95 assert len(passes_qc) == len(sum_hk) for depend in check_var.depends.split(','): if depend in sum_hk.columns: fig = plt.figure() # Passing check and qc_percent ignore_error(lambda : sum_hk[depend].ix[(nc.variables['qc_percent'][:] < .95) & passes_qc].plot(style='b.'), TypeError) # Passing check but fails overall ignore_error(lambda : sum_hk[depend].ix[(nc.variables['qc_percent'][:] > .95) & passes_qc].plot(style='k.',alpha=.2), TypeError) # Failing check and implicitly failing overall ignore_error(lambda : sum_hk[depend].ix[~passes_qc].plot(style='r.'), TypeError) plt.xlabel('Record') plt.title(depend) yield mpld3.fig_to_html(fig) plt.close() elif depend in nc.variables.keys() and (nc.variables['depend'] > .95).any(): # Recurse yield from plot_check(nc, depend, sum_hk) def get_qc_frame(qc_path): return frame def generate_plots(qc_file, cxs_file, sum_file): cxs = get_all_housekeeping(cxs_file) frame = cxs.combine_first(get_all_housekeeping(sum_file)) frame = frame.combine_first(get_qc_frame(qc_file)) make_plots_here(os.path.dirname(qc_file), frame) def generate_all_plots(ftp_dir): for qc_file, cxs_file, sum_file in files_to_update(glob(os.path.join(ftp_dir,'AE*','*B1.CXS')), update_only=False): if os.path.isfile(qc_file): generate_plots(qc_file, cxs_file, sum_file) if __name__ == '__main__': import argparse parser = argparse.ArgumentParser() parser.add_argument('ftp') args = parser.parse_args() global_ftp_dir = os.path.abspath(args.ftp) app.run('0.0.0.0', debug=True)