made a program (file_finder) to be able to search either through a single...

made a program (file_finder) to be able to search either through a single file, a directory of files, or a directory of directories of files to find the ending it's looking for. this is so each program that needs functionality doesn't need the same lines of code copy-pasted into it. diagnostic_plotter and file_finder were implemented in testing_quick_vis. when a test has >80% flags, it will automatically make a plot of the corresponding variables. a text file with a record of tests with <100% validity is also made with the tests' percent validity

testing/diagnostic_plotter.py

+import matplotlib.pyplot as plt
+import pandas as pd
+
+from aeri_tools.io.dmv import housekeeping
+
+HBB = ['HBBapexTemp', 'HBBbottomTemp', 'HBBtopTemp']
+ABB = ['ABBapexTemp', 'ABBbottomTemp', 'ABBtopTemp']
+
+check_vars = {
+    'detector_temp_check' : ['detectorTemp',],
+    'hbb_temp_outlier_check' : HBB,
+    'abb_temp_outlier_check' : ABB,
+    'bst_temp_outlier_check' : ['BBsupportStructureTemp',],
+    'air_interferometer_outlier_check' : ['airNearInterferometerTemp',],
+    'missing_data_flag_check' : ['missingDataFlag',],
+    'imaginary_radiance_check' : ['skyViewImaginaryRadiance2510_2515',],
+    'hbb_std_dev_check' : ['HBBviewStdDevRadiance985_990',],
+    'hbb_lw_nen_check' : ['LW_HBB_NEN',],
+    'hbb_sw_nen_check' : ['SW_HBB_NEN',],
+    'lw_responsivity_check' : ['LWresponsivity',],
+    'sw_responsivity_check' : ['SWresponsivity',],
+    'sky_brightness_temp_spectral_averages_ch1_check' :
+        ['surfaceLayerAirTemp675_680', 'BBsupportStructureTemp', 'airNearBBsTemp'],
+    'sky_brightness_temp_spectral_averages_ch2_check' :
+        ['surfaceLayerAirTemp2295_2300', 'BBsupportStructureTemp', 'airNearBBsTemp'],
+    'hatch_check' : ['hatchOpen', 'sceneMirrorPosition'],
+    'safing_check' : ['hatchOpen', 'sceneMirrorPosition'],
+    'encoder_check' : ['sceneMirrorPosition',],
+    'detector_check' : ['detectorTemp',],
+    'hbb_thermistor_check' : HBB,
+    'abb_thermistor_check' : ABB,
+    'hbb_stable_check' : HBB,
+    'hbb_covariance_check' : HBB,
+    'sce_temp_deviation_check' : ['SCEtemp',]
+}
+
+sum_tests = [
+    'imaginary_radiance_check', 'hbb_std_dev_check', 'hbb_lw_nen_check',
+    'hbb_sw_nen_check', 'lw_responsivity_check', 'sw_responsivity_check',
+    'sky_brightness_temp_spectral_averages_ch1_check',
+    'sky_brightness_temp_spectral_averages_ch2_check'
+]
+
+def plot(test_name, path):
+    data = housekeeping.get_all_housekeeping(path)
+
+    fig = plt.figure(figsize=(25,15))
+    for x in check_vars[test_name]:
+        plt.plot(data['Time'], data[x])
+    plt.suptitle(path, fontsize=20)
+    plt.ylabel(test_name, fontsize=18)
+    plt.legend(loc='upper right')
+    if path.endswith('B1.CXS'):
+        savename = path.split('/')[0] + '_' + path[-12:-6] + '_' + test_name + '.png'
+    elif path.endswith('.SUM'):
+        savename = path.split('/')[0] + '_' + path[-10:-4] + '_' + test_name + '.png'
+    #savename = path.split('/')[0] + '_' + path[:-6] + '_' + test_name + '.png'
+    #plt.savefig('/Users/adiebold/aeri_quality_control/testing/' + savename)
+    plt.savefig('/Users/adiebold/aeri_quality_control/testing/pngs/diagnostics/'
+                    + savename)
+    # plt.savefig('/Users/adiebold/aeri_quality_control/testing/testing_diagnostics/'
+                    # + savename)
+    # plt.show()
+    plt.close()
+
+def test_plot():
+    # pathname = '/Users/adiebold/aeri_quality_control/testing/'
+    # pathname += 'awr/AE160911/160911B1.CXS'
+    pathname = 'awr/AE160911/160911B1.CXS'
+    for v in check_vars:
+        print(v)
+        if v in sum_tests:
+            plot(v, pathname[:-6] + '.SUM')
+        else:
+            plot(v, pathname)
+
+if __name__ == '__main__':
+    test_plot()

testing/file_finder.py

+import os
+
+def traversal(path, skip_num, ending):
+    curr_num = 0
+    if os.path.isdir(path):
+        for filename_1 in os.listdir(path):
+            filename_1 = path + '/' + filename_1
+            filename_1 = filename_1.replace('//', '/')
+            if os.path.isdir(filename_1):
+                for filename_2 in os.listdir(filename_1):
+                    filename_2 = filename_1 + '/' + filename_2
+                    filename_2 = filename_2.replace('//', '/')
+                    if (os.path.isfile(filename_2) and filename_2.endswith(ending)):
+                        curr_num += 1
+                        if curr_num >= skip_num:
+                            print(curr_num, ': ', filename_2)
+                            yield filename_2
+                        else:
+                            print(curr_num, ': ', filename_2, ' -- SKIPPED')
+            elif os.path.isfile(filename_1) and filename_1.endswith(ending):
+                 curr_num += 1
+                 if curr_num >= skip_num:
+                     print(curr_num, ': ', filename_1)
+                     yield filename_1
+                 else:
+                     print(curr_num, ': ', filename_1, ' -- SKIPPED')
+    elif os.path.isfile(path):
+        if path.endswith(ending):
+            print(path)
+            yield path

testing/testing_quick_vis.py

@@ -8,6 +8,9 @@ from scipy import arange
 import numpy as np
 import pandas as pd
 
+import file_finder
+import diagnostic_plotter
+
 
 def main(filename):
 
@@ -28,6 +31,15 @@ def main(filename):
         'hbb_std_dev_check', 'hbb_lw_nen_check', 'hbb_sw_nen_check',
         'lw_responsivity_check', 'sw_responsivity_check',
         ]
+    vars_to_skip = [
+        'qc_percent', 'hatch_check', 'missing_data_flag_check', 'time'
+    ]
+    sum_tests = [
+        'imaginary_radiance_check', 'hbb_std_dev_check', 'hbb_lw_nen_check',
+        'hbb_sw_nen_check', 'lw_responsivity_check', 'sw_responsivity_check',
+        'sky_brightness_temp_spectral_averages_ch1_check',
+        'sky_brightness_temp_spectral_averages_ch2_check'
+    ]
 
     old_data = netCDF4.Dataset(filename).variables
     #accounts for times when the first or last values are not a time (NaT)
@@ -45,8 +57,28 @@ def main(filename):
 
     data = pd.DataFrame(index=range(len(old_data['time'])),
                         columns=old_data.keys())
-    for key in old_data:
-        data[key] = old_data[key]
+    #dictionary of percentages that each variable is valid
+    percents = {}
+    #text file or a report of the varialbes' validity
+    report_name = 'reports/' + filename.split('/')[0] + '_'
+    report_name += filename[-11:-5] + '.txt'
+    #create and save plots of variables when the test is <80% valid
+    #write percentages to report file
+    data['time'] = old_data['time']
+    with open(report_name, 'w') as f:
+        f.write(filename.split('/')[0] + '_' + report_name[-10:-4] + '\n\n')
+        for key in variable_order:
+            data[key] = old_data[key]
+            percents[key] = (100 * (1 - sum(data[key])
+                                / len(data[key])))
+            if percents[key] < 80 and key not in vars_to_skip:
+                if key in sum_tests:
+                    diagnostic_plotter.plot(key, filename[:-5] + '.SUM')
+                else:
+                    diagnostic_plotter.plot(key, filename[:-5] + 'B1.CXS')
+            print(key, ' = ', percents[key])
+            if percents[key] < 100:
+                f.write(key + ' = ' + '{:3.2f}'.format(percents[key]) + '%\n')
 
     #checks for any variables in the QC file not in variable_order
     for var_name in old_data:
@@ -69,9 +101,6 @@ def main(filename):
     data['time'] = (data['time']/1000000000/60/60
                     - data['time'].iloc[0]/1000000000/60/60)
     data.set_index('time', inplace=True)
-    #calculate how much of qc_percent is valid
-    qc_percent_num = (100 * (1 - sum(data['qc_percent'])
-                        / len(data['qc_percent'])))
     plt.figure(1)
     curr_plot_num = 0
 
@@ -80,18 +109,23 @@ def main(filename):
         #alter name to make it better formatted for the graph
         #use 30 spaces because that's what works best
         if '_check' in value:
-            var_name = value[:-6] + ' '*30
+            var_name = value[:-6]
         else:
-            var_name = value + ' '*30
+            var_name = value
         if 'spike' in var_name:
-            var_name = 'igm_spike' + ' '*30
-        if 'sky_brightness_temp_spectral_averages' in var_name:
+            var_name = 'igm_spike'
+        elif 'sky_brightness_temp_spectral_averages' in var_name:
             if 'ch1' in var_name:
-                var_name = 'surface_bt_ch1' + ' '*30
+                var_name = 'surface_bt_ch1'
             elif 'ch2' in var_name:
-                var_name = 'surface_bt_ch2' + ' '*30
+                var_name = 'surface_bt_ch2'
+        if percents[value] < 100:
+            var_name += ' ({:3.2f}%)'.format(percents[value])
+        var_name += ' '*50
+        '''
         if 'qc_percent' in var_name:
-            var_name = 'qc_percent ({:3.2f}%)'.format(qc_percent_num) + ' '*30
+            var_name = 'qc_percent ({:3.2f}%)'.format(percents['qc_percent']) + ' '*30
+        '''
 
         curr_plot_num += 1
         #doublecheck that value is actually a key
@@ -127,8 +161,9 @@ def main(filename):
 
     print('saving...')
     end_of_name = filename.split('/')[-1].split('.')[0] + '.png'
-    # plt.savefig('/Users/adiebold/aeri_quality_control/testing/pngs/awr/' + end_of_name)
-    plt.savefig('/Users/adiebold/aeri_quality_control/testing/testing_bst/pngs/' + end_of_name)
+    plt.savefig('/Users/adiebold/aeri_quality_control/testing/pngs/awr_new/' + end_of_name)
+    # plt.savefig('/Users/adiebold/aeri_quality_control/testing/testing_bst/pngs/' + end_of_name)
+    # plt.savefig('/Users/adiebold/aeri_quality_control/testing/' + end_of_name)
     #comment out plt.show() when doing a directory
     # plt.show()
     plt.clf()
@@ -147,9 +182,12 @@ if __name__ == '__main__':
     print(args.filepath)
 
     #amount of files to skip
-    skip_num = 0
+    skip_num = 0    #215 for sgp; 187 for awr
     curr_num = 0
     print('skip_num = ', skip_num, '\n')
+    for f in file_finder.traversal(args.filepath, skip_num, 'QC.nc'):
+        main(f)
+    '''
     if os.path.isdir(args.filepath):
         for filename_1 in os.listdir(args.filepath):
             filename_1 = args.filepath + '/' + filename_1
@@ -177,6 +215,7 @@ if __name__ == '__main__':
         if args.filepath.endswith('QC.nc'):
             print(args.filepath)
             main(args.filepath)
+    '''
 
     print('total execution time: %d minute(s), %.2f second(s)' %
             ((time.time()-start_time)//60, (time.time()-start_time)%60))