Add qc_notes column

electronic_checks.py

@@ -68,6 +68,7 @@ def test_hbb_temp_outlier_check():
         'HBBapexTemp':[0,1,10,1],
         'HBBbottomTemp':[1,1,1,1],
         'HBBtopTemp':[0,1,10,1],
+        'qc_notes':''
     })
     assert hbb_temp_outlier_check(frame, {})['hbb_temp_outlier_check'].values.tolist() == [0,0,1,0]
 
@@ -76,11 +77,13 @@ def test_abb_temp_outlier_check():
         'ABBapexTemp':[0,1,10,1],
         'ABBbottomTemp':[1,1,1,1],
         'ABBtopTemp':[0,1,10,1],
+        'qc_notes':''
     })
     assert abb_temp_outlier_check(frame, {})['abb_temp_outlier_check'].values.tolist() == [0,0,1,0]
 
 def test_calibrationambienttemp_temp_outlier_check():
     frame = pd.DataFrame({
         'calibrationAmbientTemp':[0,1,10,1],
+        'qc_notes':''
     })
     assert calibrationambienttemp_outlier_check(frame, {})['calibrationambienttemp_outlier_check'].values.tolist() == [0,0,1,0]

main.py

+import os
+from glob import glob
+import re
+#from aeri_tools.io.dmv.housekeeping import get_all_housekeeping
+
 import electronic_checks
 import global_checks
 import radiometric_checks
@@ -14,12 +19,46 @@ levels = [
     thermal_checks.CheckList()
 ]
 
-def read_frame(path):
-    pass
+def read_frame(cxs_file, sum_file):
+    return get_all_housekeeping(cxs_file).combine_first(
+        get_all_housekeeping(sum_file))
 
 def check_frame(frame, parameters):
     frame['qc_percent'] = 0
+    frame['qc_notes'] = None
     for level in levels:
         level.set_params(parameters)
         frame = level.compute(frame)
     return frame
+
+def update_all(ftp_dir, parameters=None):
+    cxs_files = glob(os.path.join(ftp_dir,'AE*','*B1.CXS'))
+    for qc_file, cxs_file, sum_file in files_to_update(cxs_files):
+        hk = read_frame(cxs_file, sum_file)
+        if parameters is None:
+            parameters = {}
+        check_frame(frame, parameters)
+
+def files_to_update(cxs_files):
+    for cxs_file in cxs_files:
+        possible_sum = os.path.join(os.path.dirname(cxs_file), cxs_file.replace('B1.CXS','.SUM'))
+        possible_qc = os.path.join(os.path.dirname(cxs_file), cxs_file.replace('B1.CXS','.qc'))
+
+        if os.path.isfile(possible_sum):
+            sum_file = possible_sum
+
+            if os.path.isfile(possible_qc):
+                qc_file = possible_qc
+                if max(os.path.getmtime(sum_file), os.path.getmtime(cxs_file)) > os.path.getmtime(qc_file):
+                    yield (qc_file, cxs_file, sum_file)
+            else:
+                yield (qc_file, cxs_file, sum_file)
+
+if __name__ == '__main__':
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument('ftp')
+
+    args = parser.parse_args()
+
+    update_all(args.ftp)

scene_checks.py

@@ -42,13 +42,15 @@ class CheckList(BaseCheckList):
 def test_hatch_check():
     frame = pd.DataFrame({
         'hatchOpen':[1,1,0],
-        'sceneMirrorPosition':[ord('H'), ord('A'), ord('S')]
+        'sceneMirrorPosition':[ord('H'), ord('A'), ord('S')],
+        'qc_notes':''
     })
     assert hatch_check(frame, {})['hatch_check'].values.tolist() == [0,0,1]
 
     frame = pd.DataFrame({
         'hatchOpen':[1,0,1],
-        'sceneMirrorPosition':[ord('H'), ord('A'), ord('S')]
+        'sceneMirrorPosition':[ord('H'), ord('A'), ord('S')],
+        'qc_notes':''
     })
     assert hatch_check(frame, {})['hatch_check'].values.tolist() == [0,0,0]
     
@@ -56,13 +58,15 @@ def test_hatch_check():
 def test_safing_check():
     frame = pd.DataFrame({
         'hatchOpen':[0,0,0],
-        'sceneMirrorPosition':[ord('H'), ord('A'), ord('S')]
+        'sceneMirrorPosition':[ord('H'), ord('A'), ord('S')],
+        'qc_notes':''
     })
     assert safing_check(frame, {})['safing_check'].values.tolist() == [0,0,0]
 
     frame = pd.DataFrame({
         'hatchOpen':[1,-3,0,0],
-        'sceneMirrorPosition':[ord('S'), ord('H'), ord('A'), ord('S')]
+        'sceneMirrorPosition':[ord('S'), ord('H'), ord('A'), ord('S')],
+        'qc_notes':''
     })
     assert safing_check(frame, {})['safing_check'].values.tolist() == [1,1,0,1]
 

state_checks.py

@@ -40,7 +40,8 @@ def test_hbb_thermistor_check():
         'HBBbottomTemp':[300,333,336],
         'HBBapexTemp':[300,333,336],
         'HBBtopTemp':[300,333,336],
-        'sceneMirrorPosition':[ord('H'), ord('A'), ord('S')]
+        'sceneMirrorPosition':[ord('H'), ord('A'), ord('S')],
+        'qc_notes':''
     }), {})
     assert all(frame['hbb_thermistor_check'] == [1,0,1])
 
@@ -49,14 +50,16 @@ def test_hbb_thermistor_check2():
         'HBBbottomTemp':[300,333,333],
         'HBBapexTemp':[300,333,333],
         'HBBtopTemp':[300,333,333],
-        'sceneMirrorPosition':[ord('H'), ord('A'), ord('S')]
+        'sceneMirrorPosition':[ord('H'), ord('A'), ord('S')],
+        'qc_notes':''
     }), {})
     assert all(frame['hbb_thermistor_check'] == [1,0,1])
 
 def test_detector_check():
     frame = detector_check(pd.DataFrame({
         'detectorTemp':[50,100],
-        'sceneMirrorPosition':[ord('H'), ord('A')]
+        'sceneMirrorPosition':[ord('H'), ord('A')],
+        'qc_notes':''
     }), {})
     assert all(frame['detector_check'] == [0,1])
 

test.py

@@ -27,6 +27,7 @@ def test_invalidate_record():
     corrupt_cal = three_scenes.copy()
     invalidated = invalidate_record(corrupt_cal, 3, 'test', .5)
     assert invalidated.iloc[2].test == .5, invalidated.test
+    assert invalidated.iloc[4].test != .5, invalidated.test
     assert invalidated.iloc[5].test == .5, invalidated.test
 
 
@@ -37,6 +38,29 @@ def test_invalidate_records():
     
     invalidated = invalidate_records(corrupt_cal, 'test')
     assert invalidated.iloc[2].test == 1, invalidated.test
+    assert invalidated.iloc[4].test == 0, invalidated.test
     assert invalidated.iloc[5].test == 1, invalidated.test
 
-three_scenes = pd.DataFrame({'sceneMirrorPosition': list(map(ord, 'HASAHSAHSAH'))})
+def test_invalidate_record_and_annotate():
+    invalidated = invalidate_record(three_scenes.copy(), 2, 'test', .5, 'test annotation')
+    assert invalidated.iloc[2].test == .5
+    assert pd.np.isnan(invalidated.iloc[3].test)
+    assert pd.np.isnan(invalidated.iloc[1].test)
+    assert invalidated.iloc[2].qc_notes == 'test annotation'
+
+    tricky_index = three_scenes.copy()
+    tricky_index.index = tricky_index.index - 1
+    invalidated = invalidate_record(tricky_index, 2, 'test', .5, 'test annotation')
+    assert invalidated.iloc[2].test == .5, invalidated.test
+    assert pd.np.isnan(invalidated.iloc[3].test)
+    assert pd.np.isnan(invalidated.iloc[1].test)
+    assert invalidated.iloc[2].qc_notes == 'test annotation'
+
+    corrupt_cal = three_scenes.copy()
+    invalidated = invalidate_record(corrupt_cal, 3, 'test', .5)
+    assert invalidated.iloc[2].test == .5, invalidated.test
+    assert invalidated.iloc[2].qc_notes == 'invalid calibration:3'
+    assert invalidated.iloc[5].test == .5, invalidated.test
+    assert invalidated.iloc[5].qc_notes == 'invalid calibration:3'
+
+three_scenes = pd.DataFrame({'sceneMirrorPosition': list(map(ord, 'HASAHSAHSAH')), 'qc_notes':''})

util.py

@@ -7,39 +7,45 @@ def invalidate_records(frame, check_name):
         invalidate_record(frame, index, check_name, percent)
     return frame
 
-def invalidate_record(frame, index, check_name, value):
+def invalidate_record(frame, index, check_name, value, annotation=''):
     frame.ix[frame.index[index], check_name] = value
+    if annotation:
+        if frame.ix[frame.index[index], 'qc_notes']:
+            frame.ix[frame.index[index], 'qc_notes'] = ','.join([frame.ix[frame.index[index],'qc_notes'], annotation])
+        else:
+            frame.ix[frame.index[index], 'qc_notes'] = annotation
+
     corrupt_view = frame.iloc[index].sceneMirrorPosition
     if corrupt_view in [ord('H'),ord('A')]:
+    
+        def invalidate_neighbor(neighbor):
+            if frame.sceneMirrorPosition.iloc[neighbor] == corrupt_view:
+                # Made one cycle, break
+                return True
+            elif frame.sceneMirrorPosition.iloc[neighbor] in [ord('H'), ord('A')]:
+                # Skip opposite calibration views
+                return
+            else:
+                # Invalidate non-calibration views
+                frame.ix[frame.index[neighbor], check_name] = value
+                previous_notes = frame.ix[frame.index[neighbor], 'qc_notes']
+                if previous_notes:
+                    frame.ix[frame.index[neighbor], 'qc_notes'] = ','.join([previous_notes, 'invalid calibration:{:d}'.format(index)])
+                else:
+                    frame.ix[frame.index[neighbor], 'qc_notes'] = 'invalid calibration:{:d}'.format(index)
 
         # Corrupt calibration view, must also invalidate neighboring scenes
         _idx = index + 1
         while _idx < len(frame):
-            if frame.sceneMirrorPosition.iloc[_idx] == corrupt_view:
-                # Made one cycle
+            if invalidate_neighbor(_idx):
                 break
-            elif frame.sceneMirrorPosition.iloc[_idx] in [ord('H'), ord('A')]:
-                # Skip opposite calibration views
-                _idx += 1
-                continue
-            else:
-                # Invalidate non-calibration views
-                frame.ix[frame.index[_idx], check_name] = value
-                _idx += 1
+            _idx += 1
 
         _idx = index - 1
         while _idx >= 0:
-            if frame.sceneMirrorPosition.iloc[_idx] == corrupt_view:
-                # Made one cycle
+            if invalidate_neighbor(_idx):
                 break
-            elif frame.sceneMirrorPosition.iloc[_idx] in [ord('H'), ord('A')]:
-                # Skip opposite calibration views
-                _idx -= 1
-                continue
-            else:
-                # Invalidate non-calibration views
-                frame.ix[frame.index[_idx], check_name] = value
-                _idx -= 1
+            _idx -= 1
 
     return frame