Add annotations

electronic_checks.py

-from util import BaseCheckList
+from util import BaseCheckList, annotate_all, invalidate_records
 import numpy as np
 import pandas as pd
 
-def hbb_temp_outlier_check(frame, parameters):
-    if not np.in1d(['HBBbottomTemp','HBBapexTemp','HBBtopTemp'], frame.columns).all():
+def find_bb_outliers(frame, parameters, bb):
+    if not np.in1d(['{}bottomTemp'.format(bb),'{}apexTemp'.format(bb),'{}topTemp'.format(bb)], frame.columns).all():
         return frame
 
     window_length = parameters.get('window_length', 100)
 
-    frame['hbb_temp_outlier_check'] = (
-        _find_6sigma_outliers(frame['HBBbottomTemp'], window_length) |
-        _find_6sigma_outliers(frame['HBBapexTemp'], window_length) |
-        _find_6sigma_outliers(frame['HBBtopTemp'], window_length)
-     ) * 1
-    return frame
-
-def abb_temp_outlier_check(frame, parameters):
-    if not np.in1d(['ABBbottomTemp','ABBapexTemp','ABBtopTemp'], frame.columns).all():
-        return frame
+    bbb_outliers = _find_6sigma_outliers(frame['{}bottomTemp'.format(bb)], window_length)
+    bba_outliers = _find_6sigma_outliers(frame['{}apexTemp'.format(bb)], window_length)
+    bbt_outliers = _find_6sigma_outliers(frame['{}topTemp'.format(bb)], window_length)
 
-    window_length = parameters.get('window_length', 100)
+    annotate_all(frame, bbb_outliers, '{} bottom temperature outlier'.format(bb))
+    annotate_all(frame, bba_outliers, '{} apex temperature outlier'.format(bb))
+    annotate_all(frame, bbt_outliers, '{} top temperature outlier'.format(bb))
 
-    frame['abb_temp_outlier_check'] = (
-        _find_6sigma_outliers(frame['ABBbottomTemp'], window_length, use_mean=True) |
-        _find_6sigma_outliers(frame['ABBapexTemp'], window_length, use_mean=True) |
-        _find_6sigma_outliers(frame['ABBtopTemp'], window_length, use_mean=True)
-     ) * 1
+    frame['{}_temp_outlier_check'.format(bb.lower())] = ( bbb_outliers | bba_outliers | bbt_outliers ) * 1
+    frame = invalidate_records(frame, '{}_temp_outlier_check'.format(bb.lower()))
     return frame
 
+def hbb_temp_outlier_check(frame, parameters):
+    return find_bb_outliers(frame, parameters, 'HBB')
+
+def abb_temp_outlier_check(frame, parameters):
+    return find_bb_outliers(frame, parameters, 'ABB')
+
 def calibrationambienttemp_outlier_check(frame, parameters):
     if 'calibrationAmbientTemp' not in frame.columns:
         return frame
     
     window_length = parameters.get('window_length', 100)
 
-    frame['calibrationambienttemp_outlier_check'] = _find_6sigma_outliers(frame['calibrationAmbientTemp'], window_length, use_mean=True) * 1
+    temp_outliers = _find_6sigma_outliers(frame['calibrationAmbientTemp'], window_length, use_mean=True)
+    frame['calibrationambienttemp_outlier_check'] = temp_outliers * 1
+    annotate_all(frame, temp_outliers, 'calibrationAmbientTemp outlier')
     return frame
 
 class CheckList(BaseCheckList):
@@ -68,7 +68,8 @@ def test_hbb_temp_outlier_check():
         'HBBapexTemp':[0,1,10,1],
         'HBBbottomTemp':[1,1,1,1],
         'HBBtopTemp':[0,1,10,1],
-        'qc_notes':''
+        'qc_notes':'',
+        'sceneMirrorPosition':[ord(x) for x in 'HASA']
     })
     assert hbb_temp_outlier_check(frame, {})['hbb_temp_outlier_check'].values.tolist() == [0,0,1,0]
 
@@ -77,13 +78,15 @@ def test_abb_temp_outlier_check():
         'ABBapexTemp':[0,1,10,1],
         'ABBbottomTemp':[1,1,1,1],
         'ABBtopTemp':[0,1,10,1],
-        'qc_notes':''
+        'qc_notes':'',
+        'sceneMirrorPosition':[ord(x) for x in 'HASA']
     })
     assert abb_temp_outlier_check(frame, {})['abb_temp_outlier_check'].values.tolist() == [0,0,1,0]
 
 def test_calibrationambienttemp_temp_outlier_check():
     frame = pd.DataFrame({
         'calibrationAmbientTemp':[0,1,10,1],
-        'qc_notes':''
+        'qc_notes':'',
+        'sceneMirrorPosition':[ord(x) for x in 'HASA']
     })
     assert calibrationambienttemp_outlier_check(frame, {})['calibrationambienttemp_outlier_check'].values.tolist() == [0,0,1,0]

global_checks.py

-from util import BaseCheckList
+from util import BaseCheckList, annotate_all
 
 def check_missing_data_flag(frame, parameters):
     if 'missingDataFlag' not in frame:
         return frame
-    frame['check_missing_data_flag'] = (frame['missingDataFlag'] == 1)*1
+    missing_data = (frame['missingDataFlag'] == 1)
+    frame['check_missing_data_flag'] = missing_data * 1
+    annotate_all(frame, missing_data, 'missing data')
     return frame
 
 class CheckList(BaseCheckList):

scene_checks.py

-from util import BaseCheckList, invalidate_records
+from util import BaseCheckList, invalidate_records, annotate_all
 import pandas as pd
 import numpy as np
 
@@ -8,9 +8,11 @@ def hatch_check(frame, parameters):
     """
     if not np.in1d(['hatchOpen','sceneMirrorPosition'], frame.columns).all():
         return frame
+    hatch_closed_during_viewing = ((frame.hatchOpen != 1) &
+        (~frame.sceneMirrorPosition.isin([ord('H'), ord('A')])))
 
-    frame['hatch_check'] = ((frame.hatchOpen != 1) &
-        (~frame.sceneMirrorPosition.isin([ord('H'), ord('A')]))) * 1
+    frame['hatch_check'] = hatch_closed_during_viewing
+    annotate_all(frame, hatch_closed_during_viewing, 'hatch closed')
 
     return frame
 
@@ -25,7 +27,7 @@ def safing_check(frame, parameters):
     hatch_closing = ((frame.hatchOpen == 1)  & ((frame.hatchOpen == -3).diff(-1) == 1)).shift(1)
     mirror_safing = (hatch_closing & frame.sceneMirrorPosition.isin([ord('H'), ord('A')]))
     frame['safing_check'] = mirror_safing * 1
-    frame.ix[mirror_safing, 'qc_notes'] = 'mirror likely safed during view'
+    annotate_all(frame, mirror_safing, 'mirror likely safed during view')
     frame = invalidate_records(frame, 'safing_check')
     
     return frame

state_checks.py

-from util import BaseCheckList, invalidate_records
+from util import BaseCheckList, invalidate_records, annotate_all
 import pandas as pd
 import numpy as np
 
@@ -8,7 +8,9 @@ def detector_check(frame, parameters):
     """
     if 'detectorTemp' not in frame.columns:
         return frame
-    frame['detector_check'] = (frame['detectorTemp'] > 90) * 1
+    detector_temp_too_high = (frame['detectorTemp'] > 90)
+    frame['detector_check'] = detector_temp_too_high * 1
+    annotate_all(frame, detector_temp_too_high, 'detector temperature too high')
     frame = invalidate_records(frame, 'detector_check')
     return frame
 
@@ -18,11 +20,22 @@ def hbb_thermistor_check(frame, parameters):
     """
     if not np.in1d(['HBBbottomTemp','HBBapexTemp','HBBtopTemp'], frame.columns).all():
         return frame
-    frame['hbb_thermistor_check'] = (
-        (abs(frame['HBBbottomTemp'] - 333) > 2) |
-        (abs(frame['HBBapexTemp'] - 333) > 2) |
-        (abs(frame['HBBtopTemp'] - 333) > 2)
-    ) * 1
+    hbbb_too_low = (frame['HBBbottomTemp'] - 333) < -2
+    hbba_too_low = (frame['HBBapexTemp'] - 333) < -2
+    hbbt_too_low = (frame['HBBtopTemp'] - 333) < -2
+    hbbb_too_high = (frame['HBBbottomTemp'] - 333) > 2
+    hbba_too_high = (frame['HBBapexTemp'] - 333) > 2
+    hbbt_too_high = (frame['HBBtopTemp'] - 333) > 2
+    hbbb_problem = hbbb_too_low | hbbb_too_high
+    hbba_problem = hbba_too_low | hbba_too_high
+    hbbt_problem = hbbt_too_low | hbbt_too_high
+    frame['hbb_thermistor_check'] = (hbbb_problem | hbba_problem | hbbt_problem) * 1
+    annotate_all(frame, hbbb_too_low, 'HBB bottom temperature too low')
+    annotate_all(frame, hbbt_too_low, 'HBB top temperature too low')
+    annotate_all(frame, hbba_too_low, 'HBB apex temperature too low')
+    annotate_all(frame, hbbb_too_high, 'HBB bottom temperature too high')
+    annotate_all(frame, hbbt_too_high, 'HBB top temperature too high')
+    annotate_all(frame, hbba_too_high, 'HBB apex temperature too high')
     frame = invalidate_records(frame, 'hbb_thermistor_check')
     return frame
 
@@ -44,6 +57,8 @@ def test_hbb_thermistor_check():
         'qc_notes':''
     }), {})
     assert all(frame['hbb_thermistor_check'] == [1,0,1])
+    assert all('HBB {} temperature too low'.format(x) in frame.iloc[0].qc_notes.split(',') for x in ['bottom','apex','top'])
+    assert all('HBB {} temperature too high'.format(x) in frame.iloc[2].qc_notes.split(',') for x in ['bottom','apex','top'])
 
 def test_hbb_thermistor_check2():
     frame = hbb_thermistor_check(pd.DataFrame({
@@ -62,4 +77,5 @@ def test_detector_check():
         'qc_notes':''
     }), {})
     assert all(frame['detector_check'] == [0,1])
+    assert frame.iloc[1].qc_notes == 'detector temperature too high'
 

util.py

@@ -2,6 +2,17 @@ from itertools import takewhile
 import numpy as np
 import pandas as pd
 
+def annotate(frame, loc, annotation):
+    notes = frame.loc[loc, 'qc_notes']
+    if type(notes) == str and len(notes) > 0:
+        frame.loc[loc, 'qc_notes'] = ','.join([notes, annotation])
+    else:
+        frame.loc[loc, 'qc_notes'] = annotation
+
+def annotate_all(frame, mask, annotation):
+    for loc in frame.index[mask]:
+        annotate(frame, loc, annotation)
+
 def invalidate_records(frame, check_name):
     for index,percent in frame.ix[frame[check_name] > 0, check_name].iteritems():
         invalidate_record(frame, index, check_name, percent)
@@ -12,10 +23,7 @@ def invalidate_record(frame, loc, check_name, value, annotation=''):
     if annotation:
         if 'qc_notes' not in frame:
             frame['qc_notes'] = None
-        if type(frame.loc[loc, 'qc_notes']) == str and len(frame.loc[loc, 'qc_notes']) > 0:
-            frame.loc[loc, 'qc_notes'] = ','.join([frame.loc[loc,'qc_notes'], annotation])
-        else:
-            frame.loc[loc, 'qc_notes'] = annotation
+        annotate(frame, loc, annotation)
 
     corrupt_view = frame.loc[loc,'sceneMirrorPosition']
     if corrupt_view in [ord('H'),ord('A')]:
@@ -30,11 +38,7 @@ def invalidate_record(frame, loc, check_name, value, annotation=''):
             else:
                 # Invalidate non-calibration views
                 frame.loc[neighbor,check_name] = value
-                previous_notes = frame.loc[neighbor,'qc_notes']
-                if type(previous_notes) == str and len(previous_notes) > 0:
-                    frame.loc[neighbor,'qc_notes'] = ','.join([previous_notes, 'invalid calibration:{:d}'.format(loc)])
-                else:
-                    frame.loc[neighbor,'qc_notes'] = 'invalid calibration:{:d}'.format(loc)
+                annotate(frame, neighbor, 'invalid calibration:{:d}'.format(loc))
 
         # Corrupt calibration view, must also invalidate neighboring scenes
         # _idx is the iloc