From 13910803d32b65dda87acb48c89a600855b29a2c Mon Sep 17 00:00:00 2001
From: Coda Phillips <cphillips@sean.ssec.wisc.edu>
Date: Thu, 30 Jun 2016 16:22:40 -0500
Subject: [PATCH] Add annotations
---
electronic_checks.py | 49 +++++++++++++++++++++++---------------------
global_checks.py | 6 ++++--
scene_checks.py | 10 +++++----
state_checks.py | 30 ++++++++++++++++++++-------
util.py | 22 ++++++++++++--------
5 files changed, 72 insertions(+), 45 deletions(-)
diff --git a/electronic_checks.py b/electronic_checks.py
index 2f65033..dd3e394 100644
--- a/electronic_checks.py
+++ b/electronic_checks.py
@@ -1,40 +1,40 @@
-from util import BaseCheckList
+from util import BaseCheckList, annotate_all, invalidate_records
import numpy as np
import pandas as pd
-def hbb_temp_outlier_check(frame, parameters):
- if not np.in1d(['HBBbottomTemp','HBBapexTemp','HBBtopTemp'], frame.columns).all():
+def find_bb_outliers(frame, parameters, bb):
+ if not np.in1d(['{}bottomTemp'.format(bb),'{}apexTemp'.format(bb),'{}topTemp'.format(bb)], frame.columns).all():
return frame
window_length = parameters.get('window_length', 100)
- frame['hbb_temp_outlier_check'] = (
- _find_6sigma_outliers(frame['HBBbottomTemp'], window_length) |
- _find_6sigma_outliers(frame['HBBapexTemp'], window_length) |
- _find_6sigma_outliers(frame['HBBtopTemp'], window_length)
- ) * 1
- return frame
-
-def abb_temp_outlier_check(frame, parameters):
- if not np.in1d(['ABBbottomTemp','ABBapexTemp','ABBtopTemp'], frame.columns).all():
- return frame
+ bbb_outliers = _find_6sigma_outliers(frame['{}bottomTemp'.format(bb)], window_length)
+ bba_outliers = _find_6sigma_outliers(frame['{}apexTemp'.format(bb)], window_length)
+ bbt_outliers = _find_6sigma_outliers(frame['{}topTemp'.format(bb)], window_length)
- window_length = parameters.get('window_length', 100)
+ annotate_all(frame, bbb_outliers, '{} bottom temperature outlier'.format(bb))
+ annotate_all(frame, bba_outliers, '{} apex temperature outlier'.format(bb))
+ annotate_all(frame, bbt_outliers, '{} top temperature outlier'.format(bb))
- frame['abb_temp_outlier_check'] = (
- _find_6sigma_outliers(frame['ABBbottomTemp'], window_length, use_mean=True) |
- _find_6sigma_outliers(frame['ABBapexTemp'], window_length, use_mean=True) |
- _find_6sigma_outliers(frame['ABBtopTemp'], window_length, use_mean=True)
- ) * 1
+ frame['{}_temp_outlier_check'.format(bb.lower())] = ( bbb_outliers | bba_outliers | bbt_outliers ) * 1
+ frame = invalidate_records(frame, '{}_temp_outlier_check'.format(bb.lower()))
return frame
+def hbb_temp_outlier_check(frame, parameters):
+ return find_bb_outliers(frame, parameters, 'HBB')
+
+def abb_temp_outlier_check(frame, parameters):
+ return find_bb_outliers(frame, parameters, 'ABB')
+
def calibrationambienttemp_outlier_check(frame, parameters):
if 'calibrationAmbientTemp' not in frame.columns:
return frame
window_length = parameters.get('window_length', 100)
- frame['calibrationambienttemp_outlier_check'] = _find_6sigma_outliers(frame['calibrationAmbientTemp'], window_length, use_mean=True) * 1
+ temp_outliers = _find_6sigma_outliers(frame['calibrationAmbientTemp'], window_length, use_mean=True)
+ frame['calibrationambienttemp_outlier_check'] = temp_outliers * 1
+ annotate_all(frame, temp_outliers, 'calibrationAmbientTemp outlier')
return frame
class CheckList(BaseCheckList):
@@ -68,7 +68,8 @@ def test_hbb_temp_outlier_check():
'HBBapexTemp':[0,1,10,1],
'HBBbottomTemp':[1,1,1,1],
'HBBtopTemp':[0,1,10,1],
- 'qc_notes':''
+ 'qc_notes':'',
+ 'sceneMirrorPosition':[ord(x) for x in 'HASA']
})
assert hbb_temp_outlier_check(frame, {})['hbb_temp_outlier_check'].values.tolist() == [0,0,1,0]
@@ -77,13 +78,15 @@ def test_abb_temp_outlier_check():
'ABBapexTemp':[0,1,10,1],
'ABBbottomTemp':[1,1,1,1],
'ABBtopTemp':[0,1,10,1],
- 'qc_notes':''
+ 'qc_notes':'',
+ 'sceneMirrorPosition':[ord(x) for x in 'HASA']
})
assert abb_temp_outlier_check(frame, {})['abb_temp_outlier_check'].values.tolist() == [0,0,1,0]
def test_calibrationambienttemp_temp_outlier_check():
frame = pd.DataFrame({
'calibrationAmbientTemp':[0,1,10,1],
- 'qc_notes':''
+ 'qc_notes':'',
+ 'sceneMirrorPosition':[ord(x) for x in 'HASA']
})
assert calibrationambienttemp_outlier_check(frame, {})['calibrationambienttemp_outlier_check'].values.tolist() == [0,0,1,0]
diff --git a/global_checks.py b/global_checks.py
index 3bc5950..e44da3c 100644
--- a/global_checks.py
+++ b/global_checks.py
@@ -1,9 +1,11 @@
-from util import BaseCheckList
+from util import BaseCheckList, annotate_all
def check_missing_data_flag(frame, parameters):
if 'missingDataFlag' not in frame:
return frame
- frame['check_missing_data_flag'] = (frame['missingDataFlag'] == 1)*1
+ missing_data = (frame['missingDataFlag'] == 1)
+ frame['check_missing_data_flag'] = missing_data * 1
+ annotate_all(frame, missing_data, 'missing data')
return frame
class CheckList(BaseCheckList):
diff --git a/scene_checks.py b/scene_checks.py
index 9f4f812..9378384 100644
--- a/scene_checks.py
+++ b/scene_checks.py
@@ -1,4 +1,4 @@
-from util import BaseCheckList, invalidate_records
+from util import BaseCheckList, invalidate_records, annotate_all
import pandas as pd
import numpy as np
@@ -8,9 +8,11 @@ def hatch_check(frame, parameters):
"""
if not np.in1d(['hatchOpen','sceneMirrorPosition'], frame.columns).all():
return frame
+ hatch_closed_during_viewing = ((frame.hatchOpen != 1) &
+ (~frame.sceneMirrorPosition.isin([ord('H'), ord('A')])))
- frame['hatch_check'] = ((frame.hatchOpen != 1) &
- (~frame.sceneMirrorPosition.isin([ord('H'), ord('A')]))) * 1
+ frame['hatch_check'] = hatch_closed_during_viewing
+ annotate_all(frame, hatch_closed_during_viewing, 'hatch closed')
return frame
@@ -25,7 +27,7 @@ def safing_check(frame, parameters):
hatch_closing = ((frame.hatchOpen == 1) & ((frame.hatchOpen == -3).diff(-1) == 1)).shift(1)
mirror_safing = (hatch_closing & frame.sceneMirrorPosition.isin([ord('H'), ord('A')]))
frame['safing_check'] = mirror_safing * 1
- frame.ix[mirror_safing, 'qc_notes'] = 'mirror likely safed during view'
+ annotate_all(frame, mirror_safing, 'mirror likely safed during view')
frame = invalidate_records(frame, 'safing_check')
return frame
diff --git a/state_checks.py b/state_checks.py
index b852123..5f2ef63 100644
--- a/state_checks.py
+++ b/state_checks.py
@@ -1,4 +1,4 @@
-from util import BaseCheckList, invalidate_records
+from util import BaseCheckList, invalidate_records, annotate_all
import pandas as pd
import numpy as np
@@ -8,7 +8,9 @@ def detector_check(frame, parameters):
"""
if 'detectorTemp' not in frame.columns:
return frame
- frame['detector_check'] = (frame['detectorTemp'] > 90) * 1
+ detector_temp_too_high = (frame['detectorTemp'] > 90)
+ frame['detector_check'] = detector_temp_too_high * 1
+ annotate_all(frame, detector_temp_too_high, 'detector temperature too high')
frame = invalidate_records(frame, 'detector_check')
return frame
@@ -18,11 +20,22 @@ def hbb_thermistor_check(frame, parameters):
"""
if not np.in1d(['HBBbottomTemp','HBBapexTemp','HBBtopTemp'], frame.columns).all():
return frame
- frame['hbb_thermistor_check'] = (
- (abs(frame['HBBbottomTemp'] - 333) > 2) |
- (abs(frame['HBBapexTemp'] - 333) > 2) |
- (abs(frame['HBBtopTemp'] - 333) > 2)
- ) * 1
+ hbbb_too_low = (frame['HBBbottomTemp'] - 333) < -2
+ hbba_too_low = (frame['HBBapexTemp'] - 333) < -2
+ hbbt_too_low = (frame['HBBtopTemp'] - 333) < -2
+ hbbb_too_high = (frame['HBBbottomTemp'] - 333) > 2
+ hbba_too_high = (frame['HBBapexTemp'] - 333) > 2
+ hbbt_too_high = (frame['HBBtopTemp'] - 333) > 2
+ hbbb_problem = hbbb_too_low | hbbb_too_high
+ hbba_problem = hbba_too_low | hbba_too_high
+ hbbt_problem = hbbt_too_low | hbbt_too_high
+ frame['hbb_thermistor_check'] = (hbbb_problem | hbba_problem | hbbt_problem) * 1
+ annotate_all(frame, hbbb_too_low, 'HBB bottom temperature too low')
+ annotate_all(frame, hbbt_too_low, 'HBB top temperature too low')
+ annotate_all(frame, hbba_too_low, 'HBB apex temperature too low')
+ annotate_all(frame, hbbb_too_high, 'HBB bottom temperature too high')
+ annotate_all(frame, hbbt_too_high, 'HBB top temperature too high')
+ annotate_all(frame, hbba_too_high, 'HBB apex temperature too high')
frame = invalidate_records(frame, 'hbb_thermistor_check')
return frame
@@ -44,6 +57,8 @@ def test_hbb_thermistor_check():
'qc_notes':''
}), {})
assert all(frame['hbb_thermistor_check'] == [1,0,1])
+ assert all('HBB {} temperature too low'.format(x) in frame.iloc[0].qc_notes.split(',') for x in ['bottom','apex','top'])
+ assert all('HBB {} temperature too high'.format(x) in frame.iloc[2].qc_notes.split(',') for x in ['bottom','apex','top'])
def test_hbb_thermistor_check2():
frame = hbb_thermistor_check(pd.DataFrame({
@@ -62,4 +77,5 @@ def test_detector_check():
'qc_notes':''
}), {})
assert all(frame['detector_check'] == [0,1])
+ assert frame.iloc[1].qc_notes == 'detector temperature too high'
diff --git a/util.py b/util.py
index e07b54a..4e32d4f 100644
--- a/util.py
+++ b/util.py
@@ -2,6 +2,17 @@ from itertools import takewhile
import numpy as np
import pandas as pd
+def annotate(frame, loc, annotation):
+ notes = frame.loc[loc, 'qc_notes']
+ if type(notes) == str and len(notes) > 0:
+ frame.loc[loc, 'qc_notes'] = ','.join([notes, annotation])
+ else:
+ frame.loc[loc, 'qc_notes'] = annotation
+
+def annotate_all(frame, mask, annotation):
+ for loc in frame.index[mask]:
+ annotate(frame, loc, annotation)
+
def invalidate_records(frame, check_name):
for index,percent in frame.ix[frame[check_name] > 0, check_name].iteritems():
invalidate_record(frame, index, check_name, percent)
@@ -12,10 +23,7 @@ def invalidate_record(frame, loc, check_name, value, annotation=''):
if annotation:
if 'qc_notes' not in frame:
frame['qc_notes'] = None
- if type(frame.loc[loc, 'qc_notes']) == str and len(frame.loc[loc, 'qc_notes']) > 0:
- frame.loc[loc, 'qc_notes'] = ','.join([frame.loc[loc,'qc_notes'], annotation])
- else:
- frame.loc[loc, 'qc_notes'] = annotation
+ annotate(frame, loc, annotation)
corrupt_view = frame.loc[loc,'sceneMirrorPosition']
if corrupt_view in [ord('H'),ord('A')]:
@@ -30,11 +38,7 @@ def invalidate_record(frame, loc, check_name, value, annotation=''):
else:
# Invalidate non-calibration views
frame.loc[neighbor,check_name] = value
- previous_notes = frame.loc[neighbor,'qc_notes']
- if type(previous_notes) == str and len(previous_notes) > 0:
- frame.loc[neighbor,'qc_notes'] = ','.join([previous_notes, 'invalid calibration:{:d}'.format(loc)])
- else:
- frame.loc[neighbor,'qc_notes'] = 'invalid calibration:{:d}'.format(loc)
+ annotate(frame, neighbor, 'invalid calibration:{:d}'.format(loc))
# Corrupt calibration view, must also invalidate neighboring scenes
# _idx is the iloc
--
GitLab