From 2c00ad8df50a7ae36ee4b0f78b4796af128ea268 Mon Sep 17 00:00:00 2001
From: Coda Phillips <cphillips@sean.ssec.wisc.edu>
Date: Thu, 30 Jun 2016 14:16:38 -0500
Subject: [PATCH] Add qc_notes column
---
electronic_checks.py | 3 +++
main.py | 43 +++++++++++++++++++++++++++++++++++++--
scene_checks.py | 12 +++++++----
state_checks.py | 9 ++++++---
test.py | 26 +++++++++++++++++++++++-
util.py | 48 +++++++++++++++++++++++++-------------------
6 files changed, 110 insertions(+), 31 deletions(-)
diff --git a/electronic_checks.py b/electronic_checks.py
index 5761ff3..2f65033 100644
--- a/electronic_checks.py
+++ b/electronic_checks.py
@@ -68,6 +68,7 @@ def test_hbb_temp_outlier_check():
'HBBapexTemp':[0,1,10,1],
'HBBbottomTemp':[1,1,1,1],
'HBBtopTemp':[0,1,10,1],
+ 'qc_notes':''
})
assert hbb_temp_outlier_check(frame, {})['hbb_temp_outlier_check'].values.tolist() == [0,0,1,0]
@@ -76,11 +77,13 @@ def test_abb_temp_outlier_check():
'ABBapexTemp':[0,1,10,1],
'ABBbottomTemp':[1,1,1,1],
'ABBtopTemp':[0,1,10,1],
+ 'qc_notes':''
})
assert abb_temp_outlier_check(frame, {})['abb_temp_outlier_check'].values.tolist() == [0,0,1,0]
def test_calibrationambienttemp_temp_outlier_check():
frame = pd.DataFrame({
'calibrationAmbientTemp':[0,1,10,1],
+ 'qc_notes':''
})
assert calibrationambienttemp_outlier_check(frame, {})['calibrationambienttemp_outlier_check'].values.tolist() == [0,0,1,0]
diff --git a/main.py b/main.py
index 497bd77..e6df9e0 100644
--- a/main.py
+++ b/main.py
@@ -1,3 +1,8 @@
+import os
+from glob import glob
+import re
+#from aeri_tools.io.dmv.housekeeping import get_all_housekeeping
+
import electronic_checks
import global_checks
import radiometric_checks
@@ -14,12 +19,46 @@ levels = [
thermal_checks.CheckList()
]
-def read_frame(path):
- pass
+def read_frame(cxs_file, sum_file):
+ return get_all_housekeeping(cxs_file).combine_first(
+ get_all_housekeeping(sum_file))
def check_frame(frame, parameters):
frame['qc_percent'] = 0
+ frame['qc_notes'] = None
for level in levels:
level.set_params(parameters)
frame = level.compute(frame)
return frame
+
+def update_all(ftp_dir, parameters=None):
+ cxs_files = glob(os.path.join(ftp_dir,'AE*','*B1.CXS'))
+ for qc_file, cxs_file, sum_file in files_to_update(cxs_files):
+ hk = read_frame(cxs_file, sum_file)
+ if parameters is None:
+ parameters = {}
+ check_frame(frame, parameters)
+
+def files_to_update(cxs_files):
+ for cxs_file in cxs_files:
+ possible_sum = os.path.join(os.path.dirname(cxs_file), cxs_file.replace('B1.CXS','.SUM'))
+ possible_qc = os.path.join(os.path.dirname(cxs_file), cxs_file.replace('B1.CXS','.qc'))
+
+ if os.path.isfile(possible_sum):
+ sum_file = possible_sum
+
+ if os.path.isfile(possible_qc):
+ qc_file = possible_qc
+ if max(os.path.getmtime(sum_file), os.path.getmtime(cxs_file)) > os.path.getmtime(qc_file):
+ yield (qc_file, cxs_file, sum_file)
+ else:
+ yield (qc_file, cxs_file, sum_file)
+
+if __name__ == '__main__':
+ import argparse
+ parser = argparse.ArgumentParser()
+ parser.add_argument('ftp')
+
+ args = parser.parse_args()
+
+ update_all(args.ftp)
diff --git a/scene_checks.py b/scene_checks.py
index 8a3fe88..e974f33 100644
--- a/scene_checks.py
+++ b/scene_checks.py
@@ -42,13 +42,15 @@ class CheckList(BaseCheckList):
def test_hatch_check():
frame = pd.DataFrame({
'hatchOpen':[1,1,0],
- 'sceneMirrorPosition':[ord('H'), ord('A'), ord('S')]
+ 'sceneMirrorPosition':[ord('H'), ord('A'), ord('S')],
+ 'qc_notes':''
})
assert hatch_check(frame, {})['hatch_check'].values.tolist() == [0,0,1]
frame = pd.DataFrame({
'hatchOpen':[1,0,1],
- 'sceneMirrorPosition':[ord('H'), ord('A'), ord('S')]
+ 'sceneMirrorPosition':[ord('H'), ord('A'), ord('S')],
+ 'qc_notes':''
})
assert hatch_check(frame, {})['hatch_check'].values.tolist() == [0,0,0]
@@ -56,13 +58,15 @@ def test_hatch_check():
def test_safing_check():
frame = pd.DataFrame({
'hatchOpen':[0,0,0],
- 'sceneMirrorPosition':[ord('H'), ord('A'), ord('S')]
+ 'sceneMirrorPosition':[ord('H'), ord('A'), ord('S')],
+ 'qc_notes':''
})
assert safing_check(frame, {})['safing_check'].values.tolist() == [0,0,0]
frame = pd.DataFrame({
'hatchOpen':[1,-3,0,0],
- 'sceneMirrorPosition':[ord('S'), ord('H'), ord('A'), ord('S')]
+ 'sceneMirrorPosition':[ord('S'), ord('H'), ord('A'), ord('S')],
+ 'qc_notes':''
})
assert safing_check(frame, {})['safing_check'].values.tolist() == [1,1,0,1]
diff --git a/state_checks.py b/state_checks.py
index df3f374..b852123 100644
--- a/state_checks.py
+++ b/state_checks.py
@@ -40,7 +40,8 @@ def test_hbb_thermistor_check():
'HBBbottomTemp':[300,333,336],
'HBBapexTemp':[300,333,336],
'HBBtopTemp':[300,333,336],
- 'sceneMirrorPosition':[ord('H'), ord('A'), ord('S')]
+ 'sceneMirrorPosition':[ord('H'), ord('A'), ord('S')],
+ 'qc_notes':''
}), {})
assert all(frame['hbb_thermistor_check'] == [1,0,1])
@@ -49,14 +50,16 @@ def test_hbb_thermistor_check2():
'HBBbottomTemp':[300,333,333],
'HBBapexTemp':[300,333,333],
'HBBtopTemp':[300,333,333],
- 'sceneMirrorPosition':[ord('H'), ord('A'), ord('S')]
+ 'sceneMirrorPosition':[ord('H'), ord('A'), ord('S')],
+ 'qc_notes':''
}), {})
assert all(frame['hbb_thermistor_check'] == [1,0,1])
def test_detector_check():
frame = detector_check(pd.DataFrame({
'detectorTemp':[50,100],
- 'sceneMirrorPosition':[ord('H'), ord('A')]
+ 'sceneMirrorPosition':[ord('H'), ord('A')],
+ 'qc_notes':''
}), {})
assert all(frame['detector_check'] == [0,1])
diff --git a/test.py b/test.py
index 5e759c1..ce085cc 100644
--- a/test.py
+++ b/test.py
@@ -27,6 +27,7 @@ def test_invalidate_record():
corrupt_cal = three_scenes.copy()
invalidated = invalidate_record(corrupt_cal, 3, 'test', .5)
assert invalidated.iloc[2].test == .5, invalidated.test
+ assert invalidated.iloc[4].test != .5, invalidated.test
assert invalidated.iloc[5].test == .5, invalidated.test
@@ -37,6 +38,29 @@ def test_invalidate_records():
invalidated = invalidate_records(corrupt_cal, 'test')
assert invalidated.iloc[2].test == 1, invalidated.test
+ assert invalidated.iloc[4].test == 0, invalidated.test
assert invalidated.iloc[5].test == 1, invalidated.test
-three_scenes = pd.DataFrame({'sceneMirrorPosition': list(map(ord, 'HASAHSAHSAH'))})
+def test_invalidate_record_and_annotate():
+ invalidated = invalidate_record(three_scenes.copy(), 2, 'test', .5, 'test annotation')
+ assert invalidated.iloc[2].test == .5
+ assert pd.np.isnan(invalidated.iloc[3].test)
+ assert pd.np.isnan(invalidated.iloc[1].test)
+ assert invalidated.iloc[2].qc_notes == 'test annotation'
+
+ tricky_index = three_scenes.copy()
+ tricky_index.index = tricky_index.index - 1
+ invalidated = invalidate_record(tricky_index, 2, 'test', .5, 'test annotation')
+ assert invalidated.iloc[2].test == .5, invalidated.test
+ assert pd.np.isnan(invalidated.iloc[3].test)
+ assert pd.np.isnan(invalidated.iloc[1].test)
+ assert invalidated.iloc[2].qc_notes == 'test annotation'
+
+ corrupt_cal = three_scenes.copy()
+ invalidated = invalidate_record(corrupt_cal, 3, 'test', .5)
+ assert invalidated.iloc[2].test == .5, invalidated.test
+ assert invalidated.iloc[2].qc_notes == 'invalid calibration:3'
+ assert invalidated.iloc[5].test == .5, invalidated.test
+ assert invalidated.iloc[5].qc_notes == 'invalid calibration:3'
+
+three_scenes = pd.DataFrame({'sceneMirrorPosition': list(map(ord, 'HASAHSAHSAH')), 'qc_notes':''})
diff --git a/util.py b/util.py
index a99954c..8269df5 100644
--- a/util.py
+++ b/util.py
@@ -7,39 +7,45 @@ def invalidate_records(frame, check_name):
invalidate_record(frame, index, check_name, percent)
return frame
-def invalidate_record(frame, index, check_name, value):
+def invalidate_record(frame, index, check_name, value, annotation=''):
frame.ix[frame.index[index], check_name] = value
+ if annotation:
+ if frame.ix[frame.index[index], 'qc_notes']:
+ frame.ix[frame.index[index], 'qc_notes'] = ','.join([frame.ix[frame.index[index],'qc_notes'], annotation])
+ else:
+ frame.ix[frame.index[index], 'qc_notes'] = annotation
+
corrupt_view = frame.iloc[index].sceneMirrorPosition
if corrupt_view in [ord('H'),ord('A')]:
+
+ def invalidate_neighbor(neighbor):
+ if frame.sceneMirrorPosition.iloc[neighbor] == corrupt_view:
+ # Made one cycle, break
+ return True
+ elif frame.sceneMirrorPosition.iloc[neighbor] in [ord('H'), ord('A')]:
+ # Skip opposite calibration views
+ return
+ else:
+ # Invalidate non-calibration views
+ frame.ix[frame.index[neighbor], check_name] = value
+ previous_notes = frame.ix[frame.index[neighbor], 'qc_notes']
+ if previous_notes:
+ frame.ix[frame.index[neighbor], 'qc_notes'] = ','.join([previous_notes, 'invalid calibration:{:d}'.format(index)])
+ else:
+ frame.ix[frame.index[neighbor], 'qc_notes'] = 'invalid calibration:{:d}'.format(index)
# Corrupt calibration view, must also invalidate neighboring scenes
_idx = index + 1
while _idx < len(frame):
- if frame.sceneMirrorPosition.iloc[_idx] == corrupt_view:
- # Made one cycle
+ if invalidate_neighbor(_idx):
break
- elif frame.sceneMirrorPosition.iloc[_idx] in [ord('H'), ord('A')]:
- # Skip opposite calibration views
- _idx += 1
- continue
- else:
- # Invalidate non-calibration views
- frame.ix[frame.index[_idx], check_name] = value
- _idx += 1
+ _idx += 1
_idx = index - 1
while _idx >= 0:
- if frame.sceneMirrorPosition.iloc[_idx] == corrupt_view:
- # Made one cycle
+ if invalidate_neighbor(_idx):
break
- elif frame.sceneMirrorPosition.iloc[_idx] in [ord('H'), ord('A')]:
- # Skip opposite calibration views
- _idx -= 1
- continue
- else:
- # Invalidate non-calibration views
- frame.ix[frame.index[_idx], check_name] = value
- _idx -= 1
+ _idx -= 1
return frame
--
GitLab