Add radiometric checks and start filtering again

radiometric_checks.py

 from util import BaseCheckList, annotate_all, _compute_robust_zscore, invalidate_records, update_variable_qc
 import pandas as pd
 import numpy as np
+import scipy.stats
 
 def imaginary_radiance_check(frame, parameters):
     if 'skyViewImaginaryRadiance2510_2515' not in frame.columns:
@@ -14,6 +15,25 @@ def imaginary_radiance_check(frame, parameters):
 
 def hbb_radiance_check(frame, parameters):
     # Std dev, nen, lw, sw
+    if not np.in1d(['HBBviewStdDevRadiance985_990','LW_HBB_NEN','SW_HBB_NEN'], frame.columns).all():
+        return frame
+    hbb_std_dist = scipy.stats.chi2.fit(frame.HBBviewStdDevRadiance985_990)
+    _, hbb_std_dist_hi = scipy.stats.chi2.interval(.995, *hbb_std_dist)
+    hbb_std_dist_problem = frame.HBBviewStdDevRadiance985_990 > hbb_std_dist_hi
+
+    lw_hbb_nen_dist = scipy.stats.chi2.fit(frame.LW_HBB_NEN)
+    _, lw_hbb_nen_hi = scipy.stats.chi2.interval(.995, *lw_hbb_nen_dist)
+    lw_hbb_nen_problem = frame.LW_HBB_NEN > lw_hbb_nen_hi
+
+    sw_hbb_nen_dist = scipy.stats.chi2.fit(frame.SW_HBB_NEN)
+    _, sw_hbb_nen_hi = scipy.stats.chi2.interval(.995, *sw_hbb_nen_dist)
+    sw_hbb_nen_problem = frame.SW_HBB_NEN > sw_hbb_nen_hi
+
+    frame['hbb_radiance_check'] = (hbb_std_dist_problem | lw_hbb_nen_problem | sw_hbb_nen_problem) * 1
+    annotate_all(frame, hbb_std_dist_problem, 'HBB radiance Std.Dev. too high')
+    annotate_all(frame, lw_hbb_nen_problem, 'LW HBB NEN too high')
+    annotate_all(frame, sw_hbb_nen_problem, 'SW HBB NEN too high')
+    frame = invalidate_records(frame, 'hbb_radiance_check')
     return frame
 
 def responsivity_check(frame, parameters):

util.py

@@ -103,7 +103,7 @@ class BaseCheckList:
 
     def compute(self, frame):
         # Filter bad records from previous level
-        filtered_frame = frame.copy()#.ix[frame.qc_percent < 1].copy()
+        filtered_frame = frame.ix[frame.qc_percent < 1].copy()
         for check in self.checks:
             original_shape = filtered_frame.shape
             filtered_frame = check(filtered_frame, self.parameters)