From 056f7f161d8b9cdaab56614801c6869eae03baf5 Mon Sep 17 00:00:00 2001
From: Coda Phillips <cphillips@sean.ssec.wisc.edu>
Date: Mon, 11 Jul 2016 15:34:49 -0500
Subject: [PATCH] Add radiometric checks and start filtering again
---
radiometric_checks.py | 20 ++++++++++++++++++++
util.py | 2 +-
2 files changed, 21 insertions(+), 1 deletion(-)
diff --git a/radiometric_checks.py b/radiometric_checks.py
index 4c1347c..545f713 100644
--- a/radiometric_checks.py
+++ b/radiometric_checks.py
@@ -1,6 +1,7 @@
from util import BaseCheckList, annotate_all, _compute_robust_zscore, invalidate_records, update_variable_qc
import pandas as pd
import numpy as np
+import scipy.stats
def imaginary_radiance_check(frame, parameters):
if 'skyViewImaginaryRadiance2510_2515' not in frame.columns:
@@ -14,6 +15,25 @@ def imaginary_radiance_check(frame, parameters):
def hbb_radiance_check(frame, parameters):
# Std dev, nen, lw, sw
+ if not np.in1d(['HBBviewStdDevRadiance985_990','LW_HBB_NEN','SW_HBB_NEN'], frame.columns).all():
+ return frame
+ hbb_std_dist = scipy.stats.chi2.fit(frame.HBBviewStdDevRadiance985_990)
+ _, hbb_std_dist_hi = scipy.stats.chi2.interval(.995, *hbb_std_dist)
+ hbb_std_dist_problem = frame.HBBviewStdDevRadiance985_990 > hbb_std_dist_hi
+
+ lw_hbb_nen_dist = scipy.stats.chi2.fit(frame.LW_HBB_NEN)
+ _, lw_hbb_nen_hi = scipy.stats.chi2.interval(.995, *lw_hbb_nen_dist)
+ lw_hbb_nen_problem = frame.LW_HBB_NEN > lw_hbb_nen_hi
+
+ sw_hbb_nen_dist = scipy.stats.chi2.fit(frame.SW_HBB_NEN)
+ _, sw_hbb_nen_hi = scipy.stats.chi2.interval(.995, *sw_hbb_nen_dist)
+ sw_hbb_nen_problem = frame.SW_HBB_NEN > sw_hbb_nen_hi
+
+ frame['hbb_radiance_check'] = (hbb_std_dist_problem | lw_hbb_nen_problem | sw_hbb_nen_problem) * 1
+ annotate_all(frame, hbb_std_dist_problem, 'HBB radiance Std.Dev. too high')
+ annotate_all(frame, lw_hbb_nen_problem, 'LW HBB NEN too high')
+ annotate_all(frame, sw_hbb_nen_problem, 'SW HBB NEN too high')
+ frame = invalidate_records(frame, 'hbb_radiance_check')
return frame
def responsivity_check(frame, parameters):
diff --git a/util.py b/util.py
index e6ee40e..8df0278 100644
--- a/util.py
+++ b/util.py
@@ -103,7 +103,7 @@ class BaseCheckList:
def compute(self, frame):
# Filter bad records from previous level
- filtered_frame = frame.copy()#.ix[frame.qc_percent < 1].copy()
+ filtered_frame = frame.ix[frame.qc_percent < 1].copy()
for check in self.checks:
original_shape = filtered_frame.shape
filtered_frame = check(filtered_frame, self.parameters)
--
GitLab