Skip to content
Snippets Groups Projects
Commit 056f7f16 authored by Coda Phillips's avatar Coda Phillips
Browse files

Add radiometric checks and start filtering again

parent c9fc42d6
No related branches found
No related tags found
No related merge requests found
Pipeline #
from util import BaseCheckList, annotate_all, _compute_robust_zscore, invalidate_records, update_variable_qc from util import BaseCheckList, annotate_all, _compute_robust_zscore, invalidate_records, update_variable_qc
import pandas as pd import pandas as pd
import numpy as np import numpy as np
import scipy.stats
def imaginary_radiance_check(frame, parameters): def imaginary_radiance_check(frame, parameters):
if 'skyViewImaginaryRadiance2510_2515' not in frame.columns: if 'skyViewImaginaryRadiance2510_2515' not in frame.columns:
...@@ -14,6 +15,25 @@ def imaginary_radiance_check(frame, parameters): ...@@ -14,6 +15,25 @@ def imaginary_radiance_check(frame, parameters):
def hbb_radiance_check(frame, parameters): def hbb_radiance_check(frame, parameters):
# Std dev, nen, lw, sw # Std dev, nen, lw, sw
if not np.in1d(['HBBviewStdDevRadiance985_990','LW_HBB_NEN','SW_HBB_NEN'], frame.columns).all():
return frame
hbb_std_dist = scipy.stats.chi2.fit(frame.HBBviewStdDevRadiance985_990)
_, hbb_std_dist_hi = scipy.stats.chi2.interval(.995, *hbb_std_dist)
hbb_std_dist_problem = frame.HBBviewStdDevRadiance985_990 > hbb_std_dist_hi
lw_hbb_nen_dist = scipy.stats.chi2.fit(frame.LW_HBB_NEN)
_, lw_hbb_nen_hi = scipy.stats.chi2.interval(.995, *lw_hbb_nen_dist)
lw_hbb_nen_problem = frame.LW_HBB_NEN > lw_hbb_nen_hi
sw_hbb_nen_dist = scipy.stats.chi2.fit(frame.SW_HBB_NEN)
_, sw_hbb_nen_hi = scipy.stats.chi2.interval(.995, *sw_hbb_nen_dist)
sw_hbb_nen_problem = frame.SW_HBB_NEN > sw_hbb_nen_hi
frame['hbb_radiance_check'] = (hbb_std_dist_problem | lw_hbb_nen_problem | sw_hbb_nen_problem) * 1
annotate_all(frame, hbb_std_dist_problem, 'HBB radiance Std.Dev. too high')
annotate_all(frame, lw_hbb_nen_problem, 'LW HBB NEN too high')
annotate_all(frame, sw_hbb_nen_problem, 'SW HBB NEN too high')
frame = invalidate_records(frame, 'hbb_radiance_check')
return frame return frame
def responsivity_check(frame, parameters): def responsivity_check(frame, parameters):
......
...@@ -103,7 +103,7 @@ class BaseCheckList: ...@@ -103,7 +103,7 @@ class BaseCheckList:
def compute(self, frame): def compute(self, frame):
# Filter bad records from previous level # Filter bad records from previous level
filtered_frame = frame.copy()#.ix[frame.qc_percent < 1].copy() filtered_frame = frame.ix[frame.qc_percent < 1].copy()
for check in self.checks: for check in self.checks:
original_shape = filtered_frame.shape original_shape = filtered_frame.shape
filtered_frame = check(filtered_frame, self.parameters) filtered_frame = check(filtered_frame, self.parameters)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment