From c6d8c3cae7b21730a05611c7a54b06ea4f86b148 Mon Sep 17 00:00:00 2001
From: Paolo Veglio <paolo.veglio@ssec.wisc.edu>
Date: Wed, 30 Nov 2022 19:29:08 +0000
Subject: [PATCH] Vis/NIR roughly added. 11-4um BTD Test implemented for more
scenes but not all.
---
preprocess_thresholds.py | 30 ++++++++++++++++++++
tests.py | 33 +++++++++++++++++++++-
thresholds.mvcm.snpp.v0.0.1.yaml | 48 +++++++++++++++++---------------
3 files changed, 88 insertions(+), 23 deletions(-)
diff --git a/preprocess_thresholds.py b/preprocess_thresholds.py
index e02d497..38b61a0 100644
--- a/preprocess_thresholds.py
+++ b/preprocess_thresholds.py
@@ -351,6 +351,36 @@ def get_pn_thresholds(data, thresholds, scene, test_name):
return out_thr
+def get_nl_thresholds(data, threshold):
+ lo_val = threshold['bt_diff_bounds'][0]
+ hi_val = threshold['bt_diff_bounds'][1]
+ lo_val_thr = threshold['nl_11_4m'][0]
+ hi_val_thr = threshold['nl_11_4m'][1]
+ conf_range = threshold['nl_11_4m'][2]
+ power = threshold['nl_11_4m'][3]
+
+ a = (data['M15-M16'].values - lo_val) / (hi_val - lo_val)
+ midpt = lo_val_thr + a*(hi_val_thr - lo_val_thr)
+ hicut = midpt - conf_range
+ locut = midpt + conf_range
+
+ idx = np.nonzero(data['M15-M16'].values > threshold['bt_diff_bounds'][0])
+ locut[idx] = threshold['nl_11_4l'][0]
+ midpt[idx] = threshold['nl_11_4l'][1]
+ hicut[idx] = threshold['nl_11_4l'][2]
+ power[idx] = threshold['nl_11_4l'][3]
+
+ idx = np.nonzero(data['M15-M16'].values < threshold['bt_diff_bounds'][1])
+ locut[idx] = threshold['nl_11_4h'][0]
+ midpt[idx] = threshold['nl_11_4h'][1]
+ hicut[idx] = threshold['nl_11_4h'][2]
+ power[idx] = threshold['nl_11_4h'][3]
+
+ out_thr = xr.DataArray(data=np.stack((locut, midpt, hicut, np.ones(data.M01.shape), power)),
+ dims=('number_of_lines', 'number_of_pixels', 'z'))
+ return out_thr
+
+
def vis_refl_thresholds(data, thresholds, scene):
locut, midpt, hicut = get_b1_thresholds(data, thresholds)
diff --git a/tests.py b/tests.py
index 19cceb0..8e39ff9 100644
--- a/tests.py
+++ b/tests.py
@@ -281,6 +281,10 @@ class CloudTests:
elif test_name == 'Surface_Temperature_Test':
thr_xr['threshold'] = pt.preproc_surf_temp(self.data, self.thresholds[self.scene_name])
thr = np.ones((5,))
+ elif (test_name == '11-4um_Oceanic_Stratus_Test' and
+ self.scene_name in ['Land_Day_Desert', 'Land_Day_Desert_Coast', 'Polar_Day_Desert',
+ 'Polar_Day_Desert_Coast']):
+ thr = np.array([self.thresholds[self.scene_name][test_name][i] for i in range(8)])
elif test_name == 'NIR_Reflectance_Test':
corr_thr = pt.preproc_nir(self.data, self.thresholds, self.scene_name)
thr_xr['threshold'] = (('number_of_lines', 'number_of_pixels', 'z'), corr_thr)
@@ -292,6 +296,9 @@ class CloudTests:
thr_xr['threshold'] = pt.get_pn_thresholds(self.data, self.thresholds, self.scene_name,
'4-12um_BTD_Thin_Cirrus_Test')
thr = np.ones((5,))
+ elif test_name == 'Vis/NIR_Ratio_Test':
+ thr_no_sunglint = np.array([self.thresholds[self.scene_name][test_name][i] for i in range(8)])
+ thr_sunglint = np.array([self.thresholds['Sun_Glint']['snglnt'][i] for i in range(8)])
else:
thr_xr['threshold'] = (('number_of_lines', 'number_of_pixels', 'z'),
np.ones((self.data[band].shape[0], self.data[band].shape[1], 5))*thr)
@@ -309,7 +316,31 @@ class CloudTests:
if test_name == 'NIR_Reflectance_Test':
pass
- if thr[4] == 1:
+ # if thr[4] == 1:
+ if (test_name == '11-4um_Oceanic_Stratus_Test' and
+ self.scene_name in ['Land_Day_Desert', 'Land_Day_Desert_Coast', 'Polar_Day_Desert',
+ 'Polar_Day_Desert_Coast']):
+ print('test running...')
+ confidence = conf.conf_test_dble(data['M15-M16'].values, thr)
+ confidence = confidence.reshape(data.M01.shape)
+ elif test_name == 'Vis/NIR_Ratio_Test':
+ vrat = data.M07.values/data.M05.values
+ print('test running...')
+ _dtr = np.pi/180.0
+ sza = data.sensor_zenith.values
+ raz = data.relative_azimuth.values
+ vza = data.sensor_zenith.values
+ cos_refang = np.sin(vza*_dtr) * np.sin(sza*_dtr) * np.cos(raz*_dtr) + \
+ np.cos(vza*_dtr) * np.cos(sza*_dtr)
+ refang = np.arccos(cos_refang) * 180./np.pi
+ idx = np.nonzero((data.solar_zenith <= 85) & (refang <= data.sunglint_angle))
+
+ confidence = conf.conf_test_dble(vrat, thr_no_sunglint)
+ confidence = confidence.reshape(data.M01.shape)
+ confidence[idx] = conf.conf_test_dble(vrat[idx], thr_sunglint)
+
+ confidence = confidence.reshape(data.M01.shape)
+ else:
print('test running...')
confidence = conf_xr.conf_test(data, band)
diff --git a/thresholds.mvcm.snpp.v0.0.1.yaml b/thresholds.mvcm.snpp.v0.0.1.yaml
index 9a1e87e..fd7af27 100644
--- a/thresholds.mvcm.snpp.v0.0.1.yaml
+++ b/thresholds.mvcm.snpp.v0.0.1.yaml
@@ -75,8 +75,15 @@ Land_Day_Desert:
coeffs: [3.5, 1.0]
cmult: 0.3
adj: 1.25
- lds11_4hi : [2.0, 0.0, -2.0, 1.00, 1.0]
- lds11_4lo : [-28.0, -26.0, -24.0, 1.00]
+ 11-4um_Oceanic_Stratus_Test: [-28.0, -26.0, -24.0, -2.0, 0.0, 2.0, 1.0, 1.0] # this preplaces lds11_4hi and
+ # lds11_4lo. The values are
+ # sorted left to right so that
+ # they follow the scheme:
+ # Lo-Mid-Hi--Hi-Mid-Lo
+ # This is the opposite of
+ # Ocean_Day
+ # lds11_4hi : [2.0, 0.0, -2.0, 1.00, 1.0]
+ # lds11_4lo : [-28.0, -26.0, -24.0, 1.00]
CO2_High_Clouds_Test: [222.0, 224.0, 226.0, 1.0, 1.0]
Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
Visible_Reflectance_Test:
@@ -98,8 +105,7 @@ Land_Day_Desert_Coast:
coeffs: [3.5, 1.0]
cmult: 0.3
adj: 1.25
- lds11_4hi_c : [2.00, 0.00, -2.00, 1.00]
- lds11_4lo_c : [-23.0, -21.0, -19.0, 1.00, 1.0]
+ 11-4um_Oceanic_Stratus_Test: [-23.0, -21.0, -19.0, -2.0, 0.0, 2.0, 1.0, 1.0]
CO2_High_Clouds_Test: [222.0, 224.0, 226.0, 1.0, 1.0]
Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
Visible_Reflectance_Test:
@@ -135,12 +141,12 @@ Ocean_Day:
locut_coeff: 0.0100
# vnir_ratio_hi : [1.001, 1.154, 1.205, 1.0, 1.0]
# vnir_ratio_lo : [0.934, 0.889, 0.837, 1.0]
- vis_nir_ratio : [0.837, 0.889, 0.934, 1.001, 1.154, 1.205, 1.0, 1.0] # This replace dovrathi and dovratlo
- # The values are sorted left to right
- # so that they follow the scheme:
- # Hi-Mid-Lo--Lo-Mid-Hi
- # Last two values are power and switch
- # to turn the test on/off
+ Vis/NIR_Ratio_Test: [0.837, 0.889, 0.934, 1.001, 1.154, 1.205, 1.0, 1.0] # This replace dovrathi and dovratlo
+ # The values are sorted left to right
+ # so that they follow the scheme:
+ # Hi-Mid-Lo--Lo-Mid-Hi
+ # Last two values are power and switch
+ # to turn the test on/off
8.6-11um_Test: [-0.50, -1.00, -1.50, 1.0, 1.0]
SST_Test:
thr: [3.000, 2.500, 2.000, 1.0, 1.0]
@@ -238,9 +244,7 @@ Polar_Day_Desert:
coeffs: [3.5, 1.0]
cmult: 0 # I NEED TO WORK ON THIS
adj: 0 # I NEED TO WORK ON THIS
- pds11_4hi : [2.00, 0.00, -2.00, 1.00]
- pds11_4lo : [-22.0, -20.0, -18.0, 1.00]
- pds11_4_pfm : 1.0
+ 11-4um_Oceanic_Stratus_Test: [-22.0, -20.0, -18.0, -2.0, 0.0, 2.0, 1.0, 1.0]
Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
Visible_Reflectance_Test:
thr: [0.326, 0.288, 0.250, 1.00, 1.0]
@@ -257,9 +261,7 @@ Polar_Day_Desert_Coast:
coeffs: [3.5, 1.0]
cmult: 0 # I NEED TO WORK ON THIS
adj: 0 # I NEED TO WORK ON THIS
- pds11_4hi_c : [2.00, 0.00, -2.00, 1.00]
- pds11_4lo_c : [-23.0, -21.0, -19.0, 1.00]
- pds11_4_c_pfm : 1.0
+ 11-4um_Oceanic_Stratus_Test: [-23.0, -21.0, -19.0, -2.0, 0.0, 2.0, 1.0, 1.0]
Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
Visible_Reflectance_Test:
thr: [0.326, 0.288, 0.250, 1.00, 1.0]
@@ -344,12 +346,11 @@ Polar_Ocean_Day:
11-4um_Oceanic_Stratus_Test: [-11.0, -9.0, -7.0, 1.0, 1.0]
11um_Test: [267.0, 270.0, 273.0, 1.0, 1.0]
Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
- pdovrathi : [1.001, 1.154, 1.205, 1.0, 1.0]
- pdovratlo : [0.934, 0.889, 0.837, 1.0]
+ Vis/NIR_Ratio_Test: [0.837, 0.889, 0.934, 1.001, 1.154, 1.205, 1.0, 1.0]
8.6-11um_Test: [-0.50, -1.00, -1.50, 1.0, 1.0]
SST_Test:
thr: [3.000, 2.500, 2.000, 1.0, 1.0]
- coeffs: [1.998, 0.938, 0.128, 1.094]
+ coeffs: [1.886, 0.938, 0.128, 1.094]
pdo_b7pfm : 1.0
pdo_b26pfm : 1.0
pdo_b2bias_adj : 0.96
@@ -440,9 +441,12 @@ Sun_Glint:
# snglntv : [1.02, 1.17]
# snglntvch : [0.97, 1.28]
# snglntvcl : [1.07, 1.12]
- snglnt : [0.97, 1.02, 1.07, 1.12, 1.17, 1.28] # This replaces snglntv, snglntvch snglntvcl
- # The values are sorted left to right such that
- # Hi-Mid-Lo--Lo-Mid-Hi
+ snglnt : [0.97, 1.02, 1.07, 1.12, 1.17, 1.28, 1.0, 1.0] # This replaces snglntv, snglntvch snglntvcl
+ # The values are sorted left to right such that
+ # Hi-Mid-Lo--Lo-Mid-Hi
+ # I also added the two 1.0 at the end to
+ # maintain consistency with the other
+ # thresholds for conf_test_dble
sg_tbdfl : 13.0
sngm09 : 0.568
sngm02vm : 0.001
--
GitLab