Skip to content
Snippets Groups Projects
Commit 1ab331e9 authored by Paolo Veglio's avatar Paolo Veglio
Browse files

another bunch of tests implemented. also first look at restoral

parent 64486b66
No related branches found
No related tags found
No related merge requests found
Expand all Hide whitespace changes
Inline Side-by-side
preprocess_thresholds.py
+ 25
− 23
View file @ 1ab331e9
...@@ -266,21 +266,23 @@ def get_b1_thresholds(data, thresholds, scene_idx): ...@@ -266,21 +266,23 @@ def get_b1_thresholds(data, thresholds, scene_idx):
return locut, midpt, hicut return locut, midpt, hicut
def get_pn_thresholds(data, thresholds, scene, test_name): def polar_night_thresholds(data, thresholds, scene, test_name, scene_idx):
thresholds = thresholds[scene] thresholds = thresholds[scene]
if ((test_name == '4-12um_BTD_Thin_Cirrus_Test') and (scene in ['Land_Night', 'Night_Snow']) or if ((test_name == '4-12um_BTD_Thin_Cirrus_Test') and (scene in ['Land_Night', 'Night_Snow']) or
(test_name == '7.3-11um_BTD_Mid_Level_Cloud_Test') and (scene == 'Land_Night')): (test_name == '7.3-11um_BTD_Mid_Level_Cloud_Test') and (scene == 'Land_Night')):
locut = thresholds[test_name]['thr'][0] * np.ones(data.M15.shape) locut = thresholds[test_name]['thr'][0] * np.ones(data.M15.values[scene_idx].ravel())
midpt = thresholds[test_name]['thr'][1] * np.ones(data.M15.shape) midpt = thresholds[test_name]['thr'][1] * np.ones(data.M15.values[scene_idx].ravel())
hicut = thresholds[test_name]['thr'][2] * np.ones(data.M15.shape) hicut = thresholds[test_name]['thr'][2] * np.ones(data.M15.values[scene_idx].ravel())
power = thresholds[test_name]['thr'][3] * np.ones(data.M15.shape) power = thresholds[test_name]['thr'][3] * np.ones(data.M15.values[scene_idx].ravel())
out_thr = xr.DataArray(data=np.dstack((locut, midpt, hicut, np.ones(data.ndvi.shape), power)), # out_thr = xr.DataArray(data=np.dstack((locut, midpt, hicut, np.ones(data.ndvi.shape), power)),
dims=('number_of_lines', 'number_of_pixels', 'z')) # dims=('number_of_lines', 'number_of_pixels', 'z'))
return out_thr out_thr = np.dstack((locut, midpt, hicut, np.ones(locut.shape), power))
rad = data.M15.values.reshape(data.M15.shape[0]*data.M15.shape[1]) return out_thr.T
rad = data.M15.values[scene_idx].ravel()
bt_bounds = thresholds[test_name]['bt11_bounds'] bt_bounds = thresholds[test_name]['bt11_bounds']
locut, midpt = np.empty(rad.shape), np.empty(rad.shape) locut, midpt = np.empty(rad.shape), np.empty(rad.shape)
...@@ -343,15 +345,16 @@ def get_pn_thresholds(data, thresholds, scene, test_name): ...@@ -343,15 +345,16 @@ def get_pn_thresholds(data, thresholds, scene, test_name):
hicut[idx] = midpt[idx] - conf_range[idx] hicut[idx] = midpt[idx] - conf_range[idx]
locut[idx] = midpt[idx] + conf_range[idx] locut[idx] = midpt[idx] + conf_range[idx]
locut = locut.reshape(data.M15.shape) # locut = locut.reshape(data.M15.shape)
midpt = midpt.reshape(data.M15.shape) # midpt = midpt.reshape(data.M15.shape)
hicut = hicut.reshape(data.M15.shape) # hicut = hicut.reshape(data.M15.shape)
power = power.reshape(data.M15.shape) # power = power.reshape(data.M15.shape)
out_thr = xr.DataArray(data=np.dstack((locut, midpt, hicut, np.ones(data.ndvi.shape), power)), # out_thr = xr.DataArray(data=np.dstack((locut, midpt, hicut, np.ones(data.ndvi.shape), power)),
dims=('number_of_lines', 'number_of_pixels', 'z')) # dims=('number_of_lines', 'number_of_pixels', 'z'))
out_thr = np.dstack((locut, midpt, hicut, np.ones(locut.shape), power))
return out_thr return out_thr.T
def get_nl_thresholds(data, threshold): def get_nl_thresholds(data, threshold):
...@@ -433,24 +436,23 @@ def vis_refl_thresholds(data, thresholds, scene, scene_idx): ...@@ -433,24 +436,23 @@ def vis_refl_thresholds(data, thresholds, scene, scene_idx):
return np.squeeze(out_thr.T), out_rad return np.squeeze(out_thr.T), out_rad
def GEMI_thresholds(data, thresholds, scene_name, scene_idx): def gemi_thresholds(data, thresholds, scene_name, scene_idx):
thresh = thresholds[scene_name]['GEMI_Test']
ndvi = data.ndvi.values[scene_idx].ravel() ndvi = data.ndvi.values[scene_idx].ravel()
gemi_thr = np.ones((ndvi.shape[0], 5)) gemi_thr = np.ones((ndvi.shape[0], 5))
idx = np.nonzero(ndvi < 0.1) idx = np.nonzero(ndvi < 0.1)
gemi_thr[idx, :3] = thresh['gemi0'][:3] gemi_thr[idx, :3] = thresholds['gemi0'][:3]
idx = np.nonzero((ndvi >= 0.1) & (data.ndvi < 0.2)) idx = np.nonzero((ndvi >= 0.1) & (ndvi < 0.2))
gemi_thr[idx, :3] = thresh['gemi1'][:3] gemi_thr[idx, :3] = thresholds['gemi1'][:3]
idx = np.nonzero((ndvi >= 0.2) & (ndvi < 0.3)) idx = np.nonzero((ndvi >= 0.2) & (ndvi < 0.3))
gemi_thr[idx, :3] = thresh['gemi2'][:3] gemi_thr[idx, :3] = thresholds['gemi2'][:3]
# thr_out = xr.DataArray(data=np.dstack((gemi_thr[:, :, 0], gemi_thr[:, :, 1], gemi_thr[:, :, 2], # thr_out = xr.DataArray(data=np.dstack((gemi_thr[:, :, 0], gemi_thr[:, :, 1], gemi_thr[:, :, 2],
# np.ones(gemi_thr[:, :, 0].shape), # np.ones(gemi_thr[:, :, 0].shape),
# np.ones(gemi_thr[:, :, 0].shape))), # np.ones(gemi_thr[:, :, 0].shape))),
# dims=('number_of_lines', 'number_of_pixels', 'z')) # dims=('number_of_lines', 'number_of_pixels', 'z'))
return gemi_thr return gemi_thr.T
def bt11_4um_preproc(data, thresholds, scene_name): def bt11_4um_preproc(data, thresholds, scene_name):
......
spectral_tests.py 0 → 100644
+ 569
− 0
View file @ 1ab331e9
This diff is collapsed.
thresholds.mvcm.snpp.v0.0.1.yaml
+ 44
− 19
View file @ 1ab331e9
...@@ -10,7 +10,9 @@ Land_Day: ...@@ -10,7 +10,9 @@ Land_Day:
adj: 1.25 adj: 1.25
perform: True perform: True
dl_ref3_tpw : -10.00 dl_ref3_tpw : -10.00
11-4um_Oceanic_Stratus_Test: [-16.0, -14.0, -12.0, 1.0, 1.0] 11-4um_Oceanic_Stratus_Test:
thr: [-16.0, -14.0, -12.0, 1.0, 1.0]
perform: True
CO2_High_Clouds_Test: [222.0, 224.0, 226.0, 1.0, 1.0] CO2_High_Clouds_Test: [222.0, 224.0, 226.0, 1.0, 1.0]
Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0] Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
Visible_Reflectance_Test: Visible_Reflectance_Test:
...@@ -34,6 +36,7 @@ Land_Night: ...@@ -34,6 +36,7 @@ Land_Night:
perform: True perform: True
4-12um_BTD_Thin_Cirrus_Test: 4-12um_BTD_Thin_Cirrus_Test:
thr: [15.0, 10.0, 5.00, 1.0, 1.0] thr: [15.0, 10.0, 5.00, 1.0, 1.0]
perform: True
CO2_High_Clouds_Test: [222.0, 224.0, 226.0, 1.0, 1.0] CO2_High_Clouds_Test: [222.0, 224.0, 226.0, 1.0, 1.0]
Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0] Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
7.3-11um_BTD_Mid_Level_Cloud_Test: 7.3-11um_BTD_Mid_Level_Cloud_Test:
...@@ -65,7 +68,9 @@ Land_Day_Coast: ...@@ -65,7 +68,9 @@ Land_Day_Coast:
adj: 1.25 adj: 1.25
perform: True perform: True
dl_ref3_tpw_t2 : -10.00 dl_ref3_tpw_t2 : -10.00
11-4um_Oceanic_Stratus_Test: [-16.0, -14.0, -12.0, 1.0, 1.0] 11-4um_Oceanic_Stratus_Test:
thr: [-16.0, -14.0, -12.0, 1.0, 1.0]
perform: True
CO2_High_Clouds_Test: [222.0, 224.0, 226.0, 1.0, 1.0] CO2_High_Clouds_Test: [222.0, 224.0, 226.0, 1.0, 1.0]
Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0] Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
Visible_Reflectance_Test: Visible_Reflectance_Test:
...@@ -85,13 +90,11 @@ Land_Day_Desert: ...@@ -85,13 +90,11 @@ Land_Day_Desert:
cmult: 0.3 cmult: 0.3
adj: 1.25 adj: 1.25
perform: True perform: True
11-4um_Oceanic_Stratus_Test: [-28.0, -26.0, -24.0, -2.0, 0.0, 2.0, 1.0, 1.0] # this replaces lds11_4hi and 11-4um_Oceanic_Stratus_Test:
# lds11_4lo. The values are thr: [-28.0, -26.0, -24.0, -2.0, 0.0, 2.0, 1.0, 1.0] # this replaces lds11_4hi and lds11_4lo. The values
# sorted left to right so that perform: True # are sorted left to right so that they follow the
# they follow the scheme: # scheme: Lo-Mid-Hi--Hi-Mid-Lo This is the opposite
# Lo-Mid-Hi--Hi-Mid-Lo # of Ocean_Day
# This is the opposite of
# Ocean_Day
# lds11_4hi : [2.0, 0.0, -2.0, 1.00, 1.0] # lds11_4hi : [2.0, 0.0, -2.0, 1.00, 1.0]
# lds11_4lo : [-28.0, -26.0, -24.0, 1.00] # lds11_4lo : [-28.0, -26.0, -24.0, 1.00]
CO2_High_Clouds_Test: [222.0, 224.0, 226.0, 1.0, 1.0] CO2_High_Clouds_Test: [222.0, 224.0, 226.0, 1.0, 1.0]
...@@ -108,6 +111,7 @@ Land_Day_Desert: ...@@ -108,6 +111,7 @@ Land_Day_Desert:
gemi0: [0.085, 0.095, 0.115, 1.00, 1.0] gemi0: [0.085, 0.095, 0.115, 1.00, 1.0]
gemi1: [0.145, 0.170, 0.220, 1.00] gemi1: [0.145, 0.170, 0.220, 1.00]
gemi2: [0.310, 0.335, 0.360, 1.00] gemi2: [0.310, 0.335, 0.360, 1.00]
perform: True
lds_ref3_tpw : -10.00 lds_ref3_tpw : -10.00
ldsbt1 : 320.0 ldsbt1 : 320.0
# lds11_12lcmult : 0.3 # lds11_12lcmult : 0.3
...@@ -120,7 +124,9 @@ Land_Day_Desert_Coast: ...@@ -120,7 +124,9 @@ Land_Day_Desert_Coast:
cmult: 0.3 cmult: 0.3
adj: 1.25 adj: 1.25
perform: True perform: True
11-4um_Oceanic_Stratus_Test: [-23.0, -21.0, -19.0, -2.0, 0.0, 2.0, 1.0, 1.0] 11-4um_Oceanic_Stratus_Test:
thr: [-23.0, -21.0, -19.0, -2.0, 0.0, 2.0, 1.0, 1.0]
perform: True
CO2_High_Clouds_Test: [222.0, 224.0, 226.0, 1.0, 1.0] CO2_High_Clouds_Test: [222.0, 224.0, 226.0, 1.0, 1.0]
Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0] Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
Visible_Reflectance_Test: Visible_Reflectance_Test:
...@@ -152,7 +158,9 @@ Ocean_Day: ...@@ -152,7 +158,9 @@ Ocean_Day:
cmult: 0.3 cmult: 0.3
adj: 1.25 adj: 1.25
perform: True perform: True
11-4um_Oceanic_Stratus_Test: [-11.0, -9.0, -7.0, 1.0, 1.0] 11-4um_Oceanic_Stratus_Test:
thr: [-11.0, -9.0, -7.0, 1.0, 1.0]
perform: True
NIR_Reflectance_Test: NIR_Reflectance_Test:
thr: [0.062, 0.043, 0.029, 1.0, 1.0] thr: [0.062, 0.043, 0.029, 1.0, 1.0]
coeffs: [1.7291, 0.0715, -0.0026, 0.000025889] coeffs: [1.7291, 0.0715, -0.0026, 0.000025889]
...@@ -205,7 +213,9 @@ Ocean_Night: ...@@ -205,7 +213,9 @@ Ocean_Night:
cmult: 0.3 cmult: 0.3
adj: 1.0 adj: 1.0
perform: True perform: True
11-4um_Oceanic_Stratus_Test: [1.25, 1.00, 0.25, 1.0, 1.0] 11-4um_Oceanic_Stratus_Test:
thr: [1.25, 1.0, 0.25, 1.0, 1.0]
perform: True
CO2_High_Clouds_Test: [222.0, 224.0, 226.0, 1.0, 1.0] CO2_High_Clouds_Test: [222.0, 224.0, 226.0, 1.0, 1.0]
Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0] Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
no86_73 : [14.0, 15.0, 16.0, 1.0, 1.0] no86_73 : [14.0, 15.0, 16.0, 1.0, 1.0]
...@@ -225,7 +235,9 @@ Polar_Day_Land: ...@@ -225,7 +235,9 @@ Polar_Day_Land:
adj: 0.3 adj: 0.3
perform: True perform: True
pdl_ref3_tpw : -10.00 pdl_ref3_tpw : -10.00
11-4um_Oceanic_Stratus_Test: [-16.0, -14.0, -12.0, 1.0, 1.0] 11-4um_Oceanic_Stratus_Test:
thr: [-16.0, -14.0, -12.0, 1.0, 1.0]
perform: True
pdlh20 : [215.0, 220.0, 225.0, 1.0, 1.0] pdlh20 : [215.0, 220.0, 225.0, 1.0, 1.0]
Visible_Reflectance_Test: Visible_Reflectance_Test:
thr: [0.207, 0.169, 0.132, 1.0, 1.0] thr: [0.207, 0.169, 0.132, 1.0, 1.0]
...@@ -268,6 +280,7 @@ Polar_Night_Land: ...@@ -268,6 +280,7 @@ Polar_Night_Land:
mid3: [0.00, 0.00, 0.00, 0.0] mid3: [0.00, 0.00, 0.00, 0.0]
high: [1.00, 0.70, 0.40, 1.0] high: [1.00, 0.70, 0.40, 1.0]
bt11_bounds : [235.0, 000.0, 000.0, 265.0] bt11_bounds : [235.0, 000.0, 000.0, 265.0]
perform: True
Polar_Day_Coast: Polar_Day_Coast:
11-12um_Cirrus_Test: 11-12um_Cirrus_Test:
...@@ -276,7 +289,9 @@ Polar_Day_Coast: ...@@ -276,7 +289,9 @@ Polar_Day_Coast:
adj: 0 # I NEED TO WORK ON THIS adj: 0 # I NEED TO WORK ON THIS
perform: True perform: True
pdl_ref3_tpw_t2 : -10.00 pdl_ref3_tpw_t2 : -10.00
11-4um_Oceanic_Stratus_Test: [-16.0, -14.0, -12.0, 1.0, 1.0] 11-4um_Oceanic_Stratus_Test:
thr: [-16.0, -14.0, -12.0, 1.0, 1.0]
perform: True
Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0] Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
Visible_Reflectance_Test: Visible_Reflectance_Test:
thr: [0.207, 0.169, 0.132, 1.0, 1.0] thr: [0.207, 0.169, 0.132, 1.0, 1.0]
...@@ -292,7 +307,9 @@ Polar_Day_Desert: ...@@ -292,7 +307,9 @@ Polar_Day_Desert:
cmult: 0 # I NEED TO WORK ON THIS cmult: 0 # I NEED TO WORK ON THIS
adj: 0 # I NEED TO WORK ON THIS adj: 0 # I NEED TO WORK ON THIS
perform: True perform: True
11-4um_Oceanic_Stratus_Test: [-22.0, -20.0, -18.0, -2.0, 0.0, 2.0, 1.0, 1.0] 11-4um_Oceanic_Stratus_Test:
thr: [-22.0, -20.0, -18.0, -2.0, 0.0, 2.0, 1.0, 1.0]
perform: True
Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0] Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
Visible_Reflectance_Test: Visible_Reflectance_Test:
thr: [0.326, 0.288, 0.250, 1.00, 1.0] thr: [0.326, 0.288, 0.250, 1.00, 1.0]
...@@ -305,6 +322,7 @@ Polar_Day_Desert: ...@@ -305,6 +322,7 @@ Polar_Day_Desert:
gemi0: [0.085, 0.110, 0.135, 1.00, 1.0] gemi0: [0.085, 0.110, 0.135, 1.00, 1.0]
gemi1: [0.170, 0.220, 0.270, 1.00] gemi1: [0.170, 0.220, 0.270, 1.00]
gemi2: [0.310, 0.335, 0.360, 1.00] gemi2: [0.310, 0.335, 0.360, 1.00]
perform: True
pds_ref3_tpw : -10.00 pds_ref3_tpw : -10.00
pdsbt1 : 320.0 pdsbt1 : 320.0
...@@ -314,7 +332,9 @@ Polar_Day_Desert_Coast: ...@@ -314,7 +332,9 @@ Polar_Day_Desert_Coast:
cmult: 0 # I NEED TO WORK ON THIS cmult: 0 # I NEED TO WORK ON THIS
adj: 0 # I NEED TO WORK ON THIS adj: 0 # I NEED TO WORK ON THIS
perform: True perform: True
11-4um_Oceanic_Stratus_Test: [-23.0, -21.0, -19.0, -2.0, 0.0, 2.0, 1.0, 1.0] 11-4um_Oceanic_Stratus_Test:
thr: [-23.0, -21.0, -19.0, -2.0, 0.0, 2.0, 1.0, 1.0]
perform: True
Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0] Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
Visible_Reflectance_Test: Visible_Reflectance_Test:
thr: [0.326, 0.288, 0.250, 1.00, 1.0] thr: [0.326, 0.288, 0.250, 1.00, 1.0]
...@@ -361,7 +381,7 @@ Polar_Night_Snow: ...@@ -361,7 +381,7 @@ Polar_Night_Snow:
mid3: [0.00, 0.00, 0.00, 0.0] # pn_4_12m3 mid3: [0.00, 0.00, 0.00, 0.0] # pn_4_12m3
high: [2.50, 2.00, 1.50, 1.0] # pn_4_12h high: [2.50, 2.00, 1.50, 1.0] # pn_4_12h
bt11_bounds: [235.0, 0.0, 0.0, 265.0] # bt_11_bounds bt11_bounds: [235.0, 0.0, 0.0, 265.0] # bt_11_bounds
pns_4_12_pfm : 1.0 perform: True
7.3-11um_BTD_Mid_Level_Cloud_Test: 7.3-11um_BTD_Mid_Level_Cloud_Test:
low: [-1.00, 0.00, 1.00, 1.0, 1.0] # pn_7_11l low: [-1.00, 0.00, 1.00, 1.0, 1.0] # pn_7_11l
mid1: [0.00, -4.50, -1.00, 1.0] # pn_7_11m1 mid1: [0.00, -4.50, -1.00, 1.0] # pn_7_11m1
...@@ -426,7 +446,9 @@ Polar_Day_Ocean: ...@@ -426,7 +446,9 @@ Polar_Day_Ocean:
locut_coeff: 0.0100 locut_coeff: 0.0100
midpt_coeff: 0.0070 midpt_coeff: 0.0070
perform: True perform: True
11-4um_Oceanic_Stratus_Test: [-11.0, -9.0, -7.0, 1.0, 1.0] 11-4um_Oceanic_Stratus_Test:
thr: [-11.0, -9.0, -7.0, 1.0, 1.0]
perform: True
Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0] Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
pdo_b26pfm : 1.0 pdo_b26pfm : 1.0
pdo_b2bias_adj : 0.96 pdo_b2bias_adj : 0.96
...@@ -454,7 +476,9 @@ Polar_Night_Ocean: ...@@ -454,7 +476,9 @@ Polar_Night_Ocean:
adj: 1.0 adj: 1.0
perform: True perform: True
pnobt1 : 280.0 pnobt1 : 280.0
11-4um_Oceanic_Stratus_Test: [1.25, 1.00, 0.25, 1.0, 1.0] 11-4um_Oceanic_Stratus_Test:
thr: [1.25, 1.0, 0.25, 1.0, 1.0]
perform: True
Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0] Water_Vapor_High_Clouds_Test: [215.0, 220.0, 225.0, 1.0, 1.0]
pno86_73 : [14.0, 15.0, 16.0, 1.0, 1.0] pno86_73 : [14.0, 15.0, 16.0, 1.0, 1.0]
11um_Variability_Test: [3.0, 6.0, 7.0, 1.0, 1.0] 11um_Variability_Test: [3.0, 6.0, 7.0, 1.0, 1.0]
...@@ -496,6 +520,7 @@ Night_Snow: ...@@ -496,6 +520,7 @@ Night_Snow:
ns11_4lo : [0.70, 0.60, 0.50, 1.0, 1.0] ns11_4lo : [0.70, 0.60, 0.50, 1.0, 1.0]
4-12um_BTD_Thin_Cirrus_Test: 4-12um_BTD_Thin_Cirrus_Test:
thr: [4.50, 4.00, 3.50, 1.0, 1.0] thr: [4.50, 4.00, 3.50, 1.0, 1.0]
perform: True
7.3-11um_BTD_Mid_Level_Cloud_Test: 7.3-11um_BTD_Mid_Level_Cloud_Test:
low: [-1.00, 0.00, 1.00, 1.0, 1.0] # pn_7_11l low: [-1.00, 0.00, 1.00, 1.0, 1.0] # pn_7_11l
mid1: [0.00, -4.50, -1.00, 1.0] # pn_7_11m1 mid1: [0.00, -4.50, -1.00, 1.0] # pn_7_11m1
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment