From 6ef03b6759357b7f9d2e2a65a5be6c2058cc7d2d Mon Sep 17 00:00:00 2001
From: tomrink <rink@ssec.wisc.edu>
Date: Fri, 3 Mar 2023 13:00:28 -0600
Subject: [PATCH] snapshot...
---
modules/util/viirs_l1b_l2.py | 65 +++++++++++++-----------------------
1 file changed, 24 insertions(+), 41 deletions(-)
diff --git a/modules/util/viirs_l1b_l2.py b/modules/util/viirs_l1b_l2.py
index e854ec58..4bef3a0b 100644
--- a/modules/util/viirs_l1b_l2.py
+++ b/modules/util/viirs_l1b_l2.py
@@ -20,6 +20,8 @@ solzen_name = group_name + 'solar_zenith'
label_params = l2_params
data_params = l2_params
+param_idx = data_params.index(group_name + target_param)
+
# range = [0.0, 1.0]
cld_prob_norm_hist = [0.34458323, 0.03729378, 0.01817725, 0.01246574, 0.00991681, 0.00826515, 0.00785976, 0.00595133,
0.00567965, 0.00579926, 0.00642895, 0.00797761, 0.01218471, 0.51741677]
@@ -31,68 +33,47 @@ cld_opd_norm_hist = [7.31926378e-01, 9.52482193e-02, 4.62747706e-02, 3.15450036e
6.50404531e-04, 1.73557144e-02]
-def keep_tile(param_s, tile):
- k = param_s.index(group_name + target_param)
- grd_k = tile[k, ].copy()
+def is_missing(p_idx, tile):
+ keep = np.invert(np.isnan(tile[p_idx, ]))
+ if np.sum(keep) / keep.size < 0.98:
+ return True
- if target_param == 'cloud_probability':
- grd_k, bflag = process_cld_prob_(grd_k)
- elif target_param == 'cld_opd_dcomp':
- grd_k, bflag = process_cld_opd_(grd_k)
- if grd_k is not None:
- tile[k, ] = grd_k
- return tile, bflag
- else:
- return None, bflag
+def keep_tile(p_idx, tile):
+ grd_k = tile[p_idx, ].copy()
+ if target_param == 'cloud_probability':
+ grd_k, bflag = process_cld_prob(grd_k)
+ elif target_param == 'cld_opd_dcomp':
+ grd_k, bflag = process_cld_opd(grd_k)
-def process_cld_prob(param_s, tile):
- k = param_s.index(group_name + 'cloud_probability')
- grd_k = tile[k, ].copy()
- grd_k = process_cld_prob_(grd_k)
if grd_k is not None:
- tile[k, ] = grd_k
+ tile[p_idx, ] = grd_k
return tile
else:
return None
-def process_cld_prob_(grd_k):
+def process_cld_prob(grd_k):
keep = np.invert(np.isnan(grd_k))
num_keep = np.sum(keep)
- if num_keep / grd_k.size < 0.98:
- return None, True
keep_clr = np.where(keep, grd_k < 0.20, False)
frac_keep = np.sum(keep_clr)/num_keep
if not (0.38 < frac_keep < 0.62):
- return None, False
- grd_k = np.where(np.invert(keep), 0, grd_k) # Convert NaN to 0
- return grd_k, False
-
-
-def process_cld_opd(param_s, tile):
- k = param_s.index(group_name + 'cld_opd_dcomp')
- grd_k = tile[k, ].copy()
- grd_k = process_cld_opd_(grd_k)
- if grd_k is not None:
- tile[k, ] = grd_k
- return tile
- else:
return None
+ grd_k = np.where(np.invert(keep), 0, grd_k) # Convert NaN to 0
+ return grd_k
-def process_cld_opd_(grd_k):
+def process_cld_opd(grd_k):
keep = np.invert(np.isnan(grd_k))
num_keep = np.sum(keep)
- if num_keep / grd_k.size < 0.98:
- return None, True
grd_k = np.where(np.invert(keep), 0, grd_k)
keep = np.where(keep, np.logical_and(0.1 < grd_k, grd_k < 158.0), False)
frac_keep = np.sum(keep)/num_keep
if frac_keep < 0.50:
- return None, False
- return grd_k, False
+ return None
+ return grd_k
def run_all(directory, out_directory, day_night='ANY', start=10):
@@ -219,9 +200,11 @@ def run(data_h5f, param_s, train_tiles, valid_tiles, num_keep_x_tiles=8, tile_wi
continue
nda = data[:, j_a:j_b, i_a:i_b]
- nda, missing_flag = keep_tile(param_s, nda)
- if not missing_flag:
- num_not_missing += 1
+ if is_missing(param_idx, nda):
+ continue
+ num_not_missing += 1
+
+ nda = keep_tile(param_idx, nda)
if nda is not None:
tiles.append(nda)
--
GitLab