From 1f2117c9609f05dff78643814be200b8ec8e5073 Mon Sep 17 00:00:00 2001
From: tomrink <rink@ssec.wisc.edu>
Date: Wed, 15 Feb 2023 13:26:20 -0600
Subject: [PATCH] snapshot...
---
modules/util/viirs_l1b_l2.py | 20 ++++----------------
1 file changed, 4 insertions(+), 16 deletions(-)
diff --git a/modules/util/viirs_l1b_l2.py b/modules/util/viirs_l1b_l2.py
index da355275..e28d928d 100644
--- a/modules/util/viirs_l1b_l2.py
+++ b/modules/util/viirs_l1b_l2.py
@@ -65,7 +65,7 @@ def process_cld_prob_(grd_k, dum):
return None
# hist_10 += np.histogram(grd_k.flatten(), range=[0.0, 1.0], bins=10)[0]
keep = np.where(keep, np.logical_and(0.1 < grd_k, grd_k < 0.90), False)
- if np.sum(keep)/num_keep < 0.25:
+ if np.sum(keep)/num_keep < 0.30:
return None
grd_k = np.where(np.invert(keep), 0, grd_k)
return grd_k
@@ -114,7 +114,6 @@ def run_all(directory, out_directory, day_night='ANY', start=10):
print('Start, number of files: ', num_files)
# hist_10 = np.zeros((10), dtype=np.int64)
- kept_cnt = 0
for idx, data_f in enumerate(data_files):
# if idx % 4 == 0: # if we want to skip some files
@@ -126,13 +125,12 @@ def run_all(directory, out_directory, day_night='ANY', start=10):
continue
try:
- total, kept = run(data_h5f, data_params, data_train_tiles, data_valid_tiles, None, num_keep_x_tiles=num_keep_x_tiles, tile_width=128, kernel_size=11, day_night=day_night)
+ run(data_h5f, data_params, data_train_tiles, data_valid_tiles, None, num_keep_x_tiles=num_keep_x_tiles, tile_width=128, kernel_size=11, day_night=day_night)
except Exception as e:
print(e)
data_h5f.close()
continue
- kept_cnt += kept
- print(data_f, kept, kept_cnt)
+ print(data_f)
f_cnt += 1
data_h5f.close()
@@ -160,6 +158,7 @@ def run_all(directory, out_directory, day_night='ANY', start=10):
total_num_train_samples += num_train_samples
total_num_valid_samples += num_valid_samples
print('total_num_train_samples, total_num_valid_samples: ', total_num_train_samples, total_num_valid_samples)
+ print('---------------------------------------------------------')
cnt += 1
@@ -202,9 +201,6 @@ def run(data_h5f, param_s, train_tiles, valid_tiles, dum, num_keep_x_tiles=8, ti
num_y_valid = int(num_keep_y_tiles * 0.1) + 1
num_y_train = num_keep_y_tiles - num_y_valid - 1
- cnt_total = 0
- cnt_kept = 0
-
for j in range(num_y_train):
j_a = j_start + j * j_skip
j_b = j_a + tile_width
@@ -213,8 +209,6 @@ def run(data_h5f, param_s, train_tiles, valid_tiles, dum, num_keep_x_tiles=8, ti
i_a = i_start + i * i_skip
i_b = i_a + tile_width
- cnt_total += 1
-
if day_night == 'DAY' and not is_day(solzen[j_a:j_b, i_a:i_b]):
continue
elif day_night == 'NIGHT' and is_day(solzen[j_a:j_b, i_a:i_b]):
@@ -224,7 +218,6 @@ def run(data_h5f, param_s, train_tiles, valid_tiles, dum, num_keep_x_tiles=8, ti
nda = keep_tile(param_s, nda, dum)
if nda is not None:
train_tiles.append(nda)
- cnt_kept += 1
j_start = num_y_train * tile_width + 2*tile_width
for j in range(num_y_valid):
@@ -235,8 +228,6 @@ def run(data_h5f, param_s, train_tiles, valid_tiles, dum, num_keep_x_tiles=8, ti
i_a = i_start + i * i_skip
i_b = i_a + tile_width
- cnt_total += 1
-
if day_night == 'DAY' and not is_day(solzen[j_a:j_b, i_a:i_b]):
continue
elif day_night == 'NIGHT' and is_day(solzen[j_a:j_b, i_a:i_b]):
@@ -246,9 +237,6 @@ def run(data_h5f, param_s, train_tiles, valid_tiles, dum, num_keep_x_tiles=8, ti
nda = keep_tile(param_s, nda, dum)
if nda is not None:
valid_tiles.append(nda)
- cnt_kept += 1
-
- return cnt_total, cnt_kept
def scan(directory):
--
GitLab