From 14020f9432670b440ec79b638a05d7e07c636b79 Mon Sep 17 00:00:00 2001
From: tomrink <rink@ssec.wisc.edu>
Date: Thu, 26 May 2022 10:59:02 -0500
Subject: [PATCH] snapshot...
---
modules/util/viirs_l1b_l2.py | 55 +++++++++++++++---------------------
1 file changed, 22 insertions(+), 33 deletions(-)
diff --git a/modules/util/viirs_l1b_l2.py b/modules/util/viirs_l1b_l2.py
index 258c5425..247a6630 100644
--- a/modules/util/viirs_l1b_l2.py
+++ b/modules/util/viirs_l1b_l2.py
@@ -9,10 +9,13 @@ emis_params = ['temp_10_4um_nom', 'temp_11_0um_nom', 'temp_12_0um_nom', 'temp_13
'temp_6_7um_nom']
l2_params = ['cloud_fraction', 'cld_temp_acha', 'cld_press_acha']
+data_params = emis_params
+label_params = l2_params
+
def run_all(directory):
- cnt = 0
+ cnt = 10
for p in os.scandir(directory):
if not p.is_dir():
continue
@@ -47,11 +50,11 @@ def run_all(directory):
# l1b_h5f.close()
# continue
- l1b_tiles = []
- l2_tiles = []
+ data_tiles = []
+ label_tiles = []
try:
- run(l1b_h5f, None, l1b_tiles, l2_tiles, factor=1)
+ run(l1b_h5f, None, data_tiles, label_tiles, factor=1)
except Exception as e:
print(e)
l1b_h5f.close()
@@ -61,15 +64,19 @@ def run_all(directory):
l1b_h5f.close()
# l2_h5f.close()
- if len(l1b_tiles) == 0 or len(l2_tiles) == 0:
+ if len(data_tiles) == 0 or len(label_tiles) == 0:
+ continue
+
+ if len(data_tiles) != len(label_tiles):
+ print('weirdness: ', l1b_f)
continue
- num = len(l1b_tiles)
+ num = len(data_tiles)
- [label_valid_tiles.append(l2_tiles[k]) for k in range(10)]
- [label_train_tiles.append(l2_tiles[k]) for k in range(10, num)]
- [data_valid_tiles.append(l1b_tiles[k]) for k in range(10)]
- [data_train_tiles.append(l1b_tiles[k]) for k in range(10, num)]
+ [label_valid_tiles.append(label_tiles[k]) for k in range(10)]
+ [label_train_tiles.append(label_tiles[k]) for k in range(10, num)]
+ [data_valid_tiles.append(data_tiles[k]) for k in range(10)]
+ [data_train_tiles.append(data_tiles[k]) for k in range(10, num)]
if f_cnt == 10:
cnt += 1
@@ -95,6 +102,9 @@ def run_all(directory):
if len(label_train_tiles) == 0 or len(data_train_tiles) == 0:
continue
+ if len(label_train_tiles) != len(data_train_tiles):
+ print('weirdness')
+ continue
label_valid = np.stack(label_valid_tiles)
label_train = np.stack(label_train_tiles)
@@ -147,9 +157,9 @@ def run(l1b_h5f, l2_h5f, l1b_tiles, l2_tiles, factor=2):
mod_data = np.stack(l1b_grd_s)
img_data = np.stack(l2_grd_s)
- num_keep_x_tiles = 5
+ num_keep_x_tiles = 3
i_skip = 3 * mod_tile_width
- i_start = int(mod_num_pixels / 2) - int((num_keep_x_tiles * 3 * 64) / 2)
+ i_start = int(mod_num_pixels / 2) - int((num_keep_x_tiles * 3 * mod_tile_width) / 2)
num_keep_y_tiles = 16
j_skip = 3 * mod_tile_width
@@ -170,27 +180,6 @@ def run(l1b_h5f, l2_h5f, l1b_tiles, l2_tiles, factor=2):
l2_tiles.append(nda)
-# def hack():
-# l1b_nda = np.load('/Users/tomrink/data/viirs_clavrx/l1b_all.npy')
-# l2_nda = np.load('/Users/tomrink/data/viirs_clavrx/l2_all.npy')
-#
-# num_samples = l1b_nda.shape[0]
-# idxs = np.arange(num_samples)
-# vld_idxs = idxs[::10]
-# trn_idxs = idxs[np.in1d(idxs, vld_idxs, invert=True)]
-#
-# l1b_trn = l1b_nda[trn_idxs]
-# l2_trn = l2_nda[trn_idxs]
-# l1b_vld = l1b_nda[vld_idxs]
-# l2_vld = l2_nda[vld_idxs]
-#
-# print(l1b_trn.shape, l2_trn.shape, l1b_vld.shape, l2_vld.shape)
-#
-# np.save('/Users/tomrink/data/viirs_clavrx/l1b_all_trn.npy', l1b_trn)
-# np.save('/Users/tomrink/data/viirs_clavrx/l1b_all_vld.npy', l1b_vld)
-# np.save('/Users/tomrink/data/viirs_clavrx/l2_all_trn.npy', l2_trn)
-# np.save('/Users/tomrink/data/viirs_clavrx/l2_all_vld.npy', l2_vld)
-
# def run_mean_std(directory):
#
# data_dct = {name: [] for name in mod_res_params}
--
GitLab