From b702f60f9e8c6df59e3ae6aac873a30adce8d8f9 Mon Sep 17 00:00:00 2001
From: tomrink <rink@ssec.wisc.edu>
Date: Tue, 24 May 2022 12:59:18 -0500
Subject: [PATCH] deal with bigger data
---
modules/util/viirs_l1b_l2.py | 57 +++++++++++++++++++++++++++++++-----
1 file changed, 49 insertions(+), 8 deletions(-)
diff --git a/modules/util/viirs_l1b_l2.py b/modules/util/viirs_l1b_l2.py
index dc76b557..9ee95a64 100644
--- a/modules/util/viirs_l1b_l2.py
+++ b/modules/util/viirs_l1b_l2.py
@@ -16,11 +16,14 @@ def run_all(directory):
for p in os.scandir(directory):
if not p.is_dir():
continue
- cnt += 1
+
l1b_files = glob.glob(directory+p.name+'/'+'clavrx_snpp_viirs*.uwssec*.nc')
- l1b_tiles = []
- l2_tiles = []
+ label_valid_tiles = []
+ label_train_tiles = []
+ data_valid_tiles = []
+ data_train_tiles = []
+ f_cnt = 0
for idx, l1b_f in enumerate(l1b_files):
# if idx % 8 == 0:
@@ -44,6 +47,9 @@ def run_all(directory):
# l1b_h5f.close()
# continue
+ l1b_tiles = []
+ l2_tiles = []
+
try:
run(l1b_h5f, None, l1b_tiles, l2_tiles, factor=1)
except Exception as e:
@@ -55,14 +61,49 @@ def run_all(directory):
l1b_h5f.close()
# l2_h5f.close()
- if len(l1b_tiles) == 0 or len(l2_tiles) == 0:
+ if len(l1b_tiles) == 0 or len(l2_tiles) == 0:
+ continue
+
+ num = len(l1b_tiles)
+
+ [label_valid_tiles.append(l2_tiles[k]) for k in range(6)]
+ [label_train_tiles.append(l2_tiles[k]) for k in range(6, num)]
+ [data_valid_tiles.append(l1b_tiles[k]) for k in range(6)]
+ [data_train_tiles.append(l1b_tiles[k]) for k in range(6, num)]
+
+ if f_cnt == 10:
+ cnt += 1
+
+ label_valid = np.stack(label_valid_tiles)
+ label_train = np.stack(label_train_tiles)
+ data_valid = np.stack(data_valid_tiles)
+ data_train = np.stack(data_train_tiles)
+
+ np.save('/data/Personal/rink/viirs_clavrx/data_train_' + str(cnt), data_train)
+ np.save('/data/Personal/rink/viirs_clavrx/data_valid_' + str(cnt), data_valid)
+ np.save('/data/Personal/rink/viirs_clavrx/label_train_' + str(cnt), label_train)
+ np.save('/data/Personal/rink/viirs_clavrx/label_valid_' + str(cnt), label_valid)
+
+ label_valid_tiles = []
+ label_train_tiles = []
+ data_valid_tiles = []
+ data_train_tiles = []
+
+ f_cnt = 0
+
+ if len(label_train_tiles) == 0 or len(data_train_tiles) == 0:
continue
- l1b_nda = np.stack(l1b_tiles)
- l2_nda = np.stack(l2_tiles)
+ label_valid = np.stack(label_valid_tiles)
+ label_train = np.stack(label_train_tiles)
+ data_valid = np.stack(data_valid_tiles)
+ data_train = np.stack(data_train_tiles)
- np.save('/data/Personal/rink/viirs_clavrx/l1b_'+str(cnt), l1b_nda)
- np.save('/data/Personal/rink/viirs_clavrx/l2_'+str(cnt), l2_nda)
+ cnt += 1
+ np.save('/data/Personal/rink/viirs_clavrx/data_train_' + str(cnt), data_train)
+ np.save('/data/Personal/rink/viirs_clavrx/data_valid_' + str(cnt), data_valid)
+ np.save('/data/Personal/rink/viirs_clavrx/label_train_' + str(cnt), label_train)
+ np.save('/data/Personal/rink/viirs_clavrx/label_valid_' + str(cnt), label_valid)
def run(l1b_h5f, l2_h5f, l1b_tiles, l2_tiles, factor=2):
--
GitLab