From 1306bc1faaae0b6561dabc7f3d25bbb467c1e4e6 Mon Sep 17 00:00:00 2001
From: tomrink <rink@ssec.wisc.edu>
Date: Wed, 3 Aug 2022 14:51:07 -0500
Subject: [PATCH] run one
---
modules/util/viirs_l1b_l2.py | 133 +++++++++++++++++++++++++----------
1 file changed, 95 insertions(+), 38 deletions(-)
diff --git a/modules/util/viirs_l1b_l2.py b/modules/util/viirs_l1b_l2.py
index dec7098f..1b7d2d6a 100644
--- a/modules/util/viirs_l1b_l2.py
+++ b/modules/util/viirs_l1b_l2.py
@@ -39,7 +39,8 @@ def run_all(directory, out_directory):
continue
print(p.name)
- data_files = glob.glob(directory + p.name+'/'+'clavrx_snpp_viirs*.uwssec*.nc')
+ data_files = glob.glob(directory + p.name+'/'+'clavrx*highres*.nc')
+ # data_files = glob.glob(directory + p.name+'/'+'clavrx_snpp_viirs*.uwssec*.nc')
# data_files = glob.glob(directory + p.name + '/' + 'VNP02MOD*.uwssec.nc')
label_valid_tiles = []
@@ -51,13 +52,12 @@ def run_all(directory, out_directory):
for idx, data_f in enumerate(data_files):
# if idx % 8 == 0: # if we want to skip some files
if True:
- w_o_ext, ext = os.path.splitext(data_f)
- # label_f = data_f
- pname, fname = os.path.split(data_f)
- toks = fname.split('.')
- label_f = pname + '/' + 'clavrx_VNP02MOD.' + toks[1]+'.'+toks[2]+'.'+toks[3]+'.'+toks[4]+'.'+'uwssec.highres.nc.level2.nc'
- if not os.path.exists(label_f):
- continue
+ # w_o_ext, ext = os.path.splitext(data_f)
+ # pname, fname = os.path.split(data_f)
+ # toks = fname.split('.')
+ # label_f = pname + '/' + 'clavrx_VNP02MOD.' + toks[1]+'.'+toks[2]+'.'+toks[3]+'.'+toks[4]+'.'+'uwssec.highres.nc.level2.nc'
+ # if not os.path.exists(label_f):
+ # continue
try:
data_h5f = h5py.File(data_f, 'r')
@@ -65,57 +65,61 @@ def run_all(directory, out_directory):
print('cant open file: ', data_f)
continue
- try:
- label_h5f = h5py.File(label_f, 'r')
- except:
- print('cant open file: ', label_f)
- data_h5f.close()
- continue
+ # try:
+ # label_h5f = h5py.File(label_f, 'r')
+ # except:
+ # print('cant open file: ', label_f)
+ # data_h5f.close()
+ # continue
data_tiles = []
- label_tiles = []
+ #label_tiles = []
try:
- run(data_h5f, label_h5f, data_tiles, label_tiles, mod_tile_width=32, kernel_size=5)
+ # run(data_h5f, label_h5f, data_tiles, label_tiles, mod_tile_width=32, kernel_size=5)
+ run_one(data_h5f, data_tiles, mod_tile_width=64, kernel_size=5)
except Exception as e:
print(e)
data_h5f.close()
- label_h5f.close()
+ #label_h5f.close()
continue
data_h5f.close()
- label_h5f.close()
+ #label_h5f.close()
- if len(data_tiles) == 0 or len(label_tiles) == 0:
- continue
+ # if len(data_tiles) == 0 or len(label_tiles) == 0:
+ # continue
+ #
+ # if len(data_tiles) != len(label_tiles):
+ # print('weirdness: ', data_f)
+ # continue
- if len(data_tiles) != len(label_tiles):
- print('weirdness: ', data_f)
+ if len(data_tiles):
continue
num = len(data_tiles)
n_vld = int(num * 0.1)
- [label_valid_tiles.append(label_tiles[k]) for k in range(n_vld)]
- [label_train_tiles.append(label_tiles[k]) for k in range(n_vld, num)]
+ # [label_valid_tiles.append(label_tiles[k]) for k in range(n_vld)]
+ # [label_train_tiles.append(label_tiles[k]) for k in range(n_vld, num)]
[data_valid_tiles.append(data_tiles[k]) for k in range(n_vld)]
[data_train_tiles.append(data_tiles[k]) for k in range(n_vld, num)]
if f_cnt == 10:
cnt += 1
- label_valid = np.stack(label_valid_tiles)
- label_train = np.stack(label_train_tiles)
+ #label_valid = np.stack(label_valid_tiles)
+ #label_train = np.stack(label_train_tiles)
data_valid = np.stack(data_valid_tiles)
data_train = np.stack(data_train_tiles)
np.save(out_directory+'data_train_' + str(cnt), data_train)
np.save(out_directory+'data_valid_' + str(cnt), data_valid)
- np.save(out_directory+'label_train_' + str(cnt), label_train)
- np.save(out_directory+'label_valid_' + str(cnt), label_valid)
+ #np.save(out_directory+'label_train_' + str(cnt), label_train)
+ #np.save(out_directory+'label_valid_' + str(cnt), label_valid)
- label_valid_tiles = []
- label_train_tiles = []
+ #label_valid_tiles = []
+ #label_train_tiles = []
data_valid_tiles = []
data_train_tiles = []
@@ -125,22 +129,25 @@ def run_all(directory, out_directory):
else:
f_cnt += 1
- if len(label_train_tiles) == 0 or len(data_train_tiles) == 0:
- continue
- if len(label_train_tiles) != len(data_train_tiles):
- print('weirdness')
+ # if len(label_train_tiles) == 0 or len(data_train_tiles) == 0:
+ # continue
+ # if len(label_train_tiles) != len(data_train_tiles):
+ # print('weirdness')
+ # continue
+
+ if len(data_train_tiles) == 0:
continue
- label_valid = np.stack(label_valid_tiles)
- label_train = np.stack(label_train_tiles)
+ #label_valid = np.stack(label_valid_tiles)
+ #label_train = np.stack(label_train_tiles)
data_valid = np.stack(data_valid_tiles)
data_train = np.stack(data_train_tiles)
cnt += 1
np.save(out_directory+'data_train_' + str(cnt), data_train)
np.save(out_directory+'data_valid_' + str(cnt), data_valid)
- np.save(out_directory+'label_train_' + str(cnt), label_train)
- np.save(out_directory+'label_valid_' + str(cnt), label_valid)
+ #np.save(out_directory+'label_train_' + str(cnt), label_train)
+ #np.save(out_directory+'label_valid_' + str(cnt), label_valid)
num_train_samples += data_train.shape[0]
num_valid_samples += data_valid.shape[0]
@@ -226,6 +233,56 @@ def run(data_h5f, label_h5f, data_tiles, label_tiles, mod_tile_width=64, kernel_
label_tiles.append(nda)
+def run_one(data_h5f, data_tiles, tile_width=64, kernel_size=9):
+
+ border = int((kernel_size - 1)/2)
+
+ param_name = data_params[0]
+
+ num_lines = data_h5f[param_name].shape[0]
+ num_pixels = data_h5f[param_name].shape[1]
+
+ grd_s = []
+
+ for param in data_params:
+ try:
+ grd = get_grid_values(data_h5f, param, 0, 0, None, num_lines, num_pixels, range_name=None)
+ grd_s.append(grd)
+ except Exception as e:
+ print(e)
+ return
+
+ data = np.stack(grd_s)
+
+ # num_keep_x_tiles = 2
+ num_keep_x_tiles = 1
+ #i_skip = num_keep_x_tiles * tile_width
+ i_skip = 1
+ j_skip = 1 * tile_width
+ #i_start = int(num_pixels / 2) - int((num_keep_x_tiles * 2 * tile_width) / 2)
+ i_start = int(num_pixels / 2) - int((tile_width) / 2)
+
+ num_keep_y_tiles = 96
+ for j in range(num_keep_y_tiles):
+ j_c = j * j_skip
+ j_m = j_c + border
+
+ for i in range(num_keep_x_tiles):
+ i_c = i * i_skip + i_start
+ i_m = i_c + border
+
+ j_stop = j_m + tile_width + border
+ if j_stop > num_lines - 1:
+ continue
+
+ i_stop = i_m + tile_width + border
+ if i_stop > num_pixels - 1:
+ continue
+
+ nda = data[:, j_m-border:j_stop, i_m-border:i_stop]
+ data_tiles.append(nda)
+
+
def scan(directory):
data_src = CLAVRx_VIIRS(directory)
--
GitLab