diff --git a/modules/util/viirs_l1b_l2.py b/modules/util/viirs_l1b_l2.py
index ac530830a234dc12ea5d3b61b0a0ea1d55943176..70d649b382595c3664fb7c368ff53a667ad39013 100644
--- a/modules/util/viirs_l1b_l2.py
+++ b/modules/util/viirs_l1b_l2.py
@@ -15,20 +15,13 @@ emis_params = ['temp_10_4um_nom', 'temp_11_0um_nom', 'temp_12_0um_nom', 'temp_13
# data_params = refl_params + emis_params
data_params = emis_params
-# l2_params = ['refl_0_65um_nom', 'temp_11_0um_nom', 'cld_temp_acha', 'cld_press_acha', 'cloud_fraction', 'cld_opd_acha', 'cld_reff_acha']
-l2_params = ['refl_0_65um_nom', 'temp_11_0um_nom', 'cld_temp_acha', 'cld_press_acha', 'cloud_fraction']
-label_params = l2_params
-data_params = l2_params
-
-# data_params = ['cloud_fraction']
-# label_params = ['cloud_fraction']
+l2_params = ['refl_0_65um_nom', 'temp_11_0um_nom', 'cld_temp_acha', 'cld_press_acha', 'cloud_fraction', 'cld_opd_acha']
-# data_params = ['observation_data/M15']
-# label_params = ['observation_data/M15_highres']
+label_params = l2_params
+#data_params = l2_params
def run_all(directory, out_directory):
- num_train_samples, num_valid_samples = 0, 0
cnt = 10
total_num_train_samples = 0
@@ -73,11 +66,11 @@ def run_all(directory, out_directory):
# continue
data_tiles = []
- #label_tiles = []
+ label_tiles = []
try:
- # run(data_h5f, label_h5f, data_tiles, label_tiles, mod_tile_width=32, kernel_size=5)
- run_one(data_h5f, data_tiles, tile_width=128, kernel_size=7)
+ run(data_h5f, data_params, data_tiles, tile_width=128, kernel_size=7)
+ run(data_h5f, label_params, label_tiles, tile_width=128, kernel_size=7)
except Exception as e:
print(e)
data_h5f.close()
@@ -87,21 +80,21 @@ def run_all(directory, out_directory):
data_h5f.close()
#label_h5f.close()
- # if len(data_tiles) == 0 or len(label_tiles) == 0:
- # continue
- #
- # if len(data_tiles) != len(label_tiles):
- # print('weirdness: ', data_f)
- # continue
+ if len(data_tiles) == 0 or len(label_tiles) == 0:
+ continue
- if len(data_tiles) == 0:
+ if len(data_tiles) != len(label_tiles):
+ print('weirdness: ', data_f)
continue
+ # if len(data_tiles) == 0:
+ # continue
+
num = len(data_tiles)
n_vld = int(num * 0.1)
- # [label_valid_tiles.append(label_tiles[k]) for k in range(n_vld)]
- # [label_train_tiles.append(label_tiles[k]) for k in range(n_vld, num)]
+ [label_valid_tiles.append(label_tiles[k]) for k in range(n_vld)]
+ [label_train_tiles.append(label_tiles[k]) for k in range(n_vld, num)]
[data_valid_tiles.append(data_tiles[k]) for k in range(n_vld)]
[data_train_tiles.append(data_tiles[k]) for k in range(n_vld, num)]
@@ -109,18 +102,18 @@ def run_all(directory, out_directory):
if f_cnt == 10:
f_cnt = 0
- #label_valid = np.stack(label_valid_tiles)
- #label_train = np.stack(label_train_tiles)
+ label_valid = np.stack(label_valid_tiles)
+ label_train = np.stack(label_train_tiles)
data_valid = np.stack(data_valid_tiles)
data_train = np.stack(data_train_tiles)
np.save(out_directory+'data_train_' + str(cnt), data_train)
np.save(out_directory+'data_valid_' + str(cnt), data_valid)
- #np.save(out_directory+'label_train_' + str(cnt), label_train)
- #np.save(out_directory+'label_valid_' + str(cnt), label_valid)
+ np.save(out_directory+'label_train_' + str(cnt), label_train)
+ np.save(out_directory+'label_valid_' + str(cnt), label_valid)
- #label_valid_tiles = []
- #label_train_tiles = []
+ label_valid_tiles = []
+ label_train_tiles = []
data_valid_tiles = []
data_train_tiles = []
@@ -136,92 +129,18 @@ def run_all(directory, out_directory):
print('total_num_train_samples, total_num_valid_samples: ', total_num_train_samples, total_num_valid_samples)
-def run(data_h5f, label_h5f, data_tiles, label_tiles, mod_tile_width=64, kernel_size=9):
- if label_h5f is None:
- label_h5f = data_h5f
-
- border = int((kernel_size - 1)/2)
-
- l1b_param_name = data_params[0]
- l2_param_name = label_params[0]
-
- mod_num_lines = data_h5f[l1b_param_name].shape[0]
- mod_num_pixels = data_h5f[l1b_param_name].shape[1]
-
- img_num_lines = label_h5f[l2_param_name].shape[0]
- img_num_pixels = label_h5f[l2_param_name].shape[1]
-
- factor = int(img_num_pixels / mod_num_pixels)
- img_tile_width = mod_tile_width * factor
-
- l1b_grd_s = []
- l2_grd_s = []
-
- for param in data_params:
- try:
- grd = get_grid_values(data_h5f, param, 0, 0, None, mod_num_lines, mod_num_pixels, range_name=None)
- l1b_grd_s.append(grd)
- except Exception as e:
- print(e)
- return
-
- for param in label_params:
- try:
- grd = get_grid_values(label_h5f, param, 0, 0, None, img_num_lines, img_num_pixels, range_name=None)
- l2_grd_s.append(grd)
- except Exception as e:
- print(e)
- return
-
- mod_data = np.stack(l1b_grd_s)
- img_data = np.stack(l2_grd_s)
-
- num_keep_x_tiles = 3
- #num_keep_x_tiles = 1
- i_skip = 3 * mod_tile_width
- #i_skip = 1
- j_skip = 1 * mod_tile_width
- i_start = int(mod_num_pixels / 2) - int((num_keep_x_tiles * 3 * mod_tile_width) / 2)
- #i_start = int(mod_num_pixels / 2) - int((mod_tile_width) / 2)
-
- num_keep_y_tiles = 96
- for j in range(num_keep_y_tiles):
- j_c = j * j_skip
- j_m = j_c + border
- j_i = j_m * factor
-
- for i in range(num_keep_x_tiles):
- i_c = i * i_skip + i_start
- i_m = i_c + border
- i_i = i_m * factor
-
- j_stop = j_m + mod_tile_width + border
- if j_stop > mod_num_lines - 1:
- continue
-
- i_stop = i_m + mod_tile_width + border
- if i_stop > mod_num_pixels - 1:
- continue
-
- nda = mod_data[:, j_m-border:j_stop, i_m-border:i_stop]
- data_tiles.append(nda)
-
- nda = img_data[:, j_i:j_i + img_tile_width, i_i:i_i + img_tile_width]
- label_tiles.append(nda)
-
-
-def run_one(data_h5f, data_tiles, tile_width=64, kernel_size=9):
+def run(data_h5f, param_s, tiles, tile_width=64, kernel_size=9):
border = int((kernel_size - 1)/2)
- param_name = data_params[0]
+ param_name = param_s[0]
num_lines = data_h5f[param_name].shape[0]
num_pixels = data_h5f[param_name].shape[1]
grd_s = []
- for param in data_params:
+ for param in param_s:
try:
grd = get_grid_values(data_h5f, param, 0, 0, None, num_lines, num_pixels, range_name=None)
# if param == 'temp_11_0um_nom' and ((np.sum(np.isnan(grd)) / grd.size) < 0.10):
@@ -259,7 +178,7 @@ def run_one(data_h5f, data_tiles, tile_width=64, kernel_size=9):
nda = data[:, j_m-border:j_stop, i_m-border:i_stop]
tmp = nda[1, :, :]
if (np.sum(np.isnan(tmp)) / tmp.size) < 0.10:
- data_tiles.append(nda)
+ tiles.append(nda)
def scan(directory):