diff --git a/modules/util/viirs_l1b_l2.py b/modules/util/viirs_l1b_l2.py index 258c5425b2283bbf22437990e061c4ad18367e3d..247a66302a2c2392f5cac699a98e841bd52f8ad9 100644 --- a/modules/util/viirs_l1b_l2.py +++ b/modules/util/viirs_l1b_l2.py @@ -9,10 +9,13 @@ emis_params = ['temp_10_4um_nom', 'temp_11_0um_nom', 'temp_12_0um_nom', 'temp_13 'temp_6_7um_nom'] l2_params = ['cloud_fraction', 'cld_temp_acha', 'cld_press_acha'] +data_params = emis_params +label_params = l2_params + def run_all(directory): - cnt = 0 + cnt = 10 for p in os.scandir(directory): if not p.is_dir(): continue @@ -47,11 +50,11 @@ def run_all(directory): # l1b_h5f.close() # continue - l1b_tiles = [] - l2_tiles = [] + data_tiles = [] + label_tiles = [] try: - run(l1b_h5f, None, l1b_tiles, l2_tiles, factor=1) + run(l1b_h5f, None, data_tiles, label_tiles, factor=1) except Exception as e: print(e) l1b_h5f.close() @@ -61,15 +64,19 @@ def run_all(directory): l1b_h5f.close() # l2_h5f.close() - if len(l1b_tiles) == 0 or len(l2_tiles) == 0: + if len(data_tiles) == 0 or len(label_tiles) == 0: + continue + + if len(data_tiles) != len(label_tiles): + print('weirdness: ', l1b_f) continue - num = len(l1b_tiles) + num = len(data_tiles) - [label_valid_tiles.append(l2_tiles[k]) for k in range(10)] - [label_train_tiles.append(l2_tiles[k]) for k in range(10, num)] - [data_valid_tiles.append(l1b_tiles[k]) for k in range(10)] - [data_train_tiles.append(l1b_tiles[k]) for k in range(10, num)] + [label_valid_tiles.append(label_tiles[k]) for k in range(10)] + [label_train_tiles.append(label_tiles[k]) for k in range(10, num)] + [data_valid_tiles.append(data_tiles[k]) for k in range(10)] + [data_train_tiles.append(data_tiles[k]) for k in range(10, num)] if f_cnt == 10: cnt += 1 @@ -95,6 +102,9 @@ def run_all(directory): if len(label_train_tiles) == 0 or len(data_train_tiles) == 0: continue + if len(label_train_tiles) != len(data_train_tiles): + print('weirdness') + continue label_valid = np.stack(label_valid_tiles) label_train = np.stack(label_train_tiles) @@ -147,9 +157,9 @@ def run(l1b_h5f, l2_h5f, l1b_tiles, l2_tiles, factor=2): mod_data = np.stack(l1b_grd_s) img_data = np.stack(l2_grd_s) - num_keep_x_tiles = 5 + num_keep_x_tiles = 3 i_skip = 3 * mod_tile_width - i_start = int(mod_num_pixels / 2) - int((num_keep_x_tiles * 3 * 64) / 2) + i_start = int(mod_num_pixels / 2) - int((num_keep_x_tiles * 3 * mod_tile_width) / 2) num_keep_y_tiles = 16 j_skip = 3 * mod_tile_width @@ -170,27 +180,6 @@ def run(l1b_h5f, l2_h5f, l1b_tiles, l2_tiles, factor=2): l2_tiles.append(nda) -# def hack(): -# l1b_nda = np.load('/Users/tomrink/data/viirs_clavrx/l1b_all.npy') -# l2_nda = np.load('/Users/tomrink/data/viirs_clavrx/l2_all.npy') -# -# num_samples = l1b_nda.shape[0] -# idxs = np.arange(num_samples) -# vld_idxs = idxs[::10] -# trn_idxs = idxs[np.in1d(idxs, vld_idxs, invert=True)] -# -# l1b_trn = l1b_nda[trn_idxs] -# l2_trn = l2_nda[trn_idxs] -# l1b_vld = l1b_nda[vld_idxs] -# l2_vld = l2_nda[vld_idxs] -# -# print(l1b_trn.shape, l2_trn.shape, l1b_vld.shape, l2_vld.shape) -# -# np.save('/Users/tomrink/data/viirs_clavrx/l1b_all_trn.npy', l1b_trn) -# np.save('/Users/tomrink/data/viirs_clavrx/l1b_all_vld.npy', l1b_vld) -# np.save('/Users/tomrink/data/viirs_clavrx/l2_all_trn.npy', l2_trn) -# np.save('/Users/tomrink/data/viirs_clavrx/l2_all_vld.npy', l2_vld) - # def run_mean_std(directory): # # data_dct = {name: [] for name in mod_res_params}