diff --git a/modules/util/viirs_l1b_l2.py b/modules/util/viirs_l1b_l2.py index 9705562c3961f649b838471d6883b4ed6e449b28..c273e65b323eff2a4419fa9038d0b9c192a0f58d 100644 --- a/modules/util/viirs_l1b_l2.py +++ b/modules/util/viirs_l1b_l2.py @@ -128,8 +128,8 @@ def run_all(directory, out_directory, day_night='ANY'): label_tiles = [] try: - run(data_h5f, data_params, data_tiles, num_keep_x_tiles=num_keep_x_tiles, tile_width=128, kernel_size=7, day_night=day_night) - # run(data_h5f, label_params, label_tiles, tile_width=128, kernel_size=7, day_night=day_night) + run(data_h5f, data_params, data_train_tiles, data_valid_tiles, num_keep_x_tiles=num_keep_x_tiles, tile_width=128, kernel_size=7, day_night=day_night) + # run(data_h5f, label_params, label_data_tiles, label_valid_tiles, tile_width=128, kernel_size=7, day_night=day_night) except Exception as e: print(e) data_h5f.close() @@ -140,23 +140,9 @@ def run_all(directory, out_directory, day_night='ANY'): data_h5f.close() # label_h5f.close() - # if len(data_tiles) == 0 or len(label_tiles) == 0: - # continue - # if len(data_tiles) != len(label_tiles): - # print('weirdness: ', data_f) - # continue - - if len(data_tiles) == 0: + if len(data_train_tiles) == 0: continue - num = len(data_tiles) - n_vld = int(num * 0.1) - - # [label_valid_tiles.append(label_tiles[k]) for k in range(n_vld)] - # [label_train_tiles.append(label_tiles[k]) for k in range(n_vld, num)] - [data_valid_tiles.append(data_tiles[k]) for k in range(n_vld)] - [data_train_tiles.append(data_tiles[k]) for k in range(n_vld+num_keep_x_tiles, num)] - f_cnt += 1 if (f_cnt % 5) == 0: num_valid_samples = 0 @@ -188,7 +174,7 @@ def run_all(directory, out_directory, day_night='ANY'): print('** total_num_train_samples, total_num_valid_samples: ', total_num_train_samples, total_num_valid_samples) -def run(data_h5f, param_s, tiles, num_keep_x_tiles=8, tile_width=64, kernel_size=9, day_night='ANY'): +def run(data_h5f, param_s, train_tiles, valid_tiles, num_keep_x_tiles=8, tile_width=64, kernel_size=9, day_night='ANY'): border = int((kernel_size - 1)/2) @@ -215,11 +201,34 @@ def run(data_h5f, param_s, tiles, num_keep_x_tiles=8, tile_width=64, kernel_size i_skip = tile_width j_skip = tile_width i_start = int(num_pixels / 2) - int((num_keep_x_tiles * tile_width) / 2) + j_start = 0 + + num_keep_y_tiles = int(num_lines / tile_width) - 2 - num_keep_y_tiles = int(num_lines / tile_width) - 1 + num_y_valid = int(num_keep_y_tiles * 0.1) + 1 + num_y_train = num_keep_y_tiles - num_y_valid - 1 + + for j in range(num_y_train): + j_a = j_start + j * j_skip + j_b = j_a + tile_width + + for i in range(num_keep_x_tiles): + i_a = i_start + i * i_skip + i_b = i_a + tile_width + + if day_night == 'DAY' and not is_day(solzen[j_a:j_b, i_a:i_b]): + continue + elif day_night == 'NIGHT' and is_day(solzen[j_a:j_b, i_a:i_b]): + continue + + nda = data[:, j_a:j_b, i_a:i_b] + if not keep_tile(param_s, nda): + continue + train_tiles.append(nda) - for j in range(num_keep_y_tiles): - j_a = j * j_skip + j_start = num_y_train * tile_width + 1 + for j in range(num_y_valid): + j_a = j_start + j * j_skip j_b = j_a + tile_width for i in range(num_keep_x_tiles): @@ -234,7 +243,7 @@ def run(data_h5f, param_s, tiles, num_keep_x_tiles=8, tile_width=64, kernel_size nda = data[:, j_a:j_b, i_a:i_b] if not keep_tile(param_s, nda): continue - tiles.append(nda) + valid_tiles.append(nda) def scan(directory):