diff --git a/modules/util/viirs_l1b_l2.py b/modules/util/viirs_l1b_l2.py index da355275e3364e728d9e8fc8ce450c72cfaea915..e28d928dacb400a9e9870ea369eb443198d60aed 100644 --- a/modules/util/viirs_l1b_l2.py +++ b/modules/util/viirs_l1b_l2.py @@ -65,7 +65,7 @@ def process_cld_prob_(grd_k, dum): return None # hist_10 += np.histogram(grd_k.flatten(), range=[0.0, 1.0], bins=10)[0] keep = np.where(keep, np.logical_and(0.1 < grd_k, grd_k < 0.90), False) - if np.sum(keep)/num_keep < 0.25: + if np.sum(keep)/num_keep < 0.30: return None grd_k = np.where(np.invert(keep), 0, grd_k) return grd_k @@ -114,7 +114,6 @@ def run_all(directory, out_directory, day_night='ANY', start=10): print('Start, number of files: ', num_files) # hist_10 = np.zeros((10), dtype=np.int64) - kept_cnt = 0 for idx, data_f in enumerate(data_files): # if idx % 4 == 0: # if we want to skip some files @@ -126,13 +125,12 @@ def run_all(directory, out_directory, day_night='ANY', start=10): continue try: - total, kept = run(data_h5f, data_params, data_train_tiles, data_valid_tiles, None, num_keep_x_tiles=num_keep_x_tiles, tile_width=128, kernel_size=11, day_night=day_night) + run(data_h5f, data_params, data_train_tiles, data_valid_tiles, None, num_keep_x_tiles=num_keep_x_tiles, tile_width=128, kernel_size=11, day_night=day_night) except Exception as e: print(e) data_h5f.close() continue - kept_cnt += kept - print(data_f, kept, kept_cnt) + print(data_f) f_cnt += 1 data_h5f.close() @@ -160,6 +158,7 @@ def run_all(directory, out_directory, day_night='ANY', start=10): total_num_train_samples += num_train_samples total_num_valid_samples += num_valid_samples print('total_num_train_samples, total_num_valid_samples: ', total_num_train_samples, total_num_valid_samples) + print('---------------------------------------------------------') cnt += 1 @@ -202,9 +201,6 @@ def run(data_h5f, param_s, train_tiles, valid_tiles, dum, num_keep_x_tiles=8, ti num_y_valid = int(num_keep_y_tiles * 0.1) + 1 num_y_train = num_keep_y_tiles - num_y_valid - 1 - cnt_total = 0 - cnt_kept = 0 - for j in range(num_y_train): j_a = j_start + j * j_skip j_b = j_a + tile_width @@ -213,8 +209,6 @@ def run(data_h5f, param_s, train_tiles, valid_tiles, dum, num_keep_x_tiles=8, ti i_a = i_start + i * i_skip i_b = i_a + tile_width - cnt_total += 1 - if day_night == 'DAY' and not is_day(solzen[j_a:j_b, i_a:i_b]): continue elif day_night == 'NIGHT' and is_day(solzen[j_a:j_b, i_a:i_b]): @@ -224,7 +218,6 @@ def run(data_h5f, param_s, train_tiles, valid_tiles, dum, num_keep_x_tiles=8, ti nda = keep_tile(param_s, nda, dum) if nda is not None: train_tiles.append(nda) - cnt_kept += 1 j_start = num_y_train * tile_width + 2*tile_width for j in range(num_y_valid): @@ -235,8 +228,6 @@ def run(data_h5f, param_s, train_tiles, valid_tiles, dum, num_keep_x_tiles=8, ti i_a = i_start + i * i_skip i_b = i_a + tile_width - cnt_total += 1 - if day_night == 'DAY' and not is_day(solzen[j_a:j_b, i_a:i_b]): continue elif day_night == 'NIGHT' and is_day(solzen[j_a:j_b, i_a:i_b]): @@ -246,9 +237,6 @@ def run(data_h5f, param_s, train_tiles, valid_tiles, dum, num_keep_x_tiles=8, ti nda = keep_tile(param_s, nda, dum) if nda is not None: valid_tiles.append(nda) - cnt_kept += 1 - - return cnt_total, cnt_kept def scan(directory):