diff --git a/modules/util/viirs_l1b_l2.py b/modules/util/viirs_l1b_l2.py index 718b5c9a2d1b6966857e33c7c08763714b5be7f0..139550da514929272203a04d1082c148eec17d66 100644 --- a/modules/util/viirs_l1b_l2.py +++ b/modules/util/viirs_l1b_l2.py @@ -20,15 +20,25 @@ solzen_name = group_name + 'solar_zenith' label_params = l2_params data_params = l2_params +# range = [0.0, 1.0] +cld_prob_norm_hist = [0.34458323, 0.03729378, 0.01817725, 0.01246574, 0.00991681, 0.00826515, 0.00785976, 0.00595133, + 0.00567965, 0.00579926, 0.00642895, 0.00797761, 0.01218471, 0.51741677] -def keep_tile(param_s, tile, hist_10): +# range = [0.0, 160.0] +cld_opd_norm_hist = [7.31926378e-01, 9.52482193e-02, 4.62747706e-02, 3.15450036e-02, 1.98358694e-02, 1.33123841e-02, + 1.03378429e-02, 7.95560979e-03, 5.77925319e-03, 4.82856215e-03, 3.31576300e-03, 2.86789405e-03, + 2.50456177e-03, 1.79184632e-03, 1.51077739e-03, 1.29144749e-03,9.20514553e-04, 7.47183923e-04, + 6.50404531e-04, 1.73557144e-02] + + +def keep_tile(param_s, tile, dum): k = param_s.index(group_name + target_param) grd_k = tile[k, ].copy() if target_param == 'cloud_probability': - grd_k = process_cld_prob_(grd_k, hist_10) + grd_k = process_cld_prob_(grd_k, dum) elif target_param == 'cld_opd_dcomp': - grd_k = process_cld_opd_(grd_k) + grd_k = process_cld_opd_(grd_k, dum) if grd_k is not None: tile[k, ] = grd_k @@ -48,14 +58,14 @@ def process_cld_prob(param_s, tile): return None -def process_cld_prob_(grd_k, hist_10): +def process_cld_prob_(grd_k, dum): keep = np.invert(np.isnan(grd_k)) num_keep = np.sum(keep) if num_keep / grd_k.size < 0.98: return None - hist_10 += np.histogram(grd_k.flatten(), range=[0.0, 1.0], bins=10)[0] - keep = np.where(keep, np.logical_and(0.05 < grd_k, grd_k < 0.95), False) - if np.sum(keep)/num_keep < 0.50: + # hist_10 += np.histogram(grd_k.flatten(), range=[0.0, 1.0], bins=10)[0] + keep = np.where(keep, np.logical_and(0.1 < grd_k, grd_k < 0.90), False) + if np.sum(keep)/num_keep < 0.30: return None grd_k = np.where(np.invert(keep), 0, grd_k) return grd_k @@ -72,7 +82,7 @@ def process_cld_opd(param_s, tile): return None -def process_cld_opd_(grd_k): +def process_cld_opd_(grd_k, dum): keep = np.invert(np.isnan(grd_k)) num_keep = np.sum(keep) if num_keep / grd_k.size < 0.98: @@ -103,7 +113,7 @@ def run_all(directory, out_directory, day_night='ANY', start=10): num_files = len(data_files) print('Start, number of files: ', num_files) - hist_10 = np.zeros((10), dtype=np.int64) + # hist_10 = np.zeros((10), dtype=np.int64) for idx, data_f in enumerate(data_files): # if idx % 4 == 0: # if we want to skip some files @@ -115,12 +125,12 @@ def run_all(directory, out_directory, day_night='ANY', start=10): continue try: - total, kept = run(data_h5f, data_params, data_train_tiles, data_valid_tiles, hist_10, num_keep_x_tiles=num_keep_x_tiles, tile_width=128, kernel_size=11, day_night=day_night) + total, kept = run(data_h5f, data_params, data_train_tiles, data_valid_tiles, None, num_keep_x_tiles=num_keep_x_tiles, tile_width=128, kernel_size=11, day_night=day_night) except Exception as e: print(e) data_h5f.close() continue - print(data_f, int(100 * (kept/total)), hist_10, (hist_10 / np.sum(hist_10))) + print(data_f, int(100 * (kept/total))) f_cnt += 1 data_h5f.close() @@ -154,7 +164,7 @@ def run_all(directory, out_directory, day_night='ANY', start=10): # tile_width: Must be even! # kernel_size: Must be odd! -def run(data_h5f, param_s, train_tiles, valid_tiles, hist_10, num_keep_x_tiles=8, tile_width=64, kernel_size=9, day_night='ANY'): +def run(data_h5f, param_s, train_tiles, valid_tiles, dum, num_keep_x_tiles=8, tile_width=64, kernel_size=9, day_night='ANY'): border = int((kernel_size - 1)/2) @@ -207,7 +217,7 @@ def run(data_h5f, param_s, train_tiles, valid_tiles, hist_10, num_keep_x_tiles=8 continue nda = data[:, j_a:j_b, i_a:i_b] - nda = keep_tile(param_s, nda, hist_10) + nda = keep_tile(param_s, nda, dum) if nda is not None: train_tiles.append(nda) cnt_kept += 1 @@ -229,7 +239,7 @@ def run(data_h5f, param_s, train_tiles, valid_tiles, hist_10, num_keep_x_tiles=8 continue nda = data[:, j_a:j_b, i_a:i_b] - nda = keep_tile(param_s, nda, hist_10) + nda = keep_tile(param_s, nda, dum) if nda is not None: valid_tiles.append(nda) cnt_kept += 1