diff --git a/modules/util/abi_surfrad.py b/modules/util/abi_surfrad.py index e3894f6d9b476d21dd1473cf4383b51bde29ed70..d1c3b5accabb101021e39a33a69c4cd82a0a7b9e 100644 --- a/modules/util/abi_surfrad.py +++ b/modules/util/abi_surfrad.py @@ -3,9 +3,9 @@ import h5py from util.util import get_grid_values, is_day import glob -target_param = 'cloud_probability' +# target_param = 'cloud_probability' # target_param = 'cld_opd_dcomp' -# target_param = 'cld_opd_dcomp_1' +target_param = 'cld_opd_dcomp_1' # target_param = 'cld_opd_dcomp_2' # target_param = 'cld_opd_dcomp_3' @@ -130,8 +130,21 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st total_num_not_missing = 0 num_skip = 3 - param_train_hist = np.zeros([14], dtype=np.int64) - param_valid_hist = np.zeros([14], dtype=np.int64) + param_train_hist = np.zeros([16], dtype=np.int64) + param_valid_hist = np.zeros([16], dtype=np.int64) + + # cloud_prob to cloud fraction + # ---------------------------- + # tile_width = 32 + # kernel_size = 5 + # factor = 4 + + tile_width = 64 + kernel_size = 7 + factor = 4 + + # hist_range = [0.0, 1.0] + hist_range = [0.0, 160.0] for idx, data_f in enumerate(valid_files): if idx % num_skip == 0: # if we want to skip some files @@ -144,8 +157,7 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st try: num_not_missing, num_snow_covered = \ run(h5f, params_m, data_tiles_m, params_i, data_tiles_i, - tile_width=32, kernel_size=5, factor=4, - # tile_width=64, kernel_size=7, factor=2, + tile_width=tile_width, kernel_size=kernel_size, factor=factor, day_night=day_night, is_snow_covered=is_snow_covered) except Exception as e: print(e) @@ -170,7 +182,7 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st np.save(out_directory + 'valid_ires_' + str(cnt), valid_i) num_valid_samples = valid_m.shape[0] - param_valid_hist += np.histogram(valid_m[param_idx_m, ], bins=14)[0] + param_valid_hist += np.histogram(valid_m[param_idx_m, ], bins=16, range=hist_range)[0] data_tiles_i = [] data_tiles_m = [] @@ -191,7 +203,7 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st np.save(out_directory + 'valid_mres_' + str(cnt), valid_m) np.save(out_directory + 'valid_ires_' + str(cnt), valid_i) num_valid_samples = valid_m.shape[0] - param_valid_hist += np.histogram(valid_m[param_idx_m, ], bins=14)[0] + param_valid_hist += np.histogram(valid_m[param_idx_m, ], bins=16, range=hist_range)[0] total_num_valid_samples += num_valid_samples print('total_num_valid_samples, total_num_not_missing: ', total_num_valid_samples, total_num_not_missing) print(param_valid_hist) @@ -216,8 +228,7 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st try: num_not_missing, num_snow_covered = \ run(h5f, params_m, data_tiles_m, params_i, data_tiles_i, - tile_width=32, kernel_size=5, factor=4, - # tile_width=64, kernel_size=7, factor=2, + tile_width=tile_width, kernel_size=kernel_size, factor=factor, day_night=day_night, is_snow_covered=is_snow_covered) except Exception as e: print(e) @@ -242,7 +253,7 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st np.save(out_directory + 'train_mres_' + str(cnt), train_m) num_train_samples = train_m.shape[0] - param_train_hist += np.histogram(train_m[param_idx_m, ], bins=14)[0] + param_train_hist += np.histogram(train_m[param_idx_m, ], bins=16, range=hist_range)[0] data_tiles_i = [] data_tiles_m = [] @@ -263,7 +274,7 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st np.save(out_directory + 'train_ires_' + str(cnt), train_i) np.save(out_directory + 'train_mres_' + str(cnt), train_m) num_train_samples = train_m.shape[0] - param_train_hist += np.histogram(train_m[param_idx_m, ], bins=14)[0] + param_train_hist += np.histogram(train_m[param_idx_m, ], bins=16, range=hist_range)[0] total_num_train_samples += num_train_samples print('total_num_train_samples, total_num_not_missing: ', total_num_train_samples, total_num_not_missing) print(param_train_hist)