diff --git a/modules/util/abi_surfrad.py b/modules/util/abi_surfrad.py index 4fe2d133c1daaf3be16345eb014f40221f10ad93..70d91ececfe52f6959e2df080c08c3cc526f840c 100644 --- a/modules/util/abi_surfrad.py +++ b/modules/util/abi_surfrad.py @@ -3,9 +3,9 @@ import h5py from util.util import get_grid_values, is_day import glob -# target_param = 'cloud_probability' +target_param = 'cloud_probability' # target_param = 'cld_opd_dcomp' -target_param = 'cld_opd_dcomp_1' +# target_param = 'cld_opd_dcomp_1' # target_param = 'cld_opd_dcomp_2' # target_param = 'cld_opd_dcomp_3' @@ -15,15 +15,17 @@ group_name_m = 'orig/' solzen_name = group_name_m + 'solar_zenith' snow_class_name = group_name_m + 'snow_class' -# params_i = [group_name_i+'temp_ch38', group_name_i+'refl_ch01', group_name_i+target_param] -params_i = [group_name_i+'temp_ch38', group_name_i+'refl_ch01', group_name_i+'temp_stddev3x3_ch31', group_name_i+'refl_stddev3x3_ch01', group_name_i+target_param] +params_i = [group_name_i+'temp_ch38', group_name_i+'refl_ch01', group_name_i+target_param] +# params_i = [group_name_i+'temp_ch38', group_name_i+'refl_ch01', group_name_i+'temp_stddev3x3_ch31', group_name_i+'refl_stddev3x3_ch01', group_name_i+target_param] -# params_m = [group_name_m+'temp_ch38', group_name_m+'refl_ch01', group_name_m+'refl_submin_ch01', group_name_m+'refl_submax_ch01', group_name_m+'refl_substddev_ch01', group_name_m+target_param] -params_m = [group_name_m+'temp_ch38', group_name_m+'refl_ch01', group_name_m+'refl_submin_ch01', group_name_m+'refl_submax_ch01', group_name_m+'refl_substddev_ch01', group_name_m+'temp_stddev3x3_ch31', group_name_m+'refl_stddev3x3_ch01', group_name_m+target_param] +params_m = [group_name_m+'temp_ch38', group_name_m+'refl_ch01', group_name_m+'refl_submin_ch01', group_name_m+'refl_submax_ch01', group_name_m+'refl_substddev_ch01', group_name_m+target_param] +# params_m = [group_name_m+'temp_ch38', group_name_m+'refl_ch01', group_name_m+'refl_submin_ch01', group_name_m+'refl_submax_ch01', group_name_m+'refl_substddev_ch01', group_name_m+'temp_stddev3x3_ch31', group_name_m+'refl_stddev3x3_ch01', group_name_m+target_param] param_idx_m = params_m.index(group_name_m + target_param) param_idx_i = params_i.index(group_name_i + target_param) +DO_WRITE_OUTFILE = False + def snow_covered(tile): return np.any(tile > 1) @@ -142,8 +144,8 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st try: num_not_missing, num_snow_covered = \ run(h5f, params_m, data_tiles_m, params_i, data_tiles_i, - # tile_width=16, kernel_size=4, factor=4, - tile_width=64, kernel_size=7, factor=2, + tile_width=32, kernel_size=5, factor=4, + # tile_width=64, kernel_size=7, factor=2, day_night=day_night, is_snow_covered=is_snow_covered) except Exception as e: print(e) @@ -163,8 +165,9 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st if len(data_tiles_m) > 0: valid_i = np.stack(data_tiles_i) valid_m = np.stack(data_tiles_m) - np.save(out_directory + 'valid_mres_' + str(cnt), valid_m) - np.save(out_directory + 'valid_ires_' + str(cnt), valid_i) + if DO_WRITE_OUTFILE: + np.save(out_directory + 'valid_mres_' + str(cnt), valid_m) + np.save(out_directory + 'valid_ires_' + str(cnt), valid_i) num_valid_samples = valid_m.shape[0] param_valid_hist += np.histogram(valid_m[param_idx_m, ], bins=20, range=[0.0, 160.0])[0] @@ -184,8 +187,9 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st if len(data_tiles_m) > 0: valid_i = np.stack(data_tiles_i) valid_m = np.stack(data_tiles_m) - np.save(out_directory + 'valid_mres_' + str(cnt), valid_m) - np.save(out_directory + 'valid_ires_' + str(cnt), valid_i) + if DO_WRITE_OUTFILE: + np.save(out_directory + 'valid_mres_' + str(cnt), valid_m) + np.save(out_directory + 'valid_ires_' + str(cnt), valid_i) num_valid_samples = valid_m.shape[0] param_valid_hist += np.histogram(valid_m[param_idx_m, ], bins=20, range=[0.0, 160.0])[0] total_num_valid_samples += num_valid_samples @@ -233,8 +237,9 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st if len(data_tiles_m) > 0: train_i = np.stack(data_tiles_i) train_m = np.stack(data_tiles_m) - np.save(out_directory + 'train_ires_' + str(cnt), train_i) - np.save(out_directory + 'train_mres_' + str(cnt), train_m) + if DO_WRITE_OUTFILE: + np.save(out_directory + 'train_ires_' + str(cnt), train_i) + np.save(out_directory + 'train_mres_' + str(cnt), train_m) num_train_samples = train_m.shape[0] param_train_hist += np.histogram(train_m[param_idx_m, ], bins=20, range=[0.0, 160.0])[0] @@ -254,8 +259,9 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st if len(data_tiles_m) > 0: train_i = np.stack(data_tiles_i) train_m = np.stack(data_tiles_m) - np.save(out_directory + 'train_ires_' + str(cnt), train_i) - np.save(out_directory + 'train_mres_' + str(cnt), train_m) + if DO_WRITE_OUTFILE: + np.save(out_directory + 'train_ires_' + str(cnt), train_i) + np.save(out_directory + 'train_mres_' + str(cnt), train_m) num_train_samples = train_m.shape[0] param_train_hist += np.histogram(train_m[param_idx_m, ], bins=20, range=[0.0, 160.0])[0] total_num_train_samples += num_train_samples