diff --git a/modules/util/abi_surfrad.py b/modules/util/abi_surfrad.py index 884eabab6069e1e8222f3c0ccbbe900541c4d0f8..51cbbc443e20c865d34a961ce804301503698a66 100644 --- a/modules/util/abi_surfrad.py +++ b/modules/util/abi_surfrad.py @@ -3,9 +3,9 @@ import h5py from util.util import get_grid_values, is_day import glob -# target_param = 'cloud_probability' +target_param = 'cloud_probability' # target_param = 'cld_opd_dcomp' -target_param = 'cld_opd_dcomp_1' +# target_param = 'cld_opd_dcomp_1' # target_param = 'cld_opd_dcomp_2' # target_param = 'cld_opd_dcomp_3' @@ -20,12 +20,23 @@ params_i = [group_name_i+'temp_ch38', group_name_i+'refl_ch01', group_name_i+tar params_m = [group_name_m+'temp_ch38', group_name_m+'refl_ch01', group_name_m+'refl_submin_ch01', group_name_m+'refl_submax_ch01', group_name_m+'refl_substddev_ch01', group_name_m+target_param, - group_name_m+'cloud_probability', group_name_m+'cloud_fraction'] param_idx_m = params_m.index(group_name_m + target_param) param_idx_i = params_i.index(group_name_i + target_param) +hist_range = [0.0, 1.0] +# hist_range = [0.0, 160.0] + +tile_width = 32 +kernel_size = 5 +factor = 4 +# tile_width = 64 +# kernel_size = 7 +# factor = 4 + +num_skip = 3 + DO_WRITE_OUTFILE = True @@ -43,8 +54,8 @@ def keep_tile(p_idx, tile): grd_k = tile[p_idx, ].copy() if target_param == 'cloud_probability': - grd_k = process_cld_prob(grd_k) - # grd_k = process_cloud_frac(grd_k) + # grd_k = process_cld_prob(grd_k) + grd_k = process_cloud_frac(grd_k) elif 'cld_opd_dcomp' in target_param: grd_k = process_cld_opd(grd_k) @@ -125,11 +136,11 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st total_num_valid_samples = 0 # path = directory + '**' + '/' + pattern - path = directory + '*_v3/2020/' + '**' + '/' + pattern + path = directory + '*_v3/202?/' + '**' + '/' + pattern all_files = glob.glob(path, recursive=True) - test_files = glob.glob(directory + '*_v3/2020/*/01/*/*.nc', recursive=True) - valid_files = glob.glob(directory + '*_v3/2020/*/0[2-6]/*/*.nc', recursive=True) + test_files = glob.glob(directory + '*_v3/202?/*/01/*/*.nc', recursive=True) + valid_files = glob.glob(directory + '*_v3/202?/*/0[2-6]/*/*.nc', recursive=True) train_files = [f for f in all_files if f not in valid_files + test_files] data_tiles_i = [] @@ -140,24 +151,10 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st print('Start, number of valid files: ', num_files) total_num_not_missing = 0 - num_skip = 3 param_train_hist = np.zeros([16], dtype=np.int64) param_valid_hist = np.zeros([16], dtype=np.int64) - # cloud_prob to cloud fraction - # ---------------------------- - tile_width = 32 - kernel_size = 5 - factor = 4 - - # tile_width = 64 - # kernel_size = 7 - # factor = 4 - - # hist_range = [0.0, 1.0] - hist_range = [0.0, 160.0] - for idx, data_f in enumerate(valid_files): if idx % num_skip == 0: # if we want to skip some files try: