diff --git a/modules/util/abi_surfrad.py b/modules/util/abi_surfrad.py index 182ccacd4df4d21e5f52fe04cb638da0711bcde0..af85d94fe6d9a2873ea8f92ffb2985aa102da316 100644 --- a/modules/util/abi_surfrad.py +++ b/modules/util/abi_surfrad.py @@ -89,9 +89,10 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st print('Start, number of valid files: ', num_files) total_num_not_missing = 0 + num_skip = 3 for idx, data_f in enumerate(valid_files): - if idx % 3 == 0: # if we want to skip some files + if idx % num_skip == 0: # if we want to skip some files try: h5f = h5py.File(data_f, 'r') except: @@ -128,7 +129,7 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st data_tiles_i = [] data_tiles_m = [] - print(' num_valid_samples, progress % : ', num_valid_samples, int((f_cnt/(num_files/4))*100)) + print(' num_valid_samples, progress % : ', num_valid_samples, int((f_cnt/(num_files/num_skip))*100)) total_num_valid_samples += num_valid_samples print('total_num_valid_samples, total_num_not_missing: ', total_num_valid_samples, total_num_not_missing) print('--------------------------------------------------') @@ -156,7 +157,7 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st print('Start, number of train files: ', num_files) for idx, data_f in enumerate(train_files): - if idx % 4 == 0: # if we want to skip some files + if idx % num_skip == 0: # if we want to skip some files try: h5f = h5py.File(data_f, 'r') except: @@ -193,7 +194,7 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st data_tiles_i = [] data_tiles_m = [] - print(' num_train_samples, progress % : ', num_train_samples, int((f_cnt/(num_files/4))*100)) + print(' num_train_samples, progress % : ', num_train_samples, int((f_cnt/(num_files/num_skip))*100)) total_num_train_samples += num_train_samples print('total_num_train_samples, total_num_not_missing: ', total_num_train_samples, total_num_not_missing) print('--------------------------------------------------')