diff --git a/modules/util/abi_surfrad.py b/modules/util/abi_surfrad.py index 450e761d221acaf540d622ff8814c308caf0438a..99c4bbef9955109581a1b5f6dbab4564c237b548 100644 --- a/modules/util/abi_surfrad.py +++ b/modules/util/abi_surfrad.py @@ -75,10 +75,12 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st total_num_valid_samples = 0 num_keep_x_tiles = 4 - path = directory + '**' + '/' + pattern + # path = directory + '**' + '/' + pattern + path = directory + '*_v3/2020/' + '**' + '/' + pattern all_files = glob.glob(path, recursive=True) - valid_files = glob.glob(directory + '*/0[1-4]/*/*.nc', recursive=True) + # valid_files = glob.glob(directory + '*/0[1-4]/*/*.nc', recursive=True) + valid_files = glob.glob(directory + '*_v3/2020/*/0[1-5]/*/*.nc', recursive=True) train_files = [f for f in all_files if f not in valid_files] data_tiles_i = [] @@ -89,7 +91,7 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st print('Start, number of valid files: ', num_files) total_num_not_missing = 0 - num_skip = 3 + num_skip = 2 for idx, data_f in enumerate(valid_files): if idx % num_skip == 0: # if we want to skip some files @@ -182,7 +184,7 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st if len(data_tiles_m) == 0: continue - if (f_cnt % 40) == 0: + if (f_cnt % 100) == 0: num_train_samples = 0 if len(data_tiles_m) > 0: train_i = np.stack(data_tiles_i)