diff --git a/modules/util/viirs_l1b_l2.py b/modules/util/viirs_l1b_l2.py index 1b6b5f7cad73f9f377f538c0457740104d1b7af9..d35edd568eb0f0589d4fdc7480be4cb91d2223a1 100644 --- a/modules/util/viirs_l1b_l2.py +++ b/modules/util/viirs_l1b_l2.py @@ -77,7 +77,7 @@ def process_cld_prob(grd_k): num_keep = np.sum(keep) keep_clr = np.where(keep, grd_k < 0.20, False) frac_keep = np.sum(keep_clr)/num_keep - if not (0.35 < frac_keep < 0.65): + if not (0.30 < frac_keep < 0.70): return None grd_k = np.where(np.invert(keep), 0, grd_k) # Convert NaN to 0 return grd_k @@ -165,7 +165,28 @@ def run_all(directory, out_directory, day_night='ANY', start=10): cnt += 1 - print('** Done, total_num_train_samples, total_num_valid_samples: ', total_num_train_samples, total_num_valid_samples) + # Write out leftover, if any + num_valid_samples = 0 + if len(data_valid_tiles) > 0: + data_valid = np.stack(data_valid_tiles) + np.save(out_directory + 'data_valid_' + str(cnt), data_valid) + num_valid_samples = data_valid.shape[0] + + num_train_samples = 0 + if len(data_train_tiles) > 0: + data_train = np.stack(data_train_tiles) + np.save(out_directory + 'data_train_' + str(cnt), data_train) + num_train_samples = data_train.shape[0] + + print(' num_train_samples, num_valid_samples, progress % : ', num_train_samples, num_valid_samples, + int((f_cnt / num_files) * 100)) + total_num_train_samples += num_train_samples + total_num_valid_samples += num_valid_samples + print('total_num_train_samples, total_num_valid_samples, total_num_not_missing: ', + total_num_train_samples, total_num_valid_samples, total_num_not_missing) + print('---------------------------------------------------------') + + print('*** Done, total_num_train_samples, total_num_valid_samples: ', total_num_train_samples, total_num_valid_samples) # tile_width: Must be even!