Skip to content
Snippets Groups Projects
Commit 4db06aba authored by tomrink's avatar tomrink
Browse files

snapshot...

parent 23883315
No related merge requests found
......@@ -3,9 +3,9 @@ import h5py
from util.util import get_grid_values, is_day
import glob
# target_param = 'cloud_probability'
target_param = 'cloud_probability'
# target_param = 'cld_opd_dcomp'
target_param = 'cld_opd_dcomp_1'
# target_param = 'cld_opd_dcomp_1'
# target_param = 'cld_opd_dcomp_2'
# target_param = 'cld_opd_dcomp_3'
......@@ -20,12 +20,23 @@ params_i = [group_name_i+'temp_ch38', group_name_i+'refl_ch01', group_name_i+tar
params_m = [group_name_m+'temp_ch38', group_name_m+'refl_ch01',
group_name_m+'refl_submin_ch01', group_name_m+'refl_submax_ch01', group_name_m+'refl_substddev_ch01',
group_name_m+target_param,
group_name_m+'cloud_probability',
group_name_m+'cloud_fraction']
param_idx_m = params_m.index(group_name_m + target_param)
param_idx_i = params_i.index(group_name_i + target_param)
hist_range = [0.0, 1.0]
# hist_range = [0.0, 160.0]
tile_width = 32
kernel_size = 5
factor = 4
# tile_width = 64
# kernel_size = 7
# factor = 4
num_skip = 3
DO_WRITE_OUTFILE = True
......@@ -43,8 +54,8 @@ def keep_tile(p_idx, tile):
grd_k = tile[p_idx, ].copy()
if target_param == 'cloud_probability':
grd_k = process_cld_prob(grd_k)
# grd_k = process_cloud_frac(grd_k)
# grd_k = process_cld_prob(grd_k)
grd_k = process_cloud_frac(grd_k)
elif 'cld_opd_dcomp' in target_param:
grd_k = process_cld_opd(grd_k)
......@@ -125,11 +136,11 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
total_num_valid_samples = 0
# path = directory + '**' + '/' + pattern
path = directory + '*_v3/2020/' + '**' + '/' + pattern
path = directory + '*_v3/202?/' + '**' + '/' + pattern
all_files = glob.glob(path, recursive=True)
test_files = glob.glob(directory + '*_v3/2020/*/01/*/*.nc', recursive=True)
valid_files = glob.glob(directory + '*_v3/2020/*/0[2-6]/*/*.nc', recursive=True)
test_files = glob.glob(directory + '*_v3/202?/*/01/*/*.nc', recursive=True)
valid_files = glob.glob(directory + '*_v3/202?/*/0[2-6]/*/*.nc', recursive=True)
train_files = [f for f in all_files if f not in valid_files + test_files]
data_tiles_i = []
......@@ -140,24 +151,10 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
print('Start, number of valid files: ', num_files)
total_num_not_missing = 0
num_skip = 3
param_train_hist = np.zeros([16], dtype=np.int64)
param_valid_hist = np.zeros([16], dtype=np.int64)
# cloud_prob to cloud fraction
# ----------------------------
tile_width = 32
kernel_size = 5
factor = 4
# tile_width = 64
# kernel_size = 7
# factor = 4
# hist_range = [0.0, 1.0]
hist_range = [0.0, 160.0]
for idx, data_f in enumerate(valid_files):
if idx % num_skip == 0: # if we want to skip some files
try:
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment