Skip to content
Snippets Groups Projects
Commit ed8686ef authored by tomrink's avatar tomrink
Browse files

snapshot...

parent f9272aa7
No related branches found
No related tags found
No related merge requests found
...@@ -65,12 +65,25 @@ def keep_tile(p_idx, tile): ...@@ -65,12 +65,25 @@ def keep_tile(p_idx, tile):
return None return None
# def process_cld_prob(grd_k):
# keep = np.invert(np.isnan(grd_k))
# num_keep = np.sum(keep)
# keep_clr = np.where(keep, grd_k < 0.20, False)
# frac_keep = np.sum(keep_clr)/num_keep
# if not (0.30 < frac_keep < 0.70):
# return None
# grd_k = np.where(np.invert(keep), 0, grd_k) # Convert NaN to 0
# return grd_k
def process_cld_prob(grd_k): def process_cld_prob(grd_k):
keep = np.invert(np.isnan(grd_k)) keep = np.invert(np.isnan(grd_k))
num_keep = np.sum(keep) num_keep = np.sum(keep)
keep_clr = np.where(keep, grd_k < 0.20, False) keep_clr = np.where(keep, grd_k < 0.20, False)
frac_keep = np.sum(keep_clr)/num_keep keep_cld = np.where(keep, grd_k > 0.80, False)
if not (0.30 < frac_keep < 0.70): frac_clr = np.sum(keep_clr)/num_keep
frac_cld = np.sum(keep_cld)/num_keep
if not (frac_clr >= 0.23 and frac_cld >= 0.23):
return None return None
grd_k = np.where(np.invert(keep), 0, grd_k) # Convert NaN to 0 grd_k = np.where(np.invert(keep), 0, grd_k) # Convert NaN to 0
return grd_k return grd_k
...@@ -166,6 +179,31 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st ...@@ -166,6 +179,31 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
cnt += 1 cnt += 1
# Write out leftover, if any. Maybe make this better someday
num_valid_samples = 0
if len(valid_tiles_m) > 0:
valid_i = np.stack(valid_tiles_i)
valid_m = np.stack(valid_tiles_m)
np.save(out_directory + 'valid_mres_' + str(cnt), valid_m)
np.save(out_directory + 'valid_ires_' + str(cnt), valid_i)
num_valid_samples = valid_m.shape[0]
num_train_samples = 0
if len(train_tiles_m) > 0:
train_i = np.stack(train_tiles_i)
train_m = np.stack(train_tiles_m)
np.save(out_directory + 'train_ires_' + str(cnt), train_i)
np.save(out_directory + 'train_mres' + str(cnt), train_m)
num_train_samples = train_m.shape[0]
print(' num_train_samples, num_valid_samples, progress % : ', num_train_samples, num_valid_samples,
int((f_cnt / num_files) * 100))
total_num_train_samples += num_train_samples
total_num_valid_samples += num_valid_samples
print('total_num_train_samples, total_num_valid_samples, total_num_not_missing: ', total_num_train_samples,
total_num_valid_samples, total_num_not_missing)
print('--------------------------------------------------')
print('** total_num_train_samples, total_num_valid_samples: ', total_num_train_samples, total_num_valid_samples) print('** total_num_train_samples, total_num_valid_samples: ', total_num_train_samples, total_num_valid_samples)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment