Skip to content
Snippets Groups Projects
Commit 9888cd95 authored by tomrink's avatar tomrink
Browse files

snapshot...

parent c4c36a83
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
modules/util/viirs_surfrad.py
+ 50
− 48
View file @ 9888cd95
......@@ -37,10 +37,11 @@ group_name_m = 'orig/'
solzen_name = group_name_m + 'solar_zenith'
label_params = [group_name_i+target_param]
data_params = [group_name_m+'temp_11_0um', group_name_m+'refl_0_65um', group_name_m+target_param]
param_idx_m = data_params.index(group_name_m + target_param)
param_idx_i = label_params.index(group_name_i + target_param)
params_i = [group_name_i+target_param]
params_m = [group_name_m+'temp_11_0um', group_name_m+'refl_0_65um', group_name_m+target_param]
param_idx_m = params_m.index(group_name_m + target_param)
param_idx_i = params_i.index(group_name_i + target_param)
def is_missing(p_idx, tile):
......@@ -97,10 +98,10 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
all_files = glob.glob(path, recursive=True)
data_files = [f for f in all_files if f not in keep_out]
label_valid_tiles = []
label_train_tiles = []
data_valid_tiles = []
data_train_tiles = []
valid_tiles_i = []
train_tiles_i = []
valid_tiles_m = []
train_tiles_m = []
f_cnt = 0
num_files = len(data_files)
......@@ -118,8 +119,8 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
continue
try:
num_not_missing = run(h5f, data_params, data_train_tiles, data_valid_tiles,
label_params, label_train_tiles, label_valid_tiles,
num_not_missing = run(h5f, params_m, train_tiles_m, valid_tiles_m,
params_i, train_tiles_i, valid_tiles_i,
num_keep_x_tiles=num_keep_x_tiles, tile_width=64, kernel_size=7, day_night=day_night)
except Exception as e:
print(e)
......@@ -131,30 +132,30 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
total_num_not_missing += num_not_missing
if len(data_train_tiles) == 0 and len(data_valid_tiles) == 0:
if len(train_tiles_m) == 0 and len(valid_tiles_m) == 0:
continue
if (f_cnt % 5) == 0:
num_valid_samples = 0
if len(data_valid_tiles) > 0:
label_valid = np.stack(label_valid_tiles)
data_valid = np.stack(data_valid_tiles)
np.save(out_directory + 'data_valid_' + str(cnt), data_valid)
np.save(out_directory + 'label_valid_' + str(cnt), label_valid)
num_valid_samples = data_valid.shape[0]
if len(valid_tiles_m) > 0:
valid_i = np.stack(valid_tiles_i)
valid_m = np.stack(valid_tiles_m)
np.save(out_directory + 'valid_mres_' + str(cnt), valid_m)
np.save(out_directory + 'valid_ires_' + str(cnt), valid_i)
num_valid_samples = valid_m.shape[0]
num_train_samples = 0
if len(data_train_tiles) > 0:
label_train = np.stack(label_train_tiles)
data_train = np.stack(data_train_tiles)
np.save(out_directory + 'label_train_' + str(cnt), label_train)
np.save(out_directory + 'data_train_' + str(cnt), data_train)
num_train_samples = data_train.shape[0]
label_valid_tiles = []
label_train_tiles = []
data_valid_tiles = []
data_train_tiles = []
if len(train_tiles_m) > 0:
train_i = np.stack(train_tiles_i)
train_m = np.stack(train_tiles_m)
np.save(out_directory + 'train_ires_' + str(cnt), train_i)
np.save(out_directory + 'train_mres' + str(cnt), train_m)
num_train_samples = train_m.shape[0]
valid_tiles_i = []
train_tiles_i = []
valid_tiles_m = []
train_tiles_m = []
print(' num_train_samples, num_valid_samples, progress % : ', num_train_samples, num_valid_samples, int((f_cnt/num_files)*100))
total_num_train_samples += num_train_samples
......@@ -170,12 +171,12 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
# tile_width: Must be even!
# kernel_size: Must be odd!
def run(h5f, param_s, train_tiles, valid_tiles, lbl_param_s, lbl_train_tiles, lbl_valid_tiles,
def run(h5f, params_m, train_tiles_m, valid_tiles_m, params_i, train_tiles_i, valid_tiles_i,
num_keep_x_tiles=8, tile_width=64, kernel_size=3, day_night='ANY'):
border = int((kernel_size - 1)/2) + 1 # Need to add for interpolation with no edge effects
param_name = param_s[0]
param_name = params_m[0]
num_lines = h5f[param_name].shape[0]
num_pixels = h5f[param_name].shape[1] # Must be even
......@@ -184,24 +185,24 @@ def run(h5f, param_s, train_tiles, valid_tiles, lbl_param_s, lbl_train_tiles, lb
solzen = get_grid_values(h5f, solzen_name, 0, 0, None, num_lines, num_pixels)
grd_s = []
for param in param_s:
for param in params_m:
try:
grd = get_grid_values(h5f, param, 0, 0, None, num_lines, num_pixels)
grd_s.append(grd)
except Exception as e:
print(e)
return
data = np.stack(grd_s)
data_m = np.stack(grd_s)
grd_s = []
for param in lbl_param_s:
for param in params_i:
try:
grd = get_grid_values(h5f, param, 0, 0, None, num_lines*2, num_pixels*2)
grd_s.append(grd)
except Exception as e:
print(e)
return
label = np.stack(grd_s)
data_i = np.stack(grd_s)
tile_width += 2 * border
......@@ -212,8 +213,8 @@ def run(h5f, param_s, train_tiles, valid_tiles, lbl_param_s, lbl_train_tiles, lb
num_y_tiles = int(num_lines / tile_width) - 1
data_tiles = []
lbl_tiles = []
data_tiles_m = []
data_tiles_i = []
num_not_missing = 0
for j in range(num_y_tiles):
......@@ -229,26 +230,27 @@ def run(h5f, param_s, train_tiles, valid_tiles, lbl_param_s, lbl_train_tiles, lb
elif day_night == 'NIGHT' and is_day(solzen[j_a:j_b, i_a:i_b]):
continue
nda = data[:, j_a:j_b, i_a:i_b]
nda_lbl = label[:, j_a*2:j_b*2, i_a*2:i_b*2]
if is_missing(param_idx_i, nda_lbl):
nda_m = data_m[:, j_a:j_b, i_a:i_b]
nda_i = data_i[:, j_a*2:j_b*2, i_a*2:i_b*2]
if is_missing(param_idx_i, nda_i):
continue
num_not_missing += 1
nda_lbl = keep_tile(param_idx_i, nda_lbl)
if nda_lbl is not None:
data_tiles.append(nda)
lbl_tiles.append(nda_lbl)
nda_i = keep_tile(param_idx_i, nda_i)
if nda_i is not None:
data_tiles_m.append(nda_m)
data_tiles_i.append(nda_i)
num_tiles = len(lbl_tiles)
num_tiles = len(data_tiles_i)
num_valid = int(num_tiles * 0.10)
num_train = num_tiles - num_valid
for k in range(num_train):
train_tiles.append(data_tiles[k])
lbl_train_tiles.append(lbl_tiles[k])
train_tiles_m.append(data_tiles_m[k])
train_tiles_i.append(data_tiles_i[k])
for k in range(num_valid):
valid_tiles.append(data_tiles[num_train + k])
lbl_valid_tiles.append(lbl_tiles[num_train + k])
valid_tiles_m.append(data_tiles_m[num_train + k])
valid_tiles_i.append(data_tiles_i[num_train + k])
return num_not_missing
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment