Skip to content
Snippets Groups Projects
Commit 162ce166 authored by tomrink's avatar tomrink
Browse files

snapshot...

parent d058f4be
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
modules/util/viirs_l1b_l2.py
+ 25
− 106
View file @ 162ce166
......@@ -15,20 +15,13 @@ emis_params = ['temp_10_4um_nom', 'temp_11_0um_nom', 'temp_12_0um_nom', 'temp_13
# data_params = refl_params + emis_params
data_params = emis_params
# l2_params = ['refl_0_65um_nom', 'temp_11_0um_nom', 'cld_temp_acha', 'cld_press_acha', 'cloud_fraction', 'cld_opd_acha', 'cld_reff_acha']
l2_params = ['refl_0_65um_nom', 'temp_11_0um_nom', 'cld_temp_acha', 'cld_press_acha', 'cloud_fraction']
label_params = l2_params
data_params = l2_params
# data_params = ['cloud_fraction']
# label_params = ['cloud_fraction']
l2_params = ['refl_0_65um_nom', 'temp_11_0um_nom', 'cld_temp_acha', 'cld_press_acha', 'cloud_fraction', 'cld_opd_acha']
# data_params = ['observation_data/M15']
# label_params = ['observation_data/M15_highres']
label_params = l2_params
#data_params = l2_params
def run_all(directory, out_directory):
num_train_samples, num_valid_samples = 0, 0
cnt = 10
total_num_train_samples = 0
......@@ -73,11 +66,11 @@ def run_all(directory, out_directory):
# continue
data_tiles = []
#label_tiles = []
label_tiles = []
try:
# run(data_h5f, label_h5f, data_tiles, label_tiles, mod_tile_width=32, kernel_size=5)
run_one(data_h5f, data_tiles, tile_width=128, kernel_size=7)
run(data_h5f, data_params, data_tiles, tile_width=128, kernel_size=7)
run(data_h5f, label_params, label_tiles, tile_width=128, kernel_size=7)
except Exception as e:
print(e)
data_h5f.close()
......@@ -87,21 +80,21 @@ def run_all(directory, out_directory):
data_h5f.close()
#label_h5f.close()
# if len(data_tiles) == 0 or len(label_tiles) == 0:
# continue
#
# if len(data_tiles) != len(label_tiles):
# print('weirdness: ', data_f)
# continue
if len(data_tiles) == 0 or len(label_tiles) == 0:
continue
if len(data_tiles) == 0:
if len(data_tiles) != len(label_tiles):
print('weirdness: ', data_f)
continue
# if len(data_tiles) == 0:
# continue
num = len(data_tiles)
n_vld = int(num * 0.1)
# [label_valid_tiles.append(label_tiles[k]) for k in range(n_vld)]
# [label_train_tiles.append(label_tiles[k]) for k in range(n_vld, num)]
[label_valid_tiles.append(label_tiles[k]) for k in range(n_vld)]
[label_train_tiles.append(label_tiles[k]) for k in range(n_vld, num)]
[data_valid_tiles.append(data_tiles[k]) for k in range(n_vld)]
[data_train_tiles.append(data_tiles[k]) for k in range(n_vld, num)]
......@@ -109,18 +102,18 @@ def run_all(directory, out_directory):
if f_cnt == 10:
f_cnt = 0
#label_valid = np.stack(label_valid_tiles)
#label_train = np.stack(label_train_tiles)
label_valid = np.stack(label_valid_tiles)
label_train = np.stack(label_train_tiles)
data_valid = np.stack(data_valid_tiles)
data_train = np.stack(data_train_tiles)
np.save(out_directory+'data_train_' + str(cnt), data_train)
np.save(out_directory+'data_valid_' + str(cnt), data_valid)
#np.save(out_directory+'label_train_' + str(cnt), label_train)
#np.save(out_directory+'label_valid_' + str(cnt), label_valid)
np.save(out_directory+'label_train_' + str(cnt), label_train)
np.save(out_directory+'label_valid_' + str(cnt), label_valid)
#label_valid_tiles = []
#label_train_tiles = []
label_valid_tiles = []
label_train_tiles = []
data_valid_tiles = []
data_train_tiles = []
......@@ -136,92 +129,18 @@ def run_all(directory, out_directory):
print('total_num_train_samples, total_num_valid_samples: ', total_num_train_samples, total_num_valid_samples)
def run(data_h5f, label_h5f, data_tiles, label_tiles, mod_tile_width=64, kernel_size=9):
if label_h5f is None:
label_h5f = data_h5f
border = int((kernel_size - 1)/2)
l1b_param_name = data_params[0]
l2_param_name = label_params[0]
mod_num_lines = data_h5f[l1b_param_name].shape[0]
mod_num_pixels = data_h5f[l1b_param_name].shape[1]
img_num_lines = label_h5f[l2_param_name].shape[0]
img_num_pixels = label_h5f[l2_param_name].shape[1]
factor = int(img_num_pixels / mod_num_pixels)
img_tile_width = mod_tile_width * factor
l1b_grd_s = []
l2_grd_s = []
for param in data_params:
try:
grd = get_grid_values(data_h5f, param, 0, 0, None, mod_num_lines, mod_num_pixels, range_name=None)
l1b_grd_s.append(grd)
except Exception as e:
print(e)
return
for param in label_params:
try:
grd = get_grid_values(label_h5f, param, 0, 0, None, img_num_lines, img_num_pixels, range_name=None)
l2_grd_s.append(grd)
except Exception as e:
print(e)
return
mod_data = np.stack(l1b_grd_s)
img_data = np.stack(l2_grd_s)
num_keep_x_tiles = 3
#num_keep_x_tiles = 1
i_skip = 3 * mod_tile_width
#i_skip = 1
j_skip = 1 * mod_tile_width
i_start = int(mod_num_pixels / 2) - int((num_keep_x_tiles * 3 * mod_tile_width) / 2)
#i_start = int(mod_num_pixels / 2) - int((mod_tile_width) / 2)
num_keep_y_tiles = 96
for j in range(num_keep_y_tiles):
j_c = j * j_skip
j_m = j_c + border
j_i = j_m * factor
for i in range(num_keep_x_tiles):
i_c = i * i_skip + i_start
i_m = i_c + border
i_i = i_m * factor
j_stop = j_m + mod_tile_width + border
if j_stop > mod_num_lines - 1:
continue
i_stop = i_m + mod_tile_width + border
if i_stop > mod_num_pixels - 1:
continue
nda = mod_data[:, j_m-border:j_stop, i_m-border:i_stop]
data_tiles.append(nda)
nda = img_data[:, j_i:j_i + img_tile_width, i_i:i_i + img_tile_width]
label_tiles.append(nda)
def run_one(data_h5f, data_tiles, tile_width=64, kernel_size=9):
def run(data_h5f, param_s, tiles, tile_width=64, kernel_size=9):
border = int((kernel_size - 1)/2)
param_name = data_params[0]
param_name = param_s[0]
num_lines = data_h5f[param_name].shape[0]
num_pixels = data_h5f[param_name].shape[1]
grd_s = []
for param in data_params:
for param in param_s:
try:
grd = get_grid_values(data_h5f, param, 0, 0, None, num_lines, num_pixels, range_name=None)
# if param == 'temp_11_0um_nom' and ((np.sum(np.isnan(grd)) / grd.size) < 0.10):
......@@ -259,7 +178,7 @@ def run_one(data_h5f, data_tiles, tile_width=64, kernel_size=9):
nda = data[:, j_m-border:j_stop, i_m-border:i_stop]
tmp = nda[1, :, :]
if (np.sum(np.isnan(tmp)) / tmp.size) < 0.10:
data_tiles.append(nda)
tiles.append(nda)
def scan(directory):
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment