Skip to content
Snippets Groups Projects
Commit 1306bc1f authored by tomrink's avatar tomrink
Browse files

run one

parent 71dc2545
Branches
No related tags found
No related merge requests found
......@@ -39,7 +39,8 @@ def run_all(directory, out_directory):
continue
print(p.name)
data_files = glob.glob(directory + p.name+'/'+'clavrx_snpp_viirs*.uwssec*.nc')
data_files = glob.glob(directory + p.name+'/'+'clavrx*highres*.nc')
# data_files = glob.glob(directory + p.name+'/'+'clavrx_snpp_viirs*.uwssec*.nc')
# data_files = glob.glob(directory + p.name + '/' + 'VNP02MOD*.uwssec.nc')
label_valid_tiles = []
......@@ -51,13 +52,12 @@ def run_all(directory, out_directory):
for idx, data_f in enumerate(data_files):
# if idx % 8 == 0: # if we want to skip some files
if True:
w_o_ext, ext = os.path.splitext(data_f)
# label_f = data_f
pname, fname = os.path.split(data_f)
toks = fname.split('.')
label_f = pname + '/' + 'clavrx_VNP02MOD.' + toks[1]+'.'+toks[2]+'.'+toks[3]+'.'+toks[4]+'.'+'uwssec.highres.nc.level2.nc'
if not os.path.exists(label_f):
continue
# w_o_ext, ext = os.path.splitext(data_f)
# pname, fname = os.path.split(data_f)
# toks = fname.split('.')
# label_f = pname + '/' + 'clavrx_VNP02MOD.' + toks[1]+'.'+toks[2]+'.'+toks[3]+'.'+toks[4]+'.'+'uwssec.highres.nc.level2.nc'
# if not os.path.exists(label_f):
# continue
try:
data_h5f = h5py.File(data_f, 'r')
......@@ -65,57 +65,61 @@ def run_all(directory, out_directory):
print('cant open file: ', data_f)
continue
try:
label_h5f = h5py.File(label_f, 'r')
except:
print('cant open file: ', label_f)
data_h5f.close()
continue
# try:
# label_h5f = h5py.File(label_f, 'r')
# except:
# print('cant open file: ', label_f)
# data_h5f.close()
# continue
data_tiles = []
label_tiles = []
#label_tiles = []
try:
run(data_h5f, label_h5f, data_tiles, label_tiles, mod_tile_width=32, kernel_size=5)
# run(data_h5f, label_h5f, data_tiles, label_tiles, mod_tile_width=32, kernel_size=5)
run_one(data_h5f, data_tiles, mod_tile_width=64, kernel_size=5)
except Exception as e:
print(e)
data_h5f.close()
label_h5f.close()
#label_h5f.close()
continue
data_h5f.close()
label_h5f.close()
#label_h5f.close()
if len(data_tiles) == 0 or len(label_tiles) == 0:
continue
# if len(data_tiles) == 0 or len(label_tiles) == 0:
# continue
#
# if len(data_tiles) != len(label_tiles):
# print('weirdness: ', data_f)
# continue
if len(data_tiles) != len(label_tiles):
print('weirdness: ', data_f)
if len(data_tiles):
continue
num = len(data_tiles)
n_vld = int(num * 0.1)
[label_valid_tiles.append(label_tiles[k]) for k in range(n_vld)]
[label_train_tiles.append(label_tiles[k]) for k in range(n_vld, num)]
# [label_valid_tiles.append(label_tiles[k]) for k in range(n_vld)]
# [label_train_tiles.append(label_tiles[k]) for k in range(n_vld, num)]
[data_valid_tiles.append(data_tiles[k]) for k in range(n_vld)]
[data_train_tiles.append(data_tiles[k]) for k in range(n_vld, num)]
if f_cnt == 10:
cnt += 1
label_valid = np.stack(label_valid_tiles)
label_train = np.stack(label_train_tiles)
#label_valid = np.stack(label_valid_tiles)
#label_train = np.stack(label_train_tiles)
data_valid = np.stack(data_valid_tiles)
data_train = np.stack(data_train_tiles)
np.save(out_directory+'data_train_' + str(cnt), data_train)
np.save(out_directory+'data_valid_' + str(cnt), data_valid)
np.save(out_directory+'label_train_' + str(cnt), label_train)
np.save(out_directory+'label_valid_' + str(cnt), label_valid)
#np.save(out_directory+'label_train_' + str(cnt), label_train)
#np.save(out_directory+'label_valid_' + str(cnt), label_valid)
label_valid_tiles = []
label_train_tiles = []
#label_valid_tiles = []
#label_train_tiles = []
data_valid_tiles = []
data_train_tiles = []
......@@ -125,22 +129,25 @@ def run_all(directory, out_directory):
else:
f_cnt += 1
if len(label_train_tiles) == 0 or len(data_train_tiles) == 0:
continue
if len(label_train_tiles) != len(data_train_tiles):
print('weirdness')
# if len(label_train_tiles) == 0 or len(data_train_tiles) == 0:
# continue
# if len(label_train_tiles) != len(data_train_tiles):
# print('weirdness')
# continue
if len(data_train_tiles) == 0:
continue
label_valid = np.stack(label_valid_tiles)
label_train = np.stack(label_train_tiles)
#label_valid = np.stack(label_valid_tiles)
#label_train = np.stack(label_train_tiles)
data_valid = np.stack(data_valid_tiles)
data_train = np.stack(data_train_tiles)
cnt += 1
np.save(out_directory+'data_train_' + str(cnt), data_train)
np.save(out_directory+'data_valid_' + str(cnt), data_valid)
np.save(out_directory+'label_train_' + str(cnt), label_train)
np.save(out_directory+'label_valid_' + str(cnt), label_valid)
#np.save(out_directory+'label_train_' + str(cnt), label_train)
#np.save(out_directory+'label_valid_' + str(cnt), label_valid)
num_train_samples += data_train.shape[0]
num_valid_samples += data_valid.shape[0]
......@@ -226,6 +233,56 @@ def run(data_h5f, label_h5f, data_tiles, label_tiles, mod_tile_width=64, kernel_
label_tiles.append(nda)
def run_one(data_h5f, data_tiles, tile_width=64, kernel_size=9):
border = int((kernel_size - 1)/2)
param_name = data_params[0]
num_lines = data_h5f[param_name].shape[0]
num_pixels = data_h5f[param_name].shape[1]
grd_s = []
for param in data_params:
try:
grd = get_grid_values(data_h5f, param, 0, 0, None, num_lines, num_pixels, range_name=None)
grd_s.append(grd)
except Exception as e:
print(e)
return
data = np.stack(grd_s)
# num_keep_x_tiles = 2
num_keep_x_tiles = 1
#i_skip = num_keep_x_tiles * tile_width
i_skip = 1
j_skip = 1 * tile_width
#i_start = int(num_pixels / 2) - int((num_keep_x_tiles * 2 * tile_width) / 2)
i_start = int(num_pixels / 2) - int((tile_width) / 2)
num_keep_y_tiles = 96
for j in range(num_keep_y_tiles):
j_c = j * j_skip
j_m = j_c + border
for i in range(num_keep_x_tiles):
i_c = i * i_skip + i_start
i_m = i_c + border
j_stop = j_m + tile_width + border
if j_stop > num_lines - 1:
continue
i_stop = i_m + tile_width + border
if i_stop > num_pixels - 1:
continue
nda = data[:, j_m-border:j_stop, i_m-border:i_stop]
data_tiles.append(nda)
def scan(directory):
data_src = CLAVRx_VIIRS(directory)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment