Skip to content
Snippets Groups Projects
Commit 1306bc1f authored by tomrink's avatar tomrink
Browse files

run one

parent 71dc2545
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
modules/util/viirs_l1b_l2.py
+ 95
− 38
View file @ 1306bc1f
......@@ -39,7 +39,8 @@ def run_all(directory, out_directory):
continue
print(p.name)
data_files = glob.glob(directory + p.name+'/'+'clavrx_snpp_viirs*.uwssec*.nc')
data_files = glob.glob(directory + p.name+'/'+'clavrx*highres*.nc')
# data_files = glob.glob(directory + p.name+'/'+'clavrx_snpp_viirs*.uwssec*.nc')
# data_files = glob.glob(directory + p.name + '/' + 'VNP02MOD*.uwssec.nc')
label_valid_tiles = []
......@@ -51,13 +52,12 @@ def run_all(directory, out_directory):
for idx, data_f in enumerate(data_files):
# if idx % 8 == 0: # if we want to skip some files
if True:
w_o_ext, ext = os.path.splitext(data_f)
# label_f = data_f
pname, fname = os.path.split(data_f)
toks = fname.split('.')
label_f = pname + '/' + 'clavrx_VNP02MOD.' + toks[1]+'.'+toks[2]+'.'+toks[3]+'.'+toks[4]+'.'+'uwssec.highres.nc.level2.nc'
if not os.path.exists(label_f):
continue
# w_o_ext, ext = os.path.splitext(data_f)
# pname, fname = os.path.split(data_f)
# toks = fname.split('.')
# label_f = pname + '/' + 'clavrx_VNP02MOD.' + toks[1]+'.'+toks[2]+'.'+toks[3]+'.'+toks[4]+'.'+'uwssec.highres.nc.level2.nc'
# if not os.path.exists(label_f):
# continue
try:
data_h5f = h5py.File(data_f, 'r')
......@@ -65,57 +65,61 @@ def run_all(directory, out_directory):
print('cant open file: ', data_f)
continue
try:
label_h5f = h5py.File(label_f, 'r')
except:
print('cant open file: ', label_f)
data_h5f.close()
continue
# try:
# label_h5f = h5py.File(label_f, 'r')
# except:
# print('cant open file: ', label_f)
# data_h5f.close()
# continue
data_tiles = []
label_tiles = []
#label_tiles = []
try:
run(data_h5f, label_h5f, data_tiles, label_tiles, mod_tile_width=32, kernel_size=5)
# run(data_h5f, label_h5f, data_tiles, label_tiles, mod_tile_width=32, kernel_size=5)
run_one(data_h5f, data_tiles, mod_tile_width=64, kernel_size=5)
except Exception as e:
print(e)
data_h5f.close()
label_h5f.close()
#label_h5f.close()
continue
data_h5f.close()
label_h5f.close()
#label_h5f.close()
if len(data_tiles) == 0 or len(label_tiles) == 0:
continue
# if len(data_tiles) == 0 or len(label_tiles) == 0:
# continue
#
# if len(data_tiles) != len(label_tiles):
# print('weirdness: ', data_f)
# continue
if len(data_tiles) != len(label_tiles):
print('weirdness: ', data_f)
if len(data_tiles):
continue
num = len(data_tiles)
n_vld = int(num * 0.1)
[label_valid_tiles.append(label_tiles[k]) for k in range(n_vld)]
[label_train_tiles.append(label_tiles[k]) for k in range(n_vld, num)]
# [label_valid_tiles.append(label_tiles[k]) for k in range(n_vld)]
# [label_train_tiles.append(label_tiles[k]) for k in range(n_vld, num)]
[data_valid_tiles.append(data_tiles[k]) for k in range(n_vld)]
[data_train_tiles.append(data_tiles[k]) for k in range(n_vld, num)]
if f_cnt == 10:
cnt += 1
label_valid = np.stack(label_valid_tiles)
label_train = np.stack(label_train_tiles)
#label_valid = np.stack(label_valid_tiles)
#label_train = np.stack(label_train_tiles)
data_valid = np.stack(data_valid_tiles)
data_train = np.stack(data_train_tiles)
np.save(out_directory+'data_train_' + str(cnt), data_train)
np.save(out_directory+'data_valid_' + str(cnt), data_valid)
np.save(out_directory+'label_train_' + str(cnt), label_train)
np.save(out_directory+'label_valid_' + str(cnt), label_valid)
#np.save(out_directory+'label_train_' + str(cnt), label_train)
#np.save(out_directory+'label_valid_' + str(cnt), label_valid)
label_valid_tiles = []
label_train_tiles = []
#label_valid_tiles = []
#label_train_tiles = []
data_valid_tiles = []
data_train_tiles = []
......@@ -125,22 +129,25 @@ def run_all(directory, out_directory):
else:
f_cnt += 1
if len(label_train_tiles) == 0 or len(data_train_tiles) == 0:
continue
if len(label_train_tiles) != len(data_train_tiles):
print('weirdness')
# if len(label_train_tiles) == 0 or len(data_train_tiles) == 0:
# continue
# if len(label_train_tiles) != len(data_train_tiles):
# print('weirdness')
# continue
if len(data_train_tiles) == 0:
continue
label_valid = np.stack(label_valid_tiles)
label_train = np.stack(label_train_tiles)
#label_valid = np.stack(label_valid_tiles)
#label_train = np.stack(label_train_tiles)
data_valid = np.stack(data_valid_tiles)
data_train = np.stack(data_train_tiles)
cnt += 1
np.save(out_directory+'data_train_' + str(cnt), data_train)
np.save(out_directory+'data_valid_' + str(cnt), data_valid)
np.save(out_directory+'label_train_' + str(cnt), label_train)
np.save(out_directory+'label_valid_' + str(cnt), label_valid)
#np.save(out_directory+'label_train_' + str(cnt), label_train)
#np.save(out_directory+'label_valid_' + str(cnt), label_valid)
num_train_samples += data_train.shape[0]
num_valid_samples += data_valid.shape[0]
......@@ -226,6 +233,56 @@ def run(data_h5f, label_h5f, data_tiles, label_tiles, mod_tile_width=64, kernel_
label_tiles.append(nda)
def run_one(data_h5f, data_tiles, tile_width=64, kernel_size=9):
border = int((kernel_size - 1)/2)
param_name = data_params[0]
num_lines = data_h5f[param_name].shape[0]
num_pixels = data_h5f[param_name].shape[1]
grd_s = []
for param in data_params:
try:
grd = get_grid_values(data_h5f, param, 0, 0, None, num_lines, num_pixels, range_name=None)
grd_s.append(grd)
except Exception as e:
print(e)
return
data = np.stack(grd_s)
# num_keep_x_tiles = 2
num_keep_x_tiles = 1
#i_skip = num_keep_x_tiles * tile_width
i_skip = 1
j_skip = 1 * tile_width
#i_start = int(num_pixels / 2) - int((num_keep_x_tiles * 2 * tile_width) / 2)
i_start = int(num_pixels / 2) - int((tile_width) / 2)
num_keep_y_tiles = 96
for j in range(num_keep_y_tiles):
j_c = j * j_skip
j_m = j_c + border
for i in range(num_keep_x_tiles):
i_c = i * i_skip + i_start
i_m = i_c + border
j_stop = j_m + tile_width + border
if j_stop > num_lines - 1:
continue
i_stop = i_m + tile_width + border
if i_stop > num_pixels - 1:
continue
nda = data[:, j_m-border:j_stop, i_m-border:i_stop]
data_tiles.append(nda)
def scan(directory):
data_src = CLAVRx_VIIRS(directory)
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment