Skip to content
Snippets Groups Projects
Commit a1953aff authored by tomrink's avatar tomrink
Browse files

minor

parent a89b297d
No related branches found
No related tags found
No related merge requests found
......@@ -20,8 +20,8 @@ def run_all(directory):
if not p.is_dir():
continue
# l1b_files = glob.glob(directory + p.name+'/'+'clavrx_snpp_viirs*.uwssec*.nc')
l1b_files = glob.glob(directory + p.name + '/' + 'VNP02MOD*.uwssec.nc')
# data_files = glob.glob(directory + p.name+'/'+'clavrx_snpp_viirs*.uwssec*.nc')
data_files = glob.glob(directory + p.name + '/' + 'VNP02MOD*.uwssec.nc')
label_valid_tiles = []
label_train_tiles = []
......@@ -29,47 +29,47 @@ def run_all(directory):
data_train_tiles = []
f_cnt = 0
for idx, l1b_f in enumerate(l1b_files):
for idx, data_f in enumerate(data_files):
# if idx % 8 == 0: # if we want to skip some files
if True:
w_o_ext, ext = os.path.splitext(l1b_f)
l2_f = w_o_ext+'.highres'+ext
if not os.path.exists(l2_f):
w_o_ext, ext = os.path.splitext(data_f)
label_f = w_o_ext+'.highres'+ext
if not os.path.exists(label_f):
continue
print(l1b_f)
print(data_f)
try:
l1b_h5f = h5py.File(l1b_f, 'r')
data_h5f = h5py.File(data_f, 'r')
except:
print('cant open file: ', l1b_f)
print('cant open file: ', data_f)
continue
try:
l2_h5f = h5py.File(l2_f, 'r')
label_h5f = h5py.File(label_f, 'r')
except:
print('cant open file: ', l2_f)
l1b_h5f.close()
print('cant open file: ', label_f)
data_h5f.close()
continue
data_tiles = []
label_tiles = []
try:
run(l1b_h5f, l2_h5f, data_tiles, label_tiles, factor=2)
run(data_h5f, label_h5f, data_tiles, label_tiles, factor=2)
except Exception as e:
print(e)
l1b_h5f.close()
l2_h5f.close()
data_h5f.close()
label_h5f.close()
continue
l1b_h5f.close()
l2_h5f.close()
data_h5f.close()
label_h5f.close()
if len(data_tiles) == 0 or len(label_tiles) == 0:
continue
if len(data_tiles) != len(label_tiles):
print('weirdness: ', l1b_f)
print('weirdness: ', data_f)
continue
num = len(data_tiles)
......@@ -119,9 +119,9 @@ def run_all(directory):
np.save('/data/Personal/rink/viirs/label_valid_' + str(cnt), label_valid)
def run(l1b_h5f, l2_h5f, l1b_tiles, l2_tiles, factor=2):
if l2_h5f is None:
l2_h5f = l1b_h5f
def run(data_h5f, label_h5f, data_tiles, label_tiles, factor=2):
if label_h5f is None:
label_h5f = data_h5f
l1b_param_name = data_params[0]
l2_param_name = label_params[0]
......@@ -129,11 +129,11 @@ def run(l1b_h5f, l2_h5f, l1b_tiles, l2_tiles, factor=2):
mod_tile_width = 64
img_tile_width = mod_tile_width * factor
mod_num_lines = l1b_h5f[l1b_param_name].shape[0]
mod_num_pixels = l1b_h5f[l1b_param_name].shape[1]
mod_num_lines = data_h5f[l1b_param_name].shape[0]
mod_num_pixels = data_h5f[l1b_param_name].shape[1]
img_num_lines = l2_h5f[l2_param_name].shape[0]
img_num_pixels = l2_h5f[l2_param_name].shape[1]
img_num_lines = label_h5f[l2_param_name].shape[0]
img_num_pixels = label_h5f[l2_param_name].shape[1]
mod_num_y_tiles = int(mod_num_lines / mod_tile_width)
mod_num_x_tiles = int(mod_num_pixels / mod_tile_width)
......@@ -143,14 +143,14 @@ def run(l1b_h5f, l2_h5f, l1b_tiles, l2_tiles, factor=2):
for param in data_params:
try:
grd = get_grid_values(l1b_h5f, param, 0, 0, None, mod_num_lines, mod_num_pixels, range_name=None)
grd = get_grid_values(data_h5f, param, 0, 0, None, mod_num_lines, mod_num_pixels, range_name=None)
l1b_grd_s.append(grd)
except:
return
for param in label_params:
try:
grd = get_grid_values(l2_h5f, param, 0, 0, None, img_num_lines, img_num_pixels, range_name=None)
grd = get_grid_values(label_h5f, param, 0, 0, None, img_num_lines, img_num_pixels, range_name=None)
l2_grd_s.append(grd)
except:
return
......@@ -175,10 +175,10 @@ def run(l1b_h5f, l2_h5f, l1b_tiles, l2_tiles, factor=2):
i_i = i_m * factor
nda = mod_data[:, j_m:j_m + mod_tile_width, i_m:i_m + mod_tile_width]
l1b_tiles.append(nda)
data_tiles.append(nda)
nda = img_data[:, j_i:j_i + img_tile_width, i_i:i_i + img_tile_width]
l2_tiles.append(nda)
label_tiles.append(nda)
# def run_mean_std(directory):
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment