Skip to content
Snippets Groups Projects
Commit a1953aff authored by tomrink's avatar tomrink
Browse files

minor

parent a89b297d
Branches
No related tags found
No related merge requests found
......@@ -20,8 +20,8 @@ def run_all(directory):
if not p.is_dir():
continue
# l1b_files = glob.glob(directory + p.name+'/'+'clavrx_snpp_viirs*.uwssec*.nc')
l1b_files = glob.glob(directory + p.name + '/' + 'VNP02MOD*.uwssec.nc')
# data_files = glob.glob(directory + p.name+'/'+'clavrx_snpp_viirs*.uwssec*.nc')
data_files = glob.glob(directory + p.name + '/' + 'VNP02MOD*.uwssec.nc')
label_valid_tiles = []
label_train_tiles = []
......@@ -29,47 +29,47 @@ def run_all(directory):
data_train_tiles = []
f_cnt = 0
for idx, l1b_f in enumerate(l1b_files):
for idx, data_f in enumerate(data_files):
# if idx % 8 == 0: # if we want to skip some files
if True:
w_o_ext, ext = os.path.splitext(l1b_f)
l2_f = w_o_ext+'.highres'+ext
if not os.path.exists(l2_f):
w_o_ext, ext = os.path.splitext(data_f)
label_f = w_o_ext+'.highres'+ext
if not os.path.exists(label_f):
continue
print(l1b_f)
print(data_f)
try:
l1b_h5f = h5py.File(l1b_f, 'r')
data_h5f = h5py.File(data_f, 'r')
except:
print('cant open file: ', l1b_f)
print('cant open file: ', data_f)
continue
try:
l2_h5f = h5py.File(l2_f, 'r')
label_h5f = h5py.File(label_f, 'r')
except:
print('cant open file: ', l2_f)
l1b_h5f.close()
print('cant open file: ', label_f)
data_h5f.close()
continue
data_tiles = []
label_tiles = []
try:
run(l1b_h5f, l2_h5f, data_tiles, label_tiles, factor=2)
run(data_h5f, label_h5f, data_tiles, label_tiles, factor=2)
except Exception as e:
print(e)
l1b_h5f.close()
l2_h5f.close()
data_h5f.close()
label_h5f.close()
continue
l1b_h5f.close()
l2_h5f.close()
data_h5f.close()
label_h5f.close()
if len(data_tiles) == 0 or len(label_tiles) == 0:
continue
if len(data_tiles) != len(label_tiles):
print('weirdness: ', l1b_f)
print('weirdness: ', data_f)
continue
num = len(data_tiles)
......@@ -119,9 +119,9 @@ def run_all(directory):
np.save('/data/Personal/rink/viirs/label_valid_' + str(cnt), label_valid)
def run(l1b_h5f, l2_h5f, l1b_tiles, l2_tiles, factor=2):
if l2_h5f is None:
l2_h5f = l1b_h5f
def run(data_h5f, label_h5f, data_tiles, label_tiles, factor=2):
if label_h5f is None:
label_h5f = data_h5f
l1b_param_name = data_params[0]
l2_param_name = label_params[0]
......@@ -129,11 +129,11 @@ def run(l1b_h5f, l2_h5f, l1b_tiles, l2_tiles, factor=2):
mod_tile_width = 64
img_tile_width = mod_tile_width * factor
mod_num_lines = l1b_h5f[l1b_param_name].shape[0]
mod_num_pixels = l1b_h5f[l1b_param_name].shape[1]
mod_num_lines = data_h5f[l1b_param_name].shape[0]
mod_num_pixels = data_h5f[l1b_param_name].shape[1]
img_num_lines = l2_h5f[l2_param_name].shape[0]
img_num_pixels = l2_h5f[l2_param_name].shape[1]
img_num_lines = label_h5f[l2_param_name].shape[0]
img_num_pixels = label_h5f[l2_param_name].shape[1]
mod_num_y_tiles = int(mod_num_lines / mod_tile_width)
mod_num_x_tiles = int(mod_num_pixels / mod_tile_width)
......@@ -143,14 +143,14 @@ def run(l1b_h5f, l2_h5f, l1b_tiles, l2_tiles, factor=2):
for param in data_params:
try:
grd = get_grid_values(l1b_h5f, param, 0, 0, None, mod_num_lines, mod_num_pixels, range_name=None)
grd = get_grid_values(data_h5f, param, 0, 0, None, mod_num_lines, mod_num_pixels, range_name=None)
l1b_grd_s.append(grd)
except:
return
for param in label_params:
try:
grd = get_grid_values(l2_h5f, param, 0, 0, None, img_num_lines, img_num_pixels, range_name=None)
grd = get_grid_values(label_h5f, param, 0, 0, None, img_num_lines, img_num_pixels, range_name=None)
l2_grd_s.append(grd)
except:
return
......@@ -175,10 +175,10 @@ def run(l1b_h5f, l2_h5f, l1b_tiles, l2_tiles, factor=2):
i_i = i_m * factor
nda = mod_data[:, j_m:j_m + mod_tile_width, i_m:i_m + mod_tile_width]
l1b_tiles.append(nda)
data_tiles.append(nda)
nda = img_data[:, j_i:j_i + img_tile_width, i_i:i_i + img_tile_width]
l2_tiles.append(nda)
label_tiles.append(nda)
# def run_mean_std(directory):
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment