Skip to content
Snippets Groups Projects
Commit a1953aff authored by tomrink's avatar tomrink
Browse files

minor

parent a89b297d
Branches
No related tags found
No related merge requests found
...@@ -20,8 +20,8 @@ def run_all(directory): ...@@ -20,8 +20,8 @@ def run_all(directory):
if not p.is_dir(): if not p.is_dir():
continue continue
# l1b_files = glob.glob(directory + p.name+'/'+'clavrx_snpp_viirs*.uwssec*.nc') # data_files = glob.glob(directory + p.name+'/'+'clavrx_snpp_viirs*.uwssec*.nc')
l1b_files = glob.glob(directory + p.name + '/' + 'VNP02MOD*.uwssec.nc') data_files = glob.glob(directory + p.name + '/' + 'VNP02MOD*.uwssec.nc')
label_valid_tiles = [] label_valid_tiles = []
label_train_tiles = [] label_train_tiles = []
...@@ -29,47 +29,47 @@ def run_all(directory): ...@@ -29,47 +29,47 @@ def run_all(directory):
data_train_tiles = [] data_train_tiles = []
f_cnt = 0 f_cnt = 0
for idx, l1b_f in enumerate(l1b_files): for idx, data_f in enumerate(data_files):
# if idx % 8 == 0: # if we want to skip some files # if idx % 8 == 0: # if we want to skip some files
if True: if True:
w_o_ext, ext = os.path.splitext(l1b_f) w_o_ext, ext = os.path.splitext(data_f)
l2_f = w_o_ext+'.highres'+ext label_f = w_o_ext+'.highres'+ext
if not os.path.exists(l2_f): if not os.path.exists(label_f):
continue continue
print(l1b_f) print(data_f)
try: try:
l1b_h5f = h5py.File(l1b_f, 'r') data_h5f = h5py.File(data_f, 'r')
except: except:
print('cant open file: ', l1b_f) print('cant open file: ', data_f)
continue continue
try: try:
l2_h5f = h5py.File(l2_f, 'r') label_h5f = h5py.File(label_f, 'r')
except: except:
print('cant open file: ', l2_f) print('cant open file: ', label_f)
l1b_h5f.close() data_h5f.close()
continue continue
data_tiles = [] data_tiles = []
label_tiles = [] label_tiles = []
try: try:
run(l1b_h5f, l2_h5f, data_tiles, label_tiles, factor=2) run(data_h5f, label_h5f, data_tiles, label_tiles, factor=2)
except Exception as e: except Exception as e:
print(e) print(e)
l1b_h5f.close() data_h5f.close()
l2_h5f.close() label_h5f.close()
continue continue
l1b_h5f.close() data_h5f.close()
l2_h5f.close() label_h5f.close()
if len(data_tiles) == 0 or len(label_tiles) == 0: if len(data_tiles) == 0 or len(label_tiles) == 0:
continue continue
if len(data_tiles) != len(label_tiles): if len(data_tiles) != len(label_tiles):
print('weirdness: ', l1b_f) print('weirdness: ', data_f)
continue continue
num = len(data_tiles) num = len(data_tiles)
...@@ -119,9 +119,9 @@ def run_all(directory): ...@@ -119,9 +119,9 @@ def run_all(directory):
np.save('/data/Personal/rink/viirs/label_valid_' + str(cnt), label_valid) np.save('/data/Personal/rink/viirs/label_valid_' + str(cnt), label_valid)
def run(l1b_h5f, l2_h5f, l1b_tiles, l2_tiles, factor=2): def run(data_h5f, label_h5f, data_tiles, label_tiles, factor=2):
if l2_h5f is None: if label_h5f is None:
l2_h5f = l1b_h5f label_h5f = data_h5f
l1b_param_name = data_params[0] l1b_param_name = data_params[0]
l2_param_name = label_params[0] l2_param_name = label_params[0]
...@@ -129,11 +129,11 @@ def run(l1b_h5f, l2_h5f, l1b_tiles, l2_tiles, factor=2): ...@@ -129,11 +129,11 @@ def run(l1b_h5f, l2_h5f, l1b_tiles, l2_tiles, factor=2):
mod_tile_width = 64 mod_tile_width = 64
img_tile_width = mod_tile_width * factor img_tile_width = mod_tile_width * factor
mod_num_lines = l1b_h5f[l1b_param_name].shape[0] mod_num_lines = data_h5f[l1b_param_name].shape[0]
mod_num_pixels = l1b_h5f[l1b_param_name].shape[1] mod_num_pixels = data_h5f[l1b_param_name].shape[1]
img_num_lines = l2_h5f[l2_param_name].shape[0] img_num_lines = label_h5f[l2_param_name].shape[0]
img_num_pixels = l2_h5f[l2_param_name].shape[1] img_num_pixels = label_h5f[l2_param_name].shape[1]
mod_num_y_tiles = int(mod_num_lines / mod_tile_width) mod_num_y_tiles = int(mod_num_lines / mod_tile_width)
mod_num_x_tiles = int(mod_num_pixels / mod_tile_width) mod_num_x_tiles = int(mod_num_pixels / mod_tile_width)
...@@ -143,14 +143,14 @@ def run(l1b_h5f, l2_h5f, l1b_tiles, l2_tiles, factor=2): ...@@ -143,14 +143,14 @@ def run(l1b_h5f, l2_h5f, l1b_tiles, l2_tiles, factor=2):
for param in data_params: for param in data_params:
try: try:
grd = get_grid_values(l1b_h5f, param, 0, 0, None, mod_num_lines, mod_num_pixels, range_name=None) grd = get_grid_values(data_h5f, param, 0, 0, None, mod_num_lines, mod_num_pixels, range_name=None)
l1b_grd_s.append(grd) l1b_grd_s.append(grd)
except: except:
return return
for param in label_params: for param in label_params:
try: try:
grd = get_grid_values(l2_h5f, param, 0, 0, None, img_num_lines, img_num_pixels, range_name=None) grd = get_grid_values(label_h5f, param, 0, 0, None, img_num_lines, img_num_pixels, range_name=None)
l2_grd_s.append(grd) l2_grd_s.append(grd)
except: except:
return return
...@@ -175,10 +175,10 @@ def run(l1b_h5f, l2_h5f, l1b_tiles, l2_tiles, factor=2): ...@@ -175,10 +175,10 @@ def run(l1b_h5f, l2_h5f, l1b_tiles, l2_tiles, factor=2):
i_i = i_m * factor i_i = i_m * factor
nda = mod_data[:, j_m:j_m + mod_tile_width, i_m:i_m + mod_tile_width] nda = mod_data[:, j_m:j_m + mod_tile_width, i_m:i_m + mod_tile_width]
l1b_tiles.append(nda) data_tiles.append(nda)
nda = img_data[:, j_i:j_i + img_tile_width, i_i:i_i + img_tile_width] nda = img_data[:, j_i:j_i + img_tile_width, i_i:i_i + img_tile_width]
l2_tiles.append(nda) label_tiles.append(nda)
# def run_mean_std(directory): # def run_mean_std(directory):
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment