Skip to content
Snippets Groups Projects
Commit 8eb4e6b8 authored by tomrink's avatar tomrink
Browse files

snapshot...

parent e162122e
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
modules/util/viirs_surfrad.py
+ 36
− 45
View file @ 8eb4e6b8
...@@ -78,7 +78,6 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st ...@@ -78,7 +78,6 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
num_files = len(data_files) num_files = len(data_files)
print('Start, number of files: ', num_files) print('Start, number of files: ', num_files)
kept_cnt = 0
for idx, data_f in enumerate(data_files): for idx, data_f in enumerate(data_files):
# if idx % 4 == 0: # if we want to skip some files # if idx % 4 == 0: # if we want to skip some files
...@@ -90,49 +89,51 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st ...@@ -90,49 +89,51 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
continue continue
try: try:
total, kept = run(h5f, data_params, data_train_tiles, data_valid_tiles, run(h5f, data_params, data_train_tiles, data_valid_tiles,
label_params, label_train_tiles, label_valid_tiles, label_params, label_train_tiles, label_valid_tiles,
num_keep_x_tiles=num_keep_x_tiles, tile_width=64, kernel_size=5, day_night=day_night) num_keep_x_tiles=num_keep_x_tiles, tile_width=64, kernel_size=5, day_night=day_night)
except Exception as e: except Exception as e:
print(e) print(e)
h5f.close() h5f.close()
continue continue
kept_cnt += kept print(data_f)
print(data_f, kept_cnt, int(100 * (kept/total)))
f_cnt += 1 f_cnt += 1
h5f.close() h5f.close()
if len(data_train_tiles) == 0: if len(data_train_tiles) == 0 and len(data_valid_tiles) == 0:
continue continue
# if (f_cnt % 5) == 0: if (f_cnt % 5) == 0:
# num_valid_samples = 0 num_valid_samples = 0
# if len(data_valid_tiles) > 0: if len(data_valid_tiles) > 0:
# label_valid = np.stack(label_valid_tiles) label_valid = np.stack(label_valid_tiles)
# data_valid = np.stack(data_valid_tiles) data_valid = np.stack(data_valid_tiles)
# np.save(out_directory + 'data_valid_' + str(cnt), data_valid) #np.save(out_directory + 'data_valid_' + str(cnt), data_valid)
# np.save(out_directory + 'label_valid_' + str(cnt), label_valid) #np.save(out_directory + 'label_valid_' + str(cnt), label_valid)
# num_valid_samples = data_valid.shape[0] num_valid_samples = data_valid.shape[0]
#
# label_train = np.stack(label_train_tiles) num_train_samples = 0
# data_train = np.stack(data_train_tiles) if len(data_train_tiles) > 0:
# np.save(out_directory + 'label_train_' + str(cnt), label_train) label_train = np.stack(label_train_tiles)
# np.save(out_directory + 'data_train_' + str(cnt), data_train) data_train = np.stack(data_train_tiles)
# num_train_samples = data_train.shape[0] #np.save(out_directory + 'label_train_' + str(cnt), label_train)
# #np.save(out_directory + 'data_train_' + str(cnt), data_train)
# label_valid_tiles = [] num_train_samples = data_train.shape[0]
# label_train_tiles = []
# data_valid_tiles = [] label_valid_tiles = []
# data_train_tiles = [] label_train_tiles = []
# data_valid_tiles = []
# print(' num_train_samples, num_valid_samples, progress % : ', num_train_samples, num_valid_samples, int((f_cnt/num_files)*100)) data_train_tiles = []
# total_num_train_samples += num_train_samples
# total_num_valid_samples += num_valid_samples print(' num_train_samples, num_valid_samples, progress % : ', num_train_samples, num_valid_samples, int((f_cnt/num_files)*100))
# print('total_num_train_samples, total_num_valid_samples: ', total_num_train_samples, total_num_valid_samples) total_num_train_samples += num_train_samples
# total_num_valid_samples += num_valid_samples
# cnt += 1 print('total_num_train_samples, total_num_valid_samples: ', total_num_train_samples, total_num_valid_samples)
# print('--------------------------------------------------')
# print('** total_num_train_samples, total_num_valid_samples: ', total_num_train_samples, total_num_valid_samples)
cnt += 1
print('** total_num_train_samples, total_num_valid_samples: ', total_num_train_samples, total_num_valid_samples)
# tile_width: Must be even! # tile_width: Must be even!
...@@ -182,8 +183,6 @@ def run(h5f, param_s, train_tiles, valid_tiles, lbl_param_s, lbl_train_tiles, lb ...@@ -182,8 +183,6 @@ def run(h5f, param_s, train_tiles, valid_tiles, lbl_param_s, lbl_train_tiles, lb
num_y_valid = int(num_keep_y_tiles * 0.1) + 1 num_y_valid = int(num_keep_y_tiles * 0.1) + 1
num_y_train = num_keep_y_tiles - num_y_valid - 1 num_y_train = num_keep_y_tiles - num_y_valid - 1
cnt_total = 0
cnt_kept = 0
for j in range(num_y_train): for j in range(num_y_train):
j_a = j_start + j * j_skip j_a = j_start + j * j_skip
j_b = j_a + tile_width j_b = j_a + tile_width
...@@ -192,8 +191,6 @@ def run(h5f, param_s, train_tiles, valid_tiles, lbl_param_s, lbl_train_tiles, lb ...@@ -192,8 +191,6 @@ def run(h5f, param_s, train_tiles, valid_tiles, lbl_param_s, lbl_train_tiles, lb
i_a = i_start + i * i_skip i_a = i_start + i * i_skip
i_b = i_a + tile_width i_b = i_a + tile_width
cnt_total += 1
if day_night == 'DAY' and not is_day(solzen[j_a:j_b, i_a:i_b]): if day_night == 'DAY' and not is_day(solzen[j_a:j_b, i_a:i_b]):
continue continue
elif day_night == 'NIGHT' and is_day(solzen[j_a:j_b, i_a:i_b]): elif day_night == 'NIGHT' and is_day(solzen[j_a:j_b, i_a:i_b]):
...@@ -210,7 +207,6 @@ def run(h5f, param_s, train_tiles, valid_tiles, lbl_param_s, lbl_train_tiles, lb ...@@ -210,7 +207,6 @@ def run(h5f, param_s, train_tiles, valid_tiles, lbl_param_s, lbl_train_tiles, lb
if nda_lbl is not None: if nda_lbl is not None:
train_tiles.append(nda) train_tiles.append(nda)
lbl_train_tiles.append(nda_lbl) lbl_train_tiles.append(nda_lbl)
cnt_kept += 1
j_start = num_y_train * tile_width + 2*tile_width j_start = num_y_train * tile_width + 2*tile_width
for j in range(num_y_valid): for j in range(num_y_valid):
...@@ -221,8 +217,6 @@ def run(h5f, param_s, train_tiles, valid_tiles, lbl_param_s, lbl_train_tiles, lb ...@@ -221,8 +217,6 @@ def run(h5f, param_s, train_tiles, valid_tiles, lbl_param_s, lbl_train_tiles, lb
i_a = i_start + i * i_skip i_a = i_start + i * i_skip
i_b = i_a + tile_width i_b = i_a + tile_width
cnt_total += 1
if day_night == 'DAY' and not is_day(solzen[j_a:j_b, i_a:i_b]): if day_night == 'DAY' and not is_day(solzen[j_a:j_b, i_a:i_b]):
continue continue
elif day_night == 'NIGHT' and is_day(solzen[j_a:j_b, i_a:i_b]): elif day_night == 'NIGHT' and is_day(solzen[j_a:j_b, i_a:i_b]):
...@@ -239,9 +233,6 @@ def run(h5f, param_s, train_tiles, valid_tiles, lbl_param_s, lbl_train_tiles, lb ...@@ -239,9 +233,6 @@ def run(h5f, param_s, train_tiles, valid_tiles, lbl_param_s, lbl_train_tiles, lb
if nda_lbl is not None: if nda_lbl is not None:
valid_tiles.append(nda) valid_tiles.append(nda)
lbl_valid_tiles.append(nda_lbl) lbl_valid_tiles.append(nda_lbl)
cnt_kept += 1
return cnt_total, cnt_kept
# def run_mean_std(directory): # def run_mean_std(directory):
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment