snapshot...

modules/util/viirs_surfrad.py

@@ -37,10 +37,11 @@ group_name_m = 'orig/'
 
 solzen_name = group_name_m + 'solar_zenith'
 
-label_params = [group_name_i+target_param]
-data_params = [group_name_m+'temp_11_0um', group_name_m+'refl_0_65um', group_name_m+target_param]
-param_idx_m = data_params.index(group_name_m + target_param)
-param_idx_i = label_params.index(group_name_i + target_param)
+params_i = [group_name_i+target_param]
+params_m = [group_name_m+'temp_11_0um', group_name_m+'refl_0_65um', group_name_m+target_param]
+
+param_idx_m = params_m.index(group_name_m + target_param)
+param_idx_i = params_i.index(group_name_i + target_param)
 
 
 def is_missing(p_idx, tile):
@@ -97,10 +98,10 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
     all_files = glob.glob(path, recursive=True)
     data_files = [f for f in all_files if f not in keep_out]
 
-    label_valid_tiles = []
-    label_train_tiles = []
-    data_valid_tiles = []
-    data_train_tiles = []
+    valid_tiles_i = []
+    train_tiles_i = []
+    valid_tiles_m = []
+    train_tiles_m = []
     f_cnt = 0
 
     num_files = len(data_files)
@@ -118,8 +119,8 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
                 continue
 
             try:
-                num_not_missing = run(h5f, data_params, data_train_tiles, data_valid_tiles,
-                                      label_params, label_train_tiles, label_valid_tiles,
+                num_not_missing = run(h5f, params_m, train_tiles_m, valid_tiles_m,
+                                      params_i, train_tiles_i, valid_tiles_i,
                                       num_keep_x_tiles=num_keep_x_tiles, tile_width=64, kernel_size=7, day_night=day_night)
             except Exception as e:
                 print(e)
@@ -131,30 +132,30 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
 
             total_num_not_missing += num_not_missing
 
-            if len(data_train_tiles) == 0 and len(data_valid_tiles) == 0:
+            if len(train_tiles_m) == 0 and len(valid_tiles_m) == 0:
                 continue
 
             if (f_cnt % 5) == 0:
                 num_valid_samples = 0
-                if len(data_valid_tiles) > 0:
-                    label_valid = np.stack(label_valid_tiles)
-                    data_valid = np.stack(data_valid_tiles)
-                    np.save(out_directory + 'data_valid_' + str(cnt), data_valid)
-                    np.save(out_directory + 'label_valid_' + str(cnt), label_valid)
-                    num_valid_samples = data_valid.shape[0]
+                if len(valid_tiles_m) > 0:
+                    valid_i = np.stack(valid_tiles_i)
+                    valid_m = np.stack(valid_tiles_m)
+                    np.save(out_directory + 'valid_mres_' + str(cnt), valid_m)
+                    np.save(out_directory + 'valid_ires_' + str(cnt), valid_i)
+                    num_valid_samples = valid_m.shape[0]
 
                 num_train_samples = 0
-                if len(data_train_tiles) > 0:
-                    label_train = np.stack(label_train_tiles)
-                    data_train = np.stack(data_train_tiles)
-                    np.save(out_directory + 'label_train_' + str(cnt), label_train)
-                    np.save(out_directory + 'data_train_' + str(cnt), data_train)
-                    num_train_samples = data_train.shape[0]
-
-                label_valid_tiles = []
-                label_train_tiles = []
-                data_valid_tiles = []
-                data_train_tiles = []
+                if len(train_tiles_m) > 0:
+                    train_i = np.stack(train_tiles_i)
+                    train_m = np.stack(train_tiles_m)
+                    np.save(out_directory + 'train_ires_' + str(cnt), train_i)
+                    np.save(out_directory + 'train_mres' + str(cnt), train_m)
+                    num_train_samples = train_m.shape[0]
+
+                valid_tiles_i = []
+                train_tiles_i = []
+                valid_tiles_m = []
+                train_tiles_m = []
 
                 print('  num_train_samples, num_valid_samples, progress % : ', num_train_samples, num_valid_samples, int((f_cnt/num_files)*100))
                 total_num_train_samples += num_train_samples
@@ -170,12 +171,12 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
 
 #  tile_width: Must be even!
 #  kernel_size: Must be odd!
-def run(h5f, param_s, train_tiles, valid_tiles, lbl_param_s, lbl_train_tiles, lbl_valid_tiles,
+def run(h5f, params_m, train_tiles_m, valid_tiles_m, params_i, train_tiles_i, valid_tiles_i,
         num_keep_x_tiles=8, tile_width=64, kernel_size=3, day_night='ANY'):
 
     border = int((kernel_size - 1)/2) + 1  # Need to add for interpolation with no edge effects
 
-    param_name = param_s[0]
+    param_name = params_m[0]
 
     num_lines = h5f[param_name].shape[0]
     num_pixels = h5f[param_name].shape[1]  # Must be even
@@ -184,24 +185,24 @@ def run(h5f, param_s, train_tiles, valid_tiles, lbl_param_s, lbl_train_tiles, lb
         solzen = get_grid_values(h5f, solzen_name, 0, 0, None, num_lines, num_pixels)
 
     grd_s = []
-    for param in param_s:
+    for param in params_m:
         try:
             grd = get_grid_values(h5f, param, 0, 0, None, num_lines, num_pixels)
             grd_s.append(grd)
         except Exception as e:
             print(e)
             return
-    data = np.stack(grd_s)
+    data_m = np.stack(grd_s)
 
     grd_s = []
-    for param in lbl_param_s:
+    for param in params_i:
         try:
             grd = get_grid_values(h5f, param, 0, 0, None, num_lines*2, num_pixels*2)
             grd_s.append(grd)
         except Exception as e:
             print(e)
             return
-    label = np.stack(grd_s)
+    data_i = np.stack(grd_s)
 
     tile_width += 2 * border
 
@@ -212,8 +213,8 @@ def run(h5f, param_s, train_tiles, valid_tiles, lbl_param_s, lbl_train_tiles, lb
 
     num_y_tiles = int(num_lines / tile_width) - 1
 
-    data_tiles = []
-    lbl_tiles = []
+    data_tiles_m = []
+    data_tiles_i = []
     num_not_missing = 0
 
     for j in range(num_y_tiles):
@@ -229,26 +230,27 @@ def run(h5f, param_s, train_tiles, valid_tiles, lbl_param_s, lbl_train_tiles, lb
             elif day_night == 'NIGHT' and is_day(solzen[j_a:j_b, i_a:i_b]):
                 continue
 
-            nda = data[:, j_a:j_b, i_a:i_b]
-            nda_lbl = label[:, j_a*2:j_b*2, i_a*2:i_b*2]
-            if is_missing(param_idx_i, nda_lbl):
+            nda_m = data_m[:, j_a:j_b, i_a:i_b]
+            nda_i = data_i[:, j_a*2:j_b*2, i_a*2:i_b*2]
+            if is_missing(param_idx_i, nda_i):
                 continue
             num_not_missing += 1
 
-            nda_lbl = keep_tile(param_idx_i, nda_lbl)
-            if nda_lbl is not None:
-                data_tiles.append(nda)
-                lbl_tiles.append(nda_lbl)
+            nda_i = keep_tile(param_idx_i, nda_i)
+            if nda_i is not None:
+                data_tiles_m.append(nda_m)
+                data_tiles_i.append(nda_i)
 
-    num_tiles = len(lbl_tiles)
+    num_tiles = len(data_tiles_i)
     num_valid = int(num_tiles * 0.10)
     num_train = num_tiles - num_valid
 
     for k in range(num_train):
-        train_tiles.append(data_tiles[k])
-        lbl_train_tiles.append(lbl_tiles[k])
+        train_tiles_m.append(data_tiles_m[k])
+        train_tiles_i.append(data_tiles_i[k])
+
     for k in range(num_valid):
-        valid_tiles.append(data_tiles[num_train + k])
-        lbl_valid_tiles.append(lbl_tiles[num_train + k])
+        valid_tiles_m.append(data_tiles_m[num_train + k])
+        valid_tiles_i.append(data_tiles_i[num_train + k])
 
     return num_not_missing