snapshot...

modules/util/viirs_l1b_l2.py

@@ -128,8 +128,8 @@ def run_all(directory, out_directory, day_night='ANY'):
             label_tiles = []
 
             try:
-                run(data_h5f, data_params, data_tiles, num_keep_x_tiles=num_keep_x_tiles, tile_width=128, kernel_size=7, day_night=day_night)
-                # run(data_h5f, label_params, label_tiles, tile_width=128, kernel_size=7, day_night=day_night)
+                run(data_h5f, data_params, data_train_tiles, data_valid_tiles, num_keep_x_tiles=num_keep_x_tiles, tile_width=128, kernel_size=7, day_night=day_night)
+                # run(data_h5f, label_params, label_data_tiles, label_valid_tiles, tile_width=128, kernel_size=7, day_night=day_night)
             except Exception as e:
                 print(e)
                 data_h5f.close()
@@ -140,23 +140,9 @@ def run_all(directory, out_directory, day_night='ANY'):
             data_h5f.close()
             # label_h5f.close()
 
-            # if len(data_tiles) == 0 or len(label_tiles) == 0:
-            #     continue
-            # if len(data_tiles) != len(label_tiles):
-            #     print('weirdness: ', data_f)
-            #     continue
-
-            if len(data_tiles) == 0:
+            if len(data_train_tiles) == 0:
                 continue
 
-            num = len(data_tiles)
-            n_vld = int(num * 0.1)
-
-            # [label_valid_tiles.append(label_tiles[k]) for k in range(n_vld)]
-            # [label_train_tiles.append(label_tiles[k]) for k in range(n_vld, num)]
-            [data_valid_tiles.append(data_tiles[k]) for k in range(n_vld)]
-            [data_train_tiles.append(data_tiles[k]) for k in range(n_vld+num_keep_x_tiles, num)]
-
             f_cnt += 1
             if (f_cnt % 5) == 0:
                 num_valid_samples = 0
@@ -188,7 +174,7 @@ def run_all(directory, out_directory, day_night='ANY'):
     print('** total_num_train_samples, total_num_valid_samples: ', total_num_train_samples, total_num_valid_samples)
 
 
-def run(data_h5f, param_s, tiles, num_keep_x_tiles=8, tile_width=64, kernel_size=9, day_night='ANY'):
+def run(data_h5f, param_s, train_tiles, valid_tiles, num_keep_x_tiles=8, tile_width=64, kernel_size=9, day_night='ANY'):
 
     border = int((kernel_size - 1)/2)
 
@@ -215,11 +201,34 @@ def run(data_h5f, param_s, tiles, num_keep_x_tiles=8, tile_width=64, kernel_size
     i_skip = tile_width
     j_skip = tile_width
     i_start = int(num_pixels / 2) - int((num_keep_x_tiles * tile_width) / 2)
+    j_start = 0
+
+    num_keep_y_tiles = int(num_lines / tile_width) - 2
 
-    num_keep_y_tiles = int(num_lines / tile_width) - 1
+    num_y_valid = int(num_keep_y_tiles * 0.1) + 1
+    num_y_train = num_keep_y_tiles - num_y_valid - 1
+
+    for j in range(num_y_train):
+        j_a = j_start + j * j_skip
+        j_b = j_a + tile_width
+
+        for i in range(num_keep_x_tiles):
+            i_a = i_start + i * i_skip
+            i_b = i_a + tile_width
+
+            if day_night == 'DAY' and not is_day(solzen[j_a:j_b, i_a:i_b]):
+                continue
+            elif day_night == 'NIGHT' and is_day(solzen[j_a:j_b, i_a:i_b]):
+                continue
+
+            nda = data[:, j_a:j_b, i_a:i_b]
+            if not keep_tile(param_s, nda):
+                continue
+            train_tiles.append(nda)
 
-    for j in range(num_keep_y_tiles):
-        j_a = j * j_skip
+    j_start = num_y_train * tile_width + 1
+    for j in range(num_y_valid):
+        j_a = j_start + j * j_skip
         j_b = j_a + tile_width
 
         for i in range(num_keep_x_tiles):
@@ -234,7 +243,7 @@ def run(data_h5f, param_s, tiles, num_keep_x_tiles=8, tile_width=64, kernel_size
             nda = data[:, j_a:j_b, i_a:i_b]
             if not keep_tile(param_s, nda):
                 continue
-            tiles.append(nda)
+            valid_tiles.append(nda)
 
 
 def scan(directory):