snapshot...

modules/util/viirs_l1b_l2.py

@@ -52,8 +52,8 @@ emis_params = ['temp_10_4um_nom', 'temp_11_0um_nom', 'temp_12_0um_nom', 'temp_13
 # data_params = emis_params
 
 # l2_params = ['temp_11_0um_nom', 'temp_12_0um_nom', 'cloud_fraction']
-l2_params = ['temp_11_0um_nom', 'temp_12_0um_nom', 'refl_0_65um_nom', 'cld_opd_dcomp']
-# l2_params = ['temp_11_0um_nom', 'temp_12_0um_nom', 'refl_0_65um_nom', 'cloud_fraction']
+# l2_params = ['temp_11_0um_nom', 'temp_12_0um_nom', 'refl_0_65um_nom', 'cld_opd_dcomp']
+l2_params = ['temp_11_0um_nom', 'temp_12_0um_nom', 'refl_0_65um_nom', 'cloud_fraction']
 
 label_params = l2_params
 data_params = l2_params
@@ -80,12 +80,13 @@ def run_all(directory, out_directory, day_night='ANY'):
     cnt = 10
     total_num_train_samples = 0
     total_num_valid_samples = 0
+    num_keep_x_tiles = 10
 
     # pattern = 'clavrx*highres*.nc'
     # pattern = 'VNP02MOD*.uwssec.nc'
     # pattern = 'clavrx_*.nc'
-    pattern = 'clavrx_VNP02MOD*.highres.nc.level2.nc'
-    # pattern = 'clavrx_snpp_viirs*.uwssec*.nc'
+    # pattern = 'clavrx_VNP02MOD*.highres.nc.level2.nc'
+    pattern = 'clavrx_snpp_viirs*.uwssec*.nc'
     path = directory + '**' + '/' + pattern
 
     data_files = glob.glob(path, recursive=True)
@@ -127,7 +128,7 @@ def run_all(directory, out_directory, day_night='ANY'):
             label_tiles = []
 
             try:
-                run(data_h5f, data_params, data_tiles, tile_width=128, kernel_size=7, day_night=day_night)
+                run(data_h5f, data_params, data_tiles, num_keep_x_tiles=num_keep_x_tiles, tile_width=128, kernel_size=7, day_night=day_night)
                 # run(data_h5f, label_params, label_tiles, tile_width=128, kernel_size=7, day_night=day_night)
             except Exception as e:
                 print(e)
@@ -154,7 +155,7 @@ def run_all(directory, out_directory, day_night='ANY'):
             # [label_valid_tiles.append(label_tiles[k]) for k in range(n_vld)]
             # [label_train_tiles.append(label_tiles[k]) for k in range(n_vld, num)]
             [data_valid_tiles.append(data_tiles[k]) for k in range(n_vld)]
-            [data_train_tiles.append(data_tiles[k]) for k in range(n_vld, num)]
+            [data_train_tiles.append(data_tiles[k]) for k in range(n_vld+num_keep_x_tiles, num)]
 
             f_cnt += 1
             if (f_cnt % 5) == 0:
@@ -187,7 +188,7 @@ def run_all(directory, out_directory, day_night='ANY'):
     print('** total_num_train_samples, total_num_valid_samples: ', total_num_train_samples, total_num_valid_samples)
 
 
-def run(data_h5f, param_s, tiles, tile_width=64, kernel_size=9, day_night='ANY'):
+def run(data_h5f, param_s, tiles, num_keep_x_tiles=8, tile_width=64, kernel_size=9, day_night='ANY'):
 
     border = int((kernel_size - 1)/2)
 
@@ -211,7 +212,6 @@ def run(data_h5f, param_s, tiles, tile_width=64, kernel_size=9, day_night='ANY')
 
     tile_width += 2 * border
 
-    num_keep_x_tiles = 8
     i_skip = tile_width
     j_skip = tile_width
     i_start = int(num_pixels / 2) - int((num_keep_x_tiles * tile_width) / 2)
@@ -232,8 +232,8 @@ def run(data_h5f, param_s, tiles, tile_width=64, kernel_size=9, day_night='ANY')
                 continue
 
             nda = data[:, j_a:j_b, i_a:i_b]
-            # if not keep_tile(param_s, nda):
-            #     continue
+            if not keep_tile(param_s, nda):
+                continue
             tiles.append(nda)