snapshot...

modules/util/abi_surfrad.py

@@ -65,11 +65,11 @@ def process_cld_prob(grd_k):
 def process_cld_opd(grd_k):
     keep = np.invert(np.isnan(grd_k))
     num_keep = np.sum(keep)
-    keep_cld = np.where(keep, np.logical_and(0.1 < grd_k, grd_k < 158.0), False)
+    keep_cld = np.where(keep, np.logical_and(2.0 < grd_k, grd_k < 158.0), False)
     # keep_cld = np.where(keep, 2.0 < grd_k, False)
     frac_cld = np.sum(keep_cld)/num_keep
     # if not (0.40 < frac_cld):
-    if not (0.25 < frac_cld):
+    if not (0.10 < frac_cld < 0.90):
         return None
     grd_k = np.where(np.invert(keep), 0, grd_k)  # Convert NaN to 0
     return grd_k
@@ -79,8 +79,6 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
     cnt = start
     total_num_train_samples = 0
     total_num_valid_samples = 0
-    # num_keep_x_tiles = 4
-    num_keep_x_tiles = 2
 
     # path = directory + '**' + '/' + pattern
     path = directory + '*_v3/2020/' + '**' + '/' + pattern
@@ -113,8 +111,8 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
             try:
                 num_not_missing, num_snow_covered = \
                     run(h5f, params_m, data_tiles_m, params_i, data_tiles_i,
-                        # num_keep_x_tiles=num_keep_x_tiles, tile_width=16, kernel_size=4, factor=4,
-                        num_keep_x_tiles=num_keep_x_tiles, tile_width=64, kernel_size=7, factor=2,
+                        # tile_width=16, kernel_size=4, factor=4,
+                        tile_width=64, kernel_size=7, factor=2,
                         day_night=day_night, is_snow_covered=is_snow_covered)
             except Exception as e:
                 print(e)
@@ -183,8 +181,8 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
             try:
                 num_not_missing, num_snow_covered = \
                     run(h5f, params_m, data_tiles_m, params_i, data_tiles_i,
-                        # num_keep_x_tiles=num_keep_x_tiles, tile_width=16, kernel_size=4, factor=4,
-                        num_keep_x_tiles=num_keep_x_tiles, tile_width=64, kernel_size=7, factor=2,
+                        # tile_width=16, kernel_size=4, factor=4,
+                        tile_width=64, kernel_size=7, factor=2,
                         day_night=day_night, is_snow_covered=is_snow_covered)
             except Exception as e:
                 print(e)
@@ -239,7 +237,7 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
 
 #  tile_width: Must be even!
 #  kernel_size: Must be odd!
-def run(h5f, params_m, data_tiles_m, params_i, data_tiles_i, num_keep_x_tiles=8, tile_width=64, kernel_size=3, factor=2,
+def run(h5f, params_m, data_tiles_m, params_i, data_tiles_i, tile_width=64, kernel_size=3, factor=2,
         day_night='ANY', is_snow_covered=None):
 
     border = int((kernel_size - 1)/2) + 1  # Need to add for interpolation with no edge effects
@@ -282,7 +280,8 @@ def run(h5f, params_m, data_tiles_m, params_i, data_tiles_i, num_keep_x_tiles=8,
     i_start = border - 1  # zero-based
     j_start = border - 1  # zero-based
 
-    num_y_tiles = int(num_lines / tile_width) - 1
+    num_y_tiles = int(num_lines / tile_width)
+    num_x_tiles = int(num_pixels / tile_width)
 
     num_not_missing = 0
     num_snow_covered = 0
@@ -291,7 +290,7 @@ def run(h5f, params_m, data_tiles_m, params_i, data_tiles_i, num_keep_x_tiles=8,
         j_a = j_start + j * j_skip
         j_b = j_a + tile_width
 
-        for i in range(num_keep_x_tiles):
+        for i in range(num_x_tiles):
             i_a = i_start + i * i_skip
             i_b = i_a + tile_width