snapshot...

modules/util/viirs_l1b_l2.py

@@ -77,7 +77,7 @@ def process_cld_prob(grd_k):
     num_keep = np.sum(keep)
     keep_clr = np.where(keep, grd_k < 0.20, False)
     frac_keep = np.sum(keep_clr)/num_keep
-    if not (0.35 < frac_keep < 0.65):
+    if not (0.30 < frac_keep < 0.70):
         return None
     grd_k = np.where(np.invert(keep), 0, grd_k)  # Convert NaN to 0
     return grd_k
@@ -165,7 +165,28 @@ def run_all(directory, out_directory, day_night='ANY', start=10):
 
                 cnt += 1
 
-    print('** Done, total_num_train_samples, total_num_valid_samples: ', total_num_train_samples, total_num_valid_samples)
+    # Write out leftover, if any
+    num_valid_samples = 0
+    if len(data_valid_tiles) > 0:
+        data_valid = np.stack(data_valid_tiles)
+        np.save(out_directory + 'data_valid_' + str(cnt), data_valid)
+        num_valid_samples = data_valid.shape[0]
+
+    num_train_samples = 0
+    if len(data_train_tiles) > 0:
+        data_train = np.stack(data_train_tiles)
+        np.save(out_directory + 'data_train_' + str(cnt), data_train)
+        num_train_samples = data_train.shape[0]
+
+    print('  num_train_samples, num_valid_samples, progress % : ', num_train_samples, num_valid_samples,
+          int((f_cnt / num_files) * 100))
+    total_num_train_samples += num_train_samples
+    total_num_valid_samples += num_valid_samples
+    print('total_num_train_samples, total_num_valid_samples, total_num_not_missing: ',
+          total_num_train_samples, total_num_valid_samples, total_num_not_missing)
+    print('---------------------------------------------------------')
+
+    print('*** Done, total_num_train_samples, total_num_valid_samples: ', total_num_train_samples, total_num_valid_samples)
 
 
 #  tile_width: Must be even!