snapshot...

modules/util/viirs_l1b_l2.py

@@ -20,15 +20,25 @@ solzen_name = group_name + 'solar_zenith'
 label_params = l2_params
 data_params = l2_params
 
+# range = [0.0, 1.0]
+cld_prob_norm_hist = [0.34458323, 0.03729378, 0.01817725, 0.01246574, 0.00991681, 0.00826515, 0.00785976, 0.00595133,
+                      0.00567965, 0.00579926, 0.00642895, 0.00797761, 0.01218471, 0.51741677]
 
-def keep_tile(param_s, tile, hist_10):
+# range = [0.0, 160.0]
+cld_opd_norm_hist = [7.31926378e-01, 9.52482193e-02, 4.62747706e-02, 3.15450036e-02, 1.98358694e-02, 1.33123841e-02,
+                     1.03378429e-02, 7.95560979e-03, 5.77925319e-03, 4.82856215e-03, 3.31576300e-03, 2.86789405e-03,
+                     2.50456177e-03, 1.79184632e-03, 1.51077739e-03, 1.29144749e-03,9.20514553e-04, 7.47183923e-04,
+                     6.50404531e-04, 1.73557144e-02]
+
+
+def keep_tile(param_s, tile, dum):
     k = param_s.index(group_name + target_param)
     grd_k = tile[k, ].copy()
 
     if target_param == 'cloud_probability':
-        grd_k = process_cld_prob_(grd_k, hist_10)
+        grd_k = process_cld_prob_(grd_k, dum)
     elif target_param == 'cld_opd_dcomp':
-        grd_k = process_cld_opd_(grd_k)
+        grd_k = process_cld_opd_(grd_k, dum)
 
     if grd_k is not None:
         tile[k, ] = grd_k
@@ -48,14 +58,14 @@ def process_cld_prob(param_s, tile):
         return None
 
 
-def process_cld_prob_(grd_k, hist_10):
+def process_cld_prob_(grd_k, dum):
     keep = np.invert(np.isnan(grd_k))
     num_keep = np.sum(keep)
     if num_keep / grd_k.size < 0.98:
         return None
-    hist_10 += np.histogram(grd_k.flatten(), range=[0.0, 1.0], bins=10)[0]
-    keep = np.where(keep, np.logical_and(0.05 < grd_k, grd_k < 0.95), False)
-    if np.sum(keep)/num_keep < 0.50:
+    # hist_10 += np.histogram(grd_k.flatten(), range=[0.0, 1.0], bins=10)[0]
+    keep = np.where(keep, np.logical_and(0.1 < grd_k, grd_k < 0.90), False)
+    if np.sum(keep)/num_keep < 0.30:
         return None
     grd_k = np.where(np.invert(keep), 0, grd_k)
     return grd_k
@@ -72,7 +82,7 @@ def process_cld_opd(param_s, tile):
         return None
 
 
-def process_cld_opd_(grd_k):
+def process_cld_opd_(grd_k, dum):
     keep = np.invert(np.isnan(grd_k))
     num_keep = np.sum(keep)
     if num_keep / grd_k.size < 0.98:
@@ -103,7 +113,7 @@ def run_all(directory, out_directory, day_night='ANY', start=10):
     num_files = len(data_files)
 
     print('Start, number of files: ', num_files)
-    hist_10 = np.zeros((10), dtype=np.int64)
+    # hist_10 = np.zeros((10), dtype=np.int64)
 
     for idx, data_f in enumerate(data_files):
         # if idx % 4 == 0:  # if we want to skip some files
@@ -115,12 +125,12 @@ def run_all(directory, out_directory, day_night='ANY', start=10):
                 continue
 
             try:
-                total, kept = run(data_h5f, data_params, data_train_tiles, data_valid_tiles, hist_10, num_keep_x_tiles=num_keep_x_tiles, tile_width=128, kernel_size=11, day_night=day_night)
+                total, kept = run(data_h5f, data_params, data_train_tiles, data_valid_tiles, None, num_keep_x_tiles=num_keep_x_tiles, tile_width=128, kernel_size=11, day_night=day_night)
             except Exception as e:
                 print(e)
                 data_h5f.close()
                 continue
-            print(data_f, int(100 * (kept/total)), hist_10, (hist_10 / np.sum(hist_10)))
+            print(data_f, int(100 * (kept/total)))
             f_cnt += 1
 
             data_h5f.close()
@@ -154,7 +164,7 @@ def run_all(directory, out_directory, day_night='ANY', start=10):
 
 #  tile_width: Must be even!
 #  kernel_size: Must be odd!
-def run(data_h5f, param_s, train_tiles, valid_tiles, hist_10, num_keep_x_tiles=8, tile_width=64, kernel_size=9, day_night='ANY'):
+def run(data_h5f, param_s, train_tiles, valid_tiles, dum, num_keep_x_tiles=8, tile_width=64, kernel_size=9, day_night='ANY'):
 
     border = int((kernel_size - 1)/2)
 
@@ -207,7 +217,7 @@ def run(data_h5f, param_s, train_tiles, valid_tiles, hist_10, num_keep_x_tiles=8
                 continue
 
             nda = data[:, j_a:j_b, i_a:i_b]
-            nda = keep_tile(param_s, nda, hist_10)
+            nda = keep_tile(param_s, nda, dum)
             if nda is not None:
                 train_tiles.append(nda)
                 cnt_kept += 1
@@ -229,7 +239,7 @@ def run(data_h5f, param_s, train_tiles, valid_tiles, hist_10, num_keep_x_tiles=8
                 continue
 
             nda = data[:, j_a:j_b, i_a:i_b]
-            nda = keep_tile(param_s, nda, hist_10)
+            nda = keep_tile(param_s, nda, dum)
             if nda is not None:
                 valid_tiles.append(nda)
                 cnt_kept += 1