snapshot...

modules/util/abi_surfrad.py

 import numpy as np
 import h5py
-from util.util import get_grid_values, get_grid_values_all, is_night, is_day, compute_lwc_iwc, get_fill_attrs
+from util.util import get_grid_values
 import glob
-import os
-from aeolus.datasource import CLAVRx_VIIRS
-from icing.moon_phase import *
-from pathlib import Path
 
 
-# target_param = 'cloud_probability'
-target_param = 'cld_opd_dcomp'
+target_param = 'cloud_probability'
+# target_param = 'cld_opd_dcomp'
 
 group_name_i = 'super/'
 group_name_m = 'orig/'
@@ -39,10 +35,10 @@ def keep_tile(param, param_s, tile):
 def process_cld_prob_(grd_k):
     keep = np.invert(np.isnan(grd_k))
     num_keep = np.sum(keep)
-    if num_keep / grd_k.size < 0.98:
-        return None
-    keep = np.where(keep, np.logical_and(0.05 < grd_k, grd_k < 0.95), False)
-    if np.sum(keep)/num_keep < 0.50:
+    # if num_keep / grd_k.size < 0.98:
+    #     return None
+    keep = np.where(keep, np.logical_and(0.10 < grd_k, grd_k < 0.90), False)
+    if np.sum(keep)/num_keep < 0.25:
         return None
     grd_k = np.where(np.invert(keep), 0, grd_k)
     return grd_k
@@ -51,8 +47,8 @@ def process_cld_prob_(grd_k):
 def process_cld_opd_(grd_k):
     keep = np.invert(np.isnan(grd_k))
     num_keep = np.sum(keep)
-    if num_keep / grd_k.size < 0.98:
-        return None
+    # if num_keep / grd_k.size < 0.98:
+    #     return None
     grd_k = np.where(np.invert(keep), 0, grd_k)
     keep = np.where(keep, np.logical_and(0.1 < grd_k, grd_k < 158.0), False)
     if np.sum(keep)/num_keep < 0.50:
@@ -62,8 +58,7 @@ def process_cld_opd_(grd_k):
 
 def run_all(directory, out_directory, pattern='clavrx_*.nc', start=10):
     cnt = start
-    total_num_train_samples = 0
-    total_num_valid_samples = 0
+    total_num_samples = 0
 
     path = directory + '**' + '/' + pattern
 
@@ -86,7 +81,7 @@ def run_all(directory, out_directory, pattern='clavrx_*.nc', start=10):
                 continue
 
             try:
-                run(h5f, data_params, data_tiles, label_params, label_tiles, tile_width=64, kernel_size=5)
+                run(h5f, data_params, data_tiles, label_params, label_tiles, kernel_size=5)
             except Exception as e:
                 print(e)
                 h5f.close()
@@ -100,30 +95,30 @@ def run_all(directory, out_directory, pattern='clavrx_*.nc', start=10):
                 continue
 
             if (f_cnt % 100) == 0:
-                num_valid_samples = 0
-
-                label = np.stack(label_tiles)
-                data = np.stack(data_tiles)
-                np.save(out_directory + 'label_' + str(cnt), label)
-                np.save(out_directory + 'data_' + str(cnt), data)
-                num_samples = data.shape[0]
+                num_samples = 0
+                if len(data_tiles) > 0:
+                    label = np.stack(label_tiles)
+                    data = np.stack(data_tiles)
+                    #np.save(out_directory + 'label_' + str(cnt), label)
+                    #np.save(out_directory + 'data_' + str(cnt), data)
+                    num_samples = data.shape[0]
 
                 label_tiles = []
                 data_tiles = []
 
-                # print('  num_train_samples, num_valid_samples, progress % : ', num_train_samples, num_valid_samples, int((f_cnt/num_files)*100))
-                # total_num_train_samples += num_samples
-                # total_num_valid_samples += num_valid_samples
-                # print('total_num_train_samples, total_num_valid_samples: ', total_num_train_samples, total_num_valid_samples)
+                print('  num_samples, progress % : ', num_samples, int((f_cnt/num_files)*100))
+                total_num_samples += num_samples
+                print('total_num_samples: ', total_num_samples)
+                print('------------------------------------------------------------')
 
                 cnt += 1
 
-    print('** total_num_train_samples, total_num_valid_samples: ', total_num_train_samples, total_num_valid_samples)
+    print('** total_num_samples: ', total_num_samples)
 
 
 #  tile_width: Must be even!
 #  kernel_size: Must be odd!
-def run(h5f, param_s, tiles, lbl_param_s, lbl_tiles, tile_width=64, kernel_size=3):
+def run(h5f, param_s, tiles, lbl_param_s, lbl_tiles, kernel_size=3):
 
     border = int((kernel_size - 1)/2) + 1  # Need to add for interpolation with no edge effects