snapshot...

modules/util/abi_surfrad.py

@@ -3,9 +3,9 @@ import h5py
 from util.util import get_grid_values, is_day
 import glob
 
-# target_param = 'cloud_probability'
+target_param = 'cloud_probability'
 # target_param = 'cld_opd_dcomp'
-target_param = 'cld_opd_dcomp_1'
+# target_param = 'cld_opd_dcomp_1'
 # target_param = 'cld_opd_dcomp_2'
 # target_param = 'cld_opd_dcomp_3'
 
@@ -20,12 +20,23 @@ params_i = [group_name_i+'temp_ch38', group_name_i+'refl_ch01', group_name_i+tar
 params_m = [group_name_m+'temp_ch38', group_name_m+'refl_ch01',
             group_name_m+'refl_submin_ch01', group_name_m+'refl_submax_ch01', group_name_m+'refl_substddev_ch01',
             group_name_m+target_param,
-            group_name_m+'cloud_probability',
             group_name_m+'cloud_fraction']
 
 param_idx_m = params_m.index(group_name_m + target_param)
 param_idx_i = params_i.index(group_name_i + target_param)
 
+hist_range = [0.0, 1.0]
+# hist_range = [0.0, 160.0]
+
+tile_width = 32
+kernel_size = 5
+factor = 4
+# tile_width = 64
+# kernel_size = 7
+# factor = 4
+
+num_skip = 3
+
 DO_WRITE_OUTFILE = True
 
 
@@ -43,8 +54,8 @@ def keep_tile(p_idx, tile):
     grd_k = tile[p_idx, ].copy()
 
     if target_param == 'cloud_probability':
-        grd_k = process_cld_prob(grd_k)
-        # grd_k = process_cloud_frac(grd_k)
+        # grd_k = process_cld_prob(grd_k)
+        grd_k = process_cloud_frac(grd_k)
     elif 'cld_opd_dcomp' in target_param:
         grd_k = process_cld_opd(grd_k)
 
@@ -125,11 +136,11 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
     total_num_valid_samples = 0
 
     # path = directory + '**' + '/' + pattern
-    path = directory + '*_v3/2020/' + '**' + '/' + pattern
+    path = directory + '*_v3/202?/' + '**' + '/' + pattern
 
     all_files = glob.glob(path, recursive=True)
-    test_files = glob.glob(directory + '*_v3/2020/*/01/*/*.nc', recursive=True)
-    valid_files = glob.glob(directory + '*_v3/2020/*/0[2-6]/*/*.nc', recursive=True)
+    test_files = glob.glob(directory + '*_v3/202?/*/01/*/*.nc', recursive=True)
+    valid_files = glob.glob(directory + '*_v3/202?/*/0[2-6]/*/*.nc', recursive=True)
     train_files = [f for f in all_files if f not in valid_files + test_files]
 
     data_tiles_i = []
@@ -140,24 +151,10 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
     print('Start, number of valid files: ', num_files)
 
     total_num_not_missing = 0
-    num_skip = 3
 
     param_train_hist = np.zeros([16], dtype=np.int64)
     param_valid_hist = np.zeros([16], dtype=np.int64)
 
-    # cloud_prob to cloud fraction
-    # ----------------------------
-    tile_width = 32
-    kernel_size = 5
-    factor = 4
-
-    # tile_width = 64
-    # kernel_size = 7
-    # factor = 4
-
-    # hist_range = [0.0, 1.0]
-    hist_range = [0.0, 160.0]
-
     for idx, data_f in enumerate(valid_files):
         if idx % num_skip == 0:  # if we want to skip some files
             try: