`snapshot...`

modules/util/abi_surfrad.py

@@ -3,9 +3,9 @@ import h5py
 from util.util import get_grid_values, is_day
 import glob
 
-# target_param = 'cloud_probability'
+target_param = 'cloud_probability'
 # target_param = 'cld_opd_dcomp'
-target_param = 'cld_opd_dcomp_1'
+# target_param = 'cld_opd_dcomp_1'
 # target_param = 'cld_opd_dcomp_2'
 # target_param = 'cld_opd_dcomp_3'
 
@@ -15,15 +15,17 @@ group_name_m = 'orig/'
 solzen_name = group_name_m + 'solar_zenith'
 snow_class_name = group_name_m + 'snow_class'
 
-# params_i = [group_name_i+'temp_ch38', group_name_i+'refl_ch01', group_name_i+target_param]
-params_i = [group_name_i+'temp_ch38', group_name_i+'refl_ch01', group_name_i+'temp_stddev3x3_ch31', group_name_i+'refl_stddev3x3_ch01', group_name_i+target_param]
+params_i = [group_name_i+'temp_ch38', group_name_i+'refl_ch01', group_name_i+target_param]
+# params_i = [group_name_i+'temp_ch38', group_name_i+'refl_ch01', group_name_i+'temp_stddev3x3_ch31', group_name_i+'refl_stddev3x3_ch01', group_name_i+target_param]
 
-# params_m = [group_name_m+'temp_ch38', group_name_m+'refl_ch01', group_name_m+'refl_submin_ch01', group_name_m+'refl_submax_ch01', group_name_m+'refl_substddev_ch01', group_name_m+target_param]
-params_m = [group_name_m+'temp_ch38', group_name_m+'refl_ch01', group_name_m+'refl_submin_ch01', group_name_m+'refl_submax_ch01', group_name_m+'refl_substddev_ch01', group_name_m+'temp_stddev3x3_ch31', group_name_m+'refl_stddev3x3_ch01', group_name_m+target_param]
+params_m = [group_name_m+'temp_ch38', group_name_m+'refl_ch01', group_name_m+'refl_submin_ch01', group_name_m+'refl_submax_ch01', group_name_m+'refl_substddev_ch01', group_name_m+target_param]
+# params_m = [group_name_m+'temp_ch38', group_name_m+'refl_ch01', group_name_m+'refl_submin_ch01', group_name_m+'refl_submax_ch01', group_name_m+'refl_substddev_ch01', group_name_m+'temp_stddev3x3_ch31', group_name_m+'refl_stddev3x3_ch01', group_name_m+target_param]
 
 param_idx_m = params_m.index(group_name_m + target_param)
 param_idx_i = params_i.index(group_name_i + target_param)
 
+DO_WRITE_OUTFILE = False
+
 
 def snow_covered(tile):
     return np.any(tile > 1)
@@ -142,8 +144,8 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
             try:
                 num_not_missing, num_snow_covered = \
                     run(h5f, params_m, data_tiles_m, params_i, data_tiles_i,
-                        # tile_width=16, kernel_size=4, factor=4,
-                        tile_width=64, kernel_size=7, factor=2,
+                        tile_width=32, kernel_size=5, factor=4,
+                        # tile_width=64, kernel_size=7, factor=2,
                         day_night=day_night, is_snow_covered=is_snow_covered)
             except Exception as e:
                 print(e)
@@ -163,8 +165,9 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
                 if len(data_tiles_m) > 0:
                     valid_i = np.stack(data_tiles_i)
                     valid_m = np.stack(data_tiles_m)
-                    np.save(out_directory + 'valid_mres_' + str(cnt), valid_m)
-                    np.save(out_directory + 'valid_ires_' + str(cnt), valid_i)
+                    if DO_WRITE_OUTFILE:
+                        np.save(out_directory + 'valid_mres_' + str(cnt), valid_m)
+                        np.save(out_directory + 'valid_ires_' + str(cnt), valid_i)
                     num_valid_samples = valid_m.shape[0]
 
                     param_valid_hist += np.histogram(valid_m[param_idx_m, ], bins=20, range=[0.0, 160.0])[0]
@@ -184,8 +187,9 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
     if len(data_tiles_m) > 0:
         valid_i = np.stack(data_tiles_i)
         valid_m = np.stack(data_tiles_m)
-        np.save(out_directory + 'valid_mres_' + str(cnt), valid_m)
-        np.save(out_directory + 'valid_ires_' + str(cnt), valid_i)
+        if DO_WRITE_OUTFILE:
+            np.save(out_directory + 'valid_mres_' + str(cnt), valid_m)
+            np.save(out_directory + 'valid_ires_' + str(cnt), valid_i)
         num_valid_samples = valid_m.shape[0]
         param_valid_hist += np.histogram(valid_m[param_idx_m, ], bins=20, range=[0.0, 160.0])[0]
     total_num_valid_samples += num_valid_samples
@@ -233,8 +237,9 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
                 if len(data_tiles_m) > 0:
                     train_i = np.stack(data_tiles_i)
                     train_m = np.stack(data_tiles_m)
-                    np.save(out_directory + 'train_ires_' + str(cnt), train_i)
-                    np.save(out_directory + 'train_mres_' + str(cnt), train_m)
+                    if DO_WRITE_OUTFILE:
+                        np.save(out_directory + 'train_ires_' + str(cnt), train_i)
+                        np.save(out_directory + 'train_mres_' + str(cnt), train_m)
                     num_train_samples = train_m.shape[0]
 
                     param_train_hist += np.histogram(train_m[param_idx_m, ], bins=20, range=[0.0, 160.0])[0]
@@ -254,8 +259,9 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
     if len(data_tiles_m) > 0:
         train_i = np.stack(data_tiles_i)
         train_m = np.stack(data_tiles_m)
-        np.save(out_directory + 'train_ires_' + str(cnt), train_i)
-        np.save(out_directory + 'train_mres_' + str(cnt), train_m)
+        if DO_WRITE_OUTFILE:
+            np.save(out_directory + 'train_ires_' + str(cnt), train_i)
+            np.save(out_directory + 'train_mres_' + str(cnt), train_m)
         num_train_samples = train_m.shape[0]
         param_train_hist += np.histogram(train_m[param_idx_m, ], bins=20, range=[0.0, 160.0])[0]
     total_num_train_samples += num_train_samples