snapshot...

modules/util/abi_surfrad.py

@@ -3,9 +3,9 @@ import h5py
 from util.util import get_grid_values, is_day
 import glob
 
-target_param = 'cloud_probability'
+# target_param = 'cloud_probability'
 # target_param = 'cld_opd_dcomp'
-# target_param = 'cld_opd_dcomp_1'
+target_param = 'cld_opd_dcomp_1'
 # target_param = 'cld_opd_dcomp_2'
 # target_param = 'cld_opd_dcomp_3'
 
@@ -130,8 +130,21 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
     total_num_not_missing = 0
     num_skip = 3
 
-    param_train_hist = np.zeros([14], dtype=np.int64)
-    param_valid_hist = np.zeros([14], dtype=np.int64)
+    param_train_hist = np.zeros([16], dtype=np.int64)
+    param_valid_hist = np.zeros([16], dtype=np.int64)
+
+    # cloud_prob to cloud fraction
+    # ----------------------------
+    # tile_width = 32
+    # kernel_size = 5
+    # factor = 4
+
+    tile_width = 64
+    kernel_size = 7
+    factor = 4
+
+    # hist_range = [0.0, 1.0]
+    hist_range = [0.0, 160.0]
 
     for idx, data_f in enumerate(valid_files):
         if idx % num_skip == 0:  # if we want to skip some files
@@ -144,8 +157,7 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
             try:
                 num_not_missing, num_snow_covered = \
                     run(h5f, params_m, data_tiles_m, params_i, data_tiles_i,
-                        tile_width=32, kernel_size=5, factor=4,
-                        # tile_width=64, kernel_size=7, factor=2,
+                        tile_width=tile_width, kernel_size=kernel_size, factor=factor,
                         day_night=day_night, is_snow_covered=is_snow_covered)
             except Exception as e:
                 print(e)
@@ -170,7 +182,7 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
                         np.save(out_directory + 'valid_ires_' + str(cnt), valid_i)
                     num_valid_samples = valid_m.shape[0]
 
-                    param_valid_hist += np.histogram(valid_m[param_idx_m, ], bins=14)[0]
+                    param_valid_hist += np.histogram(valid_m[param_idx_m, ], bins=16, range=hist_range)[0]
 
                 data_tiles_i = []
                 data_tiles_m = []
@@ -191,7 +203,7 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
             np.save(out_directory + 'valid_mres_' + str(cnt), valid_m)
             np.save(out_directory + 'valid_ires_' + str(cnt), valid_i)
         num_valid_samples = valid_m.shape[0]
-        param_valid_hist += np.histogram(valid_m[param_idx_m, ], bins=14)[0]
+        param_valid_hist += np.histogram(valid_m[param_idx_m, ], bins=16, range=hist_range)[0]
     total_num_valid_samples += num_valid_samples
     print('total_num_valid_samples, total_num_not_missing: ', total_num_valid_samples, total_num_not_missing)
     print(param_valid_hist)
@@ -216,8 +228,7 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
             try:
                 num_not_missing, num_snow_covered = \
                     run(h5f, params_m, data_tiles_m, params_i, data_tiles_i,
-                        tile_width=32, kernel_size=5, factor=4,
-                        # tile_width=64, kernel_size=7, factor=2,
+                        tile_width=tile_width, kernel_size=kernel_size, factor=factor,
                         day_night=day_night, is_snow_covered=is_snow_covered)
             except Exception as e:
                 print(e)
@@ -242,7 +253,7 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
                         np.save(out_directory + 'train_mres_' + str(cnt), train_m)
                     num_train_samples = train_m.shape[0]
 
-                    param_train_hist += np.histogram(train_m[param_idx_m, ], bins=14)[0]
+                    param_train_hist += np.histogram(train_m[param_idx_m, ], bins=16, range=hist_range)[0]
 
                 data_tiles_i = []
                 data_tiles_m = []
@@ -263,7 +274,7 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
             np.save(out_directory + 'train_ires_' + str(cnt), train_i)
             np.save(out_directory + 'train_mres_' + str(cnt), train_m)
         num_train_samples = train_m.shape[0]
-        param_train_hist += np.histogram(train_m[param_idx_m, ], bins=14)[0]
+        param_train_hist += np.histogram(train_m[param_idx_m, ], bins=16, range=hist_range)[0]
     total_num_train_samples += num_train_samples
     print('total_num_train_samples,  total_num_not_missing: ', total_num_train_samples, total_num_not_missing)
     print(param_train_hist)