snapshot...

modules/util/viirs_surfrad.py

@@ -112,6 +112,11 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
 
     total_num_not_missing = 0
 
+    hist_accum_valid_i = np.zeros(20, dtype=np.int64)
+    hist_accum_valid_m = np.zeros(20, dtype=np.int64)
+    hist_accum_train_i = np.zeros(20, dtype=np.int64)
+    hist_accum_train_m = np.zeros(20, dtype=np.int64)
+
     for idx, data_f in enumerate(data_files):
         # if idx % 4 == 0:  # if we want to skip some files
         if True:
@@ -177,6 +182,10 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
         np.save(out_directory + 'valid_mres_' + str(cnt), valid_m)
         np.save(out_directory + 'valid_ires_' + str(cnt), valid_i)
         num_valid_samples = valid_m.shape[0]
+        h, b = np.histogram(valid_i.flatten(), bins=20, range=[0.0, 160.0])
+        hist_accum_valid_i += h
+        h, b = np.histogram(valid_m.flatten(), bins=20, range=[0.0, 160.0])
+        hist_accum_valid_m += h
 
     num_train_samples = 0
     if len(train_tiles_m) > 0:
@@ -185,6 +194,10 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
         np.save(out_directory + 'train_ires_' + str(cnt), train_i)
         np.save(out_directory + 'train_mres_' + str(cnt), train_m)
         num_train_samples = train_m.shape[0]
+        h, b = np.histogram(train_i.flatten(), bins=20, range=[0.0, 160.0])
+        hist_accum_train_i += h
+        h, b = np.histogram(train_m.flatten(), bins=20, range=[0.0, 160.0])
+        hist_accum_train_m += h
 
     print('  num_train_samples, num_valid_samples, progress % : ', num_train_samples, num_valid_samples,
           int((f_cnt / num_files) * 100))
@@ -195,6 +208,11 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
     print('--------------------------------------------------')
 
     print('** total_num_train_samples, total_num_valid_samples: ', total_num_train_samples, total_num_valid_samples)
+    print('--------------------------------------------------')
+    print(hist_accum_train_i)
+    print(hist_accum_train_m)
+    print(hist_accum_valid_i)
+    print(hist_accum_valid_m)
 
 
 #  tile_width: Must be even!