snapshot...

modules/util/abi_surfrad.py

@@ -89,9 +89,10 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
     print('Start, number of valid files: ', num_files)
 
     total_num_not_missing = 0
+    num_skip = 3
 
     for idx, data_f in enumerate(valid_files):
-        if idx % 3 == 0:  # if we want to skip some files
+        if idx % num_skip == 0:  # if we want to skip some files
             try:
                 h5f = h5py.File(data_f, 'r')
             except:
@@ -128,7 +129,7 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
                 data_tiles_i = []
                 data_tiles_m = []
 
-                print('  num_valid_samples, progress % : ', num_valid_samples, int((f_cnt/(num_files/4))*100))
+                print('  num_valid_samples, progress % : ', num_valid_samples, int((f_cnt/(num_files/num_skip))*100))
                 total_num_valid_samples += num_valid_samples
                 print('total_num_valid_samples, total_num_not_missing: ', total_num_valid_samples, total_num_not_missing)
                 print('--------------------------------------------------')
@@ -156,7 +157,7 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
     print('Start, number of train files: ', num_files)
 
     for idx, data_f in enumerate(train_files):
-        if idx % 4 == 0:  # if we want to skip some files
+        if idx % num_skip == 0:  # if we want to skip some files
             try:
                 h5f = h5py.File(data_f, 'r')
             except:
@@ -193,7 +194,7 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
                 data_tiles_i = []
                 data_tiles_m = []
 
-                print('  num_train_samples, progress % : ', num_train_samples, int((f_cnt/(num_files/4))*100))
+                print('  num_train_samples, progress % : ', num_train_samples, int((f_cnt/(num_files/num_skip))*100))
                 total_num_train_samples += num_train_samples
                 print('total_num_train_samples, total_num_not_missing: ', total_num_train_samples, total_num_not_missing)
                 print('--------------------------------------------------')