snapshot...

modules/util/abi_surfrad.py

@@ -65,7 +65,7 @@ def process_cld_prob(grd_k):
 def process_cld_opd(grd_k):
     keep = np.invert(np.isnan(grd_k))
     num_keep = np.sum(keep)
-    keep_cld = np.where(keep, np.logical_and(1.0 < grd_k, grd_k < 158.0), False)
+    keep_cld = np.where(keep, np.logical_and(2.0 < grd_k, grd_k < 158.0), False)
     # keep_cld = np.where(keep, 2.0 < grd_k, False)
     frac_cld = np.sum(keep_cld)/num_keep
     # if not (0.40 < frac_cld):
@@ -84,8 +84,9 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
     path = directory + '*_v3/2020/' + '**' + '/' + pattern
 
     all_files = glob.glob(path, recursive=True)
-    valid_files = glob.glob(directory + '*_v3/2020/*/0[1-5]/*/*.nc', recursive=True)
-    train_files = [f for f in all_files if f not in valid_files]
+    test_files = glob.glob(directory + '*_v3/2020/*/01/*/*.nc', recursive=True)
+    valid_files = glob.glob(directory + '*_v3/2020/*/0[2-6]/*/*.nc', recursive=True)
+    train_files = [f for f in all_files if f not in valid_files + test_files]
 
     data_tiles_i = []
     data_tiles_m = []
@@ -95,7 +96,7 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
     print('Start, number of valid files: ', num_files)
 
     total_num_not_missing = 0
-    num_skip = 4
+    num_skip = 3
 
     param_train_hist = np.zeros([20], dtype=np.int64)
     param_valid_hist = np.zeros([20], dtype=np.int64)