From 4db06abaa67e1ef05235ad6ac2afb8259e7bd764 Mon Sep 17 00:00:00 2001
From: tomrink <rink@ssec.wisc.edu>
Date: Fri, 5 Jan 2024 13:49:57 -0600
Subject: [PATCH] snapshot...
---
modules/util/abi_surfrad.py | 41 +++++++++++++++++--------------------
1 file changed, 19 insertions(+), 22 deletions(-)
diff --git a/modules/util/abi_surfrad.py b/modules/util/abi_surfrad.py
index 884eabab..51cbbc44 100644
--- a/modules/util/abi_surfrad.py
+++ b/modules/util/abi_surfrad.py
@@ -3,9 +3,9 @@ import h5py
from util.util import get_grid_values, is_day
import glob
-# target_param = 'cloud_probability'
+target_param = 'cloud_probability'
# target_param = 'cld_opd_dcomp'
-target_param = 'cld_opd_dcomp_1'
+# target_param = 'cld_opd_dcomp_1'
# target_param = 'cld_opd_dcomp_2'
# target_param = 'cld_opd_dcomp_3'
@@ -20,12 +20,23 @@ params_i = [group_name_i+'temp_ch38', group_name_i+'refl_ch01', group_name_i+tar
params_m = [group_name_m+'temp_ch38', group_name_m+'refl_ch01',
group_name_m+'refl_submin_ch01', group_name_m+'refl_submax_ch01', group_name_m+'refl_substddev_ch01',
group_name_m+target_param,
- group_name_m+'cloud_probability',
group_name_m+'cloud_fraction']
param_idx_m = params_m.index(group_name_m + target_param)
param_idx_i = params_i.index(group_name_i + target_param)
+hist_range = [0.0, 1.0]
+# hist_range = [0.0, 160.0]
+
+tile_width = 32
+kernel_size = 5
+factor = 4
+# tile_width = 64
+# kernel_size = 7
+# factor = 4
+
+num_skip = 3
+
DO_WRITE_OUTFILE = True
@@ -43,8 +54,8 @@ def keep_tile(p_idx, tile):
grd_k = tile[p_idx, ].copy()
if target_param == 'cloud_probability':
- grd_k = process_cld_prob(grd_k)
- # grd_k = process_cloud_frac(grd_k)
+ # grd_k = process_cld_prob(grd_k)
+ grd_k = process_cloud_frac(grd_k)
elif 'cld_opd_dcomp' in target_param:
grd_k = process_cld_opd(grd_k)
@@ -125,11 +136,11 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
total_num_valid_samples = 0
# path = directory + '**' + '/' + pattern
- path = directory + '*_v3/2020/' + '**' + '/' + pattern
+ path = directory + '*_v3/202?/' + '**' + '/' + pattern
all_files = glob.glob(path, recursive=True)
- test_files = glob.glob(directory + '*_v3/2020/*/01/*/*.nc', recursive=True)
- valid_files = glob.glob(directory + '*_v3/2020/*/0[2-6]/*/*.nc', recursive=True)
+ test_files = glob.glob(directory + '*_v3/202?/*/01/*/*.nc', recursive=True)
+ valid_files = glob.glob(directory + '*_v3/202?/*/0[2-6]/*/*.nc', recursive=True)
train_files = [f for f in all_files if f not in valid_files + test_files]
data_tiles_i = []
@@ -140,24 +151,10 @@ def run_all(directory, out_directory, day_night='ANY', pattern='clavrx_*.nc', st
print('Start, number of valid files: ', num_files)
total_num_not_missing = 0
- num_skip = 3
param_train_hist = np.zeros([16], dtype=np.int64)
param_valid_hist = np.zeros([16], dtype=np.int64)
- # cloud_prob to cloud fraction
- # ----------------------------
- tile_width = 32
- kernel_size = 5
- factor = 4
-
- # tile_width = 64
- # kernel_size = 7
- # factor = 4
-
- # hist_range = [0.0, 1.0]
- hist_range = [0.0, 160.0]
-
for idx, data_f in enumerate(valid_files):
if idx % num_skip == 0: # if we want to skip some files
try:
--
GitLab