From d53282cb2507766cebc59b9815b7d768f1b22dbd Mon Sep 17 00:00:00 2001
From: tomrink <rink@ssec.wisc.edu>
Date: Fri, 30 Dec 2022 09:48:33 -0600
Subject: [PATCH] snapshot...
---
modules/deeplearning/srcnn_l1b_l2.py | 89 +++++++++++++++++++++++-----
1 file changed, 74 insertions(+), 15 deletions(-)
diff --git a/modules/deeplearning/srcnn_l1b_l2.py b/modules/deeplearning/srcnn_l1b_l2.py
index 7b9f70e1..5e123688 100644
--- a/modules/deeplearning/srcnn_l1b_l2.py
+++ b/modules/deeplearning/srcnn_l1b_l2.py
@@ -324,15 +324,66 @@ class SRCNN:
return self.get_in_mem_data_batch(idxs, False)
def get_in_mem_data_batch_eval(self, idxs):
- data = []
- for param in self.train_params:
- nda = self.data_dct[param]
- nda = normalize(nda, param, mean_std_dct)
- data.append(nda)
- data = np.stack(data)
- data = data.astype(np.float32)
- data = np.transpose(data, axes=(1, 2, 0))
+ in_file = '/home/rink/data/clavrx_snpp_day/clavrx_VNP02MOD.A2019017.1600.001.2019017214117.uwssec.highres.nc.level2.nc'
+ N = 8
+
+ slc_x = slice(2, N * 128 + 4)
+ slc_y = slice(2, N * 128 + 4)
+ slc_x_2 = slice(1, N * 128 + 6, 2)
+ slc_y_2 = slice(1, N * 128 + 6, 2)
+ x_2 = np.arange(int((N * 128) / 2) + 3)
+ y_2 = np.arange(int((N * 128) / 2) + 3)
+ t = np.arange(0, int((N * 128) / 2) + 3, 0.5)
+ s = np.arange(0, int((N * 128) / 2) + 3, 0.5)
+ x_k = slice(1, N * 128 + 3)
+ y_k = slice(1, N * 128 + 3)
+ x_128 = slice(3, N * 128 + 3)
+ y_128 = slice(3, N * 128 + 3)
+
+ sub_y, sub_x = (N * 128) + 10, (N * 128) + 10
+ y_0, x_0, = 2432 - int(sub_y / 2), 2432 - int(sub_x / 2)
+
+ h5f = h5py.File(in_file, 'r')
+
+ grd_a = get_grid_values_all(h5f, 'temp_11_0um_nom')
+ grd_a = grd_a[y_0:y_0 + sub_y, x_0:x_0 + sub_x]
+ grd_a = grd_a.copy()
+ grd_a = np.where(np.isnan(grd_a), 0, grd_a)
+ hr_grd_a = grd_a.copy()
+ hr_grd_a = hr_grd_a[y_128, x_128]
+ grd_a = grd_a[slc_y_2, slc_x_2]
+ grd_a = resample_2d_linear_one(x_2, y_2, grd_a, t, s)
+ grd_a = grd_a[y_k, x_k]
+ grd_a = normalize(grd_a, 'temp_11_0um_nom', mean_std_dct)
+ #
+ # grd_b = get_grid_values_all(h5f, 'refl_0_65um_nom')
+ # grd_b = grd_b[y_0:y_0+sub_y, x_0:x_0+sub_x]
+ # grd_b = grd_b[y_130, x_130]
+ # refl = grd_b
+ # grd_b = normalize(grd_b, 'refl_0_65um_nom', mean_std_dct)
+
+ grd_c = get_grid_values_all(h5f, label_param)
+ grd_c = grd_c[y_0:y_0 + sub_y, x_0:x_0 + sub_x]
+
+ hr_grd_c = grd_c.copy()
+ hr_grd_c = hr_grd_c[y_128, x_128]
+
+ grd_c = np.where(np.isnan(grd_c), 0, grd_c)
+ grd_c = grd_c.copy()
+ grd_c = grd_c[slc_y_2, slc_x_2]
+ grd_c = resample_2d_linear_one(x_2, y_2, grd_c, t, s)
+ grd_c = grd_c[y_k, x_k]
+
+ if label_param != 'cloud_probability':
+ grd_c = normalize(grd_c, label_param, mean_std_dct)
+
+ # data = np.stack([grd_a, grd_b, grd_c], axis=2)
+ # data = np.stack([grd_a, grd_c], axis=2)
+ data = np.stack([grd_c], axis=2)
data = np.expand_dims(data, axis=0)
+ data = data.astype(np.float32)
+
+ h5f.close()
return data
@@ -409,8 +460,7 @@ class SRCNN:
def setup_eval_pipeline(self, filename):
idxs = [0]
- self.num_data_samples = idxs.shape[0]
-
+ self.num_data_samples = 1
self.get_evaluate_dataset(idxs)
def build_srcnn(self, do_drop_out=False, do_batch_norm=False, drop_rate=0.5, factor=2):
@@ -688,11 +738,18 @@ class SRCNN:
self.reset_test_metrics()
- pred = self.model([data], training=False)
- self.test_probs = pred
- pred = pred.numpy()
- if label_param != 'cloud_probability':
- pred = denormalize(pred, label_param, mean_std_dct)
+ for data in self.eval_dataset:
+ pred = self.model([data], training=False)
+ pred = pred.numpy()
+ if label_param != 'cloud_probability':
+ pred = denormalize(pred, label_param, mean_std_dct)
+ print(pred.min(), pred.max())
+
+ # pred = self.model([data], training=False)
+ # self.test_probs = pred
+ # pred = pred.numpy()
+ # if label_param != 'cloud_probability':
+ # pred = denormalize(pred, label_param, mean_std_dct)
return pred
@@ -720,6 +777,7 @@ class SRCNN:
def run_evaluate(self, data, ckpt_dir):
data = tf.convert_to_tensor(data, dtype=tf.float32)
self.num_data_samples = 80000
+ # self.setup_eval_pipeline('clavrx_VNP02MOD.A2019017.1600.001.2019017214117.uwssec.highres.nc.level2.nc')
self.build_model()
self.build_training()
self.build_evaluation()
@@ -779,6 +837,7 @@ def run_evaluate_static(in_file, out_file, ckpt_dir):
grd_c = np.where(np.isnan(grd_c), 0, grd_c)
grd_c = grd_c.copy()
+ grd_c = smooth_2d_single(grd_c, sigma=1.0)
grd_c = grd_c[slc_y_2, slc_x_2]
grd_c = resample_2d_linear_one(x_2, y_2, grd_c, t, s)
grd_c = grd_c[y_k, x_k]
--
GitLab