From 7718854a6eaa88490ea5265c4e770e77beb9da7a Mon Sep 17 00:00:00 2001
From: tomrink <rink@ssec.wisc.edu>
Date: Thu, 5 Jan 2023 14:57:16 -0600
Subject: [PATCH] snapshot...
---
modules/deeplearning/srcnn_l1b_l2.py | 91 ----------------------------
1 file changed, 91 deletions(-)
diff --git a/modules/deeplearning/srcnn_l1b_l2.py b/modules/deeplearning/srcnn_l1b_l2.py
index 8431a6c7..57788cc9 100644
--- a/modules/deeplearning/srcnn_l1b_l2.py
+++ b/modules/deeplearning/srcnn_l1b_l2.py
@@ -314,70 +314,6 @@ class SRCNN:
def get_in_mem_data_batch_test(self, idxs):
return self.get_in_mem_data_batch(idxs, False)
- def get_in_mem_data_batch_eval(self, idxs):
- in_file = '/home/rink/data/clavrx_snpp_day/clavrx_VNP02MOD.A2019017.1600.001.2019017214117.uwssec.highres.nc.level2.nc'
- N = 8
-
- slc_x = slice(2, N * 128 + 4)
- slc_y = slice(2, N * 128 + 4)
- slc_x_2 = slice(1, N * 128 + 6, 2)
- slc_y_2 = slice(1, N * 128 + 6, 2)
- x_2 = np.arange(int((N * 128) / 2) + 3)
- y_2 = np.arange(int((N * 128) / 2) + 3)
- t = np.arange(0, int((N * 128) / 2) + 3, 0.5)
- s = np.arange(0, int((N * 128) / 2) + 3, 0.5)
- x_k = slice(1, N * 128 + 3)
- y_k = slice(1, N * 128 + 3)
- x_128 = slice(3, N * 128 + 3)
- y_128 = slice(3, N * 128 + 3)
-
- sub_y, sub_x = (N * 128) + 10, (N * 128) + 10
- y_0, x_0, = 2432 - int(sub_y / 2), 2432 - int(sub_x / 2)
-
- h5f = h5py.File(in_file, 'r')
-
- grd_a = get_grid_values_all(h5f, 'temp_11_0um_nom')
- grd_a = grd_a[y_0:y_0 + sub_y, x_0:x_0 + sub_x]
- grd_a = grd_a.copy()
- grd_a = np.where(np.isnan(grd_a), 0, grd_a)
- hr_grd_a = grd_a.copy()
- hr_grd_a = hr_grd_a[y_128, x_128]
- grd_a = grd_a[slc_y_2, slc_x_2]
- grd_a = resample_2d_linear_one(x_2, y_2, grd_a, t, s)
- grd_a = grd_a[y_k, x_k]
- grd_a = normalize(grd_a, 'temp_11_0um_nom', mean_std_dct)
- #
- # grd_b = get_grid_values_all(h5f, 'refl_0_65um_nom')
- # grd_b = grd_b[y_0:y_0+sub_y, x_0:x_0+sub_x]
- # grd_b = grd_b[y_130, x_130]
- # refl = grd_b
- # grd_b = normalize(grd_b, 'refl_0_65um_nom', mean_std_dct)
-
- grd_c = get_grid_values_all(h5f, label_param)
- grd_c = grd_c[y_0:y_0 + sub_y, x_0:x_0 + sub_x]
-
- hr_grd_c = grd_c.copy()
- hr_grd_c = hr_grd_c[y_128, x_128]
-
- grd_c = np.where(np.isnan(grd_c), 0, grd_c)
- grd_c = grd_c.copy()
- grd_c = grd_c[slc_y_2, slc_x_2]
- grd_c = resample_2d_linear_one(x_2, y_2, grd_c, t, s)
- grd_c = grd_c[y_k, x_k]
-
- if label_param != 'cloud_probability':
- grd_c = normalize(grd_c, label_param, mean_std_dct)
-
- # data = np.stack([grd_a, grd_b, grd_c], axis=2)
- # data = np.stack([grd_a, grd_c], axis=2)
- data = np.stack([grd_c], axis=2)
- data = np.expand_dims(data, axis=0)
- data = data.astype(np.float32)
-
- h5f.close()
-
- return data
-
@tf.function(input_signature=[tf.TensorSpec(None, tf.int32)])
def data_function(self, indexes):
out = tf.numpy_function(self.get_in_mem_data_batch_train, [indexes], [tf.float32, tf.float32])
@@ -388,11 +324,6 @@ class SRCNN:
out = tf.numpy_function(self.get_in_mem_data_batch_test, [indexes], [tf.float32, tf.float32])
return out
- @tf.function(input_signature=[tf.TensorSpec(None, tf.int32)])
- def data_function_evaluate(self, indexes):
- out = tf.numpy_function(self.get_in_mem_data_batch_eval, [indexes], [tf.float32])
- return out
-
def get_train_dataset(self, indexes):
indexes = list(indexes)
@@ -414,13 +345,6 @@ class SRCNN:
dataset = dataset.cache()
self.test_dataset = dataset
- def get_evaluate_dataset(self, indexes):
- indexes = list(indexes)
-
- dataset = tf.data.Dataset.from_tensor_slices(indexes)
- dataset = dataset.map(self.data_function_evaluate, num_parallel_calls=8)
- self.eval_dataset = dataset
-
def setup_pipeline(self, train_data_files, test_data_files, num_train_samples):
self.train_data_files = train_data_files
@@ -449,11 +373,6 @@ class SRCNN:
self.get_test_dataset(tst_idxs)
print('setup_test_pipeline: Done')
- def setup_eval_pipeline(self, filename):
- idxs = [0]
- self.num_data_samples = 1
- self.get_evaluate_dataset(idxs)
-
def build_srcnn(self, do_drop_out=False, do_batch_norm=False, drop_rate=0.5, factor=2):
print('build_cnn')
padding = "SAME"
@@ -731,13 +650,6 @@ class SRCNN:
self.reset_test_metrics()
- # for data in self.eval_dataset:
- # pred = self.model([data], training=False)
- # pred = pred.numpy()
- # if label_param != 'cloud_probability':
- # pred = denormalize(pred, label_param, mean_std_dct)
- # print(pred.min(), pred.max())
-
pred = self.model([data], training=False)
self.test_probs = pred
pred = pred.numpy()
@@ -749,8 +661,6 @@ class SRCNN:
def run(self, directory, ckpt_dir=None, num_data_samples=50000):
train_data_files = glob.glob(directory+'data_train_*.npy')
valid_data_files = glob.glob(directory+'data_valid_*.npy')
- # train_data_files = train_data_files[::2]
- # valid_data_files = valid_data_files[::2]
self.setup_pipeline(train_data_files, valid_data_files, num_data_samples)
self.build_model()
@@ -770,7 +680,6 @@ class SRCNN:
def run_evaluate(self, data, ckpt_dir):
data = tf.convert_to_tensor(data, dtype=tf.float32)
self.num_data_samples = 80000
- # self.setup_eval_pipeline('clavrx_VNP02MOD.A2019017.1600.001.2019017214117.uwssec.highres.nc.level2.nc')
self.build_model()
self.build_training()
self.build_evaluation()
--
GitLab