snapshot...

modules/deeplearning/srcnn_l1b_l2.py

@@ -324,15 +324,66 @@ class SRCNN:
         return self.get_in_mem_data_batch(idxs, False)
 
     def get_in_mem_data_batch_eval(self, idxs):
-        data = []
-        for param in self.train_params:
-            nda = self.data_dct[param]
-            nda = normalize(nda, param, mean_std_dct)
-            data.append(nda)
-        data = np.stack(data)
-        data = data.astype(np.float32)
-        data = np.transpose(data, axes=(1, 2, 0))
+        in_file = '/home/rink/data/clavrx_snpp_day/clavrx_VNP02MOD.A2019017.1600.001.2019017214117.uwssec.highres.nc.level2.nc'
+        N = 8
+
+        slc_x = slice(2, N * 128 + 4)
+        slc_y = slice(2, N * 128 + 4)
+        slc_x_2 = slice(1, N * 128 + 6, 2)
+        slc_y_2 = slice(1, N * 128 + 6, 2)
+        x_2 = np.arange(int((N * 128) / 2) + 3)
+        y_2 = np.arange(int((N * 128) / 2) + 3)
+        t = np.arange(0, int((N * 128) / 2) + 3, 0.5)
+        s = np.arange(0, int((N * 128) / 2) + 3, 0.5)
+        x_k = slice(1, N * 128 + 3)
+        y_k = slice(1, N * 128 + 3)
+        x_128 = slice(3, N * 128 + 3)
+        y_128 = slice(3, N * 128 + 3)
+
+        sub_y, sub_x = (N * 128) + 10, (N * 128) + 10
+        y_0, x_0, = 2432 - int(sub_y / 2), 2432 - int(sub_x / 2)
+
+        h5f = h5py.File(in_file, 'r')
+
+        grd_a = get_grid_values_all(h5f, 'temp_11_0um_nom')
+        grd_a = grd_a[y_0:y_0 + sub_y, x_0:x_0 + sub_x]
+        grd_a = grd_a.copy()
+        grd_a = np.where(np.isnan(grd_a), 0, grd_a)
+        hr_grd_a = grd_a.copy()
+        hr_grd_a = hr_grd_a[y_128, x_128]
+        grd_a = grd_a[slc_y_2, slc_x_2]
+        grd_a = resample_2d_linear_one(x_2, y_2, grd_a, t, s)
+        grd_a = grd_a[y_k, x_k]
+        grd_a = normalize(grd_a, 'temp_11_0um_nom', mean_std_dct)
+        #
+        # grd_b = get_grid_values_all(h5f, 'refl_0_65um_nom')
+        # grd_b = grd_b[y_0:y_0+sub_y, x_0:x_0+sub_x]
+        # grd_b = grd_b[y_130, x_130]
+        # refl = grd_b
+        # grd_b = normalize(grd_b, 'refl_0_65um_nom', mean_std_dct)
+
+        grd_c = get_grid_values_all(h5f, label_param)
+        grd_c = grd_c[y_0:y_0 + sub_y, x_0:x_0 + sub_x]
+
+        hr_grd_c = grd_c.copy()
+        hr_grd_c = hr_grd_c[y_128, x_128]
+
+        grd_c = np.where(np.isnan(grd_c), 0, grd_c)
+        grd_c = grd_c.copy()
+        grd_c = grd_c[slc_y_2, slc_x_2]
+        grd_c = resample_2d_linear_one(x_2, y_2, grd_c, t, s)
+        grd_c = grd_c[y_k, x_k]
+
+        if label_param != 'cloud_probability':
+            grd_c = normalize(grd_c, label_param, mean_std_dct)
+
+        # data = np.stack([grd_a, grd_b, grd_c], axis=2)
+        # data = np.stack([grd_a, grd_c], axis=2)
+        data = np.stack([grd_c], axis=2)
         data = np.expand_dims(data, axis=0)
+        data = data.astype(np.float32)
+
+        h5f.close()
 
         return data
 
@@ -409,8 +460,7 @@ class SRCNN:
 
     def setup_eval_pipeline(self, filename):
         idxs = [0]
-        self.num_data_samples = idxs.shape[0]
-
+        self.num_data_samples = 1
         self.get_evaluate_dataset(idxs)
 
     def build_srcnn(self, do_drop_out=False, do_batch_norm=False, drop_rate=0.5, factor=2):
@@ -688,11 +738,18 @@ class SRCNN:
 
         self.reset_test_metrics()
 
-        pred = self.model([data], training=False)
-        self.test_probs = pred
-        pred = pred.numpy()
-        if label_param != 'cloud_probability':
-            pred = denormalize(pred, label_param, mean_std_dct)
+        for data in self.eval_dataset:
+            pred = self.model([data], training=False)
+            pred = pred.numpy()
+            if label_param != 'cloud_probability':
+                pred = denormalize(pred, label_param, mean_std_dct)
+                print(pred.min(), pred.max())
+
+        # pred = self.model([data], training=False)
+        # self.test_probs = pred
+        # pred = pred.numpy()
+        # if label_param != 'cloud_probability':
+        #     pred = denormalize(pred, label_param, mean_std_dct)
 
         return pred
 
@@ -720,6 +777,7 @@ class SRCNN:
     def run_evaluate(self, data, ckpt_dir):
         data = tf.convert_to_tensor(data, dtype=tf.float32)
         self.num_data_samples = 80000
+        # self.setup_eval_pipeline('clavrx_VNP02MOD.A2019017.1600.001.2019017214117.uwssec.highres.nc.level2.nc')
         self.build_model()
         self.build_training()
         self.build_evaluation()
@@ -779,6 +837,7 @@ def run_evaluate_static(in_file, out_file, ckpt_dir):
 
     grd_c = np.where(np.isnan(grd_c), 0, grd_c)
     grd_c = grd_c.copy()
+    grd_c = smooth_2d_single(grd_c, sigma=1.0)
     grd_c = grd_c[slc_y_2, slc_x_2]
     grd_c = resample_2d_linear_one(x_2, y_2, grd_c, t, s)
     grd_c = grd_c[y_k, x_k]