snapshot...

modules/deeplearning/unet.py

@@ -39,39 +39,11 @@ DO_AUGMENT = True
 
 img_width = 16
 
-mean_std_dct = {}
-mean_std_file = ancillary_path+'mean_std_lo_hi_l2.pkl'
+mean_std_file = home_dir+'/viirs_emis_rad_mean_std.pkl'
 f = open(mean_std_file, 'rb')
-mean_std_dct_l2 = pickle.load(f)
+mean_std_dct = pickle.load(f)
 f.close()
 
-mean_std_file = ancillary_path+'mean_std_lo_hi_l1b.pkl'
-f = open(mean_std_file, 'rb')
-mean_std_dct_l1b = pickle.load(f)
-f.close()
-
-mean_std_dct.update(mean_std_dct_l1b)
-mean_std_dct.update(mean_std_dct_l2)
-
-# --  NIGHT L2 -----------------------------
-train_params_l2_night = ['cld_height_acha', 'cld_geo_thick', 'cld_temp_acha', 'cld_press_acha', 'supercooled_cloud_fraction',
-                         'cld_emiss_acha', 'conv_cloud_fraction', 'cld_reff_acha', 'cld_opd_acha']
-# -- DAY L2 --------------------------------
-train_params_l2_day = ['cld_height_acha', 'cld_geo_thick', 'cld_temp_acha', 'cld_press_acha', 'supercooled_cloud_fraction',
-                       'cld_emiss_acha', 'conv_cloud_fraction', 'cld_reff_dcomp', 'cld_opd_dcomp', 'iwc_dcomp', 'lwc_dcomp']
-# -- DAY L1B --------------------------------
-train_params_l1b_day = ['temp_10_4um_nom', 'temp_11_0um_nom', 'temp_12_0um_nom', 'temp_13_3um_nom', 'temp_3_75um_nom',
-                        'temp_6_2um_nom', 'temp_6_7um_nom', 'temp_7_3um_nom', 'temp_8_5um_nom', 'temp_9_7um_nom',
-                        'refl_0_47um_nom', 'refl_0_65um_nom', 'refl_0_86um_nom', 'refl_1_38um_nom', 'refl_1_60um_nom']
-# -- NIGHT L1B -------------------------------
-train_params_l1b_night = ['temp_10_4um_nom', 'temp_11_0um_nom', 'temp_12_0um_nom', 'temp_13_3um_nom', 'temp_3_75um_nom',
-                          'temp_6_2um_nom', 'temp_6_7um_nom', 'temp_7_3um_nom', 'temp_8_5um_nom', 'temp_9_7um_nom']
-# -- DAY LUNAR ---------------------------------
-# train_params_l1b = ['cld_height_acha', 'cld_geo_thick', 'cld_temp_acha', 'cld_press_acha', 'supercooled_cloud_fraction',
-#                     'cld_emiss_acha', 'conv_cloud_fraction', 'cld_reff_dcomp', 'cld_opd_dcomp', 'iwc_dcomp', 'lwc_dcomp']
-# ---------------------------------------------
-
-train_params = train_params_l1b_day + train_params_l2_day
 # -- Zero out params (Experimentation Only) ------------
 zero_out_params = ['cld_reff_dcomp', 'cld_opd_dcomp', 'iwc_dcomp', 'lwc_dcomp']
 DO_ZERO_OUT = False
@@ -335,19 +307,45 @@ class UNET:
 
     def get_in_mem_data_batch(self, idxs, is_training):
         if is_training:
-            data = self.train_data_nda[idxs,]
+            train_data = []
+            label_data = []
+            for k in idxs:
+                f = self.train_data_files[k]
+                nda = np.load(f)
+                train_data.append(nda)
+
+                f = self.train_label_files[k]
+                nda = np.load(f)
+                label_data.append(nda)
+
+            data = np.concatenate(train_data)
             data = np.expand_dims(data, axis=3)
-            label = self.train_label_nda[idxs,]
+            label = np.concatenate(label_data)
             label = np.expand_dims(label, axis=3)
         else:
-            data = self.test_data_nda[idxs,]
+            train_data = []
+            label_data = []
+            for k in idxs:
+                f = self.test_data_files[k]
+                nda = np.load(f)
+                train_data.append(nda)
+
+                f = self.test_label_files[k]
+                nda = np.load(f)
+                label_data.append(nda)
+
+            data = np.concatenate(train_data)
             data = np.expand_dims(data, axis=3)
-            label = self.test_label_nda[idxs,]
+
+            label = np.concatenate(label_data)
             label = np.expand_dims(label, axis=3)
 
         data = data.astype(np.float32)
         label = label.astype(np.float32)
 
+        normalize(data, 'M15', mean_std_dct)
+        normalize(label, 'M15', mean_std_dct)
+
         if is_training and DO_AUGMENT:
             data_ud = np.flip(data, axis=1)
             label_ud = np.flip(label, axis=1)
@@ -1113,8 +1111,10 @@ class UNET:
         self.build_evaluation()
         self.do_training()
 
-    def run_test(self, data_nda, label_nda):
-        self.setup_pipeline(data_nda, label_nda)
+    def run_test(self, directory):
+        data_files = glob.glob(directory+'mod_res*.npy')
+        label_files = [f.replace('mod', 'img') for f in data_files]
+        self.setup_pipeline_files(data_files, label_files)
         self.build_model()
         self.build_training()
         self.build_evaluation()