snapshot...

modules/deeplearning/icing_cnn.py

@@ -38,30 +38,39 @@ NOISE_TRAINING = False
 
 img_width = 16
 
-# mean_std_file = homedir+'data/icing/mean_std_no_ice.pkl'
-mean_std_file = homedir+'data/icing/mean_std_l1b_no_ice.pkl'
+mean_std_dct = {}
+mean_std_file = homedir+'data/icing/mean_std_no_ice.pkl'
 f = open(mean_std_file, 'rb')
-mean_std_dct = pickle.load(f)
+mean_std_dct_l2 = pickle.load(f)
 f.close()
 
+mean_std_file = homedir+'data/icing/mean_std_no_ice.pkl'
+f = open(mean_std_file, 'rb')
+mean_std_dct_l1b = pickle.load(f)
+f.close()
+
+mean_std_dct.update(mean_std_dct_l1b)
+mean_std_dct.update(mean_std_dct_l2)
+
 # --  NIGHT L2 -----------------------------
-# train_params = ['cld_height_acha', 'cld_geo_thick', 'cld_temp_acha', 'cld_press_acha', 'supercooled_cloud_fraction',
-#                 'cld_emiss_acha', 'conv_cloud_fraction', 'cld_reff_acha', 'cld_opd_acha']
+# train_params_l2 = ['cld_height_acha', 'cld_geo_thick', 'cld_temp_acha', 'cld_press_acha', 'supercooled_cloud_fraction',
+#                    'cld_emiss_acha', 'conv_cloud_fraction', 'cld_reff_acha', 'cld_opd_acha']
 # -- DAY L2 -------------
-#train_params = ['cld_height_acha', 'cld_geo_thick', 'cld_temp_acha', 'cld_press_acha', 'supercooled_cloud_fraction',
+train_params_l2 = ['cld_height_acha', 'cld_geo_thick', 'cld_temp_acha', 'cld_press_acha', 'supercooled_cloud_fraction',
 ##                'cld_emiss_acha', 'conv_cloud_fraction', 'cld_reff_dcomp', 'cld_opd_dcomp', 'cld_cwp_dcomp', 'iwc_dcomp', 'lwc_dcomp']
-#                'cld_emiss_acha', 'conv_cloud_fraction', 'cld_reff_dcomp', 'cld_opd_dcomp', 'iwc_dcomp', 'lwc_dcomp']
+                  'cld_emiss_acha', 'conv_cloud_fraction', 'cld_reff_dcomp', 'cld_opd_dcomp', 'iwc_dcomp', 'lwc_dcomp']
 # -- DAY L1B --------------------------------
-train_params = ['temp_10_4um_nom', 'temp_11_0um_nom', 'temp_12_0um_nom', 'temp_13_3um_nom', 'temp_3_75um_nom',
-                'temp_6_2um_nom', 'temp_6_7um_nom', 'temp_7_3um_nom', 'temp_8_5um_nom', 'temp_9_7um_nom',
-                'refl_0_47um_nom', 'refl_0_65um_nom', 'refl_0_86um_nom', 'refl_1_38um_nom', 'refl_1_60um_nom']
+train_params_l1b = ['temp_10_4um_nom', 'temp_11_0um_nom', 'temp_12_0um_nom', 'temp_13_3um_nom', 'temp_3_75um_nom',
+                    'temp_6_2um_nom', 'temp_6_7um_nom', 'temp_7_3um_nom', 'temp_8_5um_nom', 'temp_9_7um_nom',
+                    'refl_0_47um_nom', 'refl_0_65um_nom', 'refl_0_86um_nom', 'refl_1_38um_nom', 'refl_1_60um_nom']
 # -- NIGHT L1B -------------------------------
-# train_params = ['temp_10_4um_nom', 'temp_11_0um_nom', 'temp_12_0um_nom', 'temp_13_3um_nom', 'temp_3_75um_nom',
-#                 'temp_6_2um_nom', 'temp_6_7um_nom', 'temp_7_3um_nom', 'temp_8_5um_nom', 'temp_9_7um_nom']
+# train_params_l1b = ['temp_10_4um_nom', 'temp_11_0um_nom', 'temp_12_0um_nom', 'temp_13_3um_nom', 'temp_3_75um_nom',
+#                    'temp_6_2um_nom', 'temp_6_7um_nom', 'temp_7_3um_nom', 'temp_8_5um_nom', 'temp_9_7um_nom']
 # -- DAY LUNAR ---------------------
-# train_params = ['cld_height_acha', 'cld_geo_thick', 'cld_temp_acha', 'cld_press_acha', 'supercooled_cloud_fraction',
-#                 'cld_emiss_acha', 'conv_cloud_fraction', 'cld_reff_dcomp', 'cld_opd_dcomp', 'iwc_dcomp', 'lwc_dcomp']
+# train_params_l1b = ['cld_height_acha', 'cld_geo_thick', 'cld_temp_acha', 'cld_press_acha', 'supercooled_cloud_fraction',
+#                     'cld_emiss_acha', 'conv_cloud_fraction', 'cld_reff_dcomp', 'cld_opd_dcomp', 'iwc_dcomp', 'lwc_dcomp']
 # ---------------------------------------------
+train_params = train_params_l1b
 # -- Zero out params (Experimentation Only) ------------
 zero_out_params = ['cld_reff_dcomp', 'cld_opd_dcomp', 'iwc_dcomp', 'lwc_dcomp']
 DO_ZERO_OUT = False
@@ -135,6 +144,11 @@ class IcingIntensityNN:
         self.h5f_tst = None
         self.h5f_l1b = None
 
+        self.h5f_l1b_trn = None
+        self.h5f_l1b_tst = None
+        self.h5f_l2_trn = None
+        self.h5f_l2_tst = None
+
         self.logits = None
 
         self.predict_data = None
@@ -270,6 +284,21 @@ class IcingIntensityNN:
 
         return data, label
 
+    def get_parameter_data(self, param, nd_idxs, is_training):
+        if is_training:
+            if param in train_params_l1b:
+                h5f = self.h5f_l1b_trn
+            else:
+                h5f = self.h5f_l2_trn
+        else:
+            if param in train_params_l1b:
+                h5f = self.h5f_l1b_tst
+            else:
+                h5f = self.h5f_l2_tst
+
+        nda = h5f[param][nd_idxs,]
+        return nda
+
     def get_in_mem_data_batch_train(self, idxs):
         return self.get_in_mem_data_batch(idxs, True)