diff --git a/modules/deeplearning/icing_cnn.py b/modules/deeplearning/icing_cnn.py index c7213d1c6186c3d89d3b6ddc15cc4ed9a373225e..d63051373da855731b985f1c8fae042bfda1c144 100644 --- a/modules/deeplearning/icing_cnn.py +++ b/modules/deeplearning/icing_cnn.py @@ -38,30 +38,39 @@ NOISE_TRAINING = False img_width = 16 -# mean_std_file = homedir+'data/icing/mean_std_no_ice.pkl' -mean_std_file = homedir+'data/icing/mean_std_l1b_no_ice.pkl' +mean_std_dct = {} +mean_std_file = homedir+'data/icing/mean_std_no_ice.pkl' f = open(mean_std_file, 'rb') -mean_std_dct = pickle.load(f) +mean_std_dct_l2 = pickle.load(f) f.close() +mean_std_file = homedir+'data/icing/mean_std_no_ice.pkl' +f = open(mean_std_file, 'rb') +mean_std_dct_l1b = pickle.load(f) +f.close() + +mean_std_dct.update(mean_std_dct_l1b) +mean_std_dct.update(mean_std_dct_l2) + # -- NIGHT L2 ----------------------------- -# train_params = ['cld_height_acha', 'cld_geo_thick', 'cld_temp_acha', 'cld_press_acha', 'supercooled_cloud_fraction', -# 'cld_emiss_acha', 'conv_cloud_fraction', 'cld_reff_acha', 'cld_opd_acha'] +# train_params_l2 = ['cld_height_acha', 'cld_geo_thick', 'cld_temp_acha', 'cld_press_acha', 'supercooled_cloud_fraction', +# 'cld_emiss_acha', 'conv_cloud_fraction', 'cld_reff_acha', 'cld_opd_acha'] # -- DAY L2 ------------- -#train_params = ['cld_height_acha', 'cld_geo_thick', 'cld_temp_acha', 'cld_press_acha', 'supercooled_cloud_fraction', +train_params_l2 = ['cld_height_acha', 'cld_geo_thick', 'cld_temp_acha', 'cld_press_acha', 'supercooled_cloud_fraction', ## 'cld_emiss_acha', 'conv_cloud_fraction', 'cld_reff_dcomp', 'cld_opd_dcomp', 'cld_cwp_dcomp', 'iwc_dcomp', 'lwc_dcomp'] -# 'cld_emiss_acha', 'conv_cloud_fraction', 'cld_reff_dcomp', 'cld_opd_dcomp', 'iwc_dcomp', 'lwc_dcomp'] + 'cld_emiss_acha', 'conv_cloud_fraction', 'cld_reff_dcomp', 'cld_opd_dcomp', 'iwc_dcomp', 'lwc_dcomp'] # -- DAY L1B -------------------------------- -train_params = ['temp_10_4um_nom', 'temp_11_0um_nom', 'temp_12_0um_nom', 'temp_13_3um_nom', 'temp_3_75um_nom', - 'temp_6_2um_nom', 'temp_6_7um_nom', 'temp_7_3um_nom', 'temp_8_5um_nom', 'temp_9_7um_nom', - 'refl_0_47um_nom', 'refl_0_65um_nom', 'refl_0_86um_nom', 'refl_1_38um_nom', 'refl_1_60um_nom'] +train_params_l1b = ['temp_10_4um_nom', 'temp_11_0um_nom', 'temp_12_0um_nom', 'temp_13_3um_nom', 'temp_3_75um_nom', + 'temp_6_2um_nom', 'temp_6_7um_nom', 'temp_7_3um_nom', 'temp_8_5um_nom', 'temp_9_7um_nom', + 'refl_0_47um_nom', 'refl_0_65um_nom', 'refl_0_86um_nom', 'refl_1_38um_nom', 'refl_1_60um_nom'] # -- NIGHT L1B ------------------------------- -# train_params = ['temp_10_4um_nom', 'temp_11_0um_nom', 'temp_12_0um_nom', 'temp_13_3um_nom', 'temp_3_75um_nom', -# 'temp_6_2um_nom', 'temp_6_7um_nom', 'temp_7_3um_nom', 'temp_8_5um_nom', 'temp_9_7um_nom'] +# train_params_l1b = ['temp_10_4um_nom', 'temp_11_0um_nom', 'temp_12_0um_nom', 'temp_13_3um_nom', 'temp_3_75um_nom', +# 'temp_6_2um_nom', 'temp_6_7um_nom', 'temp_7_3um_nom', 'temp_8_5um_nom', 'temp_9_7um_nom'] # -- DAY LUNAR --------------------- -# train_params = ['cld_height_acha', 'cld_geo_thick', 'cld_temp_acha', 'cld_press_acha', 'supercooled_cloud_fraction', -# 'cld_emiss_acha', 'conv_cloud_fraction', 'cld_reff_dcomp', 'cld_opd_dcomp', 'iwc_dcomp', 'lwc_dcomp'] +# train_params_l1b = ['cld_height_acha', 'cld_geo_thick', 'cld_temp_acha', 'cld_press_acha', 'supercooled_cloud_fraction', +# 'cld_emiss_acha', 'conv_cloud_fraction', 'cld_reff_dcomp', 'cld_opd_dcomp', 'iwc_dcomp', 'lwc_dcomp'] # --------------------------------------------- +train_params = train_params_l1b # -- Zero out params (Experimentation Only) ------------ zero_out_params = ['cld_reff_dcomp', 'cld_opd_dcomp', 'iwc_dcomp', 'lwc_dcomp'] DO_ZERO_OUT = False @@ -135,6 +144,11 @@ class IcingIntensityNN: self.h5f_tst = None self.h5f_l1b = None + self.h5f_l1b_trn = None + self.h5f_l1b_tst = None + self.h5f_l2_trn = None + self.h5f_l2_tst = None + self.logits = None self.predict_data = None @@ -270,6 +284,21 @@ class IcingIntensityNN: return data, label + def get_parameter_data(self, param, nd_idxs, is_training): + if is_training: + if param in train_params_l1b: + h5f = self.h5f_l1b_trn + else: + h5f = self.h5f_l2_trn + else: + if param in train_params_l1b: + h5f = self.h5f_l1b_tst + else: + h5f = self.h5f_l2_tst + + nda = h5f[param][nd_idxs,] + return nda + def get_in_mem_data_batch_train(self, idxs): return self.get_in_mem_data_batch(idxs, True)