diff --git a/modules/deeplearning/unet.py b/modules/deeplearning/unet.py index 491f7065d868c1736e394e54b87e71d45606062c..88605d090311a1238888246bb63e17cc6cf9c36f 100644 --- a/modules/deeplearning/unet.py +++ b/modules/deeplearning/unet.py @@ -39,39 +39,11 @@ DO_AUGMENT = True img_width = 16 -mean_std_dct = {} -mean_std_file = ancillary_path+'mean_std_lo_hi_l2.pkl' +mean_std_file = home_dir+'/viirs_emis_rad_mean_std.pkl' f = open(mean_std_file, 'rb') -mean_std_dct_l2 = pickle.load(f) +mean_std_dct = pickle.load(f) f.close() -mean_std_file = ancillary_path+'mean_std_lo_hi_l1b.pkl' -f = open(mean_std_file, 'rb') -mean_std_dct_l1b = pickle.load(f) -f.close() - -mean_std_dct.update(mean_std_dct_l1b) -mean_std_dct.update(mean_std_dct_l2) - -# -- NIGHT L2 ----------------------------- -train_params_l2_night = ['cld_height_acha', 'cld_geo_thick', 'cld_temp_acha', 'cld_press_acha', 'supercooled_cloud_fraction', - 'cld_emiss_acha', 'conv_cloud_fraction', 'cld_reff_acha', 'cld_opd_acha'] -# -- DAY L2 -------------------------------- -train_params_l2_day = ['cld_height_acha', 'cld_geo_thick', 'cld_temp_acha', 'cld_press_acha', 'supercooled_cloud_fraction', - 'cld_emiss_acha', 'conv_cloud_fraction', 'cld_reff_dcomp', 'cld_opd_dcomp', 'iwc_dcomp', 'lwc_dcomp'] -# -- DAY L1B -------------------------------- -train_params_l1b_day = ['temp_10_4um_nom', 'temp_11_0um_nom', 'temp_12_0um_nom', 'temp_13_3um_nom', 'temp_3_75um_nom', - 'temp_6_2um_nom', 'temp_6_7um_nom', 'temp_7_3um_nom', 'temp_8_5um_nom', 'temp_9_7um_nom', - 'refl_0_47um_nom', 'refl_0_65um_nom', 'refl_0_86um_nom', 'refl_1_38um_nom', 'refl_1_60um_nom'] -# -- NIGHT L1B ------------------------------- -train_params_l1b_night = ['temp_10_4um_nom', 'temp_11_0um_nom', 'temp_12_0um_nom', 'temp_13_3um_nom', 'temp_3_75um_nom', - 'temp_6_2um_nom', 'temp_6_7um_nom', 'temp_7_3um_nom', 'temp_8_5um_nom', 'temp_9_7um_nom'] -# -- DAY LUNAR --------------------------------- -# train_params_l1b = ['cld_height_acha', 'cld_geo_thick', 'cld_temp_acha', 'cld_press_acha', 'supercooled_cloud_fraction', -# 'cld_emiss_acha', 'conv_cloud_fraction', 'cld_reff_dcomp', 'cld_opd_dcomp', 'iwc_dcomp', 'lwc_dcomp'] -# --------------------------------------------- - -train_params = train_params_l1b_day + train_params_l2_day # -- Zero out params (Experimentation Only) ------------ zero_out_params = ['cld_reff_dcomp', 'cld_opd_dcomp', 'iwc_dcomp', 'lwc_dcomp'] DO_ZERO_OUT = False @@ -335,19 +307,45 @@ class UNET: def get_in_mem_data_batch(self, idxs, is_training): if is_training: - data = self.train_data_nda[idxs,] + train_data = [] + label_data = [] + for k in idxs: + f = self.train_data_files[k] + nda = np.load(f) + train_data.append(nda) + + f = self.train_label_files[k] + nda = np.load(f) + label_data.append(nda) + + data = np.concatenate(train_data) data = np.expand_dims(data, axis=3) - label = self.train_label_nda[idxs,] + label = np.concatenate(label_data) label = np.expand_dims(label, axis=3) else: - data = self.test_data_nda[idxs,] + train_data = [] + label_data = [] + for k in idxs: + f = self.test_data_files[k] + nda = np.load(f) + train_data.append(nda) + + f = self.test_label_files[k] + nda = np.load(f) + label_data.append(nda) + + data = np.concatenate(train_data) data = np.expand_dims(data, axis=3) - label = self.test_label_nda[idxs,] + + label = np.concatenate(label_data) label = np.expand_dims(label, axis=3) data = data.astype(np.float32) label = label.astype(np.float32) + normalize(data, 'M15', mean_std_dct) + normalize(label, 'M15', mean_std_dct) + if is_training and DO_AUGMENT: data_ud = np.flip(data, axis=1) label_ud = np.flip(label, axis=1) @@ -1113,8 +1111,10 @@ class UNET: self.build_evaluation() self.do_training() - def run_test(self, data_nda, label_nda): - self.setup_pipeline(data_nda, label_nda) + def run_test(self, directory): + data_files = glob.glob(directory+'mod_res*.npy') + label_files = [f.replace('mod', 'img') for f in data_files] + self.setup_pipeline_files(data_files, label_files) self.build_model() self.build_training() self.build_evaluation()