diff --git a/modules/deeplearning/icing.py b/modules/deeplearning/icing.py index 1eb1634a2d33e0dce195d6af0e10fc860050aaa6..d97e2fbc04bab2cec2f48d5ff5b90c54018b9faa 100644 --- a/modules/deeplearning/icing.py +++ b/modules/deeplearning/icing.py @@ -31,6 +31,9 @@ CONV3D = False img_width = 16 mean_std_file = '/Users/tomrink/data/icing/fovs_mean_std_day.pkl' +f = open(mean_std_file, 'rb') +mean_std_dct = pickle.load(f) +f.close() train_params = ['cld_height_acha', 'cld_geo_thick', 'supercooled_cloud_fraction', 'cld_temp_acha', 'cld_press_acha', 'cld_reff_dcomp', 'cld_opd_dcomp', 'cld_cwp_dcomp', 'iwc_dcomp', 'lwc_dcomp'] @@ -186,7 +189,7 @@ class IcingIntensityNN: data = [] for param in train_params: nda = self.h5f[param][nd_keys, ] - # nda = normalize(nda, param) + nda = normalize(nda, param, mean_std_dct) data.append(nda) data = np.stack(data) data = data.astype(np.float32) @@ -212,11 +215,11 @@ class IcingIntensityNN: # if CACHE_DATA_IN_MEM: # self.in_mem_data_cache[key] = (nda, ndb, ndc) - return data, data, label + return data, label @tf.function(input_signature=[tf.TensorSpec(None, tf.int32)]) def data_function(self, indexes): - out = tf.numpy_function(self.get_in_mem_data_batch, [indexes], [tf.float32, tf.float32, tf.int32]) + out = tf.numpy_function(self.get_in_mem_data_batch, [indexes], [tf.float32, tf.int32]) return out def get_train_dataset(self, indexes): @@ -374,7 +377,7 @@ class IcingIntensityNN: self.logits = logits def build_training(self): - self.loss = tf.keras.losses.BinaryCrossentropy # for two-class only + self.loss = tf.keras.losses.BinaryCrossentropy() # for two-class only #self.loss = tf.keras.losses.SparseCategoricalCrossentropy() # For multi-class # decayed_learning_rate = learning_rate * decay_rate ^ (global_step / decay_steps) @@ -414,8 +417,8 @@ class IcingIntensityNN: @tf.function def train_step(self, mini_batch): - inputs = [mini_batch[0], mini_batch[1]] - labels = mini_batch[2] + inputs = [mini_batch[0]] + labels = mini_batch[1] with tf.GradientTape() as tape: pred = self.model(inputs, training=True) loss = self.loss(labels, pred) @@ -433,8 +436,8 @@ class IcingIntensityNN: @tf.function def test_step(self, mini_batch): - inputs = [mini_batch[0], mini_batch[1]] - labels = mini_batch[2] + inputs = [mini_batch[0]] + labels = mini_batch[1] pred = self.model(inputs, training=False) t_loss = self.loss(labels, pred) @@ -442,8 +445,8 @@ class IcingIntensityNN: self.test_accuracy(labels, pred) def predict(self, mini_batch): - inputs = [mini_batch[0], mini_batch[1]] - labels = mini_batch[2] + inputs = [mini_batch[0]] + labels = mini_batch[1] pred = self.model(inputs, training=False) t_loss = self.loss(labels, pred) @@ -473,8 +476,8 @@ class IcingIntensityNN: proc_batch_cnt = 0 n_samples = 0 - for data0, data1, label in self.train_dataset: - trn_ds = tf.data.Dataset.from_tensor_slices((data0, data1, label)) + for data0, label in self.train_dataset: + trn_ds = tf.data.Dataset.from_tensor_slices((data0, label)) trn_ds = trn_ds.batch(BATCH_SIZE) for mini_batch in trn_ds: if self.learningRateSchedule is not None: @@ -490,8 +493,8 @@ class IcingIntensityNN: self.test_loss.reset_states() self.test_accuracy.reset_states() - for data0_tst, data1_tst, label_tst in self.test_dataset: - tst_ds = tf.data.Dataset.from_tensor_slices((data0_tst, data1_tst, label_tst)) + for data0_tst, label_tst in self.test_dataset: + tst_ds = tf.data.Dataset.from_tensor_slices((data0_tst, label_tst)) tst_ds = tst_ds.batch(BATCH_SIZE) for mini_batch_test in tst_ds: self.test_step(mini_batch_test) @@ -517,8 +520,8 @@ class IcingIntensityNN: self.test_loss.reset_states() self.test_accuracy.reset_states() - for data0, data1, label in self.test_dataset: - ds = tf.data.Dataset.from_tensor_slices((data0, data1, label)) + for data0, label in self.test_dataset: + ds = tf.data.Dataset.from_tensor_slices((data0, label)) ds = ds.batch(BATCH_SIZE) for mini_batch in ds: self.test_step(mini_batch)