From 100113ed2f972eb7a22932ab65cc66aae7928739 Mon Sep 17 00:00:00 2001
From: tomrink <rink@ssec.wisc.edu>
Date: Mon, 19 Apr 2021 12:25:53 -0500
Subject: [PATCH] snapshot...
---
modules/deeplearning/icing.py | 35 +++++++++++++++++++----------------
1 file changed, 19 insertions(+), 16 deletions(-)
diff --git a/modules/deeplearning/icing.py b/modules/deeplearning/icing.py
index 1eb1634a..d97e2fbc 100644
--- a/modules/deeplearning/icing.py
+++ b/modules/deeplearning/icing.py
@@ -31,6 +31,9 @@ CONV3D = False
img_width = 16
mean_std_file = '/Users/tomrink/data/icing/fovs_mean_std_day.pkl'
+f = open(mean_std_file, 'rb')
+mean_std_dct = pickle.load(f)
+f.close()
train_params = ['cld_height_acha', 'cld_geo_thick', 'supercooled_cloud_fraction', 'cld_temp_acha', 'cld_press_acha',
'cld_reff_dcomp', 'cld_opd_dcomp', 'cld_cwp_dcomp', 'iwc_dcomp', 'lwc_dcomp']
@@ -186,7 +189,7 @@ class IcingIntensityNN:
data = []
for param in train_params:
nda = self.h5f[param][nd_keys, ]
- # nda = normalize(nda, param)
+ nda = normalize(nda, param, mean_std_dct)
data.append(nda)
data = np.stack(data)
data = data.astype(np.float32)
@@ -212,11 +215,11 @@ class IcingIntensityNN:
# if CACHE_DATA_IN_MEM:
# self.in_mem_data_cache[key] = (nda, ndb, ndc)
- return data, data, label
+ return data, label
@tf.function(input_signature=[tf.TensorSpec(None, tf.int32)])
def data_function(self, indexes):
- out = tf.numpy_function(self.get_in_mem_data_batch, [indexes], [tf.float32, tf.float32, tf.int32])
+ out = tf.numpy_function(self.get_in_mem_data_batch, [indexes], [tf.float32, tf.int32])
return out
def get_train_dataset(self, indexes):
@@ -374,7 +377,7 @@ class IcingIntensityNN:
self.logits = logits
def build_training(self):
- self.loss = tf.keras.losses.BinaryCrossentropy # for two-class only
+ self.loss = tf.keras.losses.BinaryCrossentropy() # for two-class only
#self.loss = tf.keras.losses.SparseCategoricalCrossentropy() # For multi-class
# decayed_learning_rate = learning_rate * decay_rate ^ (global_step / decay_steps)
@@ -414,8 +417,8 @@ class IcingIntensityNN:
@tf.function
def train_step(self, mini_batch):
- inputs = [mini_batch[0], mini_batch[1]]
- labels = mini_batch[2]
+ inputs = [mini_batch[0]]
+ labels = mini_batch[1]
with tf.GradientTape() as tape:
pred = self.model(inputs, training=True)
loss = self.loss(labels, pred)
@@ -433,8 +436,8 @@ class IcingIntensityNN:
@tf.function
def test_step(self, mini_batch):
- inputs = [mini_batch[0], mini_batch[1]]
- labels = mini_batch[2]
+ inputs = [mini_batch[0]]
+ labels = mini_batch[1]
pred = self.model(inputs, training=False)
t_loss = self.loss(labels, pred)
@@ -442,8 +445,8 @@ class IcingIntensityNN:
self.test_accuracy(labels, pred)
def predict(self, mini_batch):
- inputs = [mini_batch[0], mini_batch[1]]
- labels = mini_batch[2]
+ inputs = [mini_batch[0]]
+ labels = mini_batch[1]
pred = self.model(inputs, training=False)
t_loss = self.loss(labels, pred)
@@ -473,8 +476,8 @@ class IcingIntensityNN:
proc_batch_cnt = 0
n_samples = 0
- for data0, data1, label in self.train_dataset:
- trn_ds = tf.data.Dataset.from_tensor_slices((data0, data1, label))
+ for data0, label in self.train_dataset:
+ trn_ds = tf.data.Dataset.from_tensor_slices((data0, label))
trn_ds = trn_ds.batch(BATCH_SIZE)
for mini_batch in trn_ds:
if self.learningRateSchedule is not None:
@@ -490,8 +493,8 @@ class IcingIntensityNN:
self.test_loss.reset_states()
self.test_accuracy.reset_states()
- for data0_tst, data1_tst, label_tst in self.test_dataset:
- tst_ds = tf.data.Dataset.from_tensor_slices((data0_tst, data1_tst, label_tst))
+ for data0_tst, label_tst in self.test_dataset:
+ tst_ds = tf.data.Dataset.from_tensor_slices((data0_tst, label_tst))
tst_ds = tst_ds.batch(BATCH_SIZE)
for mini_batch_test in tst_ds:
self.test_step(mini_batch_test)
@@ -517,8 +520,8 @@ class IcingIntensityNN:
self.test_loss.reset_states()
self.test_accuracy.reset_states()
- for data0, data1, label in self.test_dataset:
- ds = tf.data.Dataset.from_tensor_slices((data0, data1, label))
+ for data0, label in self.test_dataset:
+ ds = tf.data.Dataset.from_tensor_slices((data0, label))
ds = ds.batch(BATCH_SIZE)
for mini_batch in ds:
self.test_step(mini_batch)
--
GitLab