From e124067431c67a0828cbc1d11986cb600954af7e Mon Sep 17 00:00:00 2001
From: tomrink <rink@ssec.wisc.edu>
Date: Fri, 7 May 2021 13:09:34 -0500
Subject: [PATCH] snapshot...
---
modules/deeplearning/icing_cnn.py | 35 ++++++++++++++++++++-----------
1 file changed, 23 insertions(+), 12 deletions(-)
diff --git a/modules/deeplearning/icing_cnn.py b/modules/deeplearning/icing_cnn.py
index 480801f2..f771dd28 100644
--- a/modules/deeplearning/icing_cnn.py
+++ b/modules/deeplearning/icing_cnn.py
@@ -104,8 +104,10 @@ class IcingIntensityNN:
self.handle = None
self.inner_handle = None
self.in_mem_batch = None
- self.filename = None
- self.h5f = None
+ self.filename_trn = None
+ self.h5f_trn = None
+ self.filename_tst = None
+ self.h5f_tst = None
self.h5f_l1b = None
self.logits = None
@@ -266,13 +268,22 @@ class IcingIntensityNN:
dataset = dataset.map(self.data_function, num_parallel_calls=8)
self.test_dataset = dataset
- def setup_pipeline(self, filename, trn_idxs=None, tst_idxs=None, seed=None):
- self.filename = filename
- self.h5f = h5py.File(filename, 'r')
- if trn_idxs is None and tst_idxs is None:
- time = self.h5f['time']
- num_obs = time.shape[0]
- trn_idxs, tst_idxs = split_data(num_obs, seed=seed)
+ def setup_pipeline(self, filename_trn, filename_tst, trn_idxs=None, tst_idxs=None, seed=None):
+ self.filename_trn = filename_trn
+ self.h5f_trn = h5py.File(filename_trn, 'r')
+
+ self.filename_tst = filename_tst
+ self.h5f_tst = h5py.File(filename_tst, 'r')
+
+ if trn_idxs is None:
+ time = self.h5f_trn['time']
+ trn_idxs = np.arange(time.shape[0])
+ np.random.shuffle(trn_idxs)
+
+ time = self.h5f_tst['time']
+ tst_idxs = np.arange(time.shape[0])
+ np.random.shuffle(tst_idxs)
+
self.num_data_samples = trn_idxs.shape[0]
self.get_train_dataset(trn_idxs)
@@ -406,7 +417,7 @@ class IcingIntensityNN:
else:
activation = tf.nn.softmax # For multi-class
- # Called logits, but these are actually probabilities see activation
+ # Called logits, but these are actually probabilities, see activation
logits = tf.keras.layers.Dense(NumLogits, activation=activation)(fc)
print(logits.shape)
@@ -643,9 +654,9 @@ class IcingIntensityNN:
self.test_labels = labels
self.test_preds = preds
- def run(self, filename, filename_l1b=None):
+ def run(self, filename_trn, filename_tst, filename_l1b=None):
with tf.device('/device:GPU:'+str(self.gpu_device)):
- self.setup_pipeline(filename)
+ self.setup_pipeline(filename_trn, filename_tst)
self.build_model()
self.build_training()
self.build_evaluation()
--
GitLab