snapshot...

modules/deeplearning/icing_cnn.py

@@ -104,8 +104,10 @@ class IcingIntensityNN:
         self.handle = None
         self.inner_handle = None
         self.in_mem_batch = None
-        self.filename = None
-        self.h5f = None
+        self.filename_trn = None
+        self.h5f_trn = None
+        self.filename_tst = None
+        self.h5f_tst = None
         self.h5f_l1b = None
 
         self.logits = None
@@ -266,13 +268,22 @@ class IcingIntensityNN:
         dataset = dataset.map(self.data_function, num_parallel_calls=8)
         self.test_dataset = dataset
 
-    def setup_pipeline(self, filename, trn_idxs=None, tst_idxs=None, seed=None):
-        self.filename = filename
-        self.h5f = h5py.File(filename, 'r')
-        if trn_idxs is None and tst_idxs is None:
-            time = self.h5f['time']
-            num_obs = time.shape[0]
-            trn_idxs, tst_idxs = split_data(num_obs, seed=seed)
+    def setup_pipeline(self, filename_trn, filename_tst, trn_idxs=None, tst_idxs=None, seed=None):
+        self.filename_trn = filename_trn
+        self.h5f_trn = h5py.File(filename_trn, 'r')
+
+        self.filename_tst = filename_tst
+        self.h5f_tst = h5py.File(filename_tst, 'r')
+
+        if trn_idxs is None:
+            time = self.h5f_trn['time']
+            trn_idxs = np.arange(time.shape[0])
+            np.random.shuffle(trn_idxs)
+
+            time = self.h5f_tst['time']
+            tst_idxs = np.arange(time.shape[0])
+            np.random.shuffle(tst_idxs)
+
         self.num_data_samples = trn_idxs.shape[0]
 
         self.get_train_dataset(trn_idxs)
@@ -406,7 +417,7 @@ class IcingIntensityNN:
         else:
             activation = tf.nn.softmax  # For multi-class
 
-        # Called logits, but these are actually probabilities see activation
+        # Called logits, but these are actually probabilities, see activation
         logits = tf.keras.layers.Dense(NumLogits, activation=activation)(fc)
         print(logits.shape)
         
@@ -643,9 +654,9 @@ class IcingIntensityNN:
         self.test_labels = labels
         self.test_preds = preds
 
-    def run(self, filename, filename_l1b=None):
+    def run(self, filename_trn, filename_tst, filename_l1b=None):
         with tf.device('/device:GPU:'+str(self.gpu_device)):
-            self.setup_pipeline(filename)
+            self.setup_pipeline(filename_trn, filename_tst)
             self.build_model()
             self.build_training()
             self.build_evaluation()