diff --git a/modules/deeplearning/icing_cnn.py b/modules/deeplearning/icing_cnn.py index 77a6fb45ec771cab44d804f1d95a17ac6fad6238..1f01a0881fb3a3e77e8315f142d02e6c40a5cf9b 100644 --- a/modules/deeplearning/icing_cnn.py +++ b/modules/deeplearning/icing_cnn.py @@ -598,6 +598,7 @@ class IcingIntensityNN: step = 0 total_time = 0 + last_test_loss = np.finfo(dtype=np.float) if EARLY_STOP: es = EarlyStop() @@ -673,14 +674,18 @@ class IcingIntensityNN: if TRACK_MOVING_AVERAGE: # This may not really work properly self.optimizer.assign_average_vars(self.model.trainable_variables) - ckpt_manager.save() + + tst_loss = self.test_loss.result().numpy() + if tst_loss < last_test_loss: + last_test_loss = tst_loss + ckpt_manager.save() if self.DISK_CACHE and epoch == 0: f = open(cachepath, 'wb') pickle.dump(self.in_mem_data_cache, f) f.close() - if EARLY_STOP and es.check_stop(self.test_loss.result().numpy()): + if EARLY_STOP and es.check_stop(tst_loss): break print('total time: ', total_time)