From 23972174b4de9bd66c57cec064f256f3d05c5e42 Mon Sep 17 00:00:00 2001
From: tomrink <rink@ssec.wisc.edu>
Date: Mon, 12 Jul 2021 14:14:01 -0500
Subject: [PATCH] minor
---
modules/deeplearning/icing_cnn.py | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/modules/deeplearning/icing_cnn.py b/modules/deeplearning/icing_cnn.py
index 77a6fb45..1f01a088 100644
--- a/modules/deeplearning/icing_cnn.py
+++ b/modules/deeplearning/icing_cnn.py
@@ -598,6 +598,7 @@ class IcingIntensityNN:
step = 0
total_time = 0
+ last_test_loss = np.finfo(dtype=np.float)
if EARLY_STOP:
es = EarlyStop()
@@ -673,14 +674,18 @@ class IcingIntensityNN:
if TRACK_MOVING_AVERAGE: # This may not really work properly
self.optimizer.assign_average_vars(self.model.trainable_variables)
- ckpt_manager.save()
+
+ tst_loss = self.test_loss.result().numpy()
+ if tst_loss < last_test_loss:
+ last_test_loss = tst_loss
+ ckpt_manager.save()
if self.DISK_CACHE and epoch == 0:
f = open(cachepath, 'wb')
pickle.dump(self.in_mem_data_cache, f)
f.close()
- if EARLY_STOP and es.check_stop(self.test_loss.result().numpy()):
+ if EARLY_STOP and es.check_stop(tst_loss):
break
print('total time: ', total_time)
--
GitLab