diff --git a/modules/deeplearning/icing_cnn.py b/modules/deeplearning/icing_cnn.py
index 6bc3487cd0ac6941a1a9a37f7e4518090f930f3d..7977e1adde7a597888cf81ac9d7abf62c0d5a2e0 100644
--- a/modules/deeplearning/icing_cnn.py
+++ b/modules/deeplearning/icing_cnn.py
@@ -174,6 +174,7 @@ class IcingIntensityNN:
         self.sfc = None
 
         self.in_mem_data_cache = {}
+        self.in_mem_data_cache_test = {}
 
         self.model = None
         self.optimizer = None
@@ -235,7 +236,10 @@ class IcingIntensityNN:
         # Pretty much dead, but left in here for reference (See note above)
         if CACHE_DATA_IN_MEM:
             key = frozenset(idxs)
-            tup = self.in_mem_data_cache.get(key)
+            if is_training:
+                tup = self.in_mem_data_cache.get(key)
+            else:
+                tup = self.in_mem_data_cache_test(key)
             if tup is not None:
                 return tup[0], tup[1]
 
@@ -276,7 +280,10 @@ class IcingIntensityNN:
             label = label.reshape((label.shape[0], 1))
 
         if CACHE_DATA_IN_MEM:
-            self.in_mem_data_cache[key] = (data, label)
+            if is_training:
+                self.in_mem_data_cache[key] = (data, label)
+            else:
+                self.in_mem_data_cache_test[key] = (data, label)
 
         return data, label