Skip to content
Snippets Groups Projects
Commit 8d0e7183 authored by tomrink's avatar tomrink
Browse files

snapshot...

parent f72d80ce
No related branches found
No related tags found
No related merge requests found
import tensorflow as tf import tensorflow as tf
from util.plot_cm import confusion_matrix_values from util.plot_cm import confusion_matrix_values
from util.augment import augment_image
from util.setup_cloud_fraction import logdir, modeldir, now, ancillary_path from util.setup_cloud_fraction import logdir, modeldir, now, ancillary_path
from util.util import EarlyStop, normalize, denormalize, get_grid_values_all from util.util import EarlyStop, normalize, denormalize, get_grid_values_all, make_tf_callable_generator
import glob import glob
import os, datetime import os, datetime
import numpy as np import numpy as np
...@@ -39,6 +40,9 @@ DO_SMOOTH = False ...@@ -39,6 +40,9 @@ DO_SMOOTH = False
SIGMA = 1.0 SIGMA = 1.0
DO_ZERO_OUT = False DO_ZERO_OUT = False
# CACHE_FILE = '/scratch/long/rink/cld_opd_abi_128x128_cache'
CACHE_FILE = ''
# setup scaling parameters dictionary # setup scaling parameters dictionary
mean_std_dct = {} mean_std_dct = {}
mean_std_file = ancillary_path+'mean_std_lo_hi_l2.pkl' mean_std_file = ancillary_path+'mean_std_lo_hi_l2.pkl'
...@@ -164,11 +168,11 @@ def get_label_data_5cat(grd_k): ...@@ -164,11 +168,11 @@ def get_label_data_5cat(grd_k):
grd_k[:, 0::4, 2::4] + grd_k[:, 1::4, 2::4] + grd_k[:, 2::4, 2::4] + grd_k[:, 3::4, 2::4] + \ grd_k[:, 0::4, 2::4] + grd_k[:, 1::4, 2::4] + grd_k[:, 2::4, 2::4] + grd_k[:, 3::4, 2::4] + \
grd_k[:, 0::4, 3::4] + grd_k[:, 1::4, 3::4] + grd_k[:, 2::4, 3::4] + grd_k[:, 3::4, 3::4] grd_k[:, 0::4, 3::4] + grd_k[:, 1::4, 3::4] + grd_k[:, 2::4, 3::4] + grd_k[:, 3::4, 3::4]
cat_0 = np.logical_and(s >= 0, s < 2) cat_0 = np.logical_and(s >= 0, s < 1)
cat_1 = np.logical_and(s >= 2, s < 6) cat_1 = np.logical_and(s >= 1, s < 6)
cat_2 = np.logical_and(s >= 6, s < 11) cat_2 = np.logical_and(s >= 6, s < 11)
cat_3 = np.logical_and(s >= 11, s < 15) cat_3 = np.logical_and(s >= 11, s <= 15)
cat_4 = np.logical_and(s >= 15, s <= 16) cat_4 = np.logical_and(s > 15, s <= 16)
s[cat_0] = 0 s[cat_0] = 0
s[cat_1] = 1 s[cat_1] = 1
...@@ -381,24 +385,37 @@ class SRCNN: ...@@ -381,24 +385,37 @@ class SRCNN:
out = tf.numpy_function(self.get_in_mem_data_batch_test, [indexes], [tf.float32, tf.float32]) out = tf.numpy_function(self.get_in_mem_data_batch_test, [indexes], [tf.float32, tf.float32])
return out return out
def get_train_dataset(self, indexes): def get_train_dataset(self, num_files):
indexes = list(indexes) def integer_gen(limit):
n = 0
while n < limit:
yield n
n += 1
num_gen = integer_gen(num_files)
gen = make_tf_callable_generator(num_gen)
dataset = tf.data.Dataset.from_tensor_slices(indexes) dataset = tf.data.Dataset.from_generator(gen, output_types=tf.int32)
dataset = dataset.batch(PROC_BATCH_SIZE) dataset = dataset.batch(PROC_BATCH_SIZE)
dataset = dataset.map(self.data_function, num_parallel_calls=AUTOTUNE) dataset = dataset.map(self.data_function, num_parallel_calls=8)
dataset = dataset.cache() dataset = dataset.cache(filename=CACHE_FILE)
dataset = dataset.shuffle(PROC_BATCH_BUFFER_SIZE, reshuffle_each_iteration=True)
if DO_AUGMENT: if DO_AUGMENT:
dataset = dataset.shuffle(PROC_BATCH_BUFFER_SIZE) dataset = dataset.map(augment_image(), num_parallel_calls=8)
dataset = dataset.prefetch(buffer_size=AUTOTUNE) dataset = dataset.prefetch(buffer_size=1)
self.train_dataset = dataset self.train_dataset = dataset
def get_test_dataset(self, indexes): def get_test_dataset(self, num_files):
indexes = list(indexes) def integer_gen(limit):
n = 0
while n < limit:
yield n
n += 1
num_gen = integer_gen(num_files)
gen = make_tf_callable_generator(num_gen)
dataset = tf.data.Dataset.from_tensor_slices(indexes) dataset = tf.data.Dataset.from_generator(gen, output_types=tf.int32)
dataset = dataset.batch(PROC_BATCH_SIZE) dataset = dataset.batch(PROC_BATCH_SIZE)
dataset = dataset.map(self.data_function_test, num_parallel_calls=AUTOTUNE) dataset = dataset.map(self.data_function_test, num_parallel_calls=8)
dataset = dataset.cache() dataset = dataset.cache()
self.test_dataset = dataset self.test_dataset = dataset
...@@ -408,29 +425,23 @@ class SRCNN: ...@@ -408,29 +425,23 @@ class SRCNN:
self.test_data_files = test_data_files self.test_data_files = test_data_files
self.test_label_files = test_label_files self.test_label_files = test_label_files
trn_idxs = np.arange(len(train_data_files)) self.get_train_dataset(len(train_data_files))
np.random.shuffle(trn_idxs) self.get_test_dataset(len(test_data_files))
tst_idxs = np.arange(len(test_data_files))
self.get_train_dataset(trn_idxs)
self.get_test_dataset(tst_idxs)
self.num_data_samples = num_train_samples # approximately self.num_data_samples = num_train_samples # approximately
print('datetime: ', now) print('datetime: ', now)
print('training and test data: ') print('training and test data: ')
print('---------------------------') print('---------------------------')
print('num train samples: ', self.num_data_samples) print('num train files: ', len(train_data_files))
print('BATCH SIZE: ', BATCH_SIZE) print('BATCH SIZE: ', BATCH_SIZE)
print('num test samples: ', tst_idxs.shape[0]) print('num test files: ', len(test_data_files))
print('setup_pipeline: Done') print('setup_pipeline: Done')
def setup_test_pipeline(self, test_data_files, test_label_files): def setup_test_pipeline(self, test_data_files, test_label_files):
self.test_data_files = test_data_files self.test_data_files = test_data_files
self.test_label_files = test_label_files self.test_label_files = test_label_files
tst_idxs = np.arange(len(test_data_files)) self.get_test_dataset(len(test_data_files))
self.get_test_dataset(tst_idxs)
print('setup_test_pipeline: Done') print('setup_test_pipeline: Done')
def build_srcnn(self, do_drop_out=False, do_batch_norm=False, drop_rate=0.5, factor=2): def build_srcnn(self, do_drop_out=False, do_batch_norm=False, drop_rate=0.5, factor=2):
...@@ -742,7 +753,7 @@ class SRCNN: ...@@ -742,7 +753,7 @@ class SRCNN:
self.num_data_samples = 1000 self.num_data_samples = 1000
valid_data_files = glob.glob(directory + 'valid*mres*.npy') valid_data_files = glob.glob(directory + 'valid*mres*.npy')
valid_label_files = glob.glob(directory + 'valid*ires*.npy') valid_label_files = [f.replace('mres', 'ires') for f in valid_data_files]
self.setup_test_pipeline(valid_data_files, valid_label_files) self.setup_test_pipeline(valid_data_files, valid_label_files)
self.build_model() self.build_model()
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment