diff --git a/modules/deeplearning/cnn_cld_frac.py b/modules/deeplearning/cnn_cld_frac.py
index 204df912c9f2197d5c0a5c16d3ef108adb1350e1..3ca3488b4c69ccd9a7f1466c0ef1f1595812d5a8 100644
--- a/modules/deeplearning/cnn_cld_frac.py
+++ b/modules/deeplearning/cnn_cld_frac.py
@@ -62,9 +62,11 @@ IMG_DEPTH = 1
label_param = 'cloud_probability'
params = ['temp_11_0um_nom', 'refl_0_65um_nom', label_param]
+params_i = ['refl_0_65um_nom', label_param]
data_params_half = ['temp_11_0um_nom']
data_params_full = ['refl_0_65um_nom']
+label_idx_i = params_i.index(label_param)
label_idx = params.index(label_param)
print('data_params_half: ', data_params_half)
@@ -75,8 +77,8 @@ KERNEL_SIZE = 3 # target size: (128, 128)
N = 1
if KERNEL_SIZE == 3:
- slc_x = slice(2, N*128 + 4)
- slc_y = slice(2, N*128 + 4)
+ # slc_x = slice(2, N*128 + 4)
+ # slc_y = slice(2, N*128 + 4)
slc_x_2 = slice(1, N*128 + 6, 2)
slc_y_2 = slice(1, N*128 + 6, 2)
x_2 = np.arange(int((N*128)/2) + 3)
@@ -85,6 +87,8 @@ if KERNEL_SIZE == 3:
s = np.arange(0, int((N*128)/2) + 3, 0.5)
x_k = slice(1, N*128 + 3)
y_k = slice(1, N*128 + 3)
+ slc_x = slice(1, N*128 + 3)
+ slc_y = slice(1, N*128 + 3)
x_128 = slice(2, N*128 + 2)
y_128 = slice(2, N*128 + 2)
elif KERNEL_SIZE == 5:
@@ -168,7 +172,8 @@ def build_residual_block_conv2d_down2x(x_in, num_filters, activation, padding='S
def upsample(tmp):
- tmp = tmp[:, slc_y_2, slc_x_2]
+ # tmp = tmp[:, slc_y_2, slc_x_2]
+ tmp = tmp[:, 0:66, 0:66]
tmp = resample_2d_linear(x_2, y_2, tmp, t, s)
tmp = tmp[:, y_k, x_k]
return tmp
@@ -183,6 +188,8 @@ def upsample_nearest(grd):
up[:, 0::2, 1::2] = grd[:, 0::, 0::]
up[:, 1::2, 1::2] = grd[:, 0::, 0::]
+ up = up[:, y_k, x_k]
+
return up
@@ -379,24 +386,24 @@ class SRCNN:
def get_in_mem_data_batch(self, idxs, is_training):
if is_training:
- files = self.train_data_files
+ data_files = self.train_data_files
+ label_files = self.train_label_files
else:
- files = self.test_data_files
+ data_files = self.test_data_files
+ label_files = self.test_label_files
data_s = []
+ label_s = []
for k in idxs:
- f = files[k]
- try:
- nda = np.load(f)
- except Exception:
- print(f)
- continue
+ f = data_files[k]
+ nda = np.load(f)
data_s.append(nda)
- input_data = np.concatenate(data_s)
- DO_ADD_NOISE = False
- if is_training and NOISE_TRAINING:
- DO_ADD_NOISE = True
+ f = label_files[k]
+ nda = np.load(f)
+ label_s.append(nda)
+ input_data = np.concatenate(data_s)
+ input_label = np.concatenate(label_s)
data_norm = []
for param in data_params_half:
@@ -412,11 +419,10 @@ class SRCNN:
data_norm.append(tmp)
for param in data_params_full:
- idx = params.index(param)
- tmp = input_data[:, idx, :, :]
+ idx = params_i.index(param)
+ tmp = input_label[:, idx, :, :]
tmp = tmp.copy()
tmp = np.where(np.isnan(tmp), 0, tmp)
- # Full res:
tmp = tmp[:, slc_y, slc_x]
tmp = normalize(tmp, param, mean_std_dct)
data_norm.append(tmp)
@@ -427,9 +433,7 @@ class SRCNN:
if DO_ESPCN:
tmp = tmp[:, slc_y_2, slc_x_2]
else: # Half res upsampled to full res:
- # tmp = upsample(tmp)
- tmp = upsample_mean(tmp)
- tmp = tmp[:, slc_y, slc_x]
+ tmp = upsample_nearest(tmp)
if label_param != 'cloud_probability':
tmp = normalize(tmp, label_param, mean_std_dct)
data_norm.append(tmp)
@@ -438,7 +442,7 @@ class SRCNN:
data = data.astype(np.float32)
# -----------------------------------------------------
# -----------------------------------------------------
- label = input_data[:, label_idx, :, :]
+ label = input_label[:, label_idx_i, :, :]
label = label.copy()
label = label[:, y_128, x_128]
if NumClasses == 5:
@@ -504,10 +508,11 @@ class SRCNN:
dataset = dataset.cache()
self.test_dataset = dataset
- def setup_pipeline(self, train_data_files, test_data_files, num_train_samples):
-
+ def setup_pipeline(self, train_data_files, train_label_files, test_data_files, test_label_files, num_train_samples):
self.train_data_files = train_data_files
+ self.train_label_files = train_label_files
self.test_data_files = test_data_files
+ self.test_label_files = test_label_files
trn_idxs = np.arange(len(train_data_files))
np.random.shuffle(trn_idxs)
@@ -526,8 +531,10 @@ class SRCNN:
print('num test samples: ', tst_idxs.shape[0])
print('setup_pipeline: Done')
- def setup_test_pipeline(self, test_data_files):
+ def setup_test_pipeline(self, test_data_files, test_label_files):
self.test_data_files = test_data_files
+ self.test_label_files = test_label_files
+
tst_idxs = np.arange(len(test_data_files))
self.get_test_dataset(tst_idxs)
print('setup_test_pipeline: Done')
@@ -839,19 +846,24 @@ class SRCNN:
return pred
def run(self, directory, ckpt_dir=None, num_data_samples=50000):
- train_data_files = glob.glob(directory+'data_train_*.npy')
- valid_data_files = glob.glob(directory+'data_valid_*.npy')
+ train_data_files = glob.glob(directory+'train*mres*.npy')
+ valid_data_files = glob.glob(directory+'valid*mres*.npy')
+ train_label_files = glob.glob(directory+'train*ires*.npy')
+ valid_label_files = glob.glob(directory+'valid*ires*.npy')
- self.setup_pipeline(train_data_files, valid_data_files, num_data_samples)
+ self.setup_pipeline(train_data_files, train_label_files, valid_data_files, valid_label_files, num_data_samples)
self.build_model()
self.build_training()
self.build_evaluation()
self.do_training(ckpt_dir=ckpt_dir)
def run_restore(self, directory, ckpt_dir):
- valid_data_files = glob.glob(directory + 'data_valid*.npy')
self.num_data_samples = 1000
- self.setup_test_pipeline(valid_data_files)
+
+ valid_data_files = glob.glob(directory + 'valid*mres*.npy')
+ valid_label_files = glob.glob(directory + 'valid*ires*.npy')
+ self.setup_test_pipeline(valid_data_files, valid_label_files)
+
self.build_model()
self.build_training()
self.build_evaluation()