diff --git a/modules/icing/pirep_goes.py b/modules/icing/pirep_goes.py
index 2635dd2dd445811d67a6b486d1cf7acc71633c5f..cbb1923a7c89d37f3123bf084ccdad46f3781825 100644
--- a/modules/icing/pirep_goes.py
+++ b/modules/icing/pirep_goes.py
@@ -1024,8 +1024,8 @@ def fov_extract(outfile='/home/rink/fovs_l1b_out.h5', train_params=l1b_ds_list,
h5f_expl.close()
-def tile_extract(trnfile='/home/rink/tiles_l1b_train.h5', tstfile='/home/rink/tiles_l1b_test.h5',
- train_params=l1b_ds_list, ds_types=l1b_ds_types, cld_mask_name='cloud_mask', augment=False, split=0.2):
+def tile_extract(trnfile='/home/rink/tiles_l1b_train.h5', tstfile='/home/rink/tiles_l1b_test.h5', l1B_or_l2='L1B',
+ cld_mask_name='cloud_mask', augment=False, split=0.2):
icing_int_s = []
ice_time_s = []
no_ice_time_s = []
@@ -1037,8 +1037,15 @@ def tile_extract(trnfile='/home/rink/tiles_l1b_train.h5', tstfile='/home/rink/ti
h5_s_icing = []
h5_s_no_icing = []
- icing_data_dct = {ds: [] for ds in train_params}
- no_icing_data_dct = {ds: [] for ds in train_params}
+ if l1B_or_l2 == 'L1B':
+ params = l1b_ds_list
+ param_types = l1b_ds_types
+ elif l1B_or_l2 == 'L2':
+ params = ds_list
+ param_types = ds_types
+
+ icing_data_dct = {ds: [] for ds in params}
+ no_icing_data_dct = {ds: [] for ds in params}
for fidx in range(len(icing_files)):
fname = icing_files[fidx]
@@ -1054,11 +1061,11 @@ def tile_extract(trnfile='/home/rink/tiles_l1b_train.h5', tstfile='/home/rink/ti
for i in range(num_obs):
cld_msk = f[cld_mask_name][i, 12:28, 12:28]
- for ds_name in train_params:
+ for ds_name in params:
dat = f[ds_name][i, 12:28, 12:28]
- # np.where(cld_tmp < 273.0, dat, np.nan)
- #keep = np.logical_or(cld_msk == 2, cld_msk == 3) # cloudy
- #np.where(keep, dat, np.nan)
+ if l1B_or_l2 == 'L2':
+ keep = np.logical_or(cld_msk == 2, cld_msk == 3) # cloudy
+ np.where(keep, dat, np.nan)
icing_data_dct[ds_name].append(dat)
icing_int_s.append(icing_int[i])
@@ -1068,7 +1075,7 @@ def tile_extract(trnfile='/home/rink/tiles_l1b_train.h5', tstfile='/home/rink/ti
print(fname)
- for ds_name in train_params:
+ for ds_name in params:
lst = icing_data_dct[ds_name]
icing_data_dct[ds_name] = np.stack(lst, axis=0)
icing_int_s = np.array(icing_int_s)
@@ -1091,11 +1098,11 @@ def tile_extract(trnfile='/home/rink/tiles_l1b_train.h5', tstfile='/home/rink/ti
for i in range(num_obs):
cld_msk = f[cld_mask_name][i, 12:28, 12:28]
- for ds_name in train_params:
+ for ds_name in params:
dat = f[ds_name][i, 12:28, 12:28]
- # np.where(cld_tmp < 273.0, dat, np.nan)
- #keep = np.logical_or(cld_msk == 2, cld_msk == 3) # cloudy
- #np.where(keep, dat, np.nan)
+ if l1B_or_l2 == 'L2':
+ keep = np.logical_or(cld_msk == 2, cld_msk == 3) # cloudy
+ np.where(keep, dat, np.nan)
no_icing_data_dct[ds_name].append(dat)
num_no_ice += 1
no_ice_time_s.append(times[i])
@@ -1104,7 +1111,7 @@ def tile_extract(trnfile='/home/rink/tiles_l1b_train.h5', tstfile='/home/rink/ti
print(fname)
- for ds_name in train_params:
+ for ds_name in params:
lst = no_icing_data_dct[ds_name]
no_icing_data_dct[ds_name] = np.stack(lst, axis=0)
no_icing_int_s = np.full(num_no_ice, -1)
@@ -1118,12 +1125,12 @@ def tile_extract(trnfile='/home/rink/tiles_l1b_train.h5', tstfile='/home/rink/ti
icing_lats = np.concatenate([ice_lat_s, no_ice_lat_s])
data_dct = {}
- for ds_name in train_params:
+ for ds_name in params:
data_dct[ds_name] = np.concatenate([icing_data_dct[ds_name], no_icing_data_dct[ds_name]])
# do sort -------------------------------------
ds_indexes = np.argsort(icing_times)
- for ds_name in train_params:
+ for ds_name in params:
data_dct[ds_name] = data_dct[ds_name][ds_indexes]
icing_intensity = icing_intensity[ds_indexes]
icing_times = icing_times[ds_indexes]
@@ -1138,7 +1145,7 @@ def tile_extract(trnfile='/home/rink/tiles_l1b_train.h5', tstfile='/home/rink/ti
# ---------------------------------------------
trn_data_dct = {}
- for ds_name in train_params:
+ for ds_name in params:
trn_data_dct[ds_name] = data_dct[ds_name][trn_idxs,]
trn_icing_intensity = icing_intensity[trn_idxs,]
trn_icing_times = icing_times[trn_idxs,]
@@ -1147,7 +1154,7 @@ def tile_extract(trnfile='/home/rink/tiles_l1b_train.h5', tstfile='/home/rink/ti
# Data augmentation -------------------------------------------------------------
if augment:
- trn_data_dct_aug = {ds_name: [] for ds_name in train_params}
+ trn_data_dct_aug = {ds_name: [] for ds_name in params}
trn_icing_intensity_aug = []
trn_icing_times_aug = []
trn_icing_lons_aug = []
@@ -1159,7 +1166,7 @@ def tile_extract(trnfile='/home/rink/tiles_l1b_train.h5', tstfile='/home/rink/ti
icelon = trn_icing_lons[k]
icelat = trn_icing_lats[k]
if iceint == 3 or iceint == 4 or iceint == 5 or iceint == 6:
- for ds_name in train_params:
+ for ds_name in params:
dat = trn_data_dct[ds_name]
trn_data_dct_aug[ds_name].append(np.fliplr(dat[k,]))
trn_data_dct_aug[ds_name].append(np.flipud(dat[k,]))
@@ -1181,14 +1188,14 @@ def tile_extract(trnfile='/home/rink/tiles_l1b_train.h5', tstfile='/home/rink/ti
trn_icing_lats_aug.append(icelat)
trn_icing_lats_aug.append(icelat)
- for ds_name in train_params:
+ for ds_name in params:
trn_data_dct_aug[ds_name] = np.stack(trn_data_dct_aug[ds_name])
trn_icing_intensity_aug = np.stack(trn_icing_intensity_aug)
trn_icing_times_aug = np.stack(trn_icing_times_aug)
trn_icing_lons_aug = np.stack(trn_icing_lons_aug)
trn_icing_lats_aug = np.stack(trn_icing_lats_aug)
- for ds_name in train_params:
+ for ds_name in params:
trn_data_dct[ds_name] = np.concatenate([trn_data_dct[ds_name], trn_data_dct_aug[ds_name]])
trn_icing_intensity = np.concatenate([trn_icing_intensity, trn_icing_intensity_aug])
trn_icing_times = np.concatenate([trn_icing_times, trn_icing_times_aug])
@@ -1197,17 +1204,17 @@ def tile_extract(trnfile='/home/rink/tiles_l1b_train.h5', tstfile='/home/rink/ti
# do sort
ds_indexes = np.argsort(trn_icing_times)
- for ds_name in train_params:
+ for ds_name in params:
trn_data_dct[ds_name] = trn_data_dct[ds_name][ds_indexes]
trn_icing_intensity = trn_icing_intensity[ds_indexes]
trn_icing_times = trn_icing_times[ds_indexes]
trn_icing_lons = trn_icing_lons[ds_indexes]
trn_icing_lats = trn_icing_lats[ds_indexes]
- write_file(trnfile, train_params, trn_data_dct, trn_icing_intensity, trn_icing_times, trn_icing_lons, trn_icing_lats)
+ write_file(trnfile, params, param_types, trn_data_dct, trn_icing_intensity, trn_icing_times, trn_icing_lons, trn_icing_lats)
tst_data_dct = {}
- for ds_name in train_params:
+ for ds_name in params:
tst_data_dct[ds_name] = data_dct[ds_name][tst_idxs,]
tst_icing_intensity = icing_intensity[tst_idxs,]
tst_icing_times = icing_times[tst_idxs,]
@@ -1227,14 +1234,14 @@ def tile_extract(trnfile='/home/rink/tiles_l1b_train.h5', tstfile='/home/rink/ti
# do sort
ds_indexes = np.argsort(tst_icing_times)
- for ds_name in train_params:
+ for ds_name in params:
tst_data_dct[ds_name] = tst_data_dct[ds_name][ds_indexes]
tst_icing_intensity = tst_icing_intensity[ds_indexes]
tst_icing_times = tst_icing_times[ds_indexes]
tst_icing_lons = tst_icing_lons[ds_indexes]
tst_icing_lats = tst_icing_lats[ds_indexes]
- write_file(tstfile, train_params, tst_data_dct, tst_icing_intensity, tst_icing_times, tst_icing_lons, tst_icing_lats)
+ write_file(tstfile, params, param_types, tst_data_dct, tst_icing_intensity, tst_icing_times, tst_icing_lons, tst_icing_lats)
# --- close files
for h5f in h5_s_icing:
@@ -1244,12 +1251,12 @@ def tile_extract(trnfile='/home/rink/tiles_l1b_train.h5', tstfile='/home/rink/ti
h5f.close()
-def write_file(outfile, train_params, data_dct, icing_intensity, icing_times, icing_lons, icing_lats):
+def write_file(outfile, params, param_types, data_dct, icing_intensity, icing_times, icing_lons, icing_lats):
h5f_expl = h5py.File(a_clvr_file, 'r')
h5f_out = h5py.File(outfile, 'w')
- for idx, ds_name in enumerate(train_params):
- dt = ds_types[idx]
+ for idx, ds_name in enumerate(params):
+ dt = param_types[idx]
data = data_dct[ds_name]
h5f_out.create_dataset(ds_name, data=data, dtype=dt)
@@ -1269,7 +1276,7 @@ def write_file(outfile, train_params, data_dct, icing_intensity, icing_times, ic
lat_ds.attrs.create('long_name', data='PIREP latitude')
# copy relevant attributes
- for ds_name in train_params:
+ for ds_name in params:
h5f_ds = h5f_out[ds_name]
h5f_ds.attrs.create('standard_name', data=h5f_expl[ds_name].attrs.get('standard_name'))
h5f_ds.attrs.create('long_name', data=h5f_expl[ds_name].attrs.get('long_name'))