From 4c2e88cc07d3f7bb600b948ce713a713868c3081 Mon Sep 17 00:00:00 2001
From: tomrink <rink@ssec.wisc.edu>
Date: Fri, 2 Apr 2021 15:35:53 -0500
Subject: [PATCH] snapshot...
---
modules/icing/pirep_goes.py | 95 ++++++++++++++++++++++++++++++++++---
1 file changed, 88 insertions(+), 7 deletions(-)
diff --git a/modules/icing/pirep_goes.py b/modules/icing/pirep_goes.py
index 98a488cb..c64195b0 100644
--- a/modules/icing/pirep_goes.py
+++ b/modules/icing/pirep_goes.py
@@ -44,7 +44,7 @@ icing_files = ['icing_2018010600_2018033022_QC_DAY.h5', 'icing_2018040100_201806
'icing_2018100115_2018123022_QC_DAY.h5', 'icing_2019010100_2019033023_QC_DAY.h5']
icing_l1b_files = []
-no_icing_files = ['no_icing_2018010600_2018033022_DAY.h5', 'no_icing_2018040100_2018063021_DAY.h5', 'no_icing_2018070101_2018093022_DAY.h5',
+no_icing_files = ['no_icing_2018010600_2018033022_DAY.h5', 'no_icing_2018040100_2018063022_DAY.h5', 'no_icing_2018070100_2018093022_DAY.h5',
'no_icing_2018100115_2018123022_DAY.h5', 'no_icing_2019010100_2019033023_DAY.h5']
no_icing_l1b_files = []
@@ -798,13 +798,22 @@ def apply_qc_icing_pireps(icing_alt, cld_top_hgt, cld_phase, cld_opd, cld_mask,
def fov_extract():
-
ice_times = []
- ice_int_s = []
+ icing_int_s = []
+
+ h5_s_icing = []
+ h5_s_no_icing = []
+
+ icing_data_dct = {ds: [] for ds in train_params_day}
+ no_icing_data_dct = {ds: [] for ds in train_params_day}
+ sub_indexes = np.arange(400)
+
+ num_ice = 0
for fidx in range(len(icing_files)):
fname = icing_files[fidx]
f = h5py.File(data_dir+fname, 'r')
+ h5_s_icing.append(f)
times = f['time'][:]
num_obs = len(times)
@@ -817,18 +826,37 @@ def fov_extract():
cld_top_temp = f['cld_temp_acha'][:, 10:30, 10:30]
cld_top_temp = cld_top_temp.reshape((num_obs, -1))
- ice_mask = []
for i in range(num_obs):
keep_0 = np.logical_or(cld_mask[i,] == 2, cld_mask[i,] == 3) # cloudy
keep_1 = np.invert(np.isnan(cld_top_temp[i,]))
keep = keep_0 & keep_1
keep = np.where(keep, cld_top_temp[i,] < 273.0, False)
+ k_idxs = sub_indexes[keep]
+ np.random.shuffle(k_idxs)
+ if len(k_idxs) > 10:
+ k_idxs = k_idxs[0:10]
+ else:
+ k_idxs = k_idxs[0:len(k_idxs)]
+ num_ice += len(k_idxs)
+
+ for ds_name in train_params_day:
+ dat = f[ds_name][i, 10:30, 10:30].flatten()
+ icing_data_dct[ds_name].append(dat[k_idxs])
+
+ icing_int_s.append(icing_int)
+ print(fname)
- ice_mask.append(keep)
+ for ds_name in train_params_day:
+ lst = icing_data_dct[ds_name]
+ icing_data_dct[ds_name] = np.concatenate(lst)
+ icing_int_s = np.concatenate(icing_int_s)
+
+ num_no_ice = 0
for fidx in range(len(no_icing_files)):
fname = no_icing_files[fidx]
f = h5py.File(data_dir+fname, 'r')
+ h5_s_no_icing.append(f)
times = f['time']
num_obs = len(times)
@@ -839,11 +867,64 @@ def fov_extract():
cld_top_temp = f['cld_temp_acha'][:, 10:30, 10:30]
cld_top_temp = cld_top_temp.reshape((num_obs, -1))
- no_ice_mask = []
for i in range(num_obs):
keep_0 = np.logical_or(cld_mask[i,] == 2, cld_mask[i,] == 3) # cloudy
# keep_1 = np.invert(np.isnan(cld_top_temp[i,]))
# keep = keep_0 & keep_1
# keep = np.where(keep, cld_top_temp[i,] > 273.0, False)
+ k_idxs = sub_indexes[keep_0]
+ np.random.shuffle(k_idxs)
+ if len(k_idxs) > 5:
+ k_idxs = k_idxs[0:5]
+ else:
+ k_idxs = k_idxs[0:len(k_idxs)]
+ num_no_ice += len(k_idxs)
+
+ for ds_name in train_params_day:
+ dat = f[ds_name][i, 10:30, 10:30].flatten()
+ no_icing_data_dct[ds_name].append(dat[k_idxs])
+
+ print(fname)
+
+ for ds_name in train_params_day:
+ lst = no_icing_data_dct[ds_name]
+ no_icing_data_dct[ds_name] = np.concatenate(lst)
+ no_icing_int_s = np.full(num_no_ice, -1)
+
+
+ ds_indexes = np.arange(num_ice + num_no_ice)
+ ds_indexes = np.random.shuffle(ds_indexes)
+
+ icing_intensity = np.concatenate([icing_int_s, no_icing_int_s])
+
+ data_dct = {}
+ for ds_name in train_params_day:
+ data_dct[ds_name] = np.concatenate([icing_data_dct[ds_name], no_icing_data_dct[ds_name]])
+
+ for ds_name in train_params_day:
+ data_dct[ds_name] = data_dct[ds_name][ds_indexes]
+ icing_intensity = icing_intensity[ds_indexes]
+
+
+ h5f_expl = h5py.File(a_clvr_file, 'r')
+ h5f_out = h5py.File('/Users/tomrink/fovs_out.h5', 'w')
+
+ for idx, ds_name in enumerate(train_params_day):
+ dt = ds_types[ds_list.index(ds_name)]
+ data = data_dct[ds_name]
+ h5f_out.create_dataset(ds_name, data=data, dtype=dt)
+
+ icing_int_ds = h5f_out.create_dataset('icing_intensity', data=icing_intensity, dtype='i4')
+ icing_int_ds.attrs.create('long_name', data='From PIREP. -1:No Icing, 1:Trace, 2:Light, 3:Light Moderate, 4:Moderate, 5:Moderate Severe, 6:Severe')
+
+
+
+ # --- close files
+ for h5f in h5_s_icing:
+ h5f.close()
+
+ for h5f in h5_s_no_icing:
+ h5f.close()
- no_ice_mask.append(keep_0)
\ No newline at end of file
+ h5f_out.close()
+ h5f_out.close()
\ No newline at end of file
--
GitLab