From 025afca8ec739f08b07d1081b691cebeb279c534 Mon Sep 17 00:00:00 2001
From: tomrink <rink@ssec.wisc.edu>
Date: Wed, 7 Apr 2021 22:06:56 -0500
Subject: [PATCH] snapshot...
---
modules/icing/pirep_goes.py | 57 ++++++++++++++++++++++---------------
1 file changed, 34 insertions(+), 23 deletions(-)
diff --git a/modules/icing/pirep_goes.py b/modules/icing/pirep_goes.py
index 4c63fe76..360c06c3 100644
--- a/modules/icing/pirep_goes.py
+++ b/modules/icing/pirep_goes.py
@@ -797,7 +797,7 @@ def apply_qc_icing_pireps(icing_alt, cld_top_hgt, cld_phase, cld_opd, cld_mask,
return mask, idxs
-def fov_extract(outfile='/Users/tomrink/fovs_out.h5'):
+def fov_extract(outfile='/Users/tomrink/fovs_out.h5', train_params=train_params_day):
ice_times = []
icing_int_s = []
ice_lons = []
@@ -810,8 +810,8 @@ def fov_extract(outfile='/Users/tomrink/fovs_out.h5'):
h5_s_icing = []
h5_s_no_icing = []
- icing_data_dct = {ds: [] for ds in train_params_day}
- no_icing_data_dct = {ds: [] for ds in train_params_day}
+ icing_data_dct = {ds: [] for ds in train_params}
+ no_icing_data_dct = {ds: [] for ds in train_params}
sub_indexes = np.arange(400)
@@ -847,7 +847,7 @@ def fov_extract(outfile='/Users/tomrink/fovs_out.h5'):
k_idxs = k_idxs[0:len(k_idxs)]
num_ice += len(k_idxs)
- for ds_name in train_params_day:
+ for ds_name in train_params:
dat = f[ds_name][i, 10:30, 10:30].flatten()
icing_data_dct[ds_name].append(dat[k_idxs])
@@ -858,7 +858,7 @@ def fov_extract(outfile='/Users/tomrink/fovs_out.h5'):
print(fname)
- for ds_name in train_params_day:
+ for ds_name in train_params:
lst = icing_data_dct[ds_name]
icing_data_dct[ds_name] = np.concatenate(lst)
@@ -900,13 +900,13 @@ def fov_extract(outfile='/Users/tomrink/fovs_out.h5'):
no_ice_lons.append(np.full(len(k_idxs), lons[i]))
no_ice_lats.append(np.full(len(k_idxs), lats[i]))
- for ds_name in train_params_day:
+ for ds_name in train_params:
dat = f[ds_name][i, 10:30, 10:30].flatten()
no_icing_data_dct[ds_name].append(dat[k_idxs])
print(fname)
- for ds_name in train_params_day:
+ for ds_name in train_params:
lst = no_icing_data_dct[ds_name]
no_icing_data_dct[ds_name] = np.concatenate(lst)
no_icing_int_s = np.full(num_no_ice, -1)
@@ -920,14 +920,14 @@ def fov_extract(outfile='/Users/tomrink/fovs_out.h5'):
icing_lats = np.concatenate([ice_lats, no_ice_lats])
data_dct = {}
- for ds_name in train_params_day:
+ for ds_name in train_params:
data_dct[ds_name] = np.concatenate([icing_data_dct[ds_name], no_icing_data_dct[ds_name]])
# apply shuffle indexes
ds_indexes = np.arange(num_ice + num_no_ice)
np.random.shuffle(ds_indexes)
- for ds_name in train_params_day:
+ for ds_name in train_params:
data_dct[ds_name] = data_dct[ds_name][ds_indexes]
icing_intensity = icing_intensity[ds_indexes]
icing_times = icing_times[ds_indexes]
@@ -937,7 +937,7 @@ def fov_extract(outfile='/Users/tomrink/fovs_out.h5'):
h5f_expl = h5py.File(a_clvr_file, 'r')
h5f_out = h5py.File(outfile, 'w')
- for idx, ds_name in enumerate(train_params_day):
+ for idx, ds_name in enumerate(train_params):
dt = ds_types[ds_list.index(ds_name)]
data = data_dct[ds_name]
h5f_out.create_dataset(ds_name, data=data, dtype=dt)
@@ -958,11 +958,17 @@ def fov_extract(outfile='/Users/tomrink/fovs_out.h5'):
lat_ds.attrs.create('long_name', data='PIREP latitude')
# copy relevant attributes
- for ds_name in train_params_day:
+ for ds_name in train_params:
h5f_ds = h5f_out[ds_name]
h5f_ds.attrs.create('standard_name', data=h5f_expl[ds_name].attrs.get('standard_name'))
h5f_ds.attrs.create('long_name', data=h5f_expl[ds_name].attrs.get('long_name'))
h5f_ds.attrs.create('units', data=h5f_expl[ds_name].attrs.get('units'))
+ attr = h5f_expl[ds_name].attrs.get('actual_range')
+ if attr is not None:
+ h5f_ds.attrs.create('actual_range', data=attr)
+ attr = h5f_expl[ds_name].attrs.get('flag_values')
+ if attr is not None:
+ h5f_ds.attrs.create('flag_values', data=attr)
# --- close files
for h5f in h5_s_icing:
@@ -975,7 +981,7 @@ def fov_extract(outfile='/Users/tomrink/fovs_out.h5'):
h5f_expl.close()
-def box_extract(outfile='/Users/tomrink/box_out.h5'):
+def box_extract(outfile='/Users/tomrink/box_out.h5', train_params=train_params_day):
icing_int_s = []
ice_time_s = []
no_ice_time_s = []
@@ -984,12 +990,11 @@ def box_extract(outfile='/Users/tomrink/box_out.h5'):
ice_lat_s = []
no_ice_lat_s = []
-
h5_s_icing = []
h5_s_no_icing = []
- icing_data_dct = {ds: [] for ds in train_params_day}
- no_icing_data_dct = {ds: [] for ds in train_params_day}
+ icing_data_dct = {ds: [] for ds in train_params}
+ no_icing_data_dct = {ds: [] for ds in train_params}
for fidx in range(len(icing_files)):
fname = icing_files[fidx]
@@ -1004,7 +1009,7 @@ def box_extract(outfile='/Users/tomrink/box_out.h5'):
icing_int = f['icing_intensity'][:]
for i in range(num_obs):
- for ds_name in train_params_day:
+ for ds_name in train_params:
dat = f[ds_name][i, 12:28, 12:28]
icing_data_dct[ds_name].append(dat)
icing_int_s.append(icing_int[i])
@@ -1014,7 +1019,7 @@ def box_extract(outfile='/Users/tomrink/box_out.h5'):
print(fname)
- for ds_name in train_params_day:
+ for ds_name in train_params:
lst = icing_data_dct[ds_name]
icing_data_dct[ds_name] = np.stack(lst, axis=0)
icing_int_s = np.array(icing_int_s)
@@ -1035,7 +1040,7 @@ def box_extract(outfile='/Users/tomrink/box_out.h5'):
lats = f['latitude']
for i in range(num_obs):
- for ds_name in train_params_day:
+ for ds_name in train_params:
dat = f[ds_name][i, 12:28, 12:28]
no_icing_data_dct[ds_name].append(dat)
num_no_ice += 1
@@ -1045,7 +1050,7 @@ def box_extract(outfile='/Users/tomrink/box_out.h5'):
print(fname)
- for ds_name in train_params_day:
+ for ds_name in train_params:
lst = no_icing_data_dct[ds_name]
no_icing_data_dct[ds_name] = np.stack(lst, axis=0)
no_icing_int_s = np.full(num_no_ice, -1)
@@ -1059,14 +1064,14 @@ def box_extract(outfile='/Users/tomrink/box_out.h5'):
icing_lats = np.concatenate([ice_lat_s, no_ice_lat_s])
data_dct = {}
- for ds_name in train_params_day:
+ for ds_name in train_params:
data_dct[ds_name] = np.concatenate([icing_data_dct[ds_name], no_icing_data_dct[ds_name]])
# Do shuffle
ds_indexes = np.arange(num_ice + num_no_ice)
np.random.shuffle(ds_indexes)
- for ds_name in train_params_day:
+ for ds_name in train_params:
data_dct[ds_name] = data_dct[ds_name][ds_indexes]
icing_intensity = icing_intensity[ds_indexes]
icing_times = icing_times[ds_indexes]
@@ -1076,7 +1081,7 @@ def box_extract(outfile='/Users/tomrink/box_out.h5'):
h5f_expl = h5py.File(a_clvr_file, 'r')
h5f_out = h5py.File(outfile, 'w')
- for idx, ds_name in enumerate(train_params_day):
+ for idx, ds_name in enumerate(train_params):
dt = ds_types[ds_list.index(ds_name)]
data = data_dct[ds_name]
h5f_out.create_dataset(ds_name, data=data, dtype=dt)
@@ -1097,11 +1102,17 @@ def box_extract(outfile='/Users/tomrink/box_out.h5'):
lat_ds.attrs.create('long_name', data='PIREP latitude')
# copy relevant attributes
- for ds_name in train_params_day:
+ for ds_name in train_params:
h5f_ds = h5f_out[ds_name]
h5f_ds.attrs.create('standard_name', data=h5f_expl[ds_name].attrs.get('standard_name'))
h5f_ds.attrs.create('long_name', data=h5f_expl[ds_name].attrs.get('long_name'))
h5f_ds.attrs.create('units', data=h5f_expl[ds_name].attrs.get('units'))
+ attr = h5f_expl[ds_name].attrs.get('actual_range')
+ if attr is not None:
+ h5f_ds.attrs.create('actual_range', data=attr)
+ attr = h5f_expl[ds_name].attrs.get('flag_values')
+ if attr is not None:
+ h5f_ds.attrs.create('flag_values', data=attr)
# --- close files
for h5f in h5_s_icing:
--
GitLab