diff --git a/modules/icing/pirep_goes.py b/modules/icing/pirep_goes.py
index 5c5f7bda2649e401c267d9f7152f87b3cfb95b0d..0dbb1cd378605993fe1711d0ce65264ea9e7634e 100644
--- a/modules/icing/pirep_goes.py
+++ b/modules/icing/pirep_goes.py
@@ -364,10 +364,15 @@ def process(ice_dct, no_ice_dct, neg_ice_dct):
new_neg_ice_dct = {}
ice_keys_5_6 = []
+ ice_tidx_5_6 = []
ice_keys_1 = []
+ ice_tidx_1 = []
ice_keys_4 = []
+ ice_tidx_4 = []
ice_keys_3 = []
+ ice_tidx_3 = []
ice_keys_2 = []
+ ice_tidx_2 = []
print('num keys ice, no_ice, neg_ice: ', len(ice_dct), len(no_ice_dct), len(neg_ice_dct))
no_intensity_cnt = 0
@@ -375,26 +380,33 @@ def process(ice_dct, no_ice_dct, neg_ice_dct):
for ts in list(ice_dct.keys()):
rpts = ice_dct[ts]
- for tup in rpts:
+ for idx, tup in enumerate(rpts):
num_ice_reports += 1
if tup[3] == 5 or tup[3] == 6:
ice_keys_5_6.append(ts)
+ ice_tidx_5_6.append(idx)
elif tup[3] == 1:
ice_keys_1.append(ts)
+ ice_tidx_1.append(idx)
elif tup[3] == 4:
ice_keys_4.append(ts)
+ ice_tidx_4.append(idx)
elif tup[3] == 3:
ice_keys_3.append(ts)
+ ice_tidx_3.append(idx)
elif tup[3] == 2:
ice_keys_2.append(ts)
+ ice_tidx_2.append(idx)
else:
no_intensity_cnt += 1
no_ice_keys = []
+ no_ice_tidx = []
for ts in list(no_ice_dct.keys()):
rpts = no_ice_dct[ts]
- for tup in rpts:
+ for idx, tup in enumerate(rpts):
no_ice_keys.append(ts)
+ no_ice_tidx.append(idx)
neg_ice_keys = []
for ts in list(neg_ice_dct.keys()):
@@ -406,46 +418,68 @@ def process(ice_dct, no_ice_dct, neg_ice_dct):
print('------------------------------------------------')
ice_keys_5_6 = np.array(ice_keys_5_6)
- print('5_6: ', ice_keys_5_6.shape)
+ ice_tidx_5_6 = np.array(ice_tidx_5_6)
+ print('5_6: ', ice_keys_5_6.shape[0])
ice_keys_4 = np.array(ice_keys_4)
- print('4: ', ice_keys_4.shape)
+ ice_tidx_4 = np.array(ice_tidx_4)
+ print('4: ', ice_keys_4.shape[0])
ice_keys_3 = np.array(ice_keys_3)
- print('3: ', ice_keys_3.shape)
+ ice_tidx_3 = np.array(ice_tidx_3)
+ print('3: ', ice_keys_3.shape[0])
ice_keys_2 = np.array(ice_keys_2)
- print('2: ', ice_keys_2.shape)
+ ice_tidx_2 = np.array(ice_tidx_2)
+ print('2: ', ice_keys_2.shape[0])
np.random.seed(42)
- np.random.shuffle(ice_keys_2)
- ice_keys_2 = ice_keys_2[0:70000]
+ ridxs = np.random.permutation(np.arange(ice_keys_2.shape[0]))
+ ice_keys_2 = ice_keys_2[ridxs]
+ ice_tidx_2 = ice_tidx_2[ridxs]
+ ice_keys_2 = ice_keys_2[::2]
+ ice_tidx_2 = ice_tidx_2[::2]
ice_keys_1 = np.array(ice_keys_1)
- print('1: ', ice_keys_1.shape)
+ ice_tidx_1 = np.array(ice_tidx_1)
+ print('1: ', ice_keys_1.shape[0])
print('no intensity: ', no_intensity_cnt)
ice_keys = np.concatenate([ice_keys_5_6, ice_keys_1, ice_keys_2, ice_keys_3, ice_keys_4])
- uniq_sorted_keys = np.unique(ice_keys)
- print('ice: ', ice_keys.shape, uniq_sorted_keys.shape)
+ ice_tidx = np.concatenate([ice_tidx_5_6, ice_tidx_1, ice_tidx_2, ice_tidx_3, ice_tidx_4])
- uniq_sorted_keys = uniq_sorted_keys.tolist()
- for key in uniq_sorted_keys:
- new_ice_dct[key] = ice_dct[key]
+ for idx, key in enumerate(ice_keys):
+ rpts = ice_dct[key]
+ tup = rpts[ice_tidx[idx]]
+ n_rpts = new_ice_dct.get(key)
+ if n_rpts is None:
+ n_rpts = []
+ n_rpts.append(tup)
+
+ # -----------------------------------------------------
no_ice_keys = np.array(no_ice_keys)
- print('no ice total: ', no_ice_keys.shape)
+ no_ice_tidx = np.array(no_ice_tidx)
+ print('no ice total: ', no_ice_keys.shape[0])
np.random.seed(42)
- np.random.shuffle(no_ice_keys)
- no_ice_keys = no_ice_keys[0:150000]
- uniq_sorted_no_ice = np.unique(no_ice_keys)
- print('no ice: ', no_ice_keys.shape, uniq_sorted_no_ice.shape)
-
- uniq_sorted_no_ice = uniq_sorted_no_ice.tolist()
- for key in uniq_sorted_no_ice:
- new_no_ice_dct[key] = no_ice_dct[key]
+ ridxs = np.random.permutation(np.arange(no_ice_keys.shape[0]))
+ no_ice_keys = no_ice_keys[ridxs]
+ no_ice_tidx = no_ice_tidx[ridxs]
+ no_ice_keys = no_ice_keys[::20]
+ no_ice_tidx = no_ice_tidx[::20]
+ print('no ice : ', no_ice_keys.shape[0])
+
+ for idx, key in enumerate(no_ice_keys):
+ rpts = no_ice_dct[key]
+ tup = rpts[no_ice_tidx[idx]]
+
+ n_rpts = new_no_ice_dct.get(key)
+ if n_rpts is None:
+ n_rpts = []
+ n_rpts.append(tup)
+ # -------------------------------------------------
neg_ice_keys = np.array(neg_ice_keys)
- print('neg ice total: ', neg_ice_keys.shape)
+ print('neg ice total: ', neg_ice_keys.shape[0])
np.random.seed(42)
np.random.shuffle(neg_ice_keys)
neg_ice_keys = neg_ice_keys[0:10000]