minor...

modules/icing/pirep_goes.py

@@ -364,10 +364,15 @@ def process(ice_dct, no_ice_dct, neg_ice_dct):
     new_neg_ice_dct = {}
 
     ice_keys_5_6 = []
+    ice_tidx_5_6 = []
     ice_keys_1 = []
+    ice_tidx_1 = []
     ice_keys_4 = []
+    ice_tidx_4 = []
     ice_keys_3 = []
+    ice_tidx_3 = []
     ice_keys_2 = []
+    ice_tidx_2 = []
 
     print('num keys ice, no_ice, neg_ice: ', len(ice_dct), len(no_ice_dct), len(neg_ice_dct))
     no_intensity_cnt = 0
@@ -375,26 +380,33 @@ def process(ice_dct, no_ice_dct, neg_ice_dct):
 
     for ts in list(ice_dct.keys()):
         rpts = ice_dct[ts]
-        for tup in rpts:
+        for idx, tup in enumerate(rpts):
             num_ice_reports += 1
             if tup[3] == 5 or tup[3] == 6:
                 ice_keys_5_6.append(ts)
+                ice_tidx_5_6.append(idx)
             elif tup[3] == 1:
                 ice_keys_1.append(ts)
+                ice_tidx_1.append(idx)
             elif tup[3] == 4:
                 ice_keys_4.append(ts)
+                ice_tidx_4.append(idx)
             elif tup[3] == 3:
                 ice_keys_3.append(ts)
+                ice_tidx_3.append(idx)
             elif tup[3] == 2:
                 ice_keys_2.append(ts)
+                ice_tidx_2.append(idx)
             else:
                 no_intensity_cnt += 1
 
     no_ice_keys = []
+    no_ice_tidx = []
     for ts in list(no_ice_dct.keys()):
         rpts = no_ice_dct[ts]
-        for tup in rpts:
+        for idx, tup in enumerate(rpts):
             no_ice_keys.append(ts)
+            no_ice_tidx.append(idx)
 
     neg_ice_keys = []
     for ts in list(neg_ice_dct.keys()):
@@ -406,46 +418,68 @@ def process(ice_dct, no_ice_dct, neg_ice_dct):
     print('------------------------------------------------')
 
     ice_keys_5_6 = np.array(ice_keys_5_6)
-    print('5_6: ', ice_keys_5_6.shape)
+    ice_tidx_5_6 = np.array(ice_tidx_5_6)
+    print('5_6: ', ice_keys_5_6.shape[0])
 
     ice_keys_4 = np.array(ice_keys_4)
-    print('4: ', ice_keys_4.shape)
+    ice_tidx_4 = np.array(ice_tidx_4)
+    print('4: ', ice_keys_4.shape[0])
 
     ice_keys_3 = np.array(ice_keys_3)
-    print('3: ', ice_keys_3.shape)
+    ice_tidx_3 = np.array(ice_tidx_3)
+    print('3: ', ice_keys_3.shape[0])
 
     ice_keys_2 = np.array(ice_keys_2)
-    print('2: ', ice_keys_2.shape)
+    ice_tidx_2 = np.array(ice_tidx_2)
+    print('2: ', ice_keys_2.shape[0])
     np.random.seed(42)
-    np.random.shuffle(ice_keys_2)
-    ice_keys_2 = ice_keys_2[0:70000]
+    ridxs = np.random.permutation(np.arange(ice_keys_2.shape[0]))
+    ice_keys_2 = ice_keys_2[ridxs]
+    ice_tidx_2 = ice_tidx_2[ridxs]
+    ice_keys_2 = ice_keys_2[::2]
+    ice_tidx_2 = ice_tidx_2[::2]
 
     ice_keys_1 = np.array(ice_keys_1)
-    print('1: ', ice_keys_1.shape)
+    ice_tidx_1 = np.array(ice_tidx_1)
+    print('1: ', ice_keys_1.shape[0])
     print('no intensity: ', no_intensity_cnt)
 
     ice_keys = np.concatenate([ice_keys_5_6, ice_keys_1, ice_keys_2, ice_keys_3, ice_keys_4])
-    uniq_sorted_keys = np.unique(ice_keys)
-    print('ice: ', ice_keys.shape, uniq_sorted_keys.shape)
+    ice_tidx = np.concatenate([ice_tidx_5_6, ice_tidx_1, ice_tidx_2, ice_tidx_3, ice_tidx_4])
 
-    uniq_sorted_keys = uniq_sorted_keys.tolist()
-    for key in uniq_sorted_keys:
-        new_ice_dct[key] = ice_dct[key]
+    for idx, key in enumerate(ice_keys):
+        rpts = ice_dct[key]
+        tup = rpts[ice_tidx[idx]]
 
+        n_rpts = new_ice_dct.get(key)
+        if n_rpts is None:
+            n_rpts = []
+        n_rpts.append(tup)
+
+    # -----------------------------------------------------
     no_ice_keys = np.array(no_ice_keys)
-    print('no ice total: ', no_ice_keys.shape)
+    no_ice_tidx = np.array(no_ice_tidx)
+    print('no ice total: ', no_ice_keys.shape[0])
     np.random.seed(42)
-    np.random.shuffle(no_ice_keys)
-    no_ice_keys = no_ice_keys[0:150000]
-    uniq_sorted_no_ice = np.unique(no_ice_keys)
-    print('no ice: ', no_ice_keys.shape, uniq_sorted_no_ice.shape)
-
-    uniq_sorted_no_ice = uniq_sorted_no_ice.tolist()
-    for key in uniq_sorted_no_ice:
-        new_no_ice_dct[key] = no_ice_dct[key]
+    ridxs = np.random.permutation(np.arange(no_ice_keys.shape[0]))
+    no_ice_keys = no_ice_keys[ridxs]
+    no_ice_tidx = no_ice_tidx[ridxs]
+    no_ice_keys = no_ice_keys[::20]
+    no_ice_tidx = no_ice_tidx[::20]
+    print('no ice   : ', no_ice_keys.shape[0])
+
+    for idx, key in enumerate(no_ice_keys):
+        rpts = no_ice_dct[key]
+        tup = rpts[no_ice_tidx[idx]]
+
+        n_rpts = new_no_ice_dct.get(key)
+        if n_rpts is None:
+            n_rpts = []
+        n_rpts.append(tup)
+    # -------------------------------------------------
 
     neg_ice_keys = np.array(neg_ice_keys)
-    print('neg ice total: ', neg_ice_keys.shape)
+    print('neg ice total: ', neg_ice_keys.shape[0])
     np.random.seed(42)
     np.random.shuffle(neg_ice_keys)
     neg_ice_keys = neg_ice_keys[0:10000]