snapshot...

modules/icing/pirep_goes.py

@@ -344,6 +344,93 @@ def analyze(ice_dct, no_ice_dct):
         print(dt_str[2:])
 
 
+def process_2(ice_dct, no_ice_dct, neg_ice_dct):
+    new_ice_dct = {}
+    new_no_ice_dct = {}
+    new_neg_ice_dct = {}
+
+    ice_keys_5_6 = []
+    ice_keys_1 = []
+    ice_keys_4 = []
+    ice_keys_3 = []
+    ice_keys_2 = []
+
+    for ts in list(ice_dct.keys()):
+        rpts = ice_dct[ts]
+        for tup in rpts:
+            if tup[3] == 5 or tup[3] == 6:
+                ice_keys_5_6.append(ts)
+            elif tup[3] == 1:
+                ice_keys_1.append(ts)
+            elif tup[3] == 4:
+                ice_keys_4.append(ts)
+            elif tup[3] == 3:
+                ice_keys_3.append(ts)
+            elif tup[3] == 2:
+                ice_keys_2.append(ts)
+
+    no_ice_keys = []
+    for ts in list(no_ice_dct.keys()):
+        rpts = no_ice_dct[ts]
+        for tup in rpts:
+            no_ice_keys.append(ts)
+
+    neg_ice_keys = []
+    for ts in list(neg_ice_dct.keys()):
+            rpts = neg_ice_dct[ts]
+            for tup in rpts:
+                neg_ice_keys.append(ts)
+
+    ice_keys_5_6 = np.array(ice_keys_5_6)
+    print('5_6: ', ice_keys_5_6.shape)
+
+    ice_keys_4 = np.array(ice_keys_4)
+    print('4: ', ice_keys_4.shape)
+
+    ice_keys_3 = np.array(ice_keys_3)
+    print('3: ', ice_keys_3.shape)
+
+    ice_keys_2 = np.array(ice_keys_2)
+    print('2: ', ice_keys_2.shape)
+    np.random.seed(42)
+    np.random.shuffle(ice_keys_2)
+    ice_keys_2 = ice_keys_2[0:30000]
+
+    ice_keys_1 = np.array(ice_keys_1)
+    print('1: ', ice_keys_1.shape)
+
+    ice_keys = np.concatenate([ice_keys_5_6, ice_keys_1, ice_keys_2, ice_keys_3, ice_keys_4])
+    uniq_sorted_keys = np.unique(ice_keys)
+    print(ice_keys.shape, uniq_sorted_keys.shape)
+
+    uniq_sorted_keys = uniq_sorted_keys.tolist()
+    for key in uniq_sorted_keys:
+        new_ice_dct[key] = ice_dct[key]
+
+    no_ice_keys = np.array(no_ice_keys)
+    np.random.seed(42)
+    np.random.shuffle(no_ice_keys)
+    no_ice_keys = no_ice_keys[0:50000]
+    uniq_sorted_no_ice = np.unique(no_ice_keys)
+    print(no_ice_keys.shape, uniq_sorted_no_ice.shape)
+
+    uniq_sorted_no_ice = uniq_sorted_no_ice.tolist()
+    for key in uniq_sorted_no_ice:
+        new_no_ice_dct[key] = no_ice_dct[key]
+
+    neg_ice_keys = np.array(neg_ice_keys)
+    np.random.seed(42)
+    np.random.shuffle(neg_ice_keys)
+    neg_ice_keys = neg_ice_keys[0:5000]
+    uniq_sorted_neg_ice = np.unique(neg_ice_keys)
+    print(neg_ice_keys.shape, uniq_sorted_neg_ice.shape)
+
+    for key in uniq_sorted_neg_ice:
+        new_neg_ice_dct[key] = neg_ice_dct[key]
+
+    return new_ice_dct, new_no_ice_dct, new_neg_ice_dct
+
+
 def process_1(ice_dct, no_ice_dct, neg_ice_dct):
     new_ice_dct = {}
     new_no_ice_dct = {}
@@ -365,10 +452,14 @@ def process_1(ice_dct, no_ice_dct, neg_ice_dct):
     ice_files_2 = []
     ice_times_2 = []
     ice_keys_2 = []
+    print('num keys ice, no_ice, neg_ice: ', len(ice_dct), len(no_ice_dct), len(neg_ice_dct))
+    nfound = 0
     for ts in list(ice_dct.keys()):
         try:
             ds = get_goes_datasource(ts)
             goes_file, t_0, _ = ds.get_file(ts)
+            if goes_file is not None:
+                nfound += 1
             if goes_file is not None and goes_file != last_file:
                 rpts = ice_dct[ts]
                 for tup in rpts:
@@ -395,15 +486,19 @@ def process_1(ice_dct, no_ice_dct, neg_ice_dct):
                 last_file = goes_file
         except Exception:
             continue
+    print('num ice files: ', nfound)
 
     last_file = None
     no_ice_files = []
     no_ice_times = []
     no_ice_keys = []
+    nfound = 0
     for ts in list(no_ice_dct.keys()):
         try:
             ds = get_goes_datasource(ts)
             goes_file, t_0, _ = ds.get_file(ts)
+            if goes_file is not None:
+                nfound += 1
             if goes_file is not None and goes_file != last_file:
                 rpts = no_ice_dct[ts]
                 for tup in rpts:
@@ -414,14 +509,19 @@ def process_1(ice_dct, no_ice_dct, neg_ice_dct):
         except Exception:
             continue
 
+    print('num no ice files found: ', nfound)
+
     last_file = None
     neg_ice_files = []
     neg_ice_times = []
     neg_ice_keys = []
+    nfound = 0
     for ts in list(neg_ice_dct.keys()):
         try:
             ds = get_goes_datasource(ts)
             goes_file, t_0, _ = ds.get_file(ts)
+            if goes_file is not None:
+                nfound += 1
             if goes_file is not None and goes_file != last_file:
                 rpts = neg_ice_dct[ts]
                 for tup in rpts:
@@ -432,6 +532,8 @@ def process_1(ice_dct, no_ice_dct, neg_ice_dct):
         except Exception:
             continue
 
+    print('num neg ice files found: ', nfound)
+
     ice_times_5_6 = np.array(ice_times_5_6)
     ice_keys_5_6 = np.array(ice_keys_5_6)
     print('5_6: ', ice_times_5_6.shape)