From 15736ece184f5ab3fde195d739bb4542d5cfd73e Mon Sep 17 00:00:00 2001
From: tomrink <rink@ssec.wisc.edu>
Date: Sat, 27 Mar 2021 13:22:44 -0500
Subject: [PATCH] snapshot...
---
modules/icing/pirep_goes.py | 102 ++++++++++++++++++++++++++++++++++++
1 file changed, 102 insertions(+)
diff --git a/modules/icing/pirep_goes.py b/modules/icing/pirep_goes.py
index 6bcbf860..22d2e449 100644
--- a/modules/icing/pirep_goes.py
+++ b/modules/icing/pirep_goes.py
@@ -344,6 +344,93 @@ def analyze(ice_dct, no_ice_dct):
print(dt_str[2:])
+def process_2(ice_dct, no_ice_dct, neg_ice_dct):
+ new_ice_dct = {}
+ new_no_ice_dct = {}
+ new_neg_ice_dct = {}
+
+ ice_keys_5_6 = []
+ ice_keys_1 = []
+ ice_keys_4 = []
+ ice_keys_3 = []
+ ice_keys_2 = []
+
+ for ts in list(ice_dct.keys()):
+ rpts = ice_dct[ts]
+ for tup in rpts:
+ if tup[3] == 5 or tup[3] == 6:
+ ice_keys_5_6.append(ts)
+ elif tup[3] == 1:
+ ice_keys_1.append(ts)
+ elif tup[3] == 4:
+ ice_keys_4.append(ts)
+ elif tup[3] == 3:
+ ice_keys_3.append(ts)
+ elif tup[3] == 2:
+ ice_keys_2.append(ts)
+
+ no_ice_keys = []
+ for ts in list(no_ice_dct.keys()):
+ rpts = no_ice_dct[ts]
+ for tup in rpts:
+ no_ice_keys.append(ts)
+
+ neg_ice_keys = []
+ for ts in list(neg_ice_dct.keys()):
+ rpts = neg_ice_dct[ts]
+ for tup in rpts:
+ neg_ice_keys.append(ts)
+
+ ice_keys_5_6 = np.array(ice_keys_5_6)
+ print('5_6: ', ice_keys_5_6.shape)
+
+ ice_keys_4 = np.array(ice_keys_4)
+ print('4: ', ice_keys_4.shape)
+
+ ice_keys_3 = np.array(ice_keys_3)
+ print('3: ', ice_keys_3.shape)
+
+ ice_keys_2 = np.array(ice_keys_2)
+ print('2: ', ice_keys_2.shape)
+ np.random.seed(42)
+ np.random.shuffle(ice_keys_2)
+ ice_keys_2 = ice_keys_2[0:30000]
+
+ ice_keys_1 = np.array(ice_keys_1)
+ print('1: ', ice_keys_1.shape)
+
+ ice_keys = np.concatenate([ice_keys_5_6, ice_keys_1, ice_keys_2, ice_keys_3, ice_keys_4])
+ uniq_sorted_keys = np.unique(ice_keys)
+ print(ice_keys.shape, uniq_sorted_keys.shape)
+
+ uniq_sorted_keys = uniq_sorted_keys.tolist()
+ for key in uniq_sorted_keys:
+ new_ice_dct[key] = ice_dct[key]
+
+ no_ice_keys = np.array(no_ice_keys)
+ np.random.seed(42)
+ np.random.shuffle(no_ice_keys)
+ no_ice_keys = no_ice_keys[0:50000]
+ uniq_sorted_no_ice = np.unique(no_ice_keys)
+ print(no_ice_keys.shape, uniq_sorted_no_ice.shape)
+
+ uniq_sorted_no_ice = uniq_sorted_no_ice.tolist()
+ for key in uniq_sorted_no_ice:
+ new_no_ice_dct[key] = no_ice_dct[key]
+
+ neg_ice_keys = np.array(neg_ice_keys)
+ np.random.seed(42)
+ np.random.shuffle(neg_ice_keys)
+ neg_ice_keys = neg_ice_keys[0:5000]
+ uniq_sorted_neg_ice = np.unique(neg_ice_keys)
+ print(neg_ice_keys.shape, uniq_sorted_neg_ice.shape)
+
+ for key in uniq_sorted_neg_ice:
+ new_neg_ice_dct[key] = neg_ice_dct[key]
+
+ return new_ice_dct, new_no_ice_dct, new_neg_ice_dct
+
+
def process_1(ice_dct, no_ice_dct, neg_ice_dct):
new_ice_dct = {}
new_no_ice_dct = {}
@@ -365,10 +452,14 @@ def process_1(ice_dct, no_ice_dct, neg_ice_dct):
ice_files_2 = []
ice_times_2 = []
ice_keys_2 = []
+ print('num keys ice, no_ice, neg_ice: ', len(ice_dct), len(no_ice_dct), len(neg_ice_dct))
+ nfound = 0
for ts in list(ice_dct.keys()):
try:
ds = get_goes_datasource(ts)
goes_file, t_0, _ = ds.get_file(ts)
+ if goes_file is not None:
+ nfound += 1
if goes_file is not None and goes_file != last_file:
rpts = ice_dct[ts]
for tup in rpts:
@@ -395,15 +486,19 @@ def process_1(ice_dct, no_ice_dct, neg_ice_dct):
last_file = goes_file
except Exception:
continue
+ print('num ice files: ', nfound)
last_file = None
no_ice_files = []
no_ice_times = []
no_ice_keys = []
+ nfound = 0
for ts in list(no_ice_dct.keys()):
try:
ds = get_goes_datasource(ts)
goes_file, t_0, _ = ds.get_file(ts)
+ if goes_file is not None:
+ nfound += 1
if goes_file is not None and goes_file != last_file:
rpts = no_ice_dct[ts]
for tup in rpts:
@@ -414,14 +509,19 @@ def process_1(ice_dct, no_ice_dct, neg_ice_dct):
except Exception:
continue
+ print('num no ice files found: ', nfound)
+
last_file = None
neg_ice_files = []
neg_ice_times = []
neg_ice_keys = []
+ nfound = 0
for ts in list(neg_ice_dct.keys()):
try:
ds = get_goes_datasource(ts)
goes_file, t_0, _ = ds.get_file(ts)
+ if goes_file is not None:
+ nfound += 1
if goes_file is not None and goes_file != last_file:
rpts = neg_ice_dct[ts]
for tup in rpts:
@@ -432,6 +532,8 @@ def process_1(ice_dct, no_ice_dct, neg_ice_dct):
except Exception:
continue
+ print('num neg ice files found: ', nfound)
+
ice_times_5_6 = np.array(ice_times_5_6)
ice_keys_5_6 = np.array(ice_keys_5_6)
print('5_6: ', ice_times_5_6.shape)
--
GitLab