Skip to content
Snippets Groups Projects
Commit 15736ece authored by tomrink's avatar tomrink
Browse files

snapshot...

parent 96b0c566
Branches
No related tags found
No related merge requests found
...@@ -344,6 +344,93 @@ def analyze(ice_dct, no_ice_dct): ...@@ -344,6 +344,93 @@ def analyze(ice_dct, no_ice_dct):
print(dt_str[2:]) print(dt_str[2:])
def process_2(ice_dct, no_ice_dct, neg_ice_dct):
new_ice_dct = {}
new_no_ice_dct = {}
new_neg_ice_dct = {}
ice_keys_5_6 = []
ice_keys_1 = []
ice_keys_4 = []
ice_keys_3 = []
ice_keys_2 = []
for ts in list(ice_dct.keys()):
rpts = ice_dct[ts]
for tup in rpts:
if tup[3] == 5 or tup[3] == 6:
ice_keys_5_6.append(ts)
elif tup[3] == 1:
ice_keys_1.append(ts)
elif tup[3] == 4:
ice_keys_4.append(ts)
elif tup[3] == 3:
ice_keys_3.append(ts)
elif tup[3] == 2:
ice_keys_2.append(ts)
no_ice_keys = []
for ts in list(no_ice_dct.keys()):
rpts = no_ice_dct[ts]
for tup in rpts:
no_ice_keys.append(ts)
neg_ice_keys = []
for ts in list(neg_ice_dct.keys()):
rpts = neg_ice_dct[ts]
for tup in rpts:
neg_ice_keys.append(ts)
ice_keys_5_6 = np.array(ice_keys_5_6)
print('5_6: ', ice_keys_5_6.shape)
ice_keys_4 = np.array(ice_keys_4)
print('4: ', ice_keys_4.shape)
ice_keys_3 = np.array(ice_keys_3)
print('3: ', ice_keys_3.shape)
ice_keys_2 = np.array(ice_keys_2)
print('2: ', ice_keys_2.shape)
np.random.seed(42)
np.random.shuffle(ice_keys_2)
ice_keys_2 = ice_keys_2[0:30000]
ice_keys_1 = np.array(ice_keys_1)
print('1: ', ice_keys_1.shape)
ice_keys = np.concatenate([ice_keys_5_6, ice_keys_1, ice_keys_2, ice_keys_3, ice_keys_4])
uniq_sorted_keys = np.unique(ice_keys)
print(ice_keys.shape, uniq_sorted_keys.shape)
uniq_sorted_keys = uniq_sorted_keys.tolist()
for key in uniq_sorted_keys:
new_ice_dct[key] = ice_dct[key]
no_ice_keys = np.array(no_ice_keys)
np.random.seed(42)
np.random.shuffle(no_ice_keys)
no_ice_keys = no_ice_keys[0:50000]
uniq_sorted_no_ice = np.unique(no_ice_keys)
print(no_ice_keys.shape, uniq_sorted_no_ice.shape)
uniq_sorted_no_ice = uniq_sorted_no_ice.tolist()
for key in uniq_sorted_no_ice:
new_no_ice_dct[key] = no_ice_dct[key]
neg_ice_keys = np.array(neg_ice_keys)
np.random.seed(42)
np.random.shuffle(neg_ice_keys)
neg_ice_keys = neg_ice_keys[0:5000]
uniq_sorted_neg_ice = np.unique(neg_ice_keys)
print(neg_ice_keys.shape, uniq_sorted_neg_ice.shape)
for key in uniq_sorted_neg_ice:
new_neg_ice_dct[key] = neg_ice_dct[key]
return new_ice_dct, new_no_ice_dct, new_neg_ice_dct
def process_1(ice_dct, no_ice_dct, neg_ice_dct): def process_1(ice_dct, no_ice_dct, neg_ice_dct):
new_ice_dct = {} new_ice_dct = {}
new_no_ice_dct = {} new_no_ice_dct = {}
...@@ -365,10 +452,14 @@ def process_1(ice_dct, no_ice_dct, neg_ice_dct): ...@@ -365,10 +452,14 @@ def process_1(ice_dct, no_ice_dct, neg_ice_dct):
ice_files_2 = [] ice_files_2 = []
ice_times_2 = [] ice_times_2 = []
ice_keys_2 = [] ice_keys_2 = []
print('num keys ice, no_ice, neg_ice: ', len(ice_dct), len(no_ice_dct), len(neg_ice_dct))
nfound = 0
for ts in list(ice_dct.keys()): for ts in list(ice_dct.keys()):
try: try:
ds = get_goes_datasource(ts) ds = get_goes_datasource(ts)
goes_file, t_0, _ = ds.get_file(ts) goes_file, t_0, _ = ds.get_file(ts)
if goes_file is not None:
nfound += 1
if goes_file is not None and goes_file != last_file: if goes_file is not None and goes_file != last_file:
rpts = ice_dct[ts] rpts = ice_dct[ts]
for tup in rpts: for tup in rpts:
...@@ -395,15 +486,19 @@ def process_1(ice_dct, no_ice_dct, neg_ice_dct): ...@@ -395,15 +486,19 @@ def process_1(ice_dct, no_ice_dct, neg_ice_dct):
last_file = goes_file last_file = goes_file
except Exception: except Exception:
continue continue
print('num ice files: ', nfound)
last_file = None last_file = None
no_ice_files = [] no_ice_files = []
no_ice_times = [] no_ice_times = []
no_ice_keys = [] no_ice_keys = []
nfound = 0
for ts in list(no_ice_dct.keys()): for ts in list(no_ice_dct.keys()):
try: try:
ds = get_goes_datasource(ts) ds = get_goes_datasource(ts)
goes_file, t_0, _ = ds.get_file(ts) goes_file, t_0, _ = ds.get_file(ts)
if goes_file is not None:
nfound += 1
if goes_file is not None and goes_file != last_file: if goes_file is not None and goes_file != last_file:
rpts = no_ice_dct[ts] rpts = no_ice_dct[ts]
for tup in rpts: for tup in rpts:
...@@ -414,14 +509,19 @@ def process_1(ice_dct, no_ice_dct, neg_ice_dct): ...@@ -414,14 +509,19 @@ def process_1(ice_dct, no_ice_dct, neg_ice_dct):
except Exception: except Exception:
continue continue
print('num no ice files found: ', nfound)
last_file = None last_file = None
neg_ice_files = [] neg_ice_files = []
neg_ice_times = [] neg_ice_times = []
neg_ice_keys = [] neg_ice_keys = []
nfound = 0
for ts in list(neg_ice_dct.keys()): for ts in list(neg_ice_dct.keys()):
try: try:
ds = get_goes_datasource(ts) ds = get_goes_datasource(ts)
goes_file, t_0, _ = ds.get_file(ts) goes_file, t_0, _ = ds.get_file(ts)
if goes_file is not None:
nfound += 1
if goes_file is not None and goes_file != last_file: if goes_file is not None and goes_file != last_file:
rpts = neg_ice_dct[ts] rpts = neg_ice_dct[ts]
for tup in rpts: for tup in rpts:
...@@ -432,6 +532,8 @@ def process_1(ice_dct, no_ice_dct, neg_ice_dct): ...@@ -432,6 +532,8 @@ def process_1(ice_dct, no_ice_dct, neg_ice_dct):
except Exception: except Exception:
continue continue
print('num neg ice files found: ', nfound)
ice_times_5_6 = np.array(ice_times_5_6) ice_times_5_6 = np.array(ice_times_5_6)
ice_keys_5_6 = np.array(ice_keys_5_6) ice_keys_5_6 = np.array(ice_keys_5_6)
print('5_6: ', ice_times_5_6.shape) print('5_6: ', ice_times_5_6.shape)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment