Skip to content
Snippets Groups Projects
Commit 674dd58a authored by tomrink's avatar tomrink
Browse files

snapshot...

parent f2b0a61b
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
modules/icing/pirep_goes.py
+ 108
− 39
View file @ 674dd58a
...@@ -2267,57 +2267,126 @@ def run_mean_std_3(train_file_path, check_cloudy=False, params=train_params_day) ...@@ -2267,57 +2267,126 @@ def run_mean_std_3(train_file_path, check_cloudy=False, params=train_params_day)
def split_data(times, has_test): def split_data(times, has_test):
time_idxs = np.arange(times.shape[0]) time_idxs = np.arange(times.shape[0])
time_ranges = [[get_timestamp('2018-01-01_00:00'), get_timestamp('2018-01-07_23:59')], # time_ranges = [[get_timestamp('2018-01-01_00:00'), get_timestamp('2018-01-07_23:59')],
[get_timestamp('2018-03-01_00:00'), get_timestamp('2018-03-07_23:59')], # [get_timestamp('2018-03-01_00:00'), get_timestamp('2018-03-07_23:59')],
[get_timestamp('2018-05-01_00:00'), get_timestamp('2018-05-07_23:59')], # [get_timestamp('2018-05-01_00:00'), get_timestamp('2018-05-07_23:59')],
[get_timestamp('2018-07-01_00:00'), get_timestamp('2018-07-07_23:59')], # [get_timestamp('2018-07-01_00:00'), get_timestamp('2018-07-07_23:59')],
[get_timestamp('2018-09-01_00:00'), get_timestamp('2018-09-07_23:59')], # [get_timestamp('2018-09-01_00:00'), get_timestamp('2018-09-07_23:59')],
[get_timestamp('2018-11-01_00:00'), get_timestamp('2018-11-07_23:59')], # [get_timestamp('2018-11-01_00:00'), get_timestamp('2018-11-07_23:59')],
[get_timestamp('2019-01-01_00:00'), get_timestamp('2019-01-07_23:59')], # [get_timestamp('2019-01-01_00:00'), get_timestamp('2019-01-07_23:59')],
[get_timestamp('2019-03-01_00:00'), get_timestamp('2019-03-07_23:59')], # [get_timestamp('2019-03-01_00:00'), get_timestamp('2019-03-07_23:59')],
[get_timestamp('2019-05-01_00:00'), get_timestamp('2019-05-07_23:59')], # [get_timestamp('2019-05-01_00:00'), get_timestamp('2019-05-07_23:59')],
[get_timestamp('2019-07-01_00:00'), get_timestamp('2019-07-07_23:59')], # [get_timestamp('2019-07-01_00:00'), get_timestamp('2019-07-07_23:59')],
[get_timestamp('2019-09-01_00:00'), get_timestamp('2019-09-07_23:59')], # [get_timestamp('2019-09-01_00:00'), get_timestamp('2019-09-07_23:59')],
[get_timestamp('2019-11-01_00:00'), get_timestamp('2019-11-07_23:59')], # [get_timestamp('2019-11-01_00:00'), get_timestamp('2019-11-07_23:59')],
[get_timestamp('2021-09-24_00:00'), get_timestamp('2021-10-01_23:59')], # [get_timestamp('2021-09-24_00:00'), get_timestamp('2021-10-01_23:59')],
[get_timestamp('2021-11-01_00:00'), get_timestamp('2021-11-07_23:59')], # [get_timestamp('2021-11-01_00:00'), get_timestamp('2021-11-07_23:59')],
[get_timestamp('2022-01-01_00:00'), get_timestamp('2022-01-07_23:59')], # [get_timestamp('2022-01-01_00:00'), get_timestamp('2022-01-07_23:59')],
[get_timestamp('2022-03-01_00:00'), get_timestamp('2022-03-07_23:59')], # [get_timestamp('2022-03-01_00:00'), get_timestamp('2022-03-07_23:59')],
[get_timestamp('2022-04-01_00:00'), get_timestamp('2022-04-04_23:59')]] # [get_timestamp('2022-04-01_00:00'), get_timestamp('2022-04-04_23:59')]]
keep_out = 10800 # 3 hrs time_ranges = [[get_timestamp('2018-01-01_00:00'), get_timestamp('2018-01-04_23:59')],
[get_timestamp('2018-02-01_00:00'), get_timestamp('2018-02-04_23:59')],
vld_time_idxs = [] [get_timestamp('2018-03-01_00:00'), get_timestamp('2018-03-04_23:59')],
for t_rng in time_ranges:
t_rng[0] -= keep_out
t_rng[1] += keep_out
tidxs = np.searchsorted(times, t_rng)
vld_time_idxs.append(np.arange(tidxs[0], tidxs[1], 1))
vld_time_idxs = np.concatenate(vld_time_idxs, axis=None)
time_ranges = [[get_timestamp('2018-02-01_00:00'), get_timestamp('2018-02-04_23:59')],
[get_timestamp('2018-04-01_00:00'), get_timestamp('2018-04-04_23:59')], [get_timestamp('2018-04-01_00:00'), get_timestamp('2018-04-04_23:59')],
[get_timestamp('2018-05-01_00:00'), get_timestamp('2018-05-04_23:59')],
[get_timestamp('2018-06-01_00:00'), get_timestamp('2018-06-04_23:59')], [get_timestamp('2018-06-01_00:00'), get_timestamp('2018-06-04_23:59')],
[get_timestamp('2018-07-01_00:00'), get_timestamp('2018-07-04_23:59')],
[get_timestamp('2018-08-01_00:00'), get_timestamp('2018-08-04_23:59')], [get_timestamp('2018-08-01_00:00'), get_timestamp('2018-08-04_23:59')],
[get_timestamp('2018-09-01_00:00'), get_timestamp('2018-09-04_23:59')],
[get_timestamp('2018-10-01_00:00'), get_timestamp('2018-10-04_23:59')], [get_timestamp('2018-10-01_00:00'), get_timestamp('2018-10-04_23:59')],
[get_timestamp('2018-11-01_00:00'), get_timestamp('2018-11-04_23:59')],
[get_timestamp('2018-12-01_00:00'), get_timestamp('2018-12-04_23:59')], [get_timestamp('2018-12-01_00:00'), get_timestamp('2018-12-04_23:59')],
[get_timestamp('2019-01-01_00:00'), get_timestamp('2019-01-04_23:59')],
[get_timestamp('2019-02-01_00:00'), get_timestamp('2019-02-04_23:59')], [get_timestamp('2019-02-01_00:00'), get_timestamp('2019-02-04_23:59')],
[get_timestamp('2019-03-01_00:00'), get_timestamp('2019-03-04_23:59')],
[get_timestamp('2019-04-01_00:00'), get_timestamp('2019-04-04_23:59')], [get_timestamp('2019-04-01_00:00'), get_timestamp('2019-04-04_23:59')],
[get_timestamp('2019-05-01_00:00'), get_timestamp('2019-05-04_23:59')],
[get_timestamp('2019-06-01_00:00'), get_timestamp('2019-06-04_23:59')], [get_timestamp('2019-06-01_00:00'), get_timestamp('2019-06-04_23:59')],
[get_timestamp('2019-07-01_00:00'), get_timestamp('2019-07-04_23:59')],
[get_timestamp('2019-08-01_00:00'), get_timestamp('2019-08-04_23:59')], [get_timestamp('2019-08-01_00:00'), get_timestamp('2019-08-04_23:59')],
[get_timestamp('2019-09-01_00:00'), get_timestamp('2019-09-04_23:59')],
[get_timestamp('2019-10-01_00:00'), get_timestamp('2019-10-04_23:59')], [get_timestamp('2019-10-01_00:00'), get_timestamp('2019-10-04_23:59')],
[get_timestamp('2019-11-01_00:00'), get_timestamp('2019-11-04_23:59')],
[get_timestamp('2019-12-01_00:00'), get_timestamp('2019-12-04_23:59')], [get_timestamp('2019-12-01_00:00'), get_timestamp('2019-12-04_23:59')],
[get_timestamp('2021-12-04_00:00'), get_timestamp('2021-12-08_23:59')],
[get_timestamp('2021-10-05_00:00'), get_timestamp('2021-10-10_23:59')], [get_timestamp('2022-01-01_00:00'), get_timestamp('2022-01-04_23:59')],
[get_timestamp('2021-12-01_00:00'), get_timestamp('2021-12-04_23:59')],
[get_timestamp('2022-02-01_00:00'), get_timestamp('2022-02-04_23:59')], [get_timestamp('2022-02-01_00:00'), get_timestamp('2022-02-04_23:59')],
[get_timestamp('2022-03-26_00:00'), get_timestamp('2022-03-30_23:59')], [get_timestamp('2022-03-01_00:00'), get_timestamp('2022-03-04_23:59')],
[get_timestamp('2022-04-07_00:00'), get_timestamp('2022-04-10_23:59')], [get_timestamp('2022-04-01_00:00'), get_timestamp('2022-04-04_23:59')],
[get_timestamp('2022-05-01_00:00'), get_timestamp('2022-05-06_23:59')], [get_timestamp('2022-05-01_00:00'), get_timestamp('2022-05-04_23:59')],
[get_timestamp('2022-10-03_00:00'), get_timestamp('2022-10-08_23:59')], [get_timestamp('2022-10-03_00:00'), get_timestamp('2022-10-07_23:59')],
[get_timestamp('2022-10-17_00:00'), get_timestamp('2022-10-22_20:59')], [get_timestamp('2022-11-01_00:00'), get_timestamp('2022-11-04_23:59')],
[get_timestamp('2022-11-01_00:00'), get_timestamp('2022-11-05_23:59')], [get_timestamp('2022-12-01_00:00'), get_timestamp('2022-12-04_23:59')],
[get_timestamp('2022-11-10_00:00'), get_timestamp('2022-12-01_23:59')]] [get_timestamp('2023-12-01_00:00'), get_timestamp('2023-12-04_23:59')]]
keep_out = 10800 # 3 hrs
vld_time_idxs = []
for t_rng in time_ranges:
t_rng[0] -= keep_out
t_rng[1] += keep_out
tidxs = np.searchsorted(times, t_rng)
vld_time_idxs.append(np.arange(tidxs[0], tidxs[1], 1))
vld_time_idxs = np.concatenate(vld_time_idxs, axis=None)
# time_ranges = [[get_timestamp('2018-02-01_00:00'), get_timestamp('2018-02-04_23:59')],
# [get_timestamp('2018-04-01_00:00'), get_timestamp('2018-04-04_23:59')],
# [get_timestamp('2018-06-01_00:00'), get_timestamp('2018-06-04_23:59')],
# [get_timestamp('2018-08-01_00:00'), get_timestamp('2018-08-04_23:59')],
# [get_timestamp('2018-10-01_00:00'), get_timestamp('2018-10-04_23:59')],
# [get_timestamp('2018-12-01_00:00'), get_timestamp('2018-12-04_23:59')],
# [get_timestamp('2019-02-01_00:00'), get_timestamp('2019-02-04_23:59')],
# [get_timestamp('2019-04-01_00:00'), get_timestamp('2019-04-04_23:59')],
# [get_timestamp('2019-06-01_00:00'), get_timestamp('2019-06-04_23:59')],
# [get_timestamp('2019-08-01_00:00'), get_timestamp('2019-08-04_23:59')],
# [get_timestamp('2019-10-01_00:00'), get_timestamp('2019-10-04_23:59')],
# [get_timestamp('2019-12-01_00:00'), get_timestamp('2019-12-04_23:59')],
# [get_timestamp('2021-10-05_00:00'), get_timestamp('2021-10-10_23:59')],
# [get_timestamp('2021-12-01_00:00'), get_timestamp('2021-12-04_23:59')],
# [get_timestamp('2022-02-01_00:00'), get_timestamp('2022-02-04_23:59')],
# [get_timestamp('2022-03-26_00:00'), get_timestamp('2022-03-30_23:59')],
# [get_timestamp('2022-04-07_00:00'), get_timestamp('2022-04-10_23:59')],
# [get_timestamp('2022-05-01_00:00'), get_timestamp('2022-05-06_23:59')],
# [get_timestamp('2022-10-03_00:00'), get_timestamp('2022-10-08_23:59')],
# [get_timestamp('2022-10-17_00:00'), get_timestamp('2022-10-22_20:59')],
# [get_timestamp('2022-11-01_00:00'), get_timestamp('2022-11-05_23:59')],
# [get_timestamp('2022-11-10_00:00'), get_timestamp('2022-12-01_23:59')]]
time_ranges = [[get_timestamp('2018-01-11_00:00'), get_timestamp('2018-01-12_23:59')],
[get_timestamp('2018-02-11_00:00'), get_timestamp('2018-02-13_23:59')],
[get_timestamp('2018-03-11_00:00'), get_timestamp('2018-03-13_23:59')],
[get_timestamp('2018-04-11_00:00'), get_timestamp('2018-04-13_23:59')],
[get_timestamp('2018-05-11_00:00'), get_timestamp('2018-05-13_23:59')],
[get_timestamp('2018-06-11_00:00'), get_timestamp('2018-06-13_23:59')],
[get_timestamp('2018-07-11_00:00'), get_timestamp('2018-07-13_23:59')],
[get_timestamp('2018-08-11_00:00'), get_timestamp('2018-08-13_23:59')],
[get_timestamp('2018-09-11_00:00'), get_timestamp('2018-09-13_23:59')],
[get_timestamp('2018-10-11_00:00'), get_timestamp('2018-10-13_23:59')],
[get_timestamp('2018-11-11_00:00'), get_timestamp('2018-11-13_23:59')],
[get_timestamp('2018-12-11_00:00'), get_timestamp('2018-12-13_23:59')],
[get_timestamp('2019-01-11_00:00'), get_timestamp('2019-01-13_23:59')],
[get_timestamp('2019-02-11_00:00'), get_timestamp('2019-02-13_23:59')],
[get_timestamp('2019-03-11_00:00'), get_timestamp('2019-03-13_23:59')],
[get_timestamp('2019-04-11_00:00'), get_timestamp('2019-04-13_23:59')],
[get_timestamp('2019-05-11_00:00'), get_timestamp('2019-05-13_23:59')],
[get_timestamp('2019-06-11_00:00'), get_timestamp('2019-06-13_23:59')],
[get_timestamp('2019-07-11_00:00'), get_timestamp('2019-07-13_23:59')],
[get_timestamp('2019-08-11_00:00'), get_timestamp('2019-08-13_23:59')],
[get_timestamp('2019-09-11_00:00'), get_timestamp('2019-09-13_23:59')],
[get_timestamp('2019-10-11_00:00'), get_timestamp('2019-10-13_23:59')],
[get_timestamp('2019-11-11_00:00'), get_timestamp('2019-11-13_23:59')],
[get_timestamp('2019-12-11_00:00'), get_timestamp('2019-12-13_23:59')],
[get_timestamp('2021-12-11_00:00'), get_timestamp('2021-12-13_23:59')],
[get_timestamp('2022-01-11_00:00'), get_timestamp('2022-01-13_23:59')],
[get_timestamp('2022-02-11_00:00'), get_timestamp('2022-02-13_23:59')],
[get_timestamp('2022-03-11_00:00'), get_timestamp('2022-03-13_23:59')],
[get_timestamp('2022-04-11_00:00'), get_timestamp('2022-04-13_23:59')],
[get_timestamp('2022-05-11_00:00'), get_timestamp('2022-05-13_23:59')],
[get_timestamp('2022-10-11_00:00'), get_timestamp('2022-10-13_23:59')],
[get_timestamp('2022-11-11_00:00'), get_timestamp('2022-11-13_23:59')],
[get_timestamp('2022-12-11_00:00'), get_timestamp('2022-12-13_23:59')],
[get_timestamp('2023-12-11_00:00'), get_timestamp('2023-12-13_23:59')]]
tst_time_idxs = [] tst_time_idxs = []
if has_test: if has_test:
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment