From e7459a59eb70e32072c8fa566bf0df764fda2c42 Mon Sep 17 00:00:00 2001
From: tomrink <rink@ssec.wisc.edu>
Date: Wed, 6 Oct 2021 16:32:08 -0500
Subject: [PATCH] minor
---
modules/icing/pirep_goes.py | 49 +++++++++++++++++++------------------
1 file changed, 25 insertions(+), 24 deletions(-)
diff --git a/modules/icing/pirep_goes.py b/modules/icing/pirep_goes.py
index 05486b84..9785d63e 100644
--- a/modules/icing/pirep_goes.py
+++ b/modules/icing/pirep_goes.py
@@ -18,6 +18,7 @@ from deeplearning.icing_cnn import run_evaluate_static
goes_date_format = '%Y%j%H'
goes16_directory = '/arcdata/goes/grb/goes16' # /year/date/abi/L1b/RadC
clavrx_dir = '/ships19/cloud/scratch/ICING/'
+#clavrx_dir = '/data/Personal/rink/clavrx/'
clavrx_viirs_dir = '/apollo/cloud/scratch/Satellite_Output/NASA-SNPP_VIIRS/global/2019_DNB_for_Rink_wDBfix/level2_h5/'
clavrx_test_dir = '/data/Personal/rink/clavrx/'
dir_fmt = '%Y_%m_%d_%j'
@@ -1127,7 +1128,7 @@ def check_times(ts):
def tile_extract(trnfile='/home/rink/tiles_l1b_train.h5', tstfile='/home/rink/tiles_l1b_test.h5', L1B_or_L2='L1B',
- cld_mask_name='cloud_mask', augment=False, split=0.2, check_keep_out=False):
+ cld_mask_name='cloud_mask', augment=False, do_split=True, check_keep_out=False):
icing_int_s = []
ice_time_s = []
no_ice_time_s = []
@@ -1243,12 +1244,11 @@ def tile_extract(trnfile='/home/rink/tiles_l1b_train.h5', tstfile='/home/rink/ti
icing_lons = icing_lons[ds_indexes]
icing_lats = icing_lats[ds_indexes]
- # #trn_idxs, tst_idxs = split_data(icing_intensity.shape[0], shuffle=False, perc=split)
- trn_idxs, tst_idxs = split_data(icing_times)
- # all_idxs = np.arange(icing_intensity.shape[0])
- # splt_idx = int(icing_intensity.shape[0] * (1-split))
- # trn_idxs = all_idxs[0:splt_idx]
- # tst_idxs = all_idxs[splt_idx:]
+ if do_split:
+ trn_idxs, tst_idxs = split_data(icing_times)
+ else:
+ trn_idxs = np.arange(icing_intensity.shape[0])
+ tst_idxs = None
# ---------------------------------------------
trn_data_dct = {}
@@ -1320,24 +1320,25 @@ def tile_extract(trnfile='/home/rink/tiles_l1b_train.h5', tstfile='/home/rink/ti
write_file(trnfile, params, param_types, trn_data_dct, trn_icing_intensity, trn_icing_times, trn_icing_lons, trn_icing_lats)
- tst_data_dct = {}
- for ds_name in params:
- tst_data_dct[ds_name] = data_dct[ds_name][tst_idxs,]
- tst_icing_intensity = icing_intensity[tst_idxs,]
- tst_icing_times = icing_times[tst_idxs,]
- tst_icing_lons = icing_lons[tst_idxs,]
- tst_icing_lats = icing_lats[tst_idxs,]
-
- # do sort
- ds_indexes = np.argsort(tst_icing_times)
- for ds_name in params:
- tst_data_dct[ds_name] = tst_data_dct[ds_name][ds_indexes]
- tst_icing_intensity = tst_icing_intensity[ds_indexes]
- tst_icing_times = tst_icing_times[ds_indexes]
- tst_icing_lons = tst_icing_lons[ds_indexes]
- tst_icing_lats = tst_icing_lats[ds_indexes]
+ if do_split:
+ tst_data_dct = {}
+ for ds_name in params:
+ tst_data_dct[ds_name] = data_dct[ds_name][tst_idxs,]
+ tst_icing_intensity = icing_intensity[tst_idxs,]
+ tst_icing_times = icing_times[tst_idxs,]
+ tst_icing_lons = icing_lons[tst_idxs,]
+ tst_icing_lats = icing_lats[tst_idxs,]
+
+ # do sort
+ ds_indexes = np.argsort(tst_icing_times)
+ for ds_name in params:
+ tst_data_dct[ds_name] = tst_data_dct[ds_name][ds_indexes]
+ tst_icing_intensity = tst_icing_intensity[ds_indexes]
+ tst_icing_times = tst_icing_times[ds_indexes]
+ tst_icing_lons = tst_icing_lons[ds_indexes]
+ tst_icing_lats = tst_icing_lats[ds_indexes]
- write_file(tstfile, params, param_types, tst_data_dct, tst_icing_intensity, tst_icing_times, tst_icing_lons, tst_icing_lats)
+ write_file(tstfile, params, param_types, tst_data_dct, tst_icing_intensity, tst_icing_times, tst_icing_lons, tst_icing_lats)
# --- close files
for h5f in h5_s_icing:
--
GitLab