minor

modules/icing/pirep_goes.py

@@ -18,6 +18,7 @@ from deeplearning.icing_cnn import run_evaluate_static
 goes_date_format = '%Y%j%H'
 goes16_directory = '/arcdata/goes/grb/goes16'  # /year/date/abi/L1b/RadC
 clavrx_dir = '/ships19/cloud/scratch/ICING/'
+#clavrx_dir = '/data/Personal/rink/clavrx/'
 clavrx_viirs_dir = '/apollo/cloud/scratch/Satellite_Output/NASA-SNPP_VIIRS/global/2019_DNB_for_Rink_wDBfix/level2_h5/'
 clavrx_test_dir = '/data/Personal/rink/clavrx/'
 dir_fmt = '%Y_%m_%d_%j'
@@ -1127,7 +1128,7 @@ def check_times(ts):
 
 
 def tile_extract(trnfile='/home/rink/tiles_l1b_train.h5', tstfile='/home/rink/tiles_l1b_test.h5', L1B_or_L2='L1B',
-                 cld_mask_name='cloud_mask', augment=False, split=0.2, check_keep_out=False):
+                 cld_mask_name='cloud_mask', augment=False, do_split=True, check_keep_out=False):
     icing_int_s = []
     ice_time_s = []
     no_ice_time_s = []
@@ -1243,12 +1244,11 @@ def tile_extract(trnfile='/home/rink/tiles_l1b_train.h5', tstfile='/home/rink/ti
     icing_lons = icing_lons[ds_indexes]
     icing_lats = icing_lats[ds_indexes]
 
-    # #trn_idxs, tst_idxs = split_data(icing_intensity.shape[0], shuffle=False, perc=split)
-    trn_idxs, tst_idxs = split_data(icing_times)
-    # all_idxs = np.arange(icing_intensity.shape[0])
-    # splt_idx = int(icing_intensity.shape[0] * (1-split))
-    # trn_idxs = all_idxs[0:splt_idx]
-    # tst_idxs = all_idxs[splt_idx:]
+    if do_split:
+        trn_idxs, tst_idxs = split_data(icing_times)
+    else:
+        trn_idxs = np.arange(icing_intensity.shape[0])
+        tst_idxs = None
     # ---------------------------------------------
 
     trn_data_dct = {}
@@ -1320,24 +1320,25 @@ def tile_extract(trnfile='/home/rink/tiles_l1b_train.h5', tstfile='/home/rink/ti
 
     write_file(trnfile, params, param_types, trn_data_dct, trn_icing_intensity, trn_icing_times, trn_icing_lons, trn_icing_lats)
 
-    tst_data_dct = {}
-    for ds_name in params:
-        tst_data_dct[ds_name] = data_dct[ds_name][tst_idxs,]
-    tst_icing_intensity = icing_intensity[tst_idxs,]
-    tst_icing_times = icing_times[tst_idxs,]
-    tst_icing_lons = icing_lons[tst_idxs,]
-    tst_icing_lats = icing_lats[tst_idxs,]
-
-    # do sort
-    ds_indexes = np.argsort(tst_icing_times)
-    for ds_name in params:
-        tst_data_dct[ds_name] = tst_data_dct[ds_name][ds_indexes]
-    tst_icing_intensity = tst_icing_intensity[ds_indexes]
-    tst_icing_times = tst_icing_times[ds_indexes]
-    tst_icing_lons = tst_icing_lons[ds_indexes]
-    tst_icing_lats = tst_icing_lats[ds_indexes]
+    if do_split:
+        tst_data_dct = {}
+        for ds_name in params:
+            tst_data_dct[ds_name] = data_dct[ds_name][tst_idxs,]
+        tst_icing_intensity = icing_intensity[tst_idxs,]
+        tst_icing_times = icing_times[tst_idxs,]
+        tst_icing_lons = icing_lons[tst_idxs,]
+        tst_icing_lats = icing_lats[tst_idxs,]
+
+        # do sort
+        ds_indexes = np.argsort(tst_icing_times)
+        for ds_name in params:
+            tst_data_dct[ds_name] = tst_data_dct[ds_name][ds_indexes]
+        tst_icing_intensity = tst_icing_intensity[ds_indexes]
+        tst_icing_times = tst_icing_times[ds_indexes]
+        tst_icing_lons = tst_icing_lons[ds_indexes]
+        tst_icing_lats = tst_icing_lats[ds_indexes]
 
-    write_file(tstfile, params, param_types, tst_data_dct, tst_icing_intensity, tst_icing_times, tst_icing_lons, tst_icing_lats)
+        write_file(tstfile, params, param_types, tst_data_dct, tst_icing_intensity, tst_icing_times, tst_icing_lons, tst_icing_lats)
 
     # --- close files
     for h5f in h5_s_icing: