snapshot...

modules/icing/pirep_goes.py

@@ -831,7 +831,7 @@ def apply_qc_no_icing_pireps(icing_alt, cld_top_hgt, cld_phase, cld_opd, cld_mas
     return mask, idxs, num_tested
 
 
-def fov_extract(outfile='/home/rink/fovs_l1b_out.h5', L1B_or_L2='L1B'):
+def fov_extract(trnfile='/home/rink/fovs_l1b_train.h5', tstfile='/home/rink/fovs_l1b_test.h5', L1B_or_L2='L1B'):
     ice_times = []
     icing_int_s = []
     ice_lons = []
@@ -984,41 +984,40 @@ def fov_extract(outfile='/home/rink/fovs_l1b_out.h5', L1B_or_L2='L1B'):
     icing_lons = icing_lons[ds_indexes]
     icing_lats = icing_lats[ds_indexes]
 
-    h5f_expl = h5py.File(a_clvr_file, 'r')
-    h5f_out = h5py.File(outfile, 'w')
-
-    for idx, ds_name in enumerate(params):
-        dt = param_types[params.index(ds_name)]
-        data = data_dct[ds_name]
-        h5f_out.create_dataset(ds_name, data=data, dtype=dt)
-
-    icing_int_ds = h5f_out.create_dataset('icing_intensity', data=icing_intensity, dtype='i4')
-    icing_int_ds.attrs.create('long_name', data='From PIREP. -1:No Icing, 1:Trace, 2:Light, 3:Light Moderate, 4:Moderate, 5:Moderate Severe, 6:Severe')
+    #trn_idxs, tst_idxs = split_data(icing_intensity.shape[0], shuffle=False, perc=split)
+    all_idxs = np.arange(icing_intensity.shape[0])
+    splt_idx = int(icing_intensity.shape[0] * (1-split))
+    trn_idxs = all_idxs[0:splt_idx]
+    tst_idxs = all_idxs[splt_idx:]
 
-    time_ds = h5f_out.create_dataset('time', data=icing_times, dtype='f4')
-    time_ds.attrs.create('units', data='seconds since 1970-1-1 00:00:00')
-    time_ds.attrs.create('long_name', data='PIREP time')
+    trn_data_dct = {}
+    for ds_name in params:
+        trn_data_dct[ds_name] = data_dct[ds_name][trn_idxs,]
+    trn_icing_intensity = icing_intensity[trn_idxs,]
+    trn_icing_times = icing_times[trn_idxs,]
+    trn_icing_lons = icing_lons[trn_idxs,]
+    trn_icing_lats = icing_lats[trn_idxs,]
 
-    lon_ds = h5f_out.create_dataset('longitude', data=icing_lons, dtype='f4')
-    lon_ds.attrs.create('units', data='degrees_east')
-    lon_ds.attrs.create('long_name', data='PIREP longitude')
+    write_file(trnfile, params, param_types, trn_data_dct, trn_icing_intensity, trn_icing_times, trn_icing_lons, trn_icing_lats)
 
-    lat_ds = h5f_out.create_dataset('latitude', data=icing_lats, dtype='f4')
-    lat_ds.attrs.create('units', data='degrees_north')
-    lat_ds.attrs.create('long_name', data='PIREP latitude')
+    tst_data_dct = {}
+    for ds_name in params:
+        tst_data_dct[ds_name] = data_dct[ds_name][tst_idxs,]
+    tst_icing_intensity = icing_intensity[tst_idxs,]
+    tst_icing_times = icing_times[tst_idxs,]
+    tst_icing_lons = icing_lons[tst_idxs,]
+    tst_icing_lats = icing_lats[tst_idxs,]
 
-    # copy relevant attributes
+    # do sort
+    ds_indexes = np.argsort(tst_icing_times)
     for ds_name in params:
-        h5f_ds = h5f_out[ds_name]
-        h5f_ds.attrs.create('standard_name', data=h5f_expl[ds_name].attrs.get('standard_name'))
-        h5f_ds.attrs.create('long_name', data=h5f_expl[ds_name].attrs.get('long_name'))
-        h5f_ds.attrs.create('units', data=h5f_expl[ds_name].attrs.get('units'))
-        attr = h5f_expl[ds_name].attrs.get('actual_range')
-        if attr is not None:
-            h5f_ds.attrs.create('actual_range', data=attr)
-        attr = h5f_expl[ds_name].attrs.get('flag_values')
-        if attr is not None:
-            h5f_ds.attrs.create('flag_values', data=attr)
+        tst_data_dct[ds_name] = tst_data_dct[ds_name][ds_indexes]
+    tst_icing_intensity = tst_icing_intensity[ds_indexes]
+    tst_icing_times = tst_icing_times[ds_indexes]
+    tst_icing_lons = tst_icing_lons[ds_indexes]
+    tst_icing_lats = tst_icing_lats[ds_indexes]
+
+    write_file(tstfile, params, param_types, tst_data_dct, tst_icing_intensity, tst_icing_times, tst_icing_lons, tst_icing_lats)
 
     # --- close files
     for h5f in h5_s_icing:
@@ -1027,9 +1026,6 @@ def fov_extract(outfile='/home/rink/fovs_l1b_out.h5', L1B_or_L2='L1B'):
     for h5f in h5_s_no_icing:
         h5f.close()
 
-    h5f_out.close()
-    h5f_expl.close()
-
 
 def tile_extract(trnfile='/home/rink/tiles_l1b_train.h5', tstfile='/home/rink/tiles_l1b_test.h5', L1B_or_L2='L1B',
                  cld_mask_name='cloud_mask', augment=False, split=0.2):