snapshot...

modules/icing/pirep_goes.py

@@ -800,6 +800,12 @@ def apply_qc_icing_pireps(icing_alt, cld_top_hgt, cld_phase, cld_opd, cld_mask,
 def fov_extract(outfile='/Users/tomrink/fovs_out.h5'):
     ice_times = []
     icing_int_s = []
+    ice_lons = []
+    ice_lats = []
+
+    no_ice_times = []
+    no_ice_lons = []
+    no_ice_lats = []
 
     h5_s_icing = []
     h5_s_no_icing = []
@@ -819,6 +825,8 @@ def fov_extract(outfile='/Users/tomrink/fovs_out.h5'):
         num_obs = len(times)
 
         icing_int = f['icing_intensity'][:]
+        lons = f['longitude'][:]
+        lats = f['latitude'][:]
 
         cld_mask = f['cloud_mask'][:, 10:30, 10:30]
         cld_mask = cld_mask.reshape((num_obs, -1))
@@ -843,8 +851,10 @@ def fov_extract(outfile='/Users/tomrink/fovs_out.h5'):
                 dat = f[ds_name][i, 10:30, 10:30].flatten()
                 icing_data_dct[ds_name].append(dat[k_idxs])
 
-            ice_flags = np.full(len(k_idxs), icing_int[i])
-            icing_int_s.append(ice_flags)
+            icing_int_s.append(np.full(len(k_idxs), icing_int[i]))
+            ice_times.append(np.full(len(k_idxs), times[i]))
+            ice_lons.append(np.full(len(k_idxs), lons[i]))
+            ice_lats.append(np.full(len(k_idxs), lats[i]))
 
         print(fname)
 
@@ -853,6 +863,9 @@ def fov_extract(outfile='/Users/tomrink/fovs_out.h5'):
         icing_data_dct[ds_name] = np.concatenate(lst)
 
     icing_int_s = np.concatenate(icing_int_s)
+    ice_times = np.concatenate(ice_times)
+    ice_lons = np.concatenate(ice_lons)
+    ice_lats = np.concatenate(ice_lats)
 
     num_no_ice = 0
     for fidx in range(len(no_icing_files)):
@@ -862,6 +875,8 @@ def fov_extract(outfile='/Users/tomrink/fovs_out.h5'):
 
         times = f['time']
         num_obs = len(times)
+        lons = f['longitude']
+        lats = f['latitude']
 
         cld_mask = f['cloud_mask'][:, 10:30, 10:30]
         cld_mask = cld_mask.reshape((num_obs, -1))
@@ -881,6 +896,9 @@ def fov_extract(outfile='/Users/tomrink/fovs_out.h5'):
             else:
                 k_idxs = k_idxs[0:len(k_idxs)]
             num_no_ice += len(k_idxs)
+            no_ice_times.append(np.full(len(k_idxs), times[i]))
+            no_ice_lons.append(np.full(len(k_idxs), lons[i]))
+            no_ice_lats.append(np.full(len(k_idxs), lats[i]))
 
             for ds_name in train_params_day:
                 dat = f[ds_name][i, 10:30, 10:30].flatten()
@@ -892,19 +910,29 @@ def fov_extract(outfile='/Users/tomrink/fovs_out.h5'):
         lst = no_icing_data_dct[ds_name]
         no_icing_data_dct[ds_name] = np.concatenate(lst)
     no_icing_int_s = np.full(num_no_ice, -1)
-
-    ds_indexes = np.arange(num_ice + num_no_ice)
-    np.random.shuffle(ds_indexes)
+    no_ice_times = np.concatenate(no_ice_times)
+    no_ice_lons = np.concatenate(no_ice_lons)
+    no_ice_lats = np.concatenate(no_ice_lats)
 
     icing_intensity = np.concatenate([icing_int_s, no_icing_int_s])
+    icing_times = np.concatenate([ice_times, no_ice_times])
+    icing_lons = np.concatenate([ice_lons, no_ice_lons])
+    icing_lats = np.concatenate([ice_lats, no_ice_lats])
 
     data_dct = {}
     for ds_name in train_params_day:
         data_dct[ds_name] = np.concatenate([icing_data_dct[ds_name], no_icing_data_dct[ds_name]])
 
+    # apply shuffle indexes
+    ds_indexes = np.arange(num_ice + num_no_ice)
+    np.random.shuffle(ds_indexes)
+
     for ds_name in train_params_day:
         data_dct[ds_name] = data_dct[ds_name][ds_indexes]
     icing_intensity = icing_intensity[ds_indexes]
+    icing_times = icing_times[ds_indexes]
+    icing_lons = icing_lons[ds_indexes]
+    icing_lats = icing_lats[ds_indexes]
 
     h5f_expl = h5py.File(a_clvr_file, 'r')
     h5f_out = h5py.File(outfile, 'w')
@@ -917,6 +945,18 @@ def fov_extract(outfile='/Users/tomrink/fovs_out.h5'):
     icing_int_ds = h5f_out.create_dataset('icing_intensity', data=icing_intensity, dtype='i4')
     icing_int_ds.attrs.create('long_name', data='From PIREP. -1:No Icing, 1:Trace, 2:Light, 3:Light Moderate, 4:Moderate, 5:Moderate Severe, 6:Severe')
 
+    time_ds = h5f_out.create_dataset('time', data=icing_times, dtype='f4')
+    time_ds.attrs.create('units', data='seconds since 1970-1-1 00:00:00')
+    time_ds.attrs.create('long_name', data='PIREP time')
+
+    lon_ds = h5f_out.create_dataset('longitude', data=icing_lons, dtype='f4')
+    lon_ds.attrs.create('units', data='degrees_east')
+    lon_ds.attrs.create('long_name', data='PIREP longitude')
+
+    lat_ds = h5f_out.create_dataset('latitude', data=icing_lats, dtype='f4')
+    lat_ds.attrs.create('units', data='degrees_north')
+    lat_ds.attrs.create('long_name', data='PIREP latitude')
+
     # copy relevant attributes
     for ds_name in train_params_day:
         h5f_ds = h5f_out[ds_name]
@@ -937,6 +977,13 @@ def fov_extract(outfile='/Users/tomrink/fovs_out.h5'):
 
 def box_extract(outfile='/Users/tomrink/box_out.h5'):
     icing_int_s = []
+    ice_time_s = []
+    no_ice_time_s = []
+    ice_lon_s = []
+    no_ice_lon_s = []
+    ice_lat_s = []
+    no_ice_lat_s = []
+
 
     h5_s_icing = []
     h5_s_no_icing = []
@@ -951,6 +998,8 @@ def box_extract(outfile='/Users/tomrink/box_out.h5'):
 
         times = f['time'][:]
         num_obs = len(times)
+        lons = f['longitude']
+        lats = f['latitude']
 
         icing_int = f['icing_intensity'][:]
 
@@ -959,6 +1008,9 @@ def box_extract(outfile='/Users/tomrink/box_out.h5'):
                 dat = f[ds_name][i, 12:28, 12:28]
                 icing_data_dct[ds_name].append(dat)
             icing_int_s.append(icing_int[i])
+            ice_time_s.append(times[i])
+            ice_lon_s.append(lons[i])
+            ice_lat_s.append(lats[i])
 
         print(fname)
 
@@ -966,6 +1018,9 @@ def box_extract(outfile='/Users/tomrink/box_out.h5'):
         lst = icing_data_dct[ds_name]
         icing_data_dct[ds_name] = np.stack(lst, axis=0)
     icing_int_s = np.array(icing_int_s)
+    ice_time_s = np.array(ice_time_s)
+    ice_lon_s = np.array(ice_lon_s)
+    ice_lat_s = np.array(ice_lat_s)
     num_ice = icing_int_s.shape[0]
 
     num_no_ice = 0
@@ -976,12 +1031,17 @@ def box_extract(outfile='/Users/tomrink/box_out.h5'):
 
         times = f['time']
         num_obs = len(times)
+        lons = f['longitude']
+        lats = f['latitude']
 
         for i in range(num_obs):
             for ds_name in train_params_day:
                 dat = f[ds_name][i, 12:28, 12:28]
                 no_icing_data_dct[ds_name].append(dat)
             num_no_ice += 1
+            no_ice_time_s.append(times[i])
+            no_ice_lon_s.append(lons[i])
+            no_ice_lat_s.append(lats[i])
 
         print(fname)
 
@@ -989,19 +1049,29 @@ def box_extract(outfile='/Users/tomrink/box_out.h5'):
         lst = no_icing_data_dct[ds_name]
         no_icing_data_dct[ds_name] = np.stack(lst, axis=0)
     no_icing_int_s = np.full(num_no_ice, -1)
-
-    ds_indexes = np.arange(num_ice + num_no_ice)
-    np.random.shuffle(ds_indexes)
+    no_ice_time_s = np.array(no_ice_time_s)
+    no_ice_lon_s = np.array(no_ice_lon_s)
+    no_ice_lat_s = np.array(no_ice_lat_s)
 
     icing_intensity = np.concatenate([icing_int_s, no_icing_int_s])
+    icing_times = np.concatenate([ice_time_s, no_ice_time_s])
+    icing_lons = np.concatenate([ice_lon_s, no_ice_lon_s])
+    icing_lats = np.concatenate([ice_lat_s, no_ice_lat_s])
 
     data_dct = {}
     for ds_name in train_params_day:
         data_dct[ds_name] = np.concatenate([icing_data_dct[ds_name], no_icing_data_dct[ds_name]])
 
+    # Do shuffle
+    ds_indexes = np.arange(num_ice + num_no_ice)
+    np.random.shuffle(ds_indexes)
+
     for ds_name in train_params_day:
         data_dct[ds_name] = data_dct[ds_name][ds_indexes]
     icing_intensity = icing_intensity[ds_indexes]
+    icing_times = icing_times[ds_indexes]
+    icing_lons = icing_lons[ds_indexes]
+    icing_lats = icing_lats[ds_indexes]
 
     h5f_expl = h5py.File(a_clvr_file, 'r')
     h5f_out = h5py.File(outfile, 'w')
@@ -1014,6 +1084,18 @@ def box_extract(outfile='/Users/tomrink/box_out.h5'):
     icing_int_ds = h5f_out.create_dataset('icing_intensity', data=icing_intensity, dtype='i4')
     icing_int_ds.attrs.create('long_name', data='From PIREP. -1:No Icing, 1:Trace, 2:Light, 3:Light Moderate, 4:Moderate, 5:Moderate Severe, 6:Severe')
 
+    time_ds = h5f_out.create_dataset('time', data=icing_times, dtype='f4')
+    time_ds.attrs.create('units', data='seconds since 1970-1-1 00:00:00')
+    time_ds.attrs.create('long_name', data='PIREP time')
+
+    lon_ds = h5f_out.create_dataset('longitude', data=icing_lons, dtype='f4')
+    lon_ds.attrs.create('units', data='degrees_east')
+    lon_ds.attrs.create('long_name', data='PIREP longitude')
+
+    lat_ds = h5f_out.create_dataset('latitude', data=icing_lats, dtype='f4')
+    lat_ds.attrs.create('units', data='degrees_north')
+    lat_ds.attrs.create('long_name', data='PIREP latitude')
+
     # copy relevant attributes
     for ds_name in train_params_day:
         h5f_ds = h5f_out[ds_name]