snapshot...

modules/icing/pirep_goes.py

@@ -797,7 +797,7 @@ def apply_qc_icing_pireps(icing_alt, cld_top_hgt, cld_phase, cld_opd, cld_mask,
     return mask, idxs
 
 
-def fov_extract(outfile='/Users/tomrink/fovs_out.h5'):
+def fov_extract(outfile='/Users/tomrink/fovs_out.h5', train_params=train_params_day):
     ice_times = []
     icing_int_s = []
     ice_lons = []
@@ -810,8 +810,8 @@ def fov_extract(outfile='/Users/tomrink/fovs_out.h5'):
     h5_s_icing = []
     h5_s_no_icing = []
 
-    icing_data_dct = {ds: [] for ds in train_params_day}
-    no_icing_data_dct = {ds: [] for ds in train_params_day}
+    icing_data_dct = {ds: [] for ds in train_params}
+    no_icing_data_dct = {ds: [] for ds in train_params}
 
     sub_indexes = np.arange(400)
 
@@ -847,7 +847,7 @@ def fov_extract(outfile='/Users/tomrink/fovs_out.h5'):
                 k_idxs = k_idxs[0:len(k_idxs)]
             num_ice += len(k_idxs)
 
-            for ds_name in train_params_day:
+            for ds_name in train_params:
                 dat = f[ds_name][i, 10:30, 10:30].flatten()
                 icing_data_dct[ds_name].append(dat[k_idxs])
 
@@ -858,7 +858,7 @@ def fov_extract(outfile='/Users/tomrink/fovs_out.h5'):
 
         print(fname)
 
-    for ds_name in train_params_day:
+    for ds_name in train_params:
         lst = icing_data_dct[ds_name]
         icing_data_dct[ds_name] = np.concatenate(lst)
 
@@ -900,13 +900,13 @@ def fov_extract(outfile='/Users/tomrink/fovs_out.h5'):
             no_ice_lons.append(np.full(len(k_idxs), lons[i]))
             no_ice_lats.append(np.full(len(k_idxs), lats[i]))
 
-            for ds_name in train_params_day:
+            for ds_name in train_params:
                 dat = f[ds_name][i, 10:30, 10:30].flatten()
                 no_icing_data_dct[ds_name].append(dat[k_idxs])
 
         print(fname)
 
-    for ds_name in train_params_day:
+    for ds_name in train_params:
         lst = no_icing_data_dct[ds_name]
         no_icing_data_dct[ds_name] = np.concatenate(lst)
     no_icing_int_s = np.full(num_no_ice, -1)
@@ -920,14 +920,14 @@ def fov_extract(outfile='/Users/tomrink/fovs_out.h5'):
     icing_lats = np.concatenate([ice_lats, no_ice_lats])
 
     data_dct = {}
-    for ds_name in train_params_day:
+    for ds_name in train_params:
         data_dct[ds_name] = np.concatenate([icing_data_dct[ds_name], no_icing_data_dct[ds_name]])
 
     # apply shuffle indexes
     ds_indexes = np.arange(num_ice + num_no_ice)
     np.random.shuffle(ds_indexes)
 
-    for ds_name in train_params_day:
+    for ds_name in train_params:
         data_dct[ds_name] = data_dct[ds_name][ds_indexes]
     icing_intensity = icing_intensity[ds_indexes]
     icing_times = icing_times[ds_indexes]
@@ -937,7 +937,7 @@ def fov_extract(outfile='/Users/tomrink/fovs_out.h5'):
     h5f_expl = h5py.File(a_clvr_file, 'r')
     h5f_out = h5py.File(outfile, 'w')
 
-    for idx, ds_name in enumerate(train_params_day):
+    for idx, ds_name in enumerate(train_params):
         dt = ds_types[ds_list.index(ds_name)]
         data = data_dct[ds_name]
         h5f_out.create_dataset(ds_name, data=data, dtype=dt)
@@ -958,11 +958,17 @@ def fov_extract(outfile='/Users/tomrink/fovs_out.h5'):
     lat_ds.attrs.create('long_name', data='PIREP latitude')
 
     # copy relevant attributes
-    for ds_name in train_params_day:
+    for ds_name in train_params:
         h5f_ds = h5f_out[ds_name]
         h5f_ds.attrs.create('standard_name', data=h5f_expl[ds_name].attrs.get('standard_name'))
         h5f_ds.attrs.create('long_name', data=h5f_expl[ds_name].attrs.get('long_name'))
         h5f_ds.attrs.create('units', data=h5f_expl[ds_name].attrs.get('units'))
+        attr = h5f_expl[ds_name].attrs.get('actual_range')
+        if attr is not None:
+            h5f_ds.attrs.create('actual_range', data=attr)
+        attr = h5f_expl[ds_name].attrs.get('flag_values')
+        if attr is not None:
+            h5f_ds.attrs.create('flag_values', data=attr)
 
     # --- close files
     for h5f in h5_s_icing:
@@ -975,7 +981,7 @@ def fov_extract(outfile='/Users/tomrink/fovs_out.h5'):
     h5f_expl.close()
 
 
-def box_extract(outfile='/Users/tomrink/box_out.h5'):
+def box_extract(outfile='/Users/tomrink/box_out.h5', train_params=train_params_day):
     icing_int_s = []
     ice_time_s = []
     no_ice_time_s = []
@@ -984,12 +990,11 @@ def box_extract(outfile='/Users/tomrink/box_out.h5'):
     ice_lat_s = []
     no_ice_lat_s = []
 
-
     h5_s_icing = []
     h5_s_no_icing = []
 
-    icing_data_dct = {ds: [] for ds in train_params_day}
-    no_icing_data_dct = {ds: [] for ds in train_params_day}
+    icing_data_dct = {ds: [] for ds in train_params}
+    no_icing_data_dct = {ds: [] for ds in train_params}
 
     for fidx in range(len(icing_files)):
         fname = icing_files[fidx]
@@ -1004,7 +1009,7 @@ def box_extract(outfile='/Users/tomrink/box_out.h5'):
         icing_int = f['icing_intensity'][:]
 
         for i in range(num_obs):
-            for ds_name in train_params_day:
+            for ds_name in train_params:
                 dat = f[ds_name][i, 12:28, 12:28]
                 icing_data_dct[ds_name].append(dat)
             icing_int_s.append(icing_int[i])
@@ -1014,7 +1019,7 @@ def box_extract(outfile='/Users/tomrink/box_out.h5'):
 
         print(fname)
 
-    for ds_name in train_params_day:
+    for ds_name in train_params:
         lst = icing_data_dct[ds_name]
         icing_data_dct[ds_name] = np.stack(lst, axis=0)
     icing_int_s = np.array(icing_int_s)
@@ -1035,7 +1040,7 @@ def box_extract(outfile='/Users/tomrink/box_out.h5'):
         lats = f['latitude']
 
         for i in range(num_obs):
-            for ds_name in train_params_day:
+            for ds_name in train_params:
                 dat = f[ds_name][i, 12:28, 12:28]
                 no_icing_data_dct[ds_name].append(dat)
             num_no_ice += 1
@@ -1045,7 +1050,7 @@ def box_extract(outfile='/Users/tomrink/box_out.h5'):
 
         print(fname)
 
-    for ds_name in train_params_day:
+    for ds_name in train_params:
         lst = no_icing_data_dct[ds_name]
         no_icing_data_dct[ds_name] = np.stack(lst, axis=0)
     no_icing_int_s = np.full(num_no_ice, -1)
@@ -1059,14 +1064,14 @@ def box_extract(outfile='/Users/tomrink/box_out.h5'):
     icing_lats = np.concatenate([ice_lat_s, no_ice_lat_s])
 
     data_dct = {}
-    for ds_name in train_params_day:
+    for ds_name in train_params:
         data_dct[ds_name] = np.concatenate([icing_data_dct[ds_name], no_icing_data_dct[ds_name]])
 
     # Do shuffle
     ds_indexes = np.arange(num_ice + num_no_ice)
     np.random.shuffle(ds_indexes)
 
-    for ds_name in train_params_day:
+    for ds_name in train_params:
         data_dct[ds_name] = data_dct[ds_name][ds_indexes]
     icing_intensity = icing_intensity[ds_indexes]
     icing_times = icing_times[ds_indexes]
@@ -1076,7 +1081,7 @@ def box_extract(outfile='/Users/tomrink/box_out.h5'):
     h5f_expl = h5py.File(a_clvr_file, 'r')
     h5f_out = h5py.File(outfile, 'w')
 
-    for idx, ds_name in enumerate(train_params_day):
+    for idx, ds_name in enumerate(train_params):
         dt = ds_types[ds_list.index(ds_name)]
         data = data_dct[ds_name]
         h5f_out.create_dataset(ds_name, data=data, dtype=dt)
@@ -1097,11 +1102,17 @@ def box_extract(outfile='/Users/tomrink/box_out.h5'):
     lat_ds.attrs.create('long_name', data='PIREP latitude')
 
     # copy relevant attributes
-    for ds_name in train_params_day:
+    for ds_name in train_params:
         h5f_ds = h5f_out[ds_name]
         h5f_ds.attrs.create('standard_name', data=h5f_expl[ds_name].attrs.get('standard_name'))
         h5f_ds.attrs.create('long_name', data=h5f_expl[ds_name].attrs.get('long_name'))
         h5f_ds.attrs.create('units', data=h5f_expl[ds_name].attrs.get('units'))
+        attr = h5f_expl[ds_name].attrs.get('actual_range')
+        if attr is not None:
+            h5f_ds.attrs.create('actual_range', data=attr)
+        attr = h5f_expl[ds_name].attrs.get('flag_values')
+        if attr is not None:
+            h5f_ds.attrs.create('flag_values', data=attr)
 
     # --- close files
     for h5f in h5_s_icing: