snapshot...

modules/icing/pirep_goes.py

@@ -831,7 +831,7 @@ def apply_qc_no_icing_pireps(icing_alt, cld_top_hgt, cld_phase, cld_opd, cld_mas
     return mask, idxs, num_tested
 
 
-def fov_extract(outfile='/home/rink/fovs_l1b_out.h5', train_params=l1b_ds_list, ds_types=l1b_ds_types):
+def fov_extract(outfile='/home/rink/fovs_l1b_out.h5', L1B_or_L2='L1B'):
     ice_times = []
     icing_int_s = []
     ice_lons = []
@@ -844,8 +844,15 @@ def fov_extract(outfile='/home/rink/fovs_l1b_out.h5', train_params=l1b_ds_list,
     h5_s_icing = []
     h5_s_no_icing = []
 
-    icing_data_dct = {ds: [] for ds in train_params}
-    no_icing_data_dct = {ds: [] for ds in train_params}
+    if L1B_or_L2 == 'L1B':
+        params = l1b_ds_list
+        param_types = l1b_ds_types
+    elif L1B_or_L2 == 'L2':
+        params = ds_list
+        param_types = ds_types
+
+    icing_data_dct = {ds: [] for ds in params}
+    no_icing_data_dct = {ds: [] for ds in params}
 
     sub_indexes = np.arange(400)
 
@@ -881,7 +888,7 @@ def fov_extract(outfile='/home/rink/fovs_l1b_out.h5', train_params=l1b_ds_list,
                 k_idxs = k_idxs[0:len(k_idxs)]
             num_ice += len(k_idxs)
 
-            for ds_name in train_params:
+            for ds_name in params:
                 dat = f[ds_name][i, 10:30, 10:30].flatten()
                 icing_data_dct[ds_name].append(dat[k_idxs])
 
@@ -892,7 +899,7 @@ def fov_extract(outfile='/home/rink/fovs_l1b_out.h5', train_params=l1b_ds_list,
 
         print(fname)
 
-    for ds_name in train_params:
+    for ds_name in params:
         lst = icing_data_dct[ds_name]
         icing_data_dct[ds_name] = np.concatenate(lst)
 
@@ -934,13 +941,13 @@ def fov_extract(outfile='/home/rink/fovs_l1b_out.h5', train_params=l1b_ds_list,
             no_ice_lons.append(np.full(len(k_idxs), lons[i]))
             no_ice_lats.append(np.full(len(k_idxs), lats[i]))
 
-            for ds_name in train_params:
+            for ds_name in params:
                 dat = f[ds_name][i, 10:30, 10:30].flatten()
                 no_icing_data_dct[ds_name].append(dat[k_idxs])
 
         print(fname)
 
-    for ds_name in train_params:
+    for ds_name in params:
         lst = no_icing_data_dct[ds_name]
         no_icing_data_dct[ds_name] = np.concatenate(lst)
     no_icing_int_s = np.full(num_no_ice, -1)
@@ -954,7 +961,7 @@ def fov_extract(outfile='/home/rink/fovs_l1b_out.h5', train_params=l1b_ds_list,
     icing_lats = np.concatenate([ice_lats, no_ice_lats])
 
     data_dct = {}
-    for ds_name in train_params:
+    for ds_name in params:
         data_dct[ds_name] = np.concatenate([icing_data_dct[ds_name], no_icing_data_dct[ds_name]])
 
     # apply shuffle indexes
@@ -970,7 +977,7 @@ def fov_extract(outfile='/home/rink/fovs_l1b_out.h5', train_params=l1b_ds_list,
 
     # do sort
     ds_indexes = np.argsort(icing_times)
-    for ds_name in train_params:
+    for ds_name in params:
         data_dct[ds_name] = data_dct[ds_name][ds_indexes]
     icing_intensity = icing_intensity[ds_indexes]
     icing_times = icing_times[ds_indexes]
@@ -980,8 +987,8 @@ def fov_extract(outfile='/home/rink/fovs_l1b_out.h5', train_params=l1b_ds_list,
     h5f_expl = h5py.File(a_clvr_file, 'r')
     h5f_out = h5py.File(outfile, 'w')
 
-    for idx, ds_name in enumerate(train_params):
-        dt = ds_types[ds_list.index(ds_name)]
+    for idx, ds_name in enumerate(params):
+        dt = param_types[params.index(ds_name)]
         data = data_dct[ds_name]
         h5f_out.create_dataset(ds_name, data=data, dtype=dt)
 
@@ -1001,7 +1008,7 @@ def fov_extract(outfile='/home/rink/fovs_l1b_out.h5', train_params=l1b_ds_list,
     lat_ds.attrs.create('long_name', data='PIREP latitude')
 
     # copy relevant attributes
-    for ds_name in train_params:
+    for ds_name in params:
         h5f_ds = h5f_out[ds_name]
         h5f_ds.attrs.create('standard_name', data=h5f_expl[ds_name].attrs.get('standard_name'))
         h5f_ds.attrs.create('long_name', data=h5f_expl[ds_name].attrs.get('long_name'))
@@ -1024,7 +1031,7 @@ def fov_extract(outfile='/home/rink/fovs_l1b_out.h5', train_params=l1b_ds_list,
     h5f_expl.close()
 
 
-def tile_extract(trnfile='/home/rink/tiles_l1b_train.h5', tstfile='/home/rink/tiles_l1b_test.h5', l1B_or_l2='L1B',
+def tile_extract(trnfile='/home/rink/tiles_l1b_train.h5', tstfile='/home/rink/tiles_l1b_test.h5', L1B_or_L2='L1B',
                  cld_mask_name='cloud_mask', augment=False, split=0.2):
     icing_int_s = []
     ice_time_s = []
@@ -1037,10 +1044,10 @@ def tile_extract(trnfile='/home/rink/tiles_l1b_train.h5', tstfile='/home/rink/ti
     h5_s_icing = []
     h5_s_no_icing = []
 
-    if l1B_or_l2 == 'L1B':
+    if L1B_or_L2 == 'L1B':
         params = l1b_ds_list
         param_types = l1b_ds_types
-    elif l1B_or_l2 == 'L2':
+    elif L1B_or_L2 == 'L2':
         params = ds_list
         param_types = ds_types
 
@@ -1063,7 +1070,7 @@ def tile_extract(trnfile='/home/rink/tiles_l1b_train.h5', tstfile='/home/rink/ti
             cld_msk = f[cld_mask_name][i, 12:28, 12:28]
             for ds_name in params:
                 dat = f[ds_name][i, 12:28, 12:28]
-                if l1B_or_l2 == 'L2':
+                if L1B_or_L2 == 'L2':
                     keep = np.logical_or(cld_msk == 2, cld_msk == 3)  # cloudy
                     np.where(keep, dat, np.nan)
                 icing_data_dct[ds_name].append(dat)
@@ -1100,7 +1107,7 @@ def tile_extract(trnfile='/home/rink/tiles_l1b_train.h5', tstfile='/home/rink/ti
             cld_msk = f[cld_mask_name][i, 12:28, 12:28]
             for ds_name in params:
                 dat = f[ds_name][i, 12:28, 12:28]
-                if l1B_or_l2 == 'L2':
+                if L1B_or_L2 == 'L2':
                     keep = np.logical_or(cld_msk == 2, cld_msk == 3)  # cloudy
                     np.where(keep, dat, np.nan)
                 no_icing_data_dct[ds_name].append(dat)