snapshot...

modules/icing/pirep_goes.py

@@ -49,7 +49,7 @@ no_icing_files = ['no_icing_2018010600_2018033022_DAY.h5', 'no_icing_2018040100_
 no_icing_l1b_files = []
 
 train_params_day = ['cld_height_acha', 'cld_geo_thick', 'supercooled_cloud_fraction', 'cld_temp_acha', 'solar_zenith_angle',
-                    'cld_reff_dcomp', 'cld_opd_dcomp', 'iwc_dcomp', 'lwc_dcomp', 'cloud_phase']
+                    'cld_reff_dcomp', 'cld_opd_dcomp', 'iwc_dcomp', 'lwc_dcomp', 'cloud_phase', 'cloud_mask']
 
 
 def setup():
@@ -917,6 +917,103 @@ def fov_extract(outfile='/Users/tomrink/fovs_out.h5'):
     icing_int_ds = h5f_out.create_dataset('icing_intensity', data=icing_intensity, dtype='i4')
     icing_int_ds.attrs.create('long_name', data='From PIREP. -1:No Icing, 1:Trace, 2:Light, 3:Light Moderate, 4:Moderate, 5:Moderate Severe, 6:Severe')
 
+    # copy relevant attributes
+    for ds_name in train_params_day:
+        h5f_ds = h5f_out[ds_name]
+        h5f_ds.attrs.create('standard_name', data=h5f_expl[ds_name].attrs.get('standard_name'))
+        h5f_ds.attrs.create('long_name', data=h5f_expl[ds_name].attrs.get('long_name'))
+        h5f_ds.attrs.create('units', data=h5f_expl[ds_name].attrs.get('units'))
+
+    # --- close files
+    for h5f in h5_s_icing:
+        h5f.close()
+
+    for h5f in h5_s_no_icing:
+        h5f.close()
+
+    h5f_out.close()
+    h5f_expl.close()
+
+
+def box_extract(outfile='/Users/tomrink/box_out.h5'):
+    icing_int_s = []
+
+    h5_s_icing = []
+    h5_s_no_icing = []
+
+    icing_data_dct = {ds: [] for ds in train_params_day}
+    no_icing_data_dct = {ds: [] for ds in train_params_day}
+
+    num_ice = 0
+    for fidx in range(len(icing_files)):
+        fname = icing_files[fidx]
+        f = h5py.File(data_dir+fname, 'r')
+        h5_s_icing.append(f)
+
+        times = f['time'][:]
+        num_obs = len(times)
+
+        icing_int = f['icing_intensity'][:]
+
+        for i in range(num_obs):
+            for ds_name in train_params_day:
+                dat = f[ds_name][i, 12:28, 12:28]
+                icing_data_dct[ds_name].append(dat)
+            icing_int_s.append(icing_int[i])
+
+        print(fname)
+
+    for ds_name in train_params_day:
+        lst = icing_data_dct[ds_name]
+        icing_data_dct[ds_name] = np.stack(lst, axis=0)
+    icing_int_s = np.array(icing_int_s)
+
+    num_no_ice = 0
+    for fidx in range(len(no_icing_files)):
+        fname = no_icing_files[fidx]
+        f = h5py.File(data_dir+fname, 'r')
+        h5_s_no_icing.append(f)
+
+        times = f['time']
+        num_obs = len(times)
+
+        for i in range(num_obs):
+            for ds_name in train_params_day:
+                dat = f[ds_name][i, 12:28, 12:28]
+                no_icing_data_dct[ds_name].append(dat)
+            num_no_ice += 1
+
+        print(fname)
+
+    for ds_name in train_params_day:
+        lst = no_icing_data_dct[ds_name]
+        no_icing_data_dct[ds_name] = np.stack(lst, axis=0)
+    no_icing_int_s = np.full(num_no_ice, -1)
+
+    ds_indexes = np.arange(num_ice + num_no_ice)
+    np.random.shuffle(ds_indexes)
+
+    icing_intensity = np.concatenate([icing_int_s, no_icing_int_s])
+
+    data_dct = {}
+    for ds_name in train_params_day:
+        data_dct[ds_name] = np.concatenate([icing_data_dct[ds_name], no_icing_data_dct[ds_name]])
+
+    for ds_name in train_params_day:
+        data_dct[ds_name] = data_dct[ds_name][ds_indexes]
+    icing_intensity = icing_intensity[ds_indexes]
+
+    h5f_expl = h5py.File(a_clvr_file, 'r')
+    h5f_out = h5py.File(outfile, 'w')
+
+    for idx, ds_name in enumerate(train_params_day):
+        dt = ds_types[ds_list.index(ds_name)]
+        data = data_dct[ds_name]
+        h5f_out.create_dataset(ds_name, data=data, dtype=dt)
+
+    icing_int_ds = h5f_out.create_dataset('icing_intensity', data=icing_intensity, dtype='i4')
+    icing_int_ds.attrs.create('long_name', data='From PIREP. -1:No Icing, 1:Trace, 2:Light, 3:Light Moderate, 4:Moderate, 5:Moderate Severe, 6:Severe')
+
     # copy relevant attributes
     for ds_name in train_params_day:
         h5f_ds = h5f_out[ds_name]