diff --git a/modules/icing/pirep_goes.py b/modules/icing/pirep_goes.py index cbb1923a7c89d37f3123bf084ccdad46f3781825..a88a30666d9fdc77b30d3ec3bcfbe31c22dc112e 100644 --- a/modules/icing/pirep_goes.py +++ b/modules/icing/pirep_goes.py @@ -831,7 +831,7 @@ def apply_qc_no_icing_pireps(icing_alt, cld_top_hgt, cld_phase, cld_opd, cld_mas return mask, idxs, num_tested -def fov_extract(outfile='/home/rink/fovs_l1b_out.h5', train_params=l1b_ds_list, ds_types=l1b_ds_types): +def fov_extract(outfile='/home/rink/fovs_l1b_out.h5', L1B_or_L2='L1B'): ice_times = [] icing_int_s = [] ice_lons = [] @@ -844,8 +844,15 @@ def fov_extract(outfile='/home/rink/fovs_l1b_out.h5', train_params=l1b_ds_list, h5_s_icing = [] h5_s_no_icing = [] - icing_data_dct = {ds: [] for ds in train_params} - no_icing_data_dct = {ds: [] for ds in train_params} + if L1B_or_L2 == 'L1B': + params = l1b_ds_list + param_types = l1b_ds_types + elif L1B_or_L2 == 'L2': + params = ds_list + param_types = ds_types + + icing_data_dct = {ds: [] for ds in params} + no_icing_data_dct = {ds: [] for ds in params} sub_indexes = np.arange(400) @@ -881,7 +888,7 @@ def fov_extract(outfile='/home/rink/fovs_l1b_out.h5', train_params=l1b_ds_list, k_idxs = k_idxs[0:len(k_idxs)] num_ice += len(k_idxs) - for ds_name in train_params: + for ds_name in params: dat = f[ds_name][i, 10:30, 10:30].flatten() icing_data_dct[ds_name].append(dat[k_idxs]) @@ -892,7 +899,7 @@ def fov_extract(outfile='/home/rink/fovs_l1b_out.h5', train_params=l1b_ds_list, print(fname) - for ds_name in train_params: + for ds_name in params: lst = icing_data_dct[ds_name] icing_data_dct[ds_name] = np.concatenate(lst) @@ -934,13 +941,13 @@ def fov_extract(outfile='/home/rink/fovs_l1b_out.h5', train_params=l1b_ds_list, no_ice_lons.append(np.full(len(k_idxs), lons[i])) no_ice_lats.append(np.full(len(k_idxs), lats[i])) - for ds_name in train_params: + for ds_name in params: dat = f[ds_name][i, 10:30, 10:30].flatten() no_icing_data_dct[ds_name].append(dat[k_idxs]) print(fname) - for ds_name in train_params: + for ds_name in params: lst = no_icing_data_dct[ds_name] no_icing_data_dct[ds_name] = np.concatenate(lst) no_icing_int_s = np.full(num_no_ice, -1) @@ -954,7 +961,7 @@ def fov_extract(outfile='/home/rink/fovs_l1b_out.h5', train_params=l1b_ds_list, icing_lats = np.concatenate([ice_lats, no_ice_lats]) data_dct = {} - for ds_name in train_params: + for ds_name in params: data_dct[ds_name] = np.concatenate([icing_data_dct[ds_name], no_icing_data_dct[ds_name]]) # apply shuffle indexes @@ -970,7 +977,7 @@ def fov_extract(outfile='/home/rink/fovs_l1b_out.h5', train_params=l1b_ds_list, # do sort ds_indexes = np.argsort(icing_times) - for ds_name in train_params: + for ds_name in params: data_dct[ds_name] = data_dct[ds_name][ds_indexes] icing_intensity = icing_intensity[ds_indexes] icing_times = icing_times[ds_indexes] @@ -980,8 +987,8 @@ def fov_extract(outfile='/home/rink/fovs_l1b_out.h5', train_params=l1b_ds_list, h5f_expl = h5py.File(a_clvr_file, 'r') h5f_out = h5py.File(outfile, 'w') - for idx, ds_name in enumerate(train_params): - dt = ds_types[ds_list.index(ds_name)] + for idx, ds_name in enumerate(params): + dt = param_types[params.index(ds_name)] data = data_dct[ds_name] h5f_out.create_dataset(ds_name, data=data, dtype=dt) @@ -1001,7 +1008,7 @@ def fov_extract(outfile='/home/rink/fovs_l1b_out.h5', train_params=l1b_ds_list, lat_ds.attrs.create('long_name', data='PIREP latitude') # copy relevant attributes - for ds_name in train_params: + for ds_name in params: h5f_ds = h5f_out[ds_name] h5f_ds.attrs.create('standard_name', data=h5f_expl[ds_name].attrs.get('standard_name')) h5f_ds.attrs.create('long_name', data=h5f_expl[ds_name].attrs.get('long_name')) @@ -1024,7 +1031,7 @@ def fov_extract(outfile='/home/rink/fovs_l1b_out.h5', train_params=l1b_ds_list, h5f_expl.close() -def tile_extract(trnfile='/home/rink/tiles_l1b_train.h5', tstfile='/home/rink/tiles_l1b_test.h5', l1B_or_l2='L1B', +def tile_extract(trnfile='/home/rink/tiles_l1b_train.h5', tstfile='/home/rink/tiles_l1b_test.h5', L1B_or_L2='L1B', cld_mask_name='cloud_mask', augment=False, split=0.2): icing_int_s = [] ice_time_s = [] @@ -1037,10 +1044,10 @@ def tile_extract(trnfile='/home/rink/tiles_l1b_train.h5', tstfile='/home/rink/ti h5_s_icing = [] h5_s_no_icing = [] - if l1B_or_l2 == 'L1B': + if L1B_or_L2 == 'L1B': params = l1b_ds_list param_types = l1b_ds_types - elif l1B_or_l2 == 'L2': + elif L1B_or_L2 == 'L2': params = ds_list param_types = ds_types @@ -1063,7 +1070,7 @@ def tile_extract(trnfile='/home/rink/tiles_l1b_train.h5', tstfile='/home/rink/ti cld_msk = f[cld_mask_name][i, 12:28, 12:28] for ds_name in params: dat = f[ds_name][i, 12:28, 12:28] - if l1B_or_l2 == 'L2': + if L1B_or_L2 == 'L2': keep = np.logical_or(cld_msk == 2, cld_msk == 3) # cloudy np.where(keep, dat, np.nan) icing_data_dct[ds_name].append(dat) @@ -1100,7 +1107,7 @@ def tile_extract(trnfile='/home/rink/tiles_l1b_train.h5', tstfile='/home/rink/ti cld_msk = f[cld_mask_name][i, 12:28, 12:28] for ds_name in params: dat = f[ds_name][i, 12:28, 12:28] - if l1B_or_l2 == 'L2': + if L1B_or_L2 == 'L2': keep = np.logical_or(cld_msk == 2, cld_msk == 3) # cloudy np.where(keep, dat, np.nan) no_icing_data_dct[ds_name].append(dat)