snapshot...

modules/util/viirs_l1b_l2.py

 import numpy as np
 import h5py
-from util.util import get_grid_values, get_grid_values_all, is_night, is_day, compute_lwc_iwc, get_fill_attrs
+from util.util import get_grid_values, get_grid_values_all, is_night, is_day, compute_lwc_iwc
 import glob
-import os
 from aeolus.datasource import CLAVRx_VIIRS
 from icing.moon_phase import *
-from pathlib import Path
-
-# --- CLAVRx Radiometric parameters and metadata ------------------------------------------------
-l1b_ds_list = ['temp_10_4um_nom', 'temp_11_0um_nom', 'temp_12_0um_nom', 'temp_13_3um_nom', 'temp_3_75um_nom',
-               'temp_6_2um_nom', 'temp_6_7um_nom', 'temp_7_3um_nom', 'temp_8_5um_nom', 'temp_9_7um_nom',
-               'refl_0_47um_nom', 'refl_0_65um_nom', 'refl_0_86um_nom', 'refl_1_38um_nom', 'refl_1_60um_nom']
-
-l1b_ds_types = {ds: 'f4' for ds in l1b_ds_list}
-l1b_ds_fill = {l1b_ds_list[i]: -32767 for i in range(10)}
-l1b_ds_fill.update({l1b_ds_list[i+10]: -32768 for i in range(5)})
-l1b_ds_range = {ds: 'actual_range' for ds in l1b_ds_list}
-
-# --- CLAVRx L2 parameters and metadata
-ds_list = ['cld_height_acha', 'cld_geo_thick', 'cld_press_acha', 'sensor_zenith_angle', 'supercooled_prob_acha',
-           'supercooled_cloud_fraction', 'cld_temp_acha', 'cld_opd_acha', 'solar_zenith_angle',
-           'cld_reff_acha', 'cld_reff_dcomp', 'cld_reff_dcomp_1', 'cld_reff_dcomp_2', 'cld_reff_dcomp_3',
-           'cld_opd_dcomp', 'cld_opd_dcomp_1', 'cld_opd_dcomp_2', 'cld_opd_dcomp_3', 'cld_cwp_dcomp', 'iwc_dcomp',
-           'lwc_dcomp', 'cld_emiss_acha', 'conv_cloud_fraction', 'cloud_type', 'cloud_phase', 'cloud_mask']
-
-ds_types = {ds_list[i]: 'f4' for i in range(23)}
-ds_types.update({ds_list[i+23]: 'i1' for i in range(3)})
-ds_fill = {ds_list[i]: -32768 for i in range(23)}
-ds_fill.update({ds_list[i+23]: -128 for i in range(3)})
-ds_range = {ds_list[i]: 'actual_range' for i in range(23)}
-ds_range.update({ds_list[i]: None for i in range(3)})
-
-ds_types.update(l1b_ds_types)
-ds_fill.update(l1b_ds_fill)
-ds_range.update(l1b_ds_range)
-
-ds_types.update({'temp_3_9um_nom': 'f4'})
-ds_types.update({'cloud_fraction': 'f4'})
-ds_fill.update({'temp_3_9um_nom': -32767})
-ds_fill.update({'cloud_fraction': -32768})
-ds_range.update({'temp_3_9um_nom': 'actual_range'})
-ds_range.update({'cloud_fraction': 'actual_range'})
-
-
-emis_params = ['temp_10_4um_nom', 'temp_11_0um_nom', 'temp_12_0um_nom', 'temp_13_3um_nom', 'temp_3_9um_nom',
-               'temp_6_7um_nom']
-# refl_params = ['refl_0_47um_nom', 'refl_0_65um_nom', 'refl_0_86um_nom', 'refl_1_38um_nom', 'refl_1_60um_nom']
-# data_params = refl_params + emis_params
-# data_params = emis_params
-
-#target_param = 'cloud_probability'
+
+
+# target_param = 'cloud_probability'
 target_param = 'cld_opd_dcomp'
 
 # group_name = ''
 group_name = 'super/'
-# l2_params = [group_name+'temp_11_0um_nom', group_name+'temp_12_0um_nom', group_name+'refl_0_65um_nom', group_name+target_param]
-l2_params = [group_name+'temp_11_0um', group_name+'temp_12_0um', group_name+'refl_0_65um', group_name+target_param]
+# l2_params = [group_name+'temp_11_0um_nom', group_name+'refl_0_65um_nom', group_name+target_param]
+l2_params = [group_name+'temp_11_0um', group_name+'refl_0_65um', group_name+target_param]
 
 # solzen_name = group_name + 'solar_zenith_angle'
 solzen_name = group_name + 'solar_zenith'
@@ -130,7 +87,7 @@ def run_all(directory, out_directory, day_night='ANY', start=10):
     cnt = start
     total_num_train_samples = 0
     total_num_valid_samples = 0
-    num_keep_x_tiles = 8
+    num_keep_x_tiles = 12
 
     # pattern = 'clavrx_VNP02MOD*.highres.nc.level2.nc'
     pattern = 'clavrx_*.nc'
@@ -138,8 +95,6 @@ def run_all(directory, out_directory, day_night='ANY', start=10):
 
     data_files = glob.glob(path, recursive=True)
 
-    label_valid_tiles = []
-    label_train_tiles = []
     data_valid_tiles = []
     data_train_tiles = []
     f_cnt = 0
@@ -151,38 +106,22 @@ def run_all(directory, out_directory, day_night='ANY', start=10):
     for idx, data_f in enumerate(data_files):
         # if idx % 4 == 0:  # if we want to skip some files
         if True:
-            # w_o_ext, ext = os.path.splitext(data_f)
-            # pname, fname = os.path.split(data_f)
-            # toks = fname.split('.')
-            # label_f = pname + '/' + 'clavrx_VNP02MOD.' + toks[1]+'.'+toks[2]+'.'+toks[3]+'.'+toks[4]+'.'+'uwssec.highres.nc.level2.nc'
-            # if not os.path.exists(label_f):
-            #     continue
-
             try:
                 data_h5f = h5py.File(data_f, 'r')
             except:
                 print('cant open file: ', data_f)
                 continue
 
-            # try:
-            #     label_h5f = h5py.File(label_f, 'r')
-            # except:
-            #     print('cant open file: ', label_f)
-            #     data_h5f.close()
-            #     continue
-
             try:
                 run(data_h5f, data_params, data_train_tiles, data_valid_tiles, num_keep_x_tiles=num_keep_x_tiles, tile_width=128, kernel_size=11, day_night=day_night)
             except Exception as e:
                 print(e)
                 data_h5f.close()
-                # label_h5f.close()
                 continue
             print(data_f)
             f_cnt += 1
 
             data_h5f.close()
-            # label_h5f.close()
 
             if len(data_train_tiles) == 0:
                 continue
@@ -190,20 +129,14 @@ def run_all(directory, out_directory, day_night='ANY', start=10):
             if (f_cnt % 5) == 0:
                 num_valid_samples = 0
                 if len(data_valid_tiles) > 0:
-                    # label_valid = np.stack(label_valid_tiles)
                     data_valid = np.stack(data_valid_tiles)
                     np.save(out_directory + 'data_valid_' + str(cnt), data_valid)
-                    # np.save(out_directory+'label_valid_' + str(cnt), label_valid)
                     num_valid_samples = data_valid.shape[0]
 
-                # label_train = np.stack(label_train_tiles)
-                # np.save(out_directory+'label_train_' + str(cnt), label_train)
                 data_train = np.stack(data_train_tiles)
                 np.save(out_directory+'data_train_' + str(cnt), data_train)
                 num_train_samples = data_train.shape[0]
 
-                label_valid_tiles = []
-                label_train_tiles = []
                 data_valid_tiles = []
                 data_train_tiles = []
 
@@ -233,9 +166,8 @@ def run(data_h5f, param_s, train_tiles, valid_tiles, num_keep_x_tiles=8, tile_wi
 
     grd_s = []
     for param in param_s:
-        fill_value, fill_value_name = get_fill_attrs(param)
         try:
-            grd = get_grid_values(data_h5f, param, 0, 0, None, num_lines, num_pixels, fill_value_name=fill_value_name, fill_value=fill_value)
+            grd = get_grid_values(data_h5f, param, 0, 0, None, num_lines, num_pixels)
             grd_s.append(grd)
         except Exception as e:
             print(e)