Skip to content
Snippets Groups Projects
Commit 174f72c0 authored by tomrink's avatar tomrink
Browse files

snapshot...

parent ee28eefa
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
modules/util/viirs_l1b_l2.py
+ 8
− 76
View file @ 174f72c0
import numpy as np
import h5py
from util.util import get_grid_values, get_grid_values_all, is_night, is_day, compute_lwc_iwc, get_fill_attrs
from util.util import get_grid_values, get_grid_values_all, is_night, is_day, compute_lwc_iwc
import glob
import os
from aeolus.datasource import CLAVRx_VIIRS
from icing.moon_phase import *
from pathlib import Path
# --- CLAVRx Radiometric parameters and metadata ------------------------------------------------
l1b_ds_list = ['temp_10_4um_nom', 'temp_11_0um_nom', 'temp_12_0um_nom', 'temp_13_3um_nom', 'temp_3_75um_nom',
'temp_6_2um_nom', 'temp_6_7um_nom', 'temp_7_3um_nom', 'temp_8_5um_nom', 'temp_9_7um_nom',
'refl_0_47um_nom', 'refl_0_65um_nom', 'refl_0_86um_nom', 'refl_1_38um_nom', 'refl_1_60um_nom']
l1b_ds_types = {ds: 'f4' for ds in l1b_ds_list}
l1b_ds_fill = {l1b_ds_list[i]: -32767 for i in range(10)}
l1b_ds_fill.update({l1b_ds_list[i+10]: -32768 for i in range(5)})
l1b_ds_range = {ds: 'actual_range' for ds in l1b_ds_list}
# --- CLAVRx L2 parameters and metadata
ds_list = ['cld_height_acha', 'cld_geo_thick', 'cld_press_acha', 'sensor_zenith_angle', 'supercooled_prob_acha',
'supercooled_cloud_fraction', 'cld_temp_acha', 'cld_opd_acha', 'solar_zenith_angle',
'cld_reff_acha', 'cld_reff_dcomp', 'cld_reff_dcomp_1', 'cld_reff_dcomp_2', 'cld_reff_dcomp_3',
'cld_opd_dcomp', 'cld_opd_dcomp_1', 'cld_opd_dcomp_2', 'cld_opd_dcomp_3', 'cld_cwp_dcomp', 'iwc_dcomp',
'lwc_dcomp', 'cld_emiss_acha', 'conv_cloud_fraction', 'cloud_type', 'cloud_phase', 'cloud_mask']
ds_types = {ds_list[i]: 'f4' for i in range(23)}
ds_types.update({ds_list[i+23]: 'i1' for i in range(3)})
ds_fill = {ds_list[i]: -32768 for i in range(23)}
ds_fill.update({ds_list[i+23]: -128 for i in range(3)})
ds_range = {ds_list[i]: 'actual_range' for i in range(23)}
ds_range.update({ds_list[i]: None for i in range(3)})
ds_types.update(l1b_ds_types)
ds_fill.update(l1b_ds_fill)
ds_range.update(l1b_ds_range)
ds_types.update({'temp_3_9um_nom': 'f4'})
ds_types.update({'cloud_fraction': 'f4'})
ds_fill.update({'temp_3_9um_nom': -32767})
ds_fill.update({'cloud_fraction': -32768})
ds_range.update({'temp_3_9um_nom': 'actual_range'})
ds_range.update({'cloud_fraction': 'actual_range'})
emis_params = ['temp_10_4um_nom', 'temp_11_0um_nom', 'temp_12_0um_nom', 'temp_13_3um_nom', 'temp_3_9um_nom',
'temp_6_7um_nom']
# refl_params = ['refl_0_47um_nom', 'refl_0_65um_nom', 'refl_0_86um_nom', 'refl_1_38um_nom', 'refl_1_60um_nom']
# data_params = refl_params + emis_params
# data_params = emis_params
#target_param = 'cloud_probability'
# target_param = 'cloud_probability'
target_param = 'cld_opd_dcomp'
# group_name = ''
group_name = 'super/'
# l2_params = [group_name+'temp_11_0um_nom', group_name+'temp_12_0um_nom', group_name+'refl_0_65um_nom', group_name+target_param]
l2_params = [group_name+'temp_11_0um', group_name+'temp_12_0um', group_name+'refl_0_65um', group_name+target_param]
# l2_params = [group_name+'temp_11_0um_nom', group_name+'refl_0_65um_nom', group_name+target_param]
l2_params = [group_name+'temp_11_0um', group_name+'refl_0_65um', group_name+target_param]
# solzen_name = group_name + 'solar_zenith_angle'
solzen_name = group_name + 'solar_zenith'
......@@ -130,7 +87,7 @@ def run_all(directory, out_directory, day_night='ANY', start=10):
cnt = start
total_num_train_samples = 0
total_num_valid_samples = 0
num_keep_x_tiles = 8
num_keep_x_tiles = 12
# pattern = 'clavrx_VNP02MOD*.highres.nc.level2.nc'
pattern = 'clavrx_*.nc'
......@@ -138,8 +95,6 @@ def run_all(directory, out_directory, day_night='ANY', start=10):
data_files = glob.glob(path, recursive=True)
label_valid_tiles = []
label_train_tiles = []
data_valid_tiles = []
data_train_tiles = []
f_cnt = 0
......@@ -151,38 +106,22 @@ def run_all(directory, out_directory, day_night='ANY', start=10):
for idx, data_f in enumerate(data_files):
# if idx % 4 == 0: # if we want to skip some files
if True:
# w_o_ext, ext = os.path.splitext(data_f)
# pname, fname = os.path.split(data_f)
# toks = fname.split('.')
# label_f = pname + '/' + 'clavrx_VNP02MOD.' + toks[1]+'.'+toks[2]+'.'+toks[3]+'.'+toks[4]+'.'+'uwssec.highres.nc.level2.nc'
# if not os.path.exists(label_f):
# continue
try:
data_h5f = h5py.File(data_f, 'r')
except:
print('cant open file: ', data_f)
continue
# try:
# label_h5f = h5py.File(label_f, 'r')
# except:
# print('cant open file: ', label_f)
# data_h5f.close()
# continue
try:
run(data_h5f, data_params, data_train_tiles, data_valid_tiles, num_keep_x_tiles=num_keep_x_tiles, tile_width=128, kernel_size=11, day_night=day_night)
except Exception as e:
print(e)
data_h5f.close()
# label_h5f.close()
continue
print(data_f)
f_cnt += 1
data_h5f.close()
# label_h5f.close()
if len(data_train_tiles) == 0:
continue
......@@ -190,20 +129,14 @@ def run_all(directory, out_directory, day_night='ANY', start=10):
if (f_cnt % 5) == 0:
num_valid_samples = 0
if len(data_valid_tiles) > 0:
# label_valid = np.stack(label_valid_tiles)
data_valid = np.stack(data_valid_tiles)
np.save(out_directory + 'data_valid_' + str(cnt), data_valid)
# np.save(out_directory+'label_valid_' + str(cnt), label_valid)
num_valid_samples = data_valid.shape[0]
# label_train = np.stack(label_train_tiles)
# np.save(out_directory+'label_train_' + str(cnt), label_train)
data_train = np.stack(data_train_tiles)
np.save(out_directory+'data_train_' + str(cnt), data_train)
num_train_samples = data_train.shape[0]
label_valid_tiles = []
label_train_tiles = []
data_valid_tiles = []
data_train_tiles = []
......@@ -233,9 +166,8 @@ def run(data_h5f, param_s, train_tiles, valid_tiles, num_keep_x_tiles=8, tile_wi
grd_s = []
for param in param_s:
fill_value, fill_value_name = get_fill_attrs(param)
try:
grd = get_grid_values(data_h5f, param, 0, 0, None, num_lines, num_pixels, fill_value_name=fill_value_name, fill_value=fill_value)
grd = get_grid_values(data_h5f, param, 0, 0, None, num_lines, num_pixels)
grd_s.append(grd)
except Exception as e:
print(e)
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment