From 174f72c010b6892fddf5960c7695837b703e375e Mon Sep 17 00:00:00 2001
From: tomrink <rink@ssec.wisc.edu>
Date: Thu, 2 Feb 2023 15:29:54 -0600
Subject: [PATCH] snapshot...
---
modules/util/viirs_l1b_l2.py | 84 ++++--------------------------------
1 file changed, 8 insertions(+), 76 deletions(-)
diff --git a/modules/util/viirs_l1b_l2.py b/modules/util/viirs_l1b_l2.py
index 31b35ca3..f0379b30 100644
--- a/modules/util/viirs_l1b_l2.py
+++ b/modules/util/viirs_l1b_l2.py
@@ -1,61 +1,18 @@
import numpy as np
import h5py
-from util.util import get_grid_values, get_grid_values_all, is_night, is_day, compute_lwc_iwc, get_fill_attrs
+from util.util import get_grid_values, get_grid_values_all, is_night, is_day, compute_lwc_iwc
import glob
-import os
from aeolus.datasource import CLAVRx_VIIRS
from icing.moon_phase import *
-from pathlib import Path
-
-# --- CLAVRx Radiometric parameters and metadata ------------------------------------------------
-l1b_ds_list = ['temp_10_4um_nom', 'temp_11_0um_nom', 'temp_12_0um_nom', 'temp_13_3um_nom', 'temp_3_75um_nom',
- 'temp_6_2um_nom', 'temp_6_7um_nom', 'temp_7_3um_nom', 'temp_8_5um_nom', 'temp_9_7um_nom',
- 'refl_0_47um_nom', 'refl_0_65um_nom', 'refl_0_86um_nom', 'refl_1_38um_nom', 'refl_1_60um_nom']
-
-l1b_ds_types = {ds: 'f4' for ds in l1b_ds_list}
-l1b_ds_fill = {l1b_ds_list[i]: -32767 for i in range(10)}
-l1b_ds_fill.update({l1b_ds_list[i+10]: -32768 for i in range(5)})
-l1b_ds_range = {ds: 'actual_range' for ds in l1b_ds_list}
-
-# --- CLAVRx L2 parameters and metadata
-ds_list = ['cld_height_acha', 'cld_geo_thick', 'cld_press_acha', 'sensor_zenith_angle', 'supercooled_prob_acha',
- 'supercooled_cloud_fraction', 'cld_temp_acha', 'cld_opd_acha', 'solar_zenith_angle',
- 'cld_reff_acha', 'cld_reff_dcomp', 'cld_reff_dcomp_1', 'cld_reff_dcomp_2', 'cld_reff_dcomp_3',
- 'cld_opd_dcomp', 'cld_opd_dcomp_1', 'cld_opd_dcomp_2', 'cld_opd_dcomp_3', 'cld_cwp_dcomp', 'iwc_dcomp',
- 'lwc_dcomp', 'cld_emiss_acha', 'conv_cloud_fraction', 'cloud_type', 'cloud_phase', 'cloud_mask']
-
-ds_types = {ds_list[i]: 'f4' for i in range(23)}
-ds_types.update({ds_list[i+23]: 'i1' for i in range(3)})
-ds_fill = {ds_list[i]: -32768 for i in range(23)}
-ds_fill.update({ds_list[i+23]: -128 for i in range(3)})
-ds_range = {ds_list[i]: 'actual_range' for i in range(23)}
-ds_range.update({ds_list[i]: None for i in range(3)})
-
-ds_types.update(l1b_ds_types)
-ds_fill.update(l1b_ds_fill)
-ds_range.update(l1b_ds_range)
-
-ds_types.update({'temp_3_9um_nom': 'f4'})
-ds_types.update({'cloud_fraction': 'f4'})
-ds_fill.update({'temp_3_9um_nom': -32767})
-ds_fill.update({'cloud_fraction': -32768})
-ds_range.update({'temp_3_9um_nom': 'actual_range'})
-ds_range.update({'cloud_fraction': 'actual_range'})
-
-
-emis_params = ['temp_10_4um_nom', 'temp_11_0um_nom', 'temp_12_0um_nom', 'temp_13_3um_nom', 'temp_3_9um_nom',
- 'temp_6_7um_nom']
-# refl_params = ['refl_0_47um_nom', 'refl_0_65um_nom', 'refl_0_86um_nom', 'refl_1_38um_nom', 'refl_1_60um_nom']
-# data_params = refl_params + emis_params
-# data_params = emis_params
-
-#target_param = 'cloud_probability'
+
+
+# target_param = 'cloud_probability'
target_param = 'cld_opd_dcomp'
# group_name = ''
group_name = 'super/'
-# l2_params = [group_name+'temp_11_0um_nom', group_name+'temp_12_0um_nom', group_name+'refl_0_65um_nom', group_name+target_param]
-l2_params = [group_name+'temp_11_0um', group_name+'temp_12_0um', group_name+'refl_0_65um', group_name+target_param]
+# l2_params = [group_name+'temp_11_0um_nom', group_name+'refl_0_65um_nom', group_name+target_param]
+l2_params = [group_name+'temp_11_0um', group_name+'refl_0_65um', group_name+target_param]
# solzen_name = group_name + 'solar_zenith_angle'
solzen_name = group_name + 'solar_zenith'
@@ -130,7 +87,7 @@ def run_all(directory, out_directory, day_night='ANY', start=10):
cnt = start
total_num_train_samples = 0
total_num_valid_samples = 0
- num_keep_x_tiles = 8
+ num_keep_x_tiles = 12
# pattern = 'clavrx_VNP02MOD*.highres.nc.level2.nc'
pattern = 'clavrx_*.nc'
@@ -138,8 +95,6 @@ def run_all(directory, out_directory, day_night='ANY', start=10):
data_files = glob.glob(path, recursive=True)
- label_valid_tiles = []
- label_train_tiles = []
data_valid_tiles = []
data_train_tiles = []
f_cnt = 0
@@ -151,38 +106,22 @@ def run_all(directory, out_directory, day_night='ANY', start=10):
for idx, data_f in enumerate(data_files):
# if idx % 4 == 0: # if we want to skip some files
if True:
- # w_o_ext, ext = os.path.splitext(data_f)
- # pname, fname = os.path.split(data_f)
- # toks = fname.split('.')
- # label_f = pname + '/' + 'clavrx_VNP02MOD.' + toks[1]+'.'+toks[2]+'.'+toks[3]+'.'+toks[4]+'.'+'uwssec.highres.nc.level2.nc'
- # if not os.path.exists(label_f):
- # continue
-
try:
data_h5f = h5py.File(data_f, 'r')
except:
print('cant open file: ', data_f)
continue
- # try:
- # label_h5f = h5py.File(label_f, 'r')
- # except:
- # print('cant open file: ', label_f)
- # data_h5f.close()
- # continue
-
try:
run(data_h5f, data_params, data_train_tiles, data_valid_tiles, num_keep_x_tiles=num_keep_x_tiles, tile_width=128, kernel_size=11, day_night=day_night)
except Exception as e:
print(e)
data_h5f.close()
- # label_h5f.close()
continue
print(data_f)
f_cnt += 1
data_h5f.close()
- # label_h5f.close()
if len(data_train_tiles) == 0:
continue
@@ -190,20 +129,14 @@ def run_all(directory, out_directory, day_night='ANY', start=10):
if (f_cnt % 5) == 0:
num_valid_samples = 0
if len(data_valid_tiles) > 0:
- # label_valid = np.stack(label_valid_tiles)
data_valid = np.stack(data_valid_tiles)
np.save(out_directory + 'data_valid_' + str(cnt), data_valid)
- # np.save(out_directory+'label_valid_' + str(cnt), label_valid)
num_valid_samples = data_valid.shape[0]
- # label_train = np.stack(label_train_tiles)
- # np.save(out_directory+'label_train_' + str(cnt), label_train)
data_train = np.stack(data_train_tiles)
np.save(out_directory+'data_train_' + str(cnt), data_train)
num_train_samples = data_train.shape[0]
- label_valid_tiles = []
- label_train_tiles = []
data_valid_tiles = []
data_train_tiles = []
@@ -233,9 +166,8 @@ def run(data_h5f, param_s, train_tiles, valid_tiles, num_keep_x_tiles=8, tile_wi
grd_s = []
for param in param_s:
- fill_value, fill_value_name = get_fill_attrs(param)
try:
- grd = get_grid_values(data_h5f, param, 0, 0, None, num_lines, num_pixels, fill_value_name=fill_value_name, fill_value=fill_value)
+ grd = get_grid_values(data_h5f, param, 0, 0, None, num_lines, num_pixels)
grd_s.append(grd)
except Exception as e:
print(e)
--
GitLab