From 6c644c02d6d05f5923eb78c06d901d86b1c48026 Mon Sep 17 00:00:00 2001
From: Paolo Veglio <paolo.veglio@ssec.wisc.edu>
Date: Mon, 19 Aug 2024 16:54:20 +0000
Subject: [PATCH] updated imports. added auto chunking in read_data
---
entrypoint.py | 5 +++-
mvcm/main.py | 13 +++++-----
mvcm/preprocess_thresholds.py | 9 +++++--
mvcm/read_data.py | 45 ++++++++++++++++++++++++-----------
pyproject.toml | 3 +++
setup.py | 4 ++--
6 files changed, 54 insertions(+), 25 deletions(-)
diff --git a/entrypoint.py b/entrypoint.py
index cd12319..7f089d3 100644
--- a/entrypoint.py
+++ b/entrypoint.py
@@ -1,4 +1,5 @@
"""Main call for MVCM."""
+
import argparse
from mvcm.main import main
@@ -73,7 +74,9 @@ if __name__ == "__main__":
# version=_VERSION,
# help="print version and exit",
# )
- parser.add_argument("-v", "--verbose", action="store_true", help="print verbose information")
+ parser.add_argument(
+ "-v", "--verbose", action="count", default=1, help="print verbose information"
+ )
parser.add_argument("-d", "--debug", action="store_true", help="activate debug mode")
args = parser.parse_args()
diff --git a/mvcm/main.py b/mvcm/main.py
index 7b78f3d..127107b 100644
--- a/mvcm/main.py
+++ b/mvcm/main.py
@@ -1,10 +1,5 @@
"""Main function for MVCM."""
-try:
- from ruamel import yaml as yml
-except ImportError:
- import ruamel.yaml as yml
-
import argparse
import logging
@@ -13,6 +8,7 @@ import xarray as xr
from netCDF4 import Dataset # type: ignore
from pkg_resources import get_distribution # type: ignore
from rich.logging import RichHandler
+from ruamel.yaml import YAML
import mvcm.read_data as rd
import mvcm.scene as scn
@@ -190,7 +186,7 @@ def main(
with open(threshold_file) as f:
text = f.read()
- thresholds = yml.safe_load(text)
+ thresholds = YAML(typ="safe").load(text)
# We are not processing night granules
if use_hires is True:
@@ -562,6 +558,11 @@ def main(
"M15-M16": {"dims": ("x", "y"), "data": viirs_data["M15-M16"].values},
"Ocean_Day": {"dims": ("x", "y"), "data": viirs_data.Ocean_Day.values},
"Land_Day": {"dims": ("x", "y"), "data": viirs_data.Land_Day.values},
+ "Land_Day_Desert": {"dims": ("x", "y"), "data": viirs_data.Land_Day_Desert.values},
+ "Land_Day_Desert_Coast": {
+ "dims": ("x", "y"),
+ "data": viirs_data.Land_Day_Desert_Coast.values,
+ },
"Day_Snow": {"dims": ("x", "y"), "data": viirs_data.Day_Snow.values},
"Polar_Day_Ocean": {"dims": ("x", "y"), "data": viirs_data.Polar_Day_Ocean.values},
"Polar_Day_Land": {"dims": ("x", "y"), "data": viirs_data.Polar_Day_Land.values},
diff --git a/mvcm/preprocess_thresholds.py b/mvcm/preprocess_thresholds.py
index 5d8f7d4..4260d49 100644
--- a/mvcm/preprocess_thresholds.py
+++ b/mvcm/preprocess_thresholds.py
@@ -92,10 +92,15 @@ def thresholds_11_12um(
"Land_Night": 0.3,
"Polar_Night_Land": 0.3,
"Polar_Night_Snow": 0.3,
- "Day_Snow": 0.0,
+ "Day_Snow": 0.8,
"Night_Snow": 0.3,
}
- midpt = midpt - (_coeffs[scene] * locut)
+
+ if scene in ["Day_Snow"]:
+ midpt = midpt + _coeffs[scene]
+ else:
+ midpt = midpt - (_coeffs[scene] * locut)
+
if scene in ["Polar_Night_Land", "Polar_Night_Snow", "Night_Snow"]:
hicut = np.full(m15.shape, midpt - 1.25)
idx = np.nonzero(m15 < thr_dict["bt1"])
diff --git a/mvcm/read_data.py b/mvcm/read_data.py
index 59e8ed6..86a6943 100644
--- a/mvcm/read_data.py
+++ b/mvcm/read_data.py
@@ -176,7 +176,12 @@ class ReadData(CollectInputs):
logger.error(err_msg)
raise FileNotFoundError(err_msg)
- geo_data = xr.open_dataset(self.file_name_geo, group="geolocation_data", engine="netcdf4")
+ geo_data = xr.open_dataset(
+ self.file_name_geo,
+ group="geolocation_data",
+ engine="netcdf4",
+ chunks="auto",
+ )
relazi = self.relative_azimuth_angle(
geo_data.sensor_azimuth.values, geo_data.solar_azimuth.values
@@ -214,27 +219,39 @@ class ReadData(CollectInputs):
raise FileNotFoundError(err_msg)
l1b_data = xr.open_dataset(
- self.file_name_l1b, group="observation_data", decode_cf=False, engine="netcdf4"
+ self.file_name_l1b,
+ group="observation_data",
+ decode_cf=False,
+ engine="netcdf4",
+ chunks="auto",
)
rad_data = xr.Dataset()
for band in list(l1b_data.variables):
if band in _reflectance_bands:
- if hasattr(l1b_data[band], "VCST_scale_factor"):
- scale_factor = l1b_data[band].VCST_scale_factor * l1b_data[band].bias_correction
+ if band in l1b_data:
+ if hasattr(l1b_data[band], "VCST_scale_factor"):
+ scale_factor = (
+ l1b_data[band].VCST_scale_factor * l1b_data[band].bias_correction
+ )
+ else:
+ scale_factor = l1b_data[band].scale_factor
+ rad_data[band] = (
+ self.dims,
+ l1b_data[band].values * scale_factor / np.cos(solar_zenith * _DTR),
+ )
else:
- scale_factor = l1b_data[band].scale_factor
- rad_data[band] = (
- self.dims,
- l1b_data[band].values * scale_factor / np.cos(solar_zenith * _DTR),
- )
+ logger.info(f"Reflective band {band} not found in L1b file")
elif band in _emissive_bands:
- bt_lut = f"{band}_brightness_temperature_lut"
- rad_data[band] = (
- self.dims,
- l1b_data[bt_lut].values[l1b_data[band].values],
- )
+ if band in l1b_data:
+ bt_lut = f"{band}_brightness_temperature_lut"
+ rad_data[band] = (
+ self.dims,
+ l1b_data[bt_lut].values[l1b_data[band].values],
+ )
+ else:
+ logger.info(f"Emissive band {band} not found in L1b file")
else:
pass
diff --git a/pyproject.toml b/pyproject.toml
index aca6b02..dd7f6fb 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -24,8 +24,11 @@ dependencies = [
'numpy',
'xarray',
'attrs',
+ 'netCDF4',
'rich',
+ 'ruamel.yaml',
'pre-commit',
+ 'dask',
]
dynamic = ['version']
diff --git a/setup.py b/setup.py
index c036f19..9beee38 100644
--- a/setup.py
+++ b/setup.py
@@ -32,7 +32,7 @@ sourcefiles = [
include_dirs = [
"mvcm/c_tools/include",
- "/opt/hdf4/4.2.14-gcc-8.3/include",
+ "/opt/hdf4/nocdf-4.2.14-gcc-8.3/include",
"/opt/hdfeos2/2.20-gcc-8.3/include",
"/opt/netcdf4/4.7.0-gcc-8.3/include",
"/usr/include/hdf",
@@ -53,7 +53,7 @@ extensions = [
sourcefiles,
include_dirs=include_dirs,
library_dirs=library_dirs,
- libraries=["netcdf"],
+ libraries=["netcdf", "mfhdf", "df"],
extra_compile_args=["-fcommon"],
),
]
--
GitLab