diff --git a/.gitignore b/.gitignore
index 32efc7d2c05a66900232b473951f03e769a974e3..a0ff2bef20d0e5ad82ebac1f69dc226891bc1367 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,8 +6,10 @@
*.vscode
+
ancillary.c
viz.py
+.coverage
build
dev
diff --git a/mvcm/main.py b/mvcm/main.py
index 197f175c9864c982882e4b15dc80b810bb40b852..46fb12a4a4675c85a94da902f18ab1ee46cb1702 100644
--- a/mvcm/main.py
+++ b/mvcm/main.py
@@ -6,6 +6,8 @@ except ImportError:
import ruamel.yaml as yml
import argparse
+
+# import cProfile
import functools
import logging
import time
@@ -115,8 +117,8 @@ def main(
data_path: str = _datapath,
mod02: str = "", # _fname_mod02,
mod03: str = "", # _fname_mod03,
- img02: str = "", # _fname_img02,
- img03: str = "", # _fname_img03,
+ img02: str = None, # _fname_img02,
+ img03: str = None, # _fname_img03,
threshold_file: str = _threshold_file,
geos_atm_1: str = _geos_atm_1,
geos_atm_2: str = _geos_atm_2,
diff --git a/mvcm/read_data.py b/mvcm/read_data.py
index c75f9c801b6b52871fd8476ab72501568ad6176e..3885b8573c203561af17bd13fd3f7c54c0ab3259 100644
--- a/mvcm/read_data.py
+++ b/mvcm/read_data.py
@@ -235,7 +235,7 @@ class ReadData(CollectInputs):
dataset containing all geolocation data
"""
logger.debug(f"Reading {self.file_name_geo}")
- geo_data = xr.open_dataset(self.file_name_geo, group="geolocation_data")
+ geo_data = xr.open_dataset(self.file_name_geo, group="geolocation_data", engine="netcdf4")
relazi = self.relative_azimuth_angle(
geo_data.sensor_azimuth.values, geo_data.solar_azimuth.values
@@ -266,7 +266,9 @@ class ReadData(CollectInputs):
solar zenith angle derived from the geolocation file
"""
logger.debug(f"Reading {self.file_name_l1b}")
- l1b_data = xr.open_dataset(self.file_name_l1b, group="observation_data", decode_cf=False)
+ l1b_data = xr.open_dataset(
+ self.file_name_l1b, group="observation_data", decode_cf=False, engine="netcdf4"
+ )
rad_data = xr.Dataset()
for band in list(l1b_data.variables):
diff --git a/mvcm/spectral_tests.py b/mvcm/spectral_tests.py
index 221b6475db0296b7504af2389690c5c18cc33335..ae7d1f5c97e7ca4ebac1e0ddf4230d7a7bddc0f0 100644
--- a/mvcm/spectral_tests.py
+++ b/mvcm/spectral_tests.py
@@ -1,9 +1,8 @@
"""Module defining spectral tests."""
-import functools
+import functools # noqa
import importlib
import logging
-from typing import Dict, Tuple
import numpy as np
import xarray as xr
@@ -11,13 +10,10 @@ from attrs import Factory, define, field, validators
from numpy.lib.stride_tricks import sliding_window_view
import ancillary_data as anc
-import mvcm.conf as conf
import mvcm.preprocess_thresholds as preproc
-import mvcm.restoral as restoral
import mvcm.scene as scn
+from mvcm import conf, restoral
-# _bad_data = -999.0
-# _RTD = 180.0 / np.pi
_DTR = np.pi / 180
_scene_list = [
"Ocean_Day",
@@ -41,7 +37,6 @@ _scene_list = [
"Land_Day_Desert_Coast",
"Land_Day_Coast",
]
-# # 'Desert', 'Australia']
# this is used for testing, eventually we want to remove it
importlib.reload(preproc)
@@ -50,21 +45,9 @@ importlib.reload(restoral)
logger = logging.getLogger(__name__)
-# this function has been moved to utility_functions
-"""
-def restoral_flag(bits):
- qa_bit = bits["qa_bit"]
- test_bit = bits["test_bit"]
- # the logic of this is convoluted as hell, but basically the point is:
- # if test_bit == 0 it means that a cloud it's likely been detected.
- # The return value is written this way to be consistent with the way
- # this is defined in the C code, at some point I might change it
- return np.logical_not(np.logical_and(qa_bit, np.logical_not(test_bit)))
-"""
-
@define(kw_only=True, slots=True)
-class CloudTests(object):
+class CloudTests:
"""Class that initializes the common field for the spectral tests.
Parameters
@@ -83,9 +66,9 @@ class CloudTests(object):
]
)
scene_name: str = field(validator=[validators.instance_of(str), validators.in_(_scene_list)])
- thresholds: Dict = field(
+ thresholds: dict = field(
validator=[
- validators.instance_of(Dict),
+ validators.instance_of(dict),
]
)
scene_idx: tuple = field(
@@ -106,6 +89,7 @@ class CloudTests(object):
],
)
+ @staticmethod
def run_if_test_exists_for_scene(test_name: str):
"""Skip tests if no pixels are present in given scene."""
@@ -113,12 +97,9 @@ class CloudTests(object):
@functools.wraps(func)
def wrapper(self, *args, **kwargs):
if test_name not in self.thresholds[self.scene_name]:
- # test_bit = np.zeros(args[-1].shape)
return args[-2], args[-1]
else:
kwargs["confidence"] = np.ones(self.data.latitude.shape)
- # kwargs['qa_bit'] = np.zeros(self.data.latitude.shape)
- # kwargs['test_bit'] = np.zeros(self.data.latitude.shape)
kwargs["thresholds"] = self.thresholds[self.scene_name][test_name]
return func(self, *args, **kwargs)
@@ -169,8 +150,8 @@ class CloudTests(object):
@run_if_test_exists_for_scene("11um_Test")
def test_11um(
- self, band: str, cmin: np.ndarray, bits: Dict, **kwargs
- ) -> Tuple[np.ndarray, Dict]:
+ self, band: str, cmin: np.ndarray, bits: dict, **kwargs
+ ) -> tuple[np.ndarray, dict]:
"""Perform 11um spectral test."""
threshold = kwargs["thresholds"]
@@ -191,8 +172,8 @@ class CloudTests(object):
# Also, this function is not currently used
@run_if_test_exists_for_scene("Surface_Temperature_Test")
def surface_temperature_test(
- self, band: str, viirs_data: xr.Dataset, cmin: np.ndarray, bits: Dict, **kwargs
- ) -> Tuple[np.ndarray, Dict]:
+ self, band: str, viirs_data: xr.Dataset, cmin: np.ndarray, bits: dict, **kwargs
+ ) -> tuple[np.ndarray, dict]:
"""Perform surface temperature test over land."""
threshold = kwargs["thresholds"]
@@ -218,8 +199,8 @@ class CloudTests(object):
@run_if_test_exists_for_scene("SST_Test")
def sst_test(
- self, band31: str, band32: str, cmin: np.ndarray, bits: Dict, **kwargs
- ) -> Tuple[np.ndarray, Dict]:
+ self, band31: str, band32: str, cmin: np.ndarray, bits: dict, **kwargs
+ ) -> tuple[np.ndarray, dict]:
"""Perform sea surface temperature test."""
threshold = kwargs["thresholds"]
@@ -257,8 +238,8 @@ class CloudTests(object):
@run_if_test_exists_for_scene("8.6-11um_Test")
def bt_diff_86_11um(
- self, band: str, cmin: np.ndarray, bits: Dict, **kwargs
- ) -> Tuple[np.ndarray, Dict]:
+ self, band: str, cmin: np.ndarray, bits: dict, **kwargs
+ ) -> tuple[np.ndarray, dict]:
"""Perform IR (8.6-11um) difference spectral test."""
threshold = kwargs["thresholds"]
@@ -276,8 +257,8 @@ class CloudTests(object):
@run_if_test_exists_for_scene("11-12um_Cirrus_Test")
def test_11_12um_diff(
- self, band: str, cmin: np.ndarray, bits: Dict, **kwargs
- ) -> Tuple[np.ndarray, Dict]:
+ self, band: str, cmin: np.ndarray, bits: dict, **kwargs
+ ) -> tuple[np.ndarray, dict]:
"""Perform 11-12um difference spectral test."""
threshold = kwargs["thresholds"]
@@ -313,8 +294,8 @@ class CloudTests(object):
# THIS NEEDS TO BE TESTED
@run_if_test_exists_for_scene("11-4um_BT_Difference_Test_Ocean")
def bt_difference_11_4um_test_ocean(
- self, band: str, cmin: np.ndarray, bits: Dict, **kwargs
- ) -> Tuple[np.ndarray, Dict]:
+ self, band: str, cmin: np.ndarray, bits: dict, **kwargs
+ ) -> tuple[np.ndarray, dict]:
"""Perfrom 11-4um difference spectral test over ocean."""
threshold = kwargs["thresholds"]
@@ -340,8 +321,8 @@ class CloudTests(object):
# THIS NEEDS TO BE TESTED
@run_if_test_exists_for_scene("11-4um_BT_Difference_Test_Land")
def bt_difference_11_4um_test_land(
- self, band1: str, band2: str, cmin: np.ndarray, bits: Dict, **kwargs
- ) -> Tuple[np.ndarray, Dict]:
+ self, band1: str, band2: str, cmin: np.ndarray, bits: dict, **kwargs
+ ) -> tuple[np.ndarray, dict]:
"""Perform 11-4um difference spectral test over land."""
threshold = kwargs["thresholds"]
test_name = "11-4um_BT_Difference_Test_Land" # this is temporary
@@ -455,7 +436,7 @@ class CloudTests(object):
@run_if_test_exists_for_scene("11um_Variability_Test")
def variability_11um_test(
self, band: str, cmin: np.ndarray, bits: np.ndarray, **kwargs
- ) -> Tuple[np.ndarray, np.ndarray]:
+ ) -> tuple[np.ndarray, np.ndarray]:
"""Perform 11um variability test."""
threshold = kwargs["thresholds"]
var = np.zeros([self.data.M15.shape[0], self.data.M15.shape[1], 9])
@@ -485,7 +466,7 @@ class CloudTests(object):
@run_if_test_exists_for_scene("11-4um_Oceanic_Stratus_Test")
def oceanic_stratus_11_4um_test(
self, band: str, cmin: np.ndarray, bits: np.ndarray, **kwargs
- ) -> Tuple[np.ndarray, np.ndarray]:
+ ) -> tuple[np.ndarray, np.ndarray]:
"""Perform oceanic stratus test."""
threshold = kwargs["thresholds"]
@@ -566,7 +547,7 @@ class CloudTests(object):
@run_if_test_exists_for_scene("NIR_Reflectance_Test")
def nir_reflectance_test(
self, band: str, cmin: np.ndarray, bits: np.ndarray, **kwargs
- ) -> Tuple[np.ndarray, np.ndarray]:
+ ) -> tuple[np.ndarray, np.ndarray]:
"""Perform NIR spectral test."""
threshold = kwargs["thresholds"]
@@ -593,7 +574,7 @@ class CloudTests(object):
@run_if_test_exists_for_scene("Vis/NIR_Ratio_Test")
def vis_nir_ratio_test(
self, band: str, cmin: np.ndarray, bits: np.ndarray, **kwargs
- ) -> Tuple[np.ndarray, np.ndarray]:
+ ) -> tuple[np.ndarray, np.ndarray]:
"""Perform Vis/NIR ratio test."""
threshold = kwargs["thresholds"]
temp_conf = np.ones(self.data[band].shape)
@@ -644,7 +625,7 @@ class CloudTests(object):
@run_if_test_exists_for_scene("1.6_2.1um_NIR_Reflectance_Test")
def test_16_21um_reflectance(
self, band: str, cmin: np.ndarray, bits: np.ndarray, **kwargs
- ) -> Tuple[np.ndarray, np.ndarray]:
+ ) -> tuple[np.ndarray, np.ndarray]:
"""Perform NIR spectral test."""
threshold = kwargs["thresholds"]
@@ -669,7 +650,7 @@ class CloudTests(object):
@run_if_test_exists_for_scene("Visible_Reflectance_Test")
def visible_reflectance_test(
self, band: str, cmin: np.ndarray, bits: np.ndarray, **kwargs
- ) -> Tuple[np.ndarray, np.ndarray]:
+ ) -> tuple[np.ndarray, np.ndarray]:
"""Perform visible reflectance test."""
threshold = kwargs["thresholds"]
logger.warning(
@@ -705,7 +686,7 @@ class CloudTests(object):
@run_if_test_exists_for_scene("GEMI_Test")
def gemi_test(
self, band: str, cmin: np.ndarray, bits: np.ndarray, **kwargs
- ) -> Tuple[np.ndarray, np.ndarray]:
+ ) -> tuple[np.ndarray, np.ndarray]:
"""Perform GEMI test over desert areas."""
threshold = kwargs["thresholds"]
scene_idx = tuple(
@@ -732,7 +713,7 @@ class CloudTests(object):
@run_if_test_exists_for_scene("1.38um_High_Cloud_Test")
def test_1_38um_high_clouds(
self, band: str, cmin: np.ndarray, bits: np.ndarray, **kwargs
- ) -> Tuple[np.ndarray, np.ndarray]:
+ ) -> tuple[np.ndarray, np.ndarray]:
"""Perform IR spectral test."""
threshold = kwargs["thresholds"]
@@ -771,7 +752,7 @@ class CloudTests(object):
@run_if_test_exists_for_scene("4-12um_BTD_Thin_Cirrus_Test")
def thin_cirrus_4_12um_BTD_test(
self, band: str, cmin: np.ndarray, bits: np.ndarray, **kwargs
- ) -> Tuple[np.ndarray, np.ndarray]:
+ ) -> tuple[np.ndarray, np.ndarray]:
"""Perfrom thin cirrus spectral test."""
threshold = kwargs["thresholds"]
diff --git a/pyproject.toml b/pyproject.toml
index 17728ea1771965f97cbdee9a63114b967568daa3..e2fda32ac176d91db7814dac469c3a87ac7a2c9b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -81,7 +81,6 @@ line-length = 100
# skip-string-normalization = true
[tool.ruff]
-target-version = ["py39"]
line-length = 100
select = [
"A",