diff --git a/.gitignore b/.gitignore index 32efc7d2c05a66900232b473951f03e769a974e3..a0ff2bef20d0e5ad82ebac1f69dc226891bc1367 100644 --- a/.gitignore +++ b/.gitignore @@ -6,8 +6,10 @@ *.vscode + ancillary.c viz.py +.coverage build dev diff --git a/mvcm/main.py b/mvcm/main.py index 197f175c9864c982882e4b15dc80b810bb40b852..46fb12a4a4675c85a94da902f18ab1ee46cb1702 100644 --- a/mvcm/main.py +++ b/mvcm/main.py @@ -6,6 +6,8 @@ except ImportError: import ruamel.yaml as yml import argparse + +# import cProfile import functools import logging import time @@ -115,8 +117,8 @@ def main( data_path: str = _datapath, mod02: str = "", # _fname_mod02, mod03: str = "", # _fname_mod03, - img02: str = "", # _fname_img02, - img03: str = "", # _fname_img03, + img02: str = None, # _fname_img02, + img03: str = None, # _fname_img03, threshold_file: str = _threshold_file, geos_atm_1: str = _geos_atm_1, geos_atm_2: str = _geos_atm_2, diff --git a/mvcm/read_data.py b/mvcm/read_data.py index c75f9c801b6b52871fd8476ab72501568ad6176e..3885b8573c203561af17bd13fd3f7c54c0ab3259 100644 --- a/mvcm/read_data.py +++ b/mvcm/read_data.py @@ -235,7 +235,7 @@ class ReadData(CollectInputs): dataset containing all geolocation data """ logger.debug(f"Reading {self.file_name_geo}") - geo_data = xr.open_dataset(self.file_name_geo, group="geolocation_data") + geo_data = xr.open_dataset(self.file_name_geo, group="geolocation_data", engine="netcdf4") relazi = self.relative_azimuth_angle( geo_data.sensor_azimuth.values, geo_data.solar_azimuth.values @@ -266,7 +266,9 @@ class ReadData(CollectInputs): solar zenith angle derived from the geolocation file """ logger.debug(f"Reading {self.file_name_l1b}") - l1b_data = xr.open_dataset(self.file_name_l1b, group="observation_data", decode_cf=False) + l1b_data = xr.open_dataset( + self.file_name_l1b, group="observation_data", decode_cf=False, engine="netcdf4" + ) rad_data = xr.Dataset() for band in list(l1b_data.variables): diff --git a/mvcm/spectral_tests.py b/mvcm/spectral_tests.py index 221b6475db0296b7504af2389690c5c18cc33335..ae7d1f5c97e7ca4ebac1e0ddf4230d7a7bddc0f0 100644 --- a/mvcm/spectral_tests.py +++ b/mvcm/spectral_tests.py @@ -1,9 +1,8 @@ """Module defining spectral tests.""" -import functools +import functools # noqa import importlib import logging -from typing import Dict, Tuple import numpy as np import xarray as xr @@ -11,13 +10,10 @@ from attrs import Factory, define, field, validators from numpy.lib.stride_tricks import sliding_window_view import ancillary_data as anc -import mvcm.conf as conf import mvcm.preprocess_thresholds as preproc -import mvcm.restoral as restoral import mvcm.scene as scn +from mvcm import conf, restoral -# _bad_data = -999.0 -# _RTD = 180.0 / np.pi _DTR = np.pi / 180 _scene_list = [ "Ocean_Day", @@ -41,7 +37,6 @@ _scene_list = [ "Land_Day_Desert_Coast", "Land_Day_Coast", ] -# # 'Desert', 'Australia'] # this is used for testing, eventually we want to remove it importlib.reload(preproc) @@ -50,21 +45,9 @@ importlib.reload(restoral) logger = logging.getLogger(__name__) -# this function has been moved to utility_functions -""" -def restoral_flag(bits): - qa_bit = bits["qa_bit"] - test_bit = bits["test_bit"] - # the logic of this is convoluted as hell, but basically the point is: - # if test_bit == 0 it means that a cloud it's likely been detected. - # The return value is written this way to be consistent with the way - # this is defined in the C code, at some point I might change it - return np.logical_not(np.logical_and(qa_bit, np.logical_not(test_bit))) -""" - @define(kw_only=True, slots=True) -class CloudTests(object): +class CloudTests: """Class that initializes the common field for the spectral tests. Parameters @@ -83,9 +66,9 @@ class CloudTests(object): ] ) scene_name: str = field(validator=[validators.instance_of(str), validators.in_(_scene_list)]) - thresholds: Dict = field( + thresholds: dict = field( validator=[ - validators.instance_of(Dict), + validators.instance_of(dict), ] ) scene_idx: tuple = field( @@ -106,6 +89,7 @@ class CloudTests(object): ], ) + @staticmethod def run_if_test_exists_for_scene(test_name: str): """Skip tests if no pixels are present in given scene.""" @@ -113,12 +97,9 @@ class CloudTests(object): @functools.wraps(func) def wrapper(self, *args, **kwargs): if test_name not in self.thresholds[self.scene_name]: - # test_bit = np.zeros(args[-1].shape) return args[-2], args[-1] else: kwargs["confidence"] = np.ones(self.data.latitude.shape) - # kwargs['qa_bit'] = np.zeros(self.data.latitude.shape) - # kwargs['test_bit'] = np.zeros(self.data.latitude.shape) kwargs["thresholds"] = self.thresholds[self.scene_name][test_name] return func(self, *args, **kwargs) @@ -169,8 +150,8 @@ class CloudTests(object): @run_if_test_exists_for_scene("11um_Test") def test_11um( - self, band: str, cmin: np.ndarray, bits: Dict, **kwargs - ) -> Tuple[np.ndarray, Dict]: + self, band: str, cmin: np.ndarray, bits: dict, **kwargs + ) -> tuple[np.ndarray, dict]: """Perform 11um spectral test.""" threshold = kwargs["thresholds"] @@ -191,8 +172,8 @@ class CloudTests(object): # Also, this function is not currently used @run_if_test_exists_for_scene("Surface_Temperature_Test") def surface_temperature_test( - self, band: str, viirs_data: xr.Dataset, cmin: np.ndarray, bits: Dict, **kwargs - ) -> Tuple[np.ndarray, Dict]: + self, band: str, viirs_data: xr.Dataset, cmin: np.ndarray, bits: dict, **kwargs + ) -> tuple[np.ndarray, dict]: """Perform surface temperature test over land.""" threshold = kwargs["thresholds"] @@ -218,8 +199,8 @@ class CloudTests(object): @run_if_test_exists_for_scene("SST_Test") def sst_test( - self, band31: str, band32: str, cmin: np.ndarray, bits: Dict, **kwargs - ) -> Tuple[np.ndarray, Dict]: + self, band31: str, band32: str, cmin: np.ndarray, bits: dict, **kwargs + ) -> tuple[np.ndarray, dict]: """Perform sea surface temperature test.""" threshold = kwargs["thresholds"] @@ -257,8 +238,8 @@ class CloudTests(object): @run_if_test_exists_for_scene("8.6-11um_Test") def bt_diff_86_11um( - self, band: str, cmin: np.ndarray, bits: Dict, **kwargs - ) -> Tuple[np.ndarray, Dict]: + self, band: str, cmin: np.ndarray, bits: dict, **kwargs + ) -> tuple[np.ndarray, dict]: """Perform IR (8.6-11um) difference spectral test.""" threshold = kwargs["thresholds"] @@ -276,8 +257,8 @@ class CloudTests(object): @run_if_test_exists_for_scene("11-12um_Cirrus_Test") def test_11_12um_diff( - self, band: str, cmin: np.ndarray, bits: Dict, **kwargs - ) -> Tuple[np.ndarray, Dict]: + self, band: str, cmin: np.ndarray, bits: dict, **kwargs + ) -> tuple[np.ndarray, dict]: """Perform 11-12um difference spectral test.""" threshold = kwargs["thresholds"] @@ -313,8 +294,8 @@ class CloudTests(object): # THIS NEEDS TO BE TESTED @run_if_test_exists_for_scene("11-4um_BT_Difference_Test_Ocean") def bt_difference_11_4um_test_ocean( - self, band: str, cmin: np.ndarray, bits: Dict, **kwargs - ) -> Tuple[np.ndarray, Dict]: + self, band: str, cmin: np.ndarray, bits: dict, **kwargs + ) -> tuple[np.ndarray, dict]: """Perfrom 11-4um difference spectral test over ocean.""" threshold = kwargs["thresholds"] @@ -340,8 +321,8 @@ class CloudTests(object): # THIS NEEDS TO BE TESTED @run_if_test_exists_for_scene("11-4um_BT_Difference_Test_Land") def bt_difference_11_4um_test_land( - self, band1: str, band2: str, cmin: np.ndarray, bits: Dict, **kwargs - ) -> Tuple[np.ndarray, Dict]: + self, band1: str, band2: str, cmin: np.ndarray, bits: dict, **kwargs + ) -> tuple[np.ndarray, dict]: """Perform 11-4um difference spectral test over land.""" threshold = kwargs["thresholds"] test_name = "11-4um_BT_Difference_Test_Land" # this is temporary @@ -455,7 +436,7 @@ class CloudTests(object): @run_if_test_exists_for_scene("11um_Variability_Test") def variability_11um_test( self, band: str, cmin: np.ndarray, bits: np.ndarray, **kwargs - ) -> Tuple[np.ndarray, np.ndarray]: + ) -> tuple[np.ndarray, np.ndarray]: """Perform 11um variability test.""" threshold = kwargs["thresholds"] var = np.zeros([self.data.M15.shape[0], self.data.M15.shape[1], 9]) @@ -485,7 +466,7 @@ class CloudTests(object): @run_if_test_exists_for_scene("11-4um_Oceanic_Stratus_Test") def oceanic_stratus_11_4um_test( self, band: str, cmin: np.ndarray, bits: np.ndarray, **kwargs - ) -> Tuple[np.ndarray, np.ndarray]: + ) -> tuple[np.ndarray, np.ndarray]: """Perform oceanic stratus test.""" threshold = kwargs["thresholds"] @@ -566,7 +547,7 @@ class CloudTests(object): @run_if_test_exists_for_scene("NIR_Reflectance_Test") def nir_reflectance_test( self, band: str, cmin: np.ndarray, bits: np.ndarray, **kwargs - ) -> Tuple[np.ndarray, np.ndarray]: + ) -> tuple[np.ndarray, np.ndarray]: """Perform NIR spectral test.""" threshold = kwargs["thresholds"] @@ -593,7 +574,7 @@ class CloudTests(object): @run_if_test_exists_for_scene("Vis/NIR_Ratio_Test") def vis_nir_ratio_test( self, band: str, cmin: np.ndarray, bits: np.ndarray, **kwargs - ) -> Tuple[np.ndarray, np.ndarray]: + ) -> tuple[np.ndarray, np.ndarray]: """Perform Vis/NIR ratio test.""" threshold = kwargs["thresholds"] temp_conf = np.ones(self.data[band].shape) @@ -644,7 +625,7 @@ class CloudTests(object): @run_if_test_exists_for_scene("1.6_2.1um_NIR_Reflectance_Test") def test_16_21um_reflectance( self, band: str, cmin: np.ndarray, bits: np.ndarray, **kwargs - ) -> Tuple[np.ndarray, np.ndarray]: + ) -> tuple[np.ndarray, np.ndarray]: """Perform NIR spectral test.""" threshold = kwargs["thresholds"] @@ -669,7 +650,7 @@ class CloudTests(object): @run_if_test_exists_for_scene("Visible_Reflectance_Test") def visible_reflectance_test( self, band: str, cmin: np.ndarray, bits: np.ndarray, **kwargs - ) -> Tuple[np.ndarray, np.ndarray]: + ) -> tuple[np.ndarray, np.ndarray]: """Perform visible reflectance test.""" threshold = kwargs["thresholds"] logger.warning( @@ -705,7 +686,7 @@ class CloudTests(object): @run_if_test_exists_for_scene("GEMI_Test") def gemi_test( self, band: str, cmin: np.ndarray, bits: np.ndarray, **kwargs - ) -> Tuple[np.ndarray, np.ndarray]: + ) -> tuple[np.ndarray, np.ndarray]: """Perform GEMI test over desert areas.""" threshold = kwargs["thresholds"] scene_idx = tuple( @@ -732,7 +713,7 @@ class CloudTests(object): @run_if_test_exists_for_scene("1.38um_High_Cloud_Test") def test_1_38um_high_clouds( self, band: str, cmin: np.ndarray, bits: np.ndarray, **kwargs - ) -> Tuple[np.ndarray, np.ndarray]: + ) -> tuple[np.ndarray, np.ndarray]: """Perform IR spectral test.""" threshold = kwargs["thresholds"] @@ -771,7 +752,7 @@ class CloudTests(object): @run_if_test_exists_for_scene("4-12um_BTD_Thin_Cirrus_Test") def thin_cirrus_4_12um_BTD_test( self, band: str, cmin: np.ndarray, bits: np.ndarray, **kwargs - ) -> Tuple[np.ndarray, np.ndarray]: + ) -> tuple[np.ndarray, np.ndarray]: """Perfrom thin cirrus spectral test.""" threshold = kwargs["thresholds"] diff --git a/pyproject.toml b/pyproject.toml index 17728ea1771965f97cbdee9a63114b967568daa3..e2fda32ac176d91db7814dac469c3a87ac7a2c9b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,6 @@ line-length = 100 # skip-string-normalization = true [tool.ruff] -target-version = ["py39"] line-length = 100 select = [ "A",