"""Create reference files.""" import logging import os import numpy as np import xarray as xr from rich.logging import RichHandler from ruamel.yaml import YAML import mvcm.spectral_tests as tst _LOG_FORMAT = "%(message)s" logging.basicConfig(level="NOTSET", datefmt="[%X]", format=_LOG_FORMAT, handlers=[RichHandler()]) logger = logging.getLogger(__name__) _scene_list = [ "Land_Day", "Land_Day_Coast", "Land_Day_Desert", "Land_Day_Desert_Coast", "Ocean_Day", "Ocean_Night", "Polar_Day_Ocean", "Polar_Night_Ocean", "Polar_Day_Land", "Polar_Day_Coast", "Polar_Day_Desert", "Polar_Day_Desert_Coast", "Polar_Day_Snow", "Land_Night", "Polar_Night_Land", "Polar_Night_Snow", "Day_Snow", "Night_Snow", ] _threshold_file = "/home/pveglio/mvcm/tests/fixtures/thresholds.mvcm.snpp.v0.0.1.yaml" _test_path = "/ships19/hercules/pveglio/mvcm_git_tests" def main() -> None: """Create reference file.""" with open(_threshold_file) as f: text = f.read() thresholds = YAML(typ="safe").load(text) output_dict: dict = {} for scene_name in _scene_list: logger.info(f"Running tests for {scene_name} \n") if os.path.isfile(f"{_test_path}/ref_confidence_{scene_name}.nc"): logger.info(f"Skipping {scene_name}, file already exists") continue if not os.path.isfile(f"{_test_path}/test_scene_{scene_name}.nc"): logger.info(f"Skipping {scene_name}, test file not present in {_test_path}") continue viirs_data = xr.open_dataset(f"{_test_path}/test_scene_{scene_name}.nc") bits = { "test": np.zeros(viirs_data.M15.shape, dtype=np.int8), "qa": np.zeros(viirs_data.M15.shape, dtype=np.int8), } if np.all(viirs_data[scene_name].values == 0): logger.info("Skipping, no pixels in scene.") continue my_scene = tst.CloudTests(data=viirs_data, scene_name=scene_name, thresholds=thresholds) # type: ignore # 11um Test logger.info(f"Running 11um test for {scene_name}") confidence, bits = my_scene.test_11um("M15", np.ones(viirs_data.M15.shape), bits) bits, output_dict = update_and_clean( viirs_data.M15.shape, confidence, bits, output_dict, "11um" ) # surface_temperature_test() is not performed right now # SST Test logger.info(f"Running SST test for {scene_name} \n") confidence, bits = my_scene.sst_test("M15", "M16", np.ones(viirs_data.M15.shape), bits) bits, output_dict = update_and_clean( viirs_data.M15.shape, confidence, bits, output_dict, "sst" ) # 8.6 - 11 um Test logger.info(f"Running 8.6 - 11 um test for {scene_name} \n") confidence, bits = my_scene.bt_diff_86_11um("M14-M15", np.ones(viirs_data.M15.shape), bits) bits, output_dict = update_and_clean( viirs_data.M15.shape, confidence, bits, output_dict, "86_11um_diff" ) # 11 - 12um Test logger.info(f"Running 11 - 12um test for {scene_name} \n") confidence, bits = my_scene.test_11_12um_diff( "M15-M16", np.ones(viirs_data.M15.shape), bits ) bits, output_dict = update_and_clean( viirs_data.M15.shape, confidence, bits, output_dict, "11_12um_diff" ) # 11um Variability Test logger.info(f"Running 11um variability test for {scene_name} \n") confidence, bits = my_scene.variability_11um_test( "M15", np.ones(viirs_data.M15.shape), bits ) bits, output_dict = update_and_clean( viirs_data.M15.shape, confidence, bits, output_dict, "variability_11um", ) # 11 - 4um Test Ocean logger.info(f"Running 11 - 4um test for {scene_name} \n") confidence, bits = my_scene.bt_difference_11_4um_test_ocean( "M15-M12", np.ones(viirs_data.M15.shape), bits ) bits, output_dict = update_and_clean( viirs_data.M15.shape, confidence, bits, output_dict, "11_4um_ocean" ) # 11 - 4um Test Land # confidence, bits = my_scene.bt_difference_11_4um_test_land( # "M15-M12", np.ones(viirs_data.M15.shape), bits # ) # bits, output_dict = update_and_clean( # viirs_data.M15.shape, confidence, bits, output_dict, "11_4um_land" # ) # 11 - 4um Oceanic Stratus Test logger.info(f"Running 11 - 4um Oceanic Stratus test for {scene_name} \n") confidence, bits = my_scene.oceanic_stratus_11_4um_test( "M15-M12", np.ones(viirs_data.M15.shape), bits ) bits, output_dict = update_and_clean( viirs_data.M15.shape, confidence, bits, output_dict, "oceanic_stratus_11_4um", ) # NIR Reflectance Test logger.info(f"Running nir reflectance test for {scene_name} \n") confidence, bits = my_scene.nir_reflectance_test("M07", np.ones(viirs_data.M15.shape), bits) bits, output_dict = update_and_clean( viirs_data.M15.shape, confidence, bits, output_dict, "nir_reflectance", ) # Vis/NIR Ratio test logger.info(f"Running vis/nir ratio test for {scene_name} \n") confidence, bits = my_scene.vis_nir_ratio_test( "M07-M05ratio", np.ones(viirs_data.M15.shape), bits ) bits, output_dict = update_and_clean( viirs_data.M15.shape, confidence, bits, output_dict, "vis_nir_ratio" ) # 1.6 - 2.1um Test logger.info(f"Running 1.6 - 2.1um test for {scene_name} \n") confidence, bits = my_scene.test_16_21um_reflectance( "M10", np.ones(viirs_data.M15.shape), bits ) bits, output_dict = update_and_clean( viirs_data.M15.shape, confidence, bits, output_dict, "16_21um_reflectance", ) # Visible Reflectance Test logger.info(f"Running visible reflectance test for {scene_name} \n") confidence, bits = my_scene.visible_reflectance_test( "M05", np.ones(viirs_data.M15.shape), bits ) bits, output_dict = update_and_clean( viirs_data.M15.shape, confidence, bits, output_dict, "vis_refl" ) # GEMI Test logger.info(f"Running GEMI test for {scene_name} \n") confidence, bits = my_scene.gemi_test("GEMI", np.ones(viirs_data.M15.shape), bits) bits, output_dict = update_and_clean( viirs_data.M15.shape, confidence, bits, output_dict, "gemi" ) # 1.38 High Clouds Test logger.info(f"Running 1.38um high clouds test for {scene_name} \n") confidence, bits = my_scene.test_1_38um_high_clouds( "M09", np.ones(viirs_data.M15.shape), bits ) bits, output_dict = update_and_clean( viirs_data.M15.shape, confidence, bits, output_dict, "high_clouds" ) # 4 -12um Thin Cirrus Test logger.info(f"Running 4-12um thin cirrus test for {scene_name} \n") confidence, bits = my_scene.thin_cirrus_4_12um_BTD_test( "M13-M16", np.ones(viirs_data.M15.shape), bits ) bits, output_dict = update_and_clean( viirs_data.M15.shape, confidence, bits, output_dict, "thin_cirrus" ) logger.info(f"Saving reference file for scene {scene_name} \n") output = xr.Dataset.from_dict(output_dict) output.to_netcdf(f"{_test_path}/ref_confidence_{scene_name}.nc") def update_and_clean( shape: tuple, confidence: np.ndarray, bits: dict, output_dict: dict, test_name: str ) -> tuple[dict, dict]: """Update and clean output dictionary.""" output_dict[f"confidence_{test_name}"] = {"dims": ("x", "y"), "data": confidence} output_dict[f"qa_bit_{test_name}"] = {"dims": ("x", "y"), "data": bits["qa"]} output_dict[f"test_bit_{test_name}"] = {"dims": ("x", "y"), "data": bits["test"]} bits = { "test": np.zeros(shape, dtype=np.int8), "qa": np.zeros(shape, dtype=np.int8), } return bits, output_dict if __name__ == "__main__": main()