changed data type to reduce memory footprint

mvcm/main.py

@@ -96,6 +96,8 @@ _mod_bands = [
     "M16",
 ]
 
+_img_bands = ["I01", "I02", "I03", "I04", "I05"]
+
 
 def timer(func):
     """Compute elapsed time."""
@@ -210,13 +212,22 @@ def main(
     # We are not processing night granules
     if use_hires is True:
         with xr.open_dataset(file_names["IMG02"], group="observation_data") as vnp02:
-            for b in _mod_bands:
+            for b in _img_bands:
                 try:
                     vnp02[b]
                 except KeyError:
                     logger.info(f"Band {b} not found in file. No output will be written.")
                     return
             logger.info(f"All bands found in file {file_names['IMG02']}. The code will run.")
+    else:
+        with xr.open_dataset(file_names["MOD02"], group="observation_data") as vnp02:
+            for b in _mod_bands:
+                try:
+                    vnp02[b]
+                except KeyError:
+                    logger.info(f"Band {b} not found in file. No output will be written.")
+                    return
+            logger.info(f"All bands found in file {file_names['MOD02']}. The code will run.")
 
     with Dataset(file_names["MOD03"]) as f:
         # time_coverage_start = f.getncattr("time_coverage_start")
@@ -236,7 +247,7 @@ def main(
         log_level=LOG_LEVELS[verbose_level],
     )
 
-    cmin_3d = np.ones((18, viirs_data.M11.shape[0], viirs_data.M11.shape[1]))
+    cmin_3d = np.ones((18, viirs_data.M11.shape[0], viirs_data.M11.shape[1]), dtype=np.float32)
 
     ##########################################################
     _bitlist = [
@@ -267,8 +278,8 @@ def main(
 
     for b in _bitlist:
         bits[b] = {
-            "test": np.zeros(viirs_data.M11.shape),
-            "qa": np.zeros(viirs_data.M11.shape),
+            "test": np.zeros(viirs_data.M11.shape, dtype=np.int8),
+            "qa": np.zeros(viirs_data.M11.shape, dtype=np.int8),
         }
     i = 0
 
@@ -282,7 +293,7 @@ def main(
         np.zeros(viirs_data.M11.shape) * np.nan, dims=viirs_data.M11.dims
     )
 
-    scene_types = np.zeros(viirs_data.M11.shape)
+    scene_types = np.zeros(viirs_data.M11.shape, dtype=np.int8)
     for _scene_i, scene_name in enumerate(_scene_list):
         scene_types[viirs_data[scene_name].values == 1] = i
 
@@ -295,11 +306,11 @@ def main(
         my_scene = tst.CloudTests(data=viirs_data, scene_name=scene_name, thresholds=thresholds)
 
         # Initialize the confidence arrays for the various test groups
-        cmin_g1 = np.ones(viirs_data.M11.shape)
-        cmin_g2 = np.ones(viirs_data.M11.shape)
-        cmin_g3 = np.ones(viirs_data.M11.shape)
-        cmin_g4 = np.ones(viirs_data.M11.shape)
-        cmin_g5 = np.ones(viirs_data.M11.shape)
+        cmin_g1 = np.ones(viirs_data.M11.shape, dtype=np.float32)
+        cmin_g2 = np.ones(viirs_data.M11.shape, dtype=np.float32)
+        cmin_g3 = np.ones(viirs_data.M11.shape, dtype=np.float32)
+        cmin_g4 = np.ones(viirs_data.M11.shape, dtype=np.float32)
+        cmin_g5 = np.ones(viirs_data.M11.shape, dtype=np.float32)
         # cmin_temp = np.ones(viirs_data.M11.shape)
 
         if use_hires is True:
@@ -570,6 +581,14 @@ def main(
         "M15-M16": {"dims": ("x", "y"), "data": viirs_data["M15-M16"].values},
         # "Land_Day": {"dims": ("x", "y"), "data": viirs_data.Land_Day.values},
         # "Land_Day_Desert": {"dims": ("x", "y"), "data": viirs_data.Land_Day_Desert.values},
+        "Ocean_Day": {"dims": ("x", "y"), "data": viirs_data.Ocean_Day.values},
+        "Polar_Day_Ocean": {"dims": ("x", "y"), "data": viirs_data.Polar_Day_Ocean.values},
+        "Polar_Day_Land": {"dims": ("x", "y"), "data": viirs_data.Polar_Day_Land.values},
+        "Polar_Day_Desert": {"dims": ("x", "y"), "data": viirs_data.Polar_Day_Desert.values},
+        "Polar_Day_Desert_Coast": {
+            "dims": ("x", "y"),
+            "data": viirs_data.Polar_Day_Desert_Coast.values,
+        },
         "elevation": {"dims": ("x", "y"), "data": viirs_data.height.values},
         "scene_type": {"dims": ("x", "y"), "data": scene_types},
         # 'thr': {'dims': ('x', 'y'),

mvcm/preprocess_thresholds.py

 """Preprocessing thresholds module."""
 
-import logging
-from typing import Dict
+import logging  # noqa
 
 import numpy as np
 import xarray as xr
 from numpy.lib.stride_tricks import sliding_window_view
 
 import ancillary_data as anc
-import mvcm.utils as utils
+from mvcm import utils
 
 # _dtr = np.pi / 180
 _DTR = np.pi / 180
@@ -43,7 +42,7 @@ def prepare_11_12um_thresholds(thresholds: dict, dim1: int) -> dict:
 
 
 def thresholds_11_12um(
-    data: xr.Dataset, thresholds: Dict, scene: str, scene_idx: tuple
+    data: xr.Dataset, thresholds: dict, scene: str, scene_idx: tuple
 ) -> np.ndarray:
     """Compute 11-12um Test thresholds."""
     cosvza = np.cos(data.sensor_zenith.values[scene_idx].ravel() * _DTR)
@@ -132,7 +131,8 @@ def thresholds_NIR(data, thresholds, scene, test_name, scene_idx):
         band_n = 7
         vzcpow = thresholds["VZA_correction"]["vzcpow"][2]
     else:
-        raise ValueError("Test name not valid")
+        err_msg = "Test name not valid"
+        raise ValueError(err_msg)
 
     refang = data.sunglint_angle.values[scene_idx].ravel()
     sunglint_thresholds = thresholds["Sun_Glint"]

mvcm/scene.py

 """Functions that define scene type."""
+
 import logging
 from typing import Dict
 
@@ -154,7 +155,7 @@ def find_scene(data: xr.Dataset, sunglint_angle: float) -> Dict:
     # tmp[day == 1] = day
     # tmp[day == 0] = night
 
-    scene_flag = {flg: np.zeros((dim1, dim2)) for flg in _flags}
+    scene_flag = {flg: np.zeros((dim1, dim2), dtype=np.int8) for flg in _flags}
 
     scene_flag["day"][sza <= 85] = 1
     scene_flag["visusd"][sza <= 85] = 1
@@ -172,8 +173,8 @@ def find_scene(data: xr.Dataset, sunglint_angle: float) -> Dict:
     eco[idx] = 14
 
     # start by defining everything as land
-    scene_flag["land"] = np.ones((dim1, dim2))
-    scene_flag["water"] = np.zeros((dim1, dim2))
+    scene_flag["land"] = np.ones((dim1, dim2), dtype=np.int8)
+    scene_flag["water"] = np.zeros((dim1, dim2), dtype=np.int8)
 
     # Fix-up for missing ecosystem data in eastern Greenland and north-eastern Siberia.
     # Without this, these regions become completely "coast".
@@ -292,7 +293,7 @@ def find_scene(data: xr.Dataset, sunglint_angle: float) -> Dict:
         | (eco == 71)
         | (eco == 50)
     )
-    scene_flag["vrused"] = np.ones((dim1, dim2))
+    scene_flag["vrused"] = np.ones((dim1, dim2), dtype=np.int8)
     scene_flag["vrused"][idx] = 0
 
     snow_fraction = data["geos_snow_fraction"]
@@ -417,12 +418,13 @@ def find_scene(data: xr.Dataset, sunglint_angle: float) -> Dict:
 def scene_id(scene_flag):
     """Define scene type."""
     dim1, dim2 = scene_flag["day"].shape[0], scene_flag["day"].shape[1]
-    scene = {scn: np.zeros((dim1, dim2)) for scn in _scene_list}
+    scene = {scn: np.zeros((dim1, dim2), dtype=np.int8) for scn in _scene_list}
 
     # Ocean Day
     idx = np.nonzero(
         (scene_flag["water"] == 1)
         & (scene_flag["day"] == 1)
+        & (scene_flag["polar"] == 0)
         & ((scene_flag["ice"] == 0) | (scene_flag["snow"] == 0))
     )  # &
     # (scene_flag['polar'] == 0) & (scene_flag['antarctica'] == 0) &
@@ -433,6 +435,7 @@ def scene_id(scene_flag):
     idx = np.nonzero(
         (scene_flag["water"] == 1)
         & (scene_flag["night"] == 1)
+        & (scene_flag["polar"] == 0)
         & ((scene_flag["ice"] == 0) | (scene_flag["snow"] == 0))
     )  # &
     # (scene_flag['polar'] == 0) & (scene_flag['antarctica'] == 0) &