bump version

mvcm/__init__.py

 """Set up version."""
 
-__version__ = "0.2.17"
+__version__ = "0.2.18"

mvcm/main_prep_only.py

@@ -23,6 +23,7 @@ logger = logging.getLogger(__name__)
 LOG_LEVELS = ["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"]
 
 proc = psutil.Process(os.getpid())
+_VERSION = "0.0.0"
 
 
 def main(

mvcm/main_tests_only.py

@@ -25,6 +25,7 @@ _LOG_FORMAT = "%(message)s"
 logging.basicConfig(level="NOTSET", datefmt="[%X]", format=_LOG_FORMAT, handlers=[RichHandler()])
 logger = logging.getLogger(__name__)
 LOG_LEVELS = ["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"]
+_VERSION = "0.0.0"
 
 proc = psutil.Process(os.getpid())
 

mvcm/read_data.py

@@ -785,10 +785,58 @@ def get_data(
     viirs_data.update(ancillary.pack_data())
     logger.info("Ancillary data added to the dataset")
     logger.info(f"read_data: Memory usage: {proc.memory_info().rss / 1e6} MB")
-
-    scene_xr, pixels_flags = scn.scene_id(
-        viirs_data, geo_data.sunglint_angle, thresholds["Snow_Mask"]
+    viirs_data.to_netcdf("tmp_viirs_data.nc")
+    # scene_xr, pixels_flags = scn.scene_id(
+    #     viirs_data, geo_data.sunglint_angle, thresholds["Snow_Mask"]
+    # )
+
+    viirs_data = xr.open_dataset("tmp_viirs_data.nc", chunks="auto")
+    logger.info(f"pre_slicing: Memory usage: {proc.memory_info().rss / 1e6} MB")
+    scene_xr_1, pixels_flags_1 = scn.scene_id(
+        viirs_data.isel(number_of_lines=slice(0, 3232), number_of_pixels=slice(0, 3200)),
+        geo_data.sunglint_angle.isel(
+            number_of_lines=slice(0, 3232), number_of_pixels=slice(0, 3200)
+        ),
+        thresholds["Snow_Mask"],
+    )
+    logger.info(f"slice 1: Memory usage: {proc.memory_info().rss / 1e6} MB")
+    scene_xr_2, pixels_flags_2 = scn.scene_id(
+        viirs_data.isel(number_of_lines=slice(3232, 6464), number_of_pixels=slice(0, 3200)),
+        geo_data.sunglint_angle.isel(
+            number_of_lines=slice(3232, 6464), number_of_pixels=slice(0, 3200)
+        ),
+        thresholds["Snow_Mask"],
+    )
+    logger.info(f"slice 2: Memory usage: {proc.memory_info().rss / 1e6} MB")
+    scene_xr_tmp_1 = xr.concat([scene_xr_1, scene_xr_2], dim="number_of_lines")
+    pixels_flags_tmp_1 = xr.concat([pixels_flags_1, pixels_flags_2], dim="number_of_lines")
+    logger.info(f"concat 1: Memory usage: {proc.memory_info().rss / 1e6} MB")
+
+    scene_xr_1, pixels_flags_1 = scn.scene_id(
+        viirs_data.isel(number_of_lines=slice(0, 3232), number_of_pixels=slice(3200, 6400)),
+        geo_data.sunglint_angle.isel(
+            number_of_lines=slice(0, 3232), number_of_pixels=slice(3200, 6400)
+        ),
+        thresholds["Snow_Mask"],
+    )
+    logger.info(f"slice 3: Memory usage: {proc.memory_info().rss / 1e6} MB")
+    scene_xr_2, pixels_flags_2 = scn.scene_id(
+        viirs_data.isel(number_of_lines=slice(3232, 6464), number_of_pixels=slice(3200, 6400)),
+        geo_data.sunglint_angle.isel(
+            number_of_lines=slice(3232, 6464), number_of_pixels=slice(3200, 6400)
+        ),
+        thresholds["Snow_Mask"],
     )
+    logger.info(f"slice 4: Memory usage: {proc.memory_info().rss / 1e6} MB")
+    scene_xr_tmp_2 = xr.concat([scene_xr_1, scene_xr_2], dim="number_of_lines")
+    pixels_flags_tmp_2 = xr.concat([pixels_flags_1, pixels_flags_2], dim="number_of_lines")
+    logger.info(f"concat 2: Memory usage: {proc.memory_info().rss / 1e6} MB")
+
+    scene_xr = xr.concat([scene_xr_tmp_1, scene_xr_tmp_2], dim="number_of_pixels")
+    pixels_flags = xr.concat([pixels_flags_tmp_1, pixels_flags_tmp_2], dim="number_of_pixels")
+    logger.info(f"concat 3: Memory usage: {proc.memory_info().rss / 1e6} MB")
+
+    logger.info(f"read_data: Memory usage: {proc.memory_info().rss / 1e6} MB")
 
     input_data = xr.Dataset(viirs_data, coords=scene_xr).chunk()
     input_data.drop_vars(["latitude", "longitude"])