From f64f151d08239417e48f10a0f566ee6fd642f05b Mon Sep 17 00:00:00 2001
From: Paolo Veglio <paolo.veglio@ssec.wisc.edu>
Date: Thu, 7 Nov 2024 17:16:26 +0000
Subject: [PATCH] bump version
---
mvcm/__init__.py | 2 +-
mvcm/main_prep_only.py | 1 +
mvcm/main_tests_only.py | 1 +
mvcm/read_data.py | 54 ++++++++++++++++++++++++++++++++++++++---
4 files changed, 54 insertions(+), 4 deletions(-)
diff --git a/mvcm/__init__.py b/mvcm/__init__.py
index 37371e9..81c4cee 100644
--- a/mvcm/__init__.py
+++ b/mvcm/__init__.py
@@ -1,3 +1,3 @@
"""Set up version."""
-__version__ = "0.2.17"
+__version__ = "0.2.18"
diff --git a/mvcm/main_prep_only.py b/mvcm/main_prep_only.py
index eb24c3c..3edaac9 100644
--- a/mvcm/main_prep_only.py
+++ b/mvcm/main_prep_only.py
@@ -23,6 +23,7 @@ logger = logging.getLogger(__name__)
LOG_LEVELS = ["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"]
proc = psutil.Process(os.getpid())
+_VERSION = "0.0.0"
def main(
diff --git a/mvcm/main_tests_only.py b/mvcm/main_tests_only.py
index 517bc45..58e1735 100644
--- a/mvcm/main_tests_only.py
+++ b/mvcm/main_tests_only.py
@@ -25,6 +25,7 @@ _LOG_FORMAT = "%(message)s"
logging.basicConfig(level="NOTSET", datefmt="[%X]", format=_LOG_FORMAT, handlers=[RichHandler()])
logger = logging.getLogger(__name__)
LOG_LEVELS = ["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"]
+_VERSION = "0.0.0"
proc = psutil.Process(os.getpid())
diff --git a/mvcm/read_data.py b/mvcm/read_data.py
index 07199a2..041aa8e 100644
--- a/mvcm/read_data.py
+++ b/mvcm/read_data.py
@@ -785,10 +785,58 @@ def get_data(
viirs_data.update(ancillary.pack_data())
logger.info("Ancillary data added to the dataset")
logger.info(f"read_data: Memory usage: {proc.memory_info().rss / 1e6} MB")
-
- scene_xr, pixels_flags = scn.scene_id(
- viirs_data, geo_data.sunglint_angle, thresholds["Snow_Mask"]
+ viirs_data.to_netcdf("tmp_viirs_data.nc")
+ # scene_xr, pixels_flags = scn.scene_id(
+ # viirs_data, geo_data.sunglint_angle, thresholds["Snow_Mask"]
+ # )
+
+ viirs_data = xr.open_dataset("tmp_viirs_data.nc", chunks="auto")
+ logger.info(f"pre_slicing: Memory usage: {proc.memory_info().rss / 1e6} MB")
+ scene_xr_1, pixels_flags_1 = scn.scene_id(
+ viirs_data.isel(number_of_lines=slice(0, 3232), number_of_pixels=slice(0, 3200)),
+ geo_data.sunglint_angle.isel(
+ number_of_lines=slice(0, 3232), number_of_pixels=slice(0, 3200)
+ ),
+ thresholds["Snow_Mask"],
+ )
+ logger.info(f"slice 1: Memory usage: {proc.memory_info().rss / 1e6} MB")
+ scene_xr_2, pixels_flags_2 = scn.scene_id(
+ viirs_data.isel(number_of_lines=slice(3232, 6464), number_of_pixels=slice(0, 3200)),
+ geo_data.sunglint_angle.isel(
+ number_of_lines=slice(3232, 6464), number_of_pixels=slice(0, 3200)
+ ),
+ thresholds["Snow_Mask"],
+ )
+ logger.info(f"slice 2: Memory usage: {proc.memory_info().rss / 1e6} MB")
+ scene_xr_tmp_1 = xr.concat([scene_xr_1, scene_xr_2], dim="number_of_lines")
+ pixels_flags_tmp_1 = xr.concat([pixels_flags_1, pixels_flags_2], dim="number_of_lines")
+ logger.info(f"concat 1: Memory usage: {proc.memory_info().rss / 1e6} MB")
+
+ scene_xr_1, pixels_flags_1 = scn.scene_id(
+ viirs_data.isel(number_of_lines=slice(0, 3232), number_of_pixels=slice(3200, 6400)),
+ geo_data.sunglint_angle.isel(
+ number_of_lines=slice(0, 3232), number_of_pixels=slice(3200, 6400)
+ ),
+ thresholds["Snow_Mask"],
+ )
+ logger.info(f"slice 3: Memory usage: {proc.memory_info().rss / 1e6} MB")
+ scene_xr_2, pixels_flags_2 = scn.scene_id(
+ viirs_data.isel(number_of_lines=slice(3232, 6464), number_of_pixels=slice(3200, 6400)),
+ geo_data.sunglint_angle.isel(
+ number_of_lines=slice(3232, 6464), number_of_pixels=slice(3200, 6400)
+ ),
+ thresholds["Snow_Mask"],
)
+ logger.info(f"slice 4: Memory usage: {proc.memory_info().rss / 1e6} MB")
+ scene_xr_tmp_2 = xr.concat([scene_xr_1, scene_xr_2], dim="number_of_lines")
+ pixels_flags_tmp_2 = xr.concat([pixels_flags_1, pixels_flags_2], dim="number_of_lines")
+ logger.info(f"concat 2: Memory usage: {proc.memory_info().rss / 1e6} MB")
+
+ scene_xr = xr.concat([scene_xr_tmp_1, scene_xr_tmp_2], dim="number_of_pixels")
+ pixels_flags = xr.concat([pixels_flags_tmp_1, pixels_flags_tmp_2], dim="number_of_pixels")
+ logger.info(f"concat 3: Memory usage: {proc.memory_info().rss / 1e6} MB")
+
+ logger.info(f"read_data: Memory usage: {proc.memory_info().rss / 1e6} MB")
input_data = xr.Dataset(viirs_data, coords=scene_xr).chunk()
input_data.drop_vars(["latitude", "longitude"])
--
GitLab