diff --git a/mvcm/__init__.py b/mvcm/__init__.py index 37371e9116b70493ebb75c20ef6fa05eb4611515..81c4cee4e9e5b06e8131d76e5cab083934fce443 100644 --- a/mvcm/__init__.py +++ b/mvcm/__init__.py @@ -1,3 +1,3 @@ """Set up version.""" -__version__ = "0.2.17" +__version__ = "0.2.18" diff --git a/mvcm/main_prep_only.py b/mvcm/main_prep_only.py index eb24c3cf174b5ae64a236b0393089666c77149b4..3edaac97d7da9cf56dd2e562547ad3efc26843da 100644 --- a/mvcm/main_prep_only.py +++ b/mvcm/main_prep_only.py @@ -23,6 +23,7 @@ logger = logging.getLogger(__name__) LOG_LEVELS = ["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"] proc = psutil.Process(os.getpid()) +_VERSION = "0.0.0" def main( diff --git a/mvcm/main_tests_only.py b/mvcm/main_tests_only.py index 517bc45285739c7ceeeac06340a8473e95eabfc1..58e1735c2022b63044b376c699cfc6bf8e5d9748 100644 --- a/mvcm/main_tests_only.py +++ b/mvcm/main_tests_only.py @@ -25,6 +25,7 @@ _LOG_FORMAT = "%(message)s" logging.basicConfig(level="NOTSET", datefmt="[%X]", format=_LOG_FORMAT, handlers=[RichHandler()]) logger = logging.getLogger(__name__) LOG_LEVELS = ["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"] +_VERSION = "0.0.0" proc = psutil.Process(os.getpid()) diff --git a/mvcm/read_data.py b/mvcm/read_data.py index 07199a2c79a5aafc4f0f067c92b99121569225e4..041aa8ef24c6d5fb073b9237e11a37fa1c0c89c5 100644 --- a/mvcm/read_data.py +++ b/mvcm/read_data.py @@ -785,10 +785,58 @@ def get_data( viirs_data.update(ancillary.pack_data()) logger.info("Ancillary data added to the dataset") logger.info(f"read_data: Memory usage: {proc.memory_info().rss / 1e6} MB") - - scene_xr, pixels_flags = scn.scene_id( - viirs_data, geo_data.sunglint_angle, thresholds["Snow_Mask"] + viirs_data.to_netcdf("tmp_viirs_data.nc") + # scene_xr, pixels_flags = scn.scene_id( + # viirs_data, geo_data.sunglint_angle, thresholds["Snow_Mask"] + # ) + + viirs_data = xr.open_dataset("tmp_viirs_data.nc", chunks="auto") + logger.info(f"pre_slicing: Memory usage: {proc.memory_info().rss / 1e6} MB") + scene_xr_1, pixels_flags_1 = scn.scene_id( + viirs_data.isel(number_of_lines=slice(0, 3232), number_of_pixels=slice(0, 3200)), + geo_data.sunglint_angle.isel( + number_of_lines=slice(0, 3232), number_of_pixels=slice(0, 3200) + ), + thresholds["Snow_Mask"], + ) + logger.info(f"slice 1: Memory usage: {proc.memory_info().rss / 1e6} MB") + scene_xr_2, pixels_flags_2 = scn.scene_id( + viirs_data.isel(number_of_lines=slice(3232, 6464), number_of_pixels=slice(0, 3200)), + geo_data.sunglint_angle.isel( + number_of_lines=slice(3232, 6464), number_of_pixels=slice(0, 3200) + ), + thresholds["Snow_Mask"], + ) + logger.info(f"slice 2: Memory usage: {proc.memory_info().rss / 1e6} MB") + scene_xr_tmp_1 = xr.concat([scene_xr_1, scene_xr_2], dim="number_of_lines") + pixels_flags_tmp_1 = xr.concat([pixels_flags_1, pixels_flags_2], dim="number_of_lines") + logger.info(f"concat 1: Memory usage: {proc.memory_info().rss / 1e6} MB") + + scene_xr_1, pixels_flags_1 = scn.scene_id( + viirs_data.isel(number_of_lines=slice(0, 3232), number_of_pixels=slice(3200, 6400)), + geo_data.sunglint_angle.isel( + number_of_lines=slice(0, 3232), number_of_pixels=slice(3200, 6400) + ), + thresholds["Snow_Mask"], + ) + logger.info(f"slice 3: Memory usage: {proc.memory_info().rss / 1e6} MB") + scene_xr_2, pixels_flags_2 = scn.scene_id( + viirs_data.isel(number_of_lines=slice(3232, 6464), number_of_pixels=slice(3200, 6400)), + geo_data.sunglint_angle.isel( + number_of_lines=slice(3232, 6464), number_of_pixels=slice(3200, 6400) + ), + thresholds["Snow_Mask"], ) + logger.info(f"slice 4: Memory usage: {proc.memory_info().rss / 1e6} MB") + scene_xr_tmp_2 = xr.concat([scene_xr_1, scene_xr_2], dim="number_of_lines") + pixels_flags_tmp_2 = xr.concat([pixels_flags_1, pixels_flags_2], dim="number_of_lines") + logger.info(f"concat 2: Memory usage: {proc.memory_info().rss / 1e6} MB") + + scene_xr = xr.concat([scene_xr_tmp_1, scene_xr_tmp_2], dim="number_of_pixels") + pixels_flags = xr.concat([pixels_flags_tmp_1, pixels_flags_tmp_2], dim="number_of_pixels") + logger.info(f"concat 3: Memory usage: {proc.memory_info().rss / 1e6} MB") + + logger.info(f"read_data: Memory usage: {proc.memory_info().rss / 1e6} MB") input_data = xr.Dataset(viirs_data, coords=scene_xr).chunk() input_data.drop_vars(["latitude", "longitude"])