From 8d9143b0af973f26f96997424ccc342fdeafcd2e Mon Sep 17 00:00:00 2001
From: Paolo Veglio <paolo.veglio@ssec.wisc.edu>
Date: Thu, 6 Feb 2025 16:08:57 +0000
Subject: [PATCH] fixed hires dimensions not being correct at night
---
mvcm/main_prep_only.py | 2 +-
mvcm/read_data.py | 78 ++++++++++++++++--------------------------
2 files changed, 30 insertions(+), 50 deletions(-)
diff --git a/mvcm/main_prep_only.py b/mvcm/main_prep_only.py
index b505640..3517cb5 100644
--- a/mvcm/main_prep_only.py
+++ b/mvcm/main_prep_only.py
@@ -144,7 +144,7 @@ def main(
viirs_m_bands = SensorConstants.VIIRS_VIS_BANDS + SensorConstants.VIIRS_IR_BANDS
- with Dataset(file_names["MOD03"]) as f:
+ with Dataset(file_names["IMG03"]) as f:
attrs = {attr: f.getncattr(attr) for attr in f.ncattrs()}
lat_shape = f["geolocation_data/latitude"][:].shape
diff --git a/mvcm/read_data.py b/mvcm/read_data.py
index a3a33e9..84b29f3 100644
--- a/mvcm/read_data.py
+++ b/mvcm/read_data.py
@@ -814,56 +814,36 @@ def get_data(
logger.info(f"read_data: Memory usage: {proc.memory_info().rss / 1e6} MB")
viirs_data.to_netcdf("tmp_viirs_data.nc")
- xdim = len(viirs_data.number_of_lines)
- ydim = len(viirs_data.number_of_pixels)
-
- viirs_data = xr.open_dataset("tmp_viirs_data.nc", chunks="auto")
- logger.info(f"pre_slicing: Memory usage: {proc.memory_info().rss / 1e6} MB")
- scene_xr_1, pixels_flags_1 = scn.scene_id(
- viirs_data.isel(
- number_of_lines=slice(0, int(xdim / 2)), number_of_pixels=slice(0, int(ydim / 2))
- ),
+ # xdim = len(viirs_data.number_of_lines)
+ # # ydim = len(viirs_data.number_of_pixels)
+ #
+ # n_slices = 4
+ # scene_xr, pixel_flags = scn.scene_id(
+ # viirs_data.isel(
+ # number_of_lines=slice(0, int(xdim / n_slices)),
+ # ),
+ # thresholds["Sun_Glint"]["bounds"][3],
+ # thresholds["Snow_Mask"],
+ # )
+ # for i in range(1, n_slices):
+ # slice_start = int(xdim / n_slices) * i
+ # slice_end = int(xdim / n_slices) * (i + 1)
+ # scene_xr_tmp, pixel_flags_tmp = scn.scene_id(
+ # viirs_data.isel(
+ # number_of_lines=slice(slice_start, slice_end),
+ # ),
+ # thresholds["Sun_Glint"]["bounds"][3],
+ # thresholds["Snow_Mask"],
+ # )
+ # logger.info(f"slice size: {scene_xr.Land_Day.shape}, {scene_xr_tmp.Land_Day.shape}")
+ # scene_xr = xr.concat([scene_xr, scene_xr_tmp], dim="number_of_lines")
+ # pixel_flags = xr.concat([pixel_flags, pixel_flags_tmp], dim="number_of_lines")
+
+ scene_xr, pixel_flags = scn.scene_id(
+ viirs_data,
thresholds["Sun_Glint"]["bounds"][3],
thresholds["Snow_Mask"],
)
- logger.info(f"slice 1: Memory usage: {proc.memory_info().rss / 1e6} MB")
- scene_xr_2, pixels_flags_2 = scn.scene_id(
- viirs_data.isel(
- number_of_lines=slice(int(xdim / 2), xdim), number_of_pixels=slice(0, int(ydim / 2))
- ),
- thresholds["Sun_Glint"]["bounds"][3],
- thresholds["Snow_Mask"],
- )
- logger.info(f"slice 2: Memory usage: {proc.memory_info().rss / 1e6} MB")
- scene_xr_tmp_1 = xr.concat([scene_xr_1, scene_xr_2], dim="number_of_lines")
- pixels_flags_tmp_1 = xr.concat([pixels_flags_1, pixels_flags_2], dim="number_of_lines")
- logger.info(f"concat 1: Memory usage: {proc.memory_info().rss / 1e6} MB")
-
- scene_xr_1, pixels_flags_1 = scn.scene_id(
- viirs_data.isel(
- number_of_lines=slice(0, int(xdim / 2)), number_of_pixels=slice(int(ydim / 2), ydim)
- ),
- thresholds["Sun_Glint"]["bounds"][3],
- thresholds["Snow_Mask"],
- )
- logger.info(f"slice 3: Memory usage: {proc.memory_info().rss / 1e6} MB")
- scene_xr_2, pixels_flags_2 = scn.scene_id(
- viirs_data.isel(
- number_of_lines=slice(int(xdim / 2), xdim), number_of_pixels=slice(int(ydim / 2), ydim)
- ),
- thresholds["Sun_Glint"]["bounds"][3],
- thresholds["Snow_Mask"],
- )
- logger.info(f"slice 4: Memory usage: {proc.memory_info().rss / 1e6} MB")
- scene_xr_tmp_2 = xr.concat([scene_xr_1, scene_xr_2], dim="number_of_lines")
- pixels_flags_tmp_2 = xr.concat([pixels_flags_1, pixels_flags_2], dim="number_of_lines")
- logger.info(f"concat 2: Memory usage: {proc.memory_info().rss / 1e6} MB")
-
- scene_xr = xr.concat([scene_xr_tmp_1, scene_xr_tmp_2], dim="number_of_pixels")
- pixels_flags = xr.concat([pixels_flags_tmp_1, pixels_flags_tmp_2], dim="number_of_pixels")
- logger.info(f"concat 3: Memory usage: {proc.memory_info().rss / 1e6} MB")
-
- logger.info(f"read_data: Memory usage: {proc.memory_info().rss / 1e6} MB")
input_data = xr.Dataset(viirs_data, coords=scene_xr).chunk()
input_data.drop_vars(["latitude", "longitude"])
@@ -871,6 +851,6 @@ def get_data(
logger.info(f"read_data: Memory usage: {proc.memory_info().rss / 1e6} MB")
# input_data.to_netcdf("input_data.nc")
- # pixels_flags.to_netcdf("pixels_flags.nc")
+ # pixel_flags.to_netcdf("pixels_flags.nc")
- return input_data, pixels_flags
+ return input_data, pixel_flags
--
GitLab