fixed hires dimensions not being correct at night

mvcm/main_prep_only.py

@@ -144,7 +144,7 @@ def main(
 
     viirs_m_bands = SensorConstants.VIIRS_VIS_BANDS + SensorConstants.VIIRS_IR_BANDS
 
-    with Dataset(file_names["MOD03"]) as f:
+    with Dataset(file_names["IMG03"]) as f:
         attrs = {attr: f.getncattr(attr) for attr in f.ncattrs()}
         lat_shape = f["geolocation_data/latitude"][:].shape
 

mvcm/read_data.py

@@ -814,56 +814,36 @@ def get_data(
     logger.info(f"read_data: Memory usage: {proc.memory_info().rss / 1e6} MB")
     viirs_data.to_netcdf("tmp_viirs_data.nc")
 
-    xdim = len(viirs_data.number_of_lines)
-    ydim = len(viirs_data.number_of_pixels)
-
-    viirs_data = xr.open_dataset("tmp_viirs_data.nc", chunks="auto")
-    logger.info(f"pre_slicing: Memory usage: {proc.memory_info().rss / 1e6} MB")
-    scene_xr_1, pixels_flags_1 = scn.scene_id(
-        viirs_data.isel(
-            number_of_lines=slice(0, int(xdim / 2)), number_of_pixels=slice(0, int(ydim / 2))
-        ),
+    # xdim = len(viirs_data.number_of_lines)
+    # # ydim = len(viirs_data.number_of_pixels)
+    #
+    # n_slices = 4
+    # scene_xr, pixel_flags = scn.scene_id(
+    #     viirs_data.isel(
+    #         number_of_lines=slice(0, int(xdim / n_slices)),
+    #     ),
+    #     thresholds["Sun_Glint"]["bounds"][3],
+    #     thresholds["Snow_Mask"],
+    # )
+    # for i in range(1, n_slices):
+    #     slice_start = int(xdim / n_slices) * i
+    #     slice_end = int(xdim / n_slices) * (i + 1)
+    #     scene_xr_tmp, pixel_flags_tmp = scn.scene_id(
+    #         viirs_data.isel(
+    #             number_of_lines=slice(slice_start, slice_end),
+    #         ),
+    #         thresholds["Sun_Glint"]["bounds"][3],
+    #         thresholds["Snow_Mask"],
+    #     )
+    #     logger.info(f"slice size: {scene_xr.Land_Day.shape}, {scene_xr_tmp.Land_Day.shape}")
+    #     scene_xr = xr.concat([scene_xr, scene_xr_tmp], dim="number_of_lines")
+    #     pixel_flags = xr.concat([pixel_flags, pixel_flags_tmp], dim="number_of_lines")
+
+    scene_xr, pixel_flags = scn.scene_id(
+        viirs_data,
         thresholds["Sun_Glint"]["bounds"][3],
         thresholds["Snow_Mask"],
     )
-    logger.info(f"slice 1: Memory usage: {proc.memory_info().rss / 1e6} MB")
-    scene_xr_2, pixels_flags_2 = scn.scene_id(
-        viirs_data.isel(
-            number_of_lines=slice(int(xdim / 2), xdim), number_of_pixels=slice(0, int(ydim / 2))
-        ),
-        thresholds["Sun_Glint"]["bounds"][3],
-        thresholds["Snow_Mask"],
-    )
-    logger.info(f"slice 2: Memory usage: {proc.memory_info().rss / 1e6} MB")
-    scene_xr_tmp_1 = xr.concat([scene_xr_1, scene_xr_2], dim="number_of_lines")
-    pixels_flags_tmp_1 = xr.concat([pixels_flags_1, pixels_flags_2], dim="number_of_lines")
-    logger.info(f"concat 1: Memory usage: {proc.memory_info().rss / 1e6} MB")
-
-    scene_xr_1, pixels_flags_1 = scn.scene_id(
-        viirs_data.isel(
-            number_of_lines=slice(0, int(xdim / 2)), number_of_pixels=slice(int(ydim / 2), ydim)
-        ),
-        thresholds["Sun_Glint"]["bounds"][3],
-        thresholds["Snow_Mask"],
-    )
-    logger.info(f"slice 3: Memory usage: {proc.memory_info().rss / 1e6} MB")
-    scene_xr_2, pixels_flags_2 = scn.scene_id(
-        viirs_data.isel(
-            number_of_lines=slice(int(xdim / 2), xdim), number_of_pixels=slice(int(ydim / 2), ydim)
-        ),
-        thresholds["Sun_Glint"]["bounds"][3],
-        thresholds["Snow_Mask"],
-    )
-    logger.info(f"slice 4: Memory usage: {proc.memory_info().rss / 1e6} MB")
-    scene_xr_tmp_2 = xr.concat([scene_xr_1, scene_xr_2], dim="number_of_lines")
-    pixels_flags_tmp_2 = xr.concat([pixels_flags_1, pixels_flags_2], dim="number_of_lines")
-    logger.info(f"concat 2: Memory usage: {proc.memory_info().rss / 1e6} MB")
-
-    scene_xr = xr.concat([scene_xr_tmp_1, scene_xr_tmp_2], dim="number_of_pixels")
-    pixels_flags = xr.concat([pixels_flags_tmp_1, pixels_flags_tmp_2], dim="number_of_pixels")
-    logger.info(f"concat 3: Memory usage: {proc.memory_info().rss / 1e6} MB")
-
-    logger.info(f"read_data: Memory usage: {proc.memory_info().rss / 1e6} MB")
 
     input_data = xr.Dataset(viirs_data, coords=scene_xr).chunk()
     input_data.drop_vars(["latitude", "longitude"])
@@ -871,6 +851,6 @@ def get_data(
     logger.info(f"read_data: Memory usage: {proc.memory_info().rss / 1e6} MB")
 
     # input_data.to_netcdf("input_data.nc")
-    # pixels_flags.to_netcdf("pixels_flags.nc")
+    # pixel_flags.to_netcdf("pixels_flags.nc")
 
-    return input_data, pixels_flags
+    return input_data, pixel_flags