diff --git a/mvcm/main_prep_only.py b/mvcm/main_prep_only.py
index 3517cb5630c9be8b3d7b52ac5d6096f9053ffc67..4cf7d79d9eb617cf581ab8c5d9bcbde6cef73817 100644
--- a/mvcm/main_prep_only.py
+++ b/mvcm/main_prep_only.py
@@ -51,6 +51,7 @@ def main(
hires_only: bool = False,
mvcm_threshold_file: str | None,
use_sipsanc: bool = False,
+ run_night: bool = False,
) -> None:
"""MVCM main function.
@@ -102,6 +103,8 @@ def main(
Path to the MVCM threshold file
use_sipsanc: bool
Use sipsanc tool for ancillary files
+ run_night: bool
+ Run MVCM also for night granules
Returns
-------
None
@@ -144,7 +147,10 @@ def main(
viirs_m_bands = SensorConstants.VIIRS_VIS_BANDS + SensorConstants.VIIRS_IR_BANDS
- with Dataset(file_names["IMG03"]) as f:
+ geo_filename_key = "MOD03"
+ if use_hires is True:
+ geo_filename_key = "IMG03"
+ with Dataset(file_names[geo_filename_key]) as f:
attrs = {attr: f.getncattr(attr) for attr in f.ncattrs()}
lat_shape = f["geolocation_data/latitude"][:].shape
@@ -154,25 +160,26 @@ def main(
# We are not processing night granules for now
if use_hires is True:
with xr.open_dataset(file_names["IMG02"], group="observation_data") as vnp02:
- for b in SensorConstants.VIIRS_IMG_BANDS:
- try:
- vnp02[b]
- except KeyError:
- logger.info(f"Band {b} not found in file. Hires data will be fill values.")
- attrs = {"version": _VERSION, "satellite": "VIIRS", "sensor": "VIIRS"}
- save_output(None, attrs, out_file, compression=5, debug=debug, shape=lat_shape)
- return
+ if (
+ not set(SensorConstants.VIIRS_IMG_BANDS).issubset(list(vnp02.variables))
+ and not run_night
+ ):
+ logger.info(
+ "One or more bands missing from L1b file. Hires data will be fill values."
+ )
+ attrs = {"version": _VERSION, "satellite": "VIIRS", "sensor": "VIIRS"}
+ save_output(None, attrs, out_file, compression=5, debug=debug, shape=lat_shape)
+ return
logger.info(f"All bands found in file {file_names['IMG02']}. The code will run.")
else:
with xr.open_dataset(file_names["MOD02"], group="observation_data") as vnp02:
- for b in viirs_m_bands:
- try:
- vnp02[b]
- except KeyError:
- logger.info(f"Band {b} not found in file. Hires data will be fill values.")
- attrs = {"version": _VERSION, "satellite": "VIIRS", "sensor": "VIIRS"}
- save_output(None, attrs, out_file, compression=5, debug=debug, shape=lat_shape)
- return
+ if not set(viirs_m_bands).issubset(list(vnp02.variables)) and not run_night:
+ logger.info(
+ "One or more bands missing from L1b file. Hires data will be fill values."
+ )
+ attrs = {"version": _VERSION, "satellite": "VIIRS", "sensor": "VIIRS"}
+ save_output(None, attrs, out_file, compression=5, debug=debug, shape=lat_shape)
+ return
logger.info(f"All bands found in file {file_names['MOD02']}. The code will run.")
logger.info(f"Memory usage #2: {proc.memory_info().rss / 1e6} MB")
@@ -253,6 +260,7 @@ if __name__ == "__main__":
parser.add_argument(
"--sipsanc", action="store_true", help="use sipsanc tool for ancillary files"
)
+ parser.add_argument("--night", action="store_true", help="run MVCM also for night granules.")
args = parser.parse_args()
@@ -275,7 +283,8 @@ if __name__ == "__main__":
out_file = args.out
verbose = args.verbose or False
debug = args.debug or False
- sipsanc = args.sipsan or False
+ sipsanc = args.sipsanc or False
+ run_night = args.night or False
main(
satellite=satellite,
@@ -301,6 +310,7 @@ if __name__ == "__main__":
hires_only=args.hires_only,
mvcm_threshold_file=args.mvcm_thresholds,
use_sipsanc=sipsanc,
+ run_night=run_night,
)
# tracemalloc.stop()
diff --git a/mvcm/read_data.py b/mvcm/read_data.py
index 84b29f37c91dad7b7f627f883ec656f48d05d704..47ac6870239cacc970823d21dcbb0e434c96ce55 100644
--- a/mvcm/read_data.py
+++ b/mvcm/read_data.py
@@ -218,6 +218,7 @@ class ReadData(CollectInputs):
/ np.cos(solar_zenith * ConstantsNamespace.DTR),
)
else:
+ rad_data[band] = (self.dims, np.full_like(solar_zenith, np.nan))
logger.info(f"Reflective band {band} not found in L1b file")
elif band in SensorConstants.EMISSIVE_BANDS:
@@ -228,6 +229,7 @@ class ReadData(CollectInputs):
l1b_data[bt_lut].values[l1b_data[band].values],
)
else:
+ rad_data[band] = (self.dims, np.full_like(solar_zenith, np.nan))
logger.info(f"Emissive band {band} not found in L1b file")
else:
pass
@@ -761,11 +763,13 @@ def get_data(
geo_data_mod = viirs.read_viirs_geo()
viirs_data_mod = viirs.read_viirs_l1b(geo_data_mod.solar_zenith.values)
+ geo_fname = "MOD03"
if hires is True:
geo_data = viirs_hires.read_viirs_geo()
viirs_data_img = viirs_hires.read_viirs_l1b(geo_data.solar_zenith.values)
viirs_data = viirs_hires.preprocess_viirs(geo_data, viirs_data_mod, viirs_data_img)
res = 1
+ geo_fname = "IMG03"
with xr.open_dataset(file_names["IMG02"]) as vnp02:
time_str = dt.strftime(dt.strptime(vnp02.time_coverage_start, fmt_in), fmt_out) # noqa
else:
@@ -780,7 +784,7 @@ def get_data(
"sipsanc",
"regrid",
"all",
- f"{file_names['IMG03']}",
+ f"{file_names[geo_fname]}",
"--output",
f"ancillary_{time_str}.nc",
]
diff --git a/mvcm/write_output.py b/mvcm/write_output.py
index d7e8213716f83cdabc9c2d93d24f6bb4a7c0bb29..5b868f3e28614798e8d33fbb5e55123cbe2544b1 100644
--- a/mvcm/write_output.py
+++ b/mvcm/write_output.py
@@ -6,6 +6,7 @@ from subprocess import run
import numpy as np
import xarray as xr
+from numpy.core.multiarray import dtype
_attributes_list = [
"orbit_number",
@@ -86,46 +87,54 @@ def save_output(
},
"sensor_azimuth": {
"dims": ("number_of_lines_hires", "number_of_pixels_hires"),
- "data": data.sensor_azimuth.values,
+ "data": np.array(data.sensor_azimuth.values * 100, dtype="int16"),
"attrs": {
"units": "degrees",
"long_name": "Sensor azimuth at pixel locations",
- "_FillValue": -999.9,
- "valid_min": -180.0,
- "valid_max": 180.0,
+ "_FillValue": -9999,
+ "valid_min": -18000,
+ "valid_max": 18000,
+ "scale_factor": 0.01,
+ "add_offset": 0.0,
},
},
"sensor_zenith": {
"dims": ("number_of_lines_hires", "number_of_pixels_hires"),
- "data": data.sensor_zenith.values,
+ "data": np.array(data.sensor_zenith.values * 100, dtype="int16"),
"attrs": {
"units": "degrees",
"long_name": "Sensor zenith at pixel locations",
- "_FillValue": -999.9,
- "valid_min": 0.0,
- "valid_max": 180.0,
+ "_FillValue": -9999,
+ "valid_min": 0,
+ "valid_max": 18000,
+ "scale_factor": 0.01,
+ "add_offset": 0.0,
},
},
"solar_azimuth": {
"dims": ("number_of_lines_hires", "number_of_pixels_hires"),
- "data": data.solar_azimuth.values,
+ "data": np.array(data.solar_azimuth.values * 100, dtype="int16"),
"attrs": {
"units": "degrees",
"long_name": "Solar azimuth at pixel locations",
- "_FillValue": -999.9,
- "valid_min": -180.0,
- "valid_max": 180.0,
+ "_FillValue": -9999,
+ "valid_min": -18000,
+ "valid_max": 18000,
+ "scale_factor": 0.01,
+ "add_offset": 0.0,
},
},
"solar_zenith": {
"dims": ("number_of_lines_hires", "number_of_pixels_hires"),
- "data": data.solar_zenith.values,
+ "data": np.array(data.solar_zenith.values * 100, dtype="int16"),
"attrs": {
"units": "degrees",
"long_name": "Solar zenith at pixel locations",
- "_FillValue": -999.9,
- "valid_min": 0.0,
- "valid_max": 180.0,
+ "_FillValue": -9999,
+ "valid_min": 0,
+ "valid_max": 18000,
+ "scale_factor": 0.01,
+ "add_offset": 0.0,
},
},
}
@@ -134,7 +143,7 @@ def save_output(
geophysical_data = {
"Cloud_Mask": {
"dims": ("byte_segment", "number_of_lines_hires", "number_of_pixels_hires"),
- "data": data.cloud_mask.values,
+ "data": np.array(data.cloud_mask.values, dtype="uint8"),
"attrs": {
"units": "none",
"long_name": "VIIRS Cloud Maks and Spectral Test Results",
@@ -145,7 +154,7 @@ def save_output(
},
"Quality_Assurance": {
"dims": ("number_of_lines_hires", "number_of_pixels_hires", "QA_dimension"),
- "data": data.quality_assurance.values,
+ "data": np.array(data.quality_assurance.values, dtype="uint8"),
"attrs": {
"units": "none",
"long_name": "Quality Assurance for VIIRS Cloud Mask",
@@ -156,7 +165,7 @@ def save_output(
},
"Clear_Sky_Confidence": {
"dims": ("number_of_lines_hires", "number_of_pixels_hires"),
- "data": data.confidence.values,
+ "data": np.array(data.confidence.values, dtype="float32"),
"attrs": {
"units": "none",
"long_name": "VIIRS Clear Sky Confidence",
@@ -167,7 +176,7 @@ def save_output(
},
"Integer_Cloud_Mask": {
"dims": ("number_of_lines_hires", "number_of_pixels_hires"),
- "data": data.integer_cloud_mask.values,
+ "data": np.array(data.integer_cloud_mask.values, dtype="int8"),
"attrs": {
"units": "none",
"long_name": "VIIRS Integer Cloud Mask",
@@ -185,7 +194,7 @@ def save_output(
spi_data = {
"Cloud_Mask_SPI": {
"dims": ("number_of_lines", "number_of_pixels", "SPI_nbands"),
- "data": data.spi.values,
+ "data": np.array(data.spi.values, dtype="float32"),
"attrs": {
"units": "none",
"long_name": "Cloud Mask SPI",