diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ccec9c9a2565069aeecba61b0c88abda8fb3659a..f41623d421735766e9314792f8b03c0597ee5051 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,43 +1,36 @@ exclude: '^$' fail_fast: false repos: - - repo: https://github.com/psf/black - rev: 24.4.2 + - repo: https://gitlab.com/vojko.pribudic.foss/pre-commit-update + rev: v0.8.0 hooks: - - id: black - language_version: python3 - exclude: versioneer.py - args: - - --target-version=py38 - - repo: https://github.com/pycqa/isort - rev: 5.13.2 - hooks: - - id: isort - language_version: python3 + - id: pre-commit-update + args: [ --dry-run ] - repo: https://github.com/astral-sh/ruff-pre-commit - # Ruff version. - rev: 'v0.5.5' + rev: 'v0.12.1' hooks: - id: ruff + args: ["--fix"] + - id: ruff-format - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - id: check-yaml args: [--unsafe] - repo: https://github.com/PyCQA/bandit - rev: '1.7.9' # Update me! + rev: '1.8.5' # Update me! hooks: - id: bandit args: [--ini, .bandit] - repo: https://github.com/pre-commit/mirrors-mypy - rev: 'v1.11.0' # Use the sha / tag you want to point at + rev: 'v1.16.1' # Use the sha / tag you want to point at hooks: - id: mypy additional_dependencies: - types-docutils - - types-pkg-resources + - types-setuptools - types-PyYAML - types-requests ci: diff --git a/aossceilo/CONFIG.py b/aossceilo/CONFIG.py index 86ecb6a0a03e8ac33f535ba58894167c0524b739..02ad901a22dbfc448622d59b2bc47c345ad3ce6e 100644 --- a/aossceilo/CONFIG.py +++ b/aossceilo/CONFIG.py @@ -16,18 +16,14 @@ from datetime import datetime, timedelta from metobscommon.util import CONFIG as c -CEILO_INCOMING_DIR = os.environ.get( - "CEILO_INCOMING_DIR", "/beach/incoming/Instrument_Data/METOBS/RIG/Ceilo/raw" -) +CEILO_INCOMING_DIR = os.environ.get("CEILO_INCOMING_DIR", "/beach/incoming/Instrument_Data/METOBS/RIG/Ceilo/raw") CEILO_PRAW_DIR = os.environ.get("CEILO_PRAW_DIR", "/beach/raw/aoss/ceilo") CEILO_CACHE_DIR = os.environ.get("CEILO_CACHE_DIR", "/beach/cache/aoss/ceilo") CEILO_LATEST_DIR = os.environ.get("CEILO_LATEST_DIR", "/beach/cache/aoss/ceilo") CEILO_DIR_FORMAT = os.environ.get("CEILO_DIR_FORMAT", "%Y/%m") CEILO_ASCII_LOC = os.environ.get("CEILO_ASCII_LOC", "/beach/cache/aoss/ceilo") CEILO_NC_LOC = os.environ.get("CEILO_NC_LOC", "/beach/cache/aoss/ceilo") -CEILO_IMG_LOC = os.environ.get( - "CEILO_IMG_LOC", "http://metobs.ssec.wisc.edu/pub/cache/aoss/ceilo" -) +CEILO_IMG_LOC = os.environ.get("CEILO_IMG_LOC", "http://metobs.ssec.wisc.edu/pub/cache/aoss/ceilo") inst = "ceilo" RE_DIGITS = re.compile(r"\d+") @@ -45,9 +41,7 @@ def get_praw_dir(when=None): def get_sraw_dir(when=None): """Return raw directory for specified date and data_type.""" - raise NotImplementedError( - "This function is not used anymore, there should only be one primary storage location" - ) + raise NotImplementedError("This function is not used anymore, there should only be one primary storage location") # when = when or datetime.now() # return os.path.join( CEILO_SRAW_DIR, when.strftime( CEILO_DIR_FORMAT ) ) @@ -114,16 +108,12 @@ def get_img_filename(begin, end, ptype=1, tag="", site="rig", description=""): def get_quicklook_filename(begin, end, ptype=1, site="rig", description=""): """Generate filename for a quicklook.""" - return get_img_filename( - begin, end, ptype, tag="", site=site, description=description - ) + return get_img_filename(begin, end, ptype, tag="", site=site, description=description) def get_thumbnail_filename(begin, end, ptype=1, site="rig", description=""): """Generate filename for a quicklook thumbnail.""" - return get_img_filename( - begin, end, ptype, tag="tn", site=site, description=description - ) + return get_img_filename(begin, end, ptype, tag="tn", site=site, description=description) def get_img_url(begin, end, ptype=1, tag="", site="rig", description=""): @@ -132,9 +122,7 @@ def get_img_url(begin, end, ptype=1, tag="", site="rig", description=""): CEILO_IMG_LOC, "img", begin.strftime(CEILO_DIR_FORMAT), - get_img_filename( - begin, end, ptype, tag=tag, site=site, description=description - ), + get_img_filename(begin, end, ptype, tag=tag, site=site, description=description), ) diff --git a/aossceilo/ingest.py b/aossceilo/ingest.py index fd4e90a80888f37dfc14b1c3aef7148e494014a6..e028a30da1fab08440f054ecc68c2d294f7a7ae6 100755 --- a/aossceilo/ingest.py +++ b/aossceilo/ingest.py @@ -7,6 +7,7 @@ of each message. No validation of message data is done. The output should match the legacy output written by the older Java software. """ + import logging import os import re @@ -66,9 +67,7 @@ def init_ceilo(portdev): When this completes the instrument should be in autosend mode and generating messages. """ - port = serial.Serial( - port=portdev, baudrate=2400, bytesize=7, parity="E", stopbits=1, timeout=1 - ) + port = serial.Serial(port=portdev, baudrate=2400, bytesize=7, parity="E", stopbits=1, timeout=1) init_commands = ( "\r\r\r", "OPEN\r\n", @@ -88,9 +87,7 @@ def init_ceilo(portdev): port.close() - return serial.Serial( - port=portdev, baudrate=2400, bytesize=7, parity="E", stopbits=1, timeout=7.5 - ) + return serial.Serial(port=portdev, baudrate=2400, bytesize=7, parity="E", stopbits=1, timeout=7.5) def read_cfg(cfgfile): @@ -117,9 +114,7 @@ def main(): "error": logging.ERROR, } parser.add_argument("-v", dest="loglvl", choices=levels.keys(), default="info") - parser.add_argument( - "--log-dir", help="Base directory where log files will be written" - ) + parser.add_argument("--log-dir", help="Base directory where log files will be written") parser.add_argument("-o", dest="outdir", default=".") parser.add_argument( "-f", diff --git a/aossceilo/level_b1/nc.py b/aossceilo/level_b1/nc.py index f38e8e34fde3172c21d51e9bd693d3e5f8d5bbd0..1379a31b35c0adfeb52a6e4f3f892075f3f993a2 100644 --- a/aossceilo/level_b1/nc.py +++ b/aossceilo/level_b1/nc.py @@ -70,9 +70,7 @@ ATTR_TYPE_MAP = { def _get_value(var, value): - return ( - var._FillValue if numpy.isnan(value) and hasattr(var, "_FillValue") else value - ) + return var._FillValue if numpy.isnan(value) and hasattr(var, "_FillValue") else value def create_nc(input_files, out_files): @@ -80,12 +78,7 @@ def create_nc(input_files, out_files): messages = message.load_messages(map(os.path.realpath, input_files)) # raise FileNotFoundError(f"{messages} No messages were found in the input files") - ncml = fromstring( - importlib.resources.files(__name__) - .joinpath("ceilo.ncml") - .open("r", encoding="utf-8") - .read() - ) + ncml = fromstring(importlib.resources.files(__name__).joinpath("ceilo.ncml").open("r", encoding="utf-8").read()) for f in out_files: _create_one_nc(messages, ncml, f) @@ -93,13 +86,9 @@ def create_nc(input_files, out_files): def _create_one_nc(messages, ncml, out_file): - now = datetime.datetime.strptime( - os.path.basename(out_file).split(".")[1], "%Y-%m-%d" - ) + now = datetime.datetime.strptime(os.path.basename(out_file).split(".")[1], "%Y-%m-%d") # get bounds of messages - mask = (messages >= _compareMessage(now)) & ( - messages <= _compareMessage(now + datetime.timedelta(days=1)) - ) + mask = (messages >= _compareMessage(now)) & (messages <= _compareMessage(now + datetime.timedelta(days=1))) if not mask.any(): LOG.info(f"No files found for date range {now}") @@ -129,9 +118,7 @@ def _create_one_nc(messages, ncml, out_file): ) var[:] = base - times = numpy.array( - tuple(numpy.int64(timegm(m.stamp.timetuple())) for m in messages[mask]) - ) + times = numpy.array(tuple(numpy.int64(timegm(m.stamp.timetuple())) for m in messages[mask])) met_data = get_message_met_data(nc, messages[mask]) hk_data = get_message_hk_data(nc, messages[mask]) @@ -367,9 +354,7 @@ def main(): parser = argparse.ArgumentParser( description="Script for creating and manipulating Viasala CT25K Ceilometer NetCDF files." ) - parser.add_argument( - "input", type=str, nargs="+", help="Input level 1 ASCII filepaths" - ) + parser.add_argument("input", type=str, nargs="+", help="Input level 1 ASCII filepaths") parser.add_argument( "-o", "--output-files", diff --git a/aossceilo/message.py b/aossceilo/message.py index 9560acb12c88161737d7a348f80ec75b77879204..6f30aa9e577ac4b611280b236636d84123dac489 100644 --- a/aossceilo/message.py +++ b/aossceilo/message.py @@ -144,9 +144,7 @@ class Message2(object): @raises MessageError: If this instance cannot be created due to an error parsing. """ - assert len(lines) == self.NUM_LINES, ( - "A Message2 must contain %s lines" % self.NUM_LINES - ) + assert len(lines) == self.NUM_LINES, "A Message2 must contain %s lines" % self.NUM_LINES self._epoch = timegm(stamp.utctimetuple()) @@ -185,9 +183,7 @@ class Message2(object): meas_params = lines[2].split() if len(meas_params) < 10: - LOG.warn( - "Invalid measurement parameters for message with time %s", self.epoch - ) + LOG.warn("Invalid measurement parameters for message with time %s", self.epoch) self._scale = _int(meas_params[0]) self._measurement_mode = _str(meas_params[1]) self._laser_pulse_energy = _float(meas_params[2]) diff --git a/aossceilo/nc.py b/aossceilo/nc.py index 7565d6d908fd97832f9cde3068038bf8e0272e15..947440de870b48f37475f4a2d217df8dc93c4dd4 100644 --- a/aossceilo/nc.py +++ b/aossceilo/nc.py @@ -100,12 +100,8 @@ def _get_message_met_data(nc, messages): backscatter=array([zeros(256) for idx in range(len(messages))]), ) for i in range(len(messages)): - data["alt_highest_signal"][i] = _get_value( - nc.var("alt_highest_signal"), messages[i].alt_highest_signal - ) - data["vertical_visibility"][i] = _get_value( - nc.var("vertical_visibility"), messages[i].vertical_visibility - ) + data["alt_highest_signal"][i] = _get_value(nc.var("alt_highest_signal"), messages[i].alt_highest_signal) + data["vertical_visibility"][i] = _get_value(nc.var("vertical_visibility"), messages[i].vertical_visibility) data["first_cbh"][i] = _get_value(nc.var("first_cbh"), messages[i].first_cbh) data["second_cbh"][i] = _get_value(nc.var("second_cbh"), messages[i].second_cbh) data["third_cbh"][i] = _get_value(nc.var("third_cbh"), messages[i].third_cbh) @@ -143,9 +139,7 @@ def _get_message_hk_data(nc, messages): data = {} str_len = nc.dim("string_len").inq()[1] data["detection_status"] = array([m.detection_status for m in messages], int32) - data["range"] = array( - range(15, 256 * 30, 30) - ) # center of range buckets. Buckets are 30m + data["range"] = array(range(15, 256 * 30, 30)) # center of range buckets. Buckets are 30m # so start at 15 and count 256 buckets. for var_name in str_names: data[var_name] = array(zeros((len(messages), str_len))) @@ -154,9 +148,7 @@ def _get_message_hk_data(nc, messages): for i in range(len(messages)): for var_name in var_names: - data[var_name][i] = _get_value( - nc.var(var_name), getattr(messages[i], var_name) - ) + data[var_name][i] = _get_value(nc.var(var_name), getattr(messages[i], var_name)) for var_name in str_names: s = getattr(messages[i], var_name) data[var_name][i][:] = map(ord, list(s.ljust(str_len))) @@ -177,12 +169,7 @@ def create(filename, lat, lon): # # TODO: eliminate the use of deprecated missing_value # - ncml = ( - importlib.resources.files(__name__) - .joinpath("ceilo.ncml") - .open("r", encoding="utf-8") - .read() - ) + ncml = importlib.resources.files(__name__).joinpath("ceilo.ncml").open("r", encoding="utf-8").read() nc = create_nc_from_ncml(filename, ncml) # @@ -194,9 +181,7 @@ def create(filename, lat, lon): return nc -def make_ceilo_files( - begin, basedir=None, end=None, loc=ssec_loc, site="rig", description="" -): +def make_ceilo_files(begin, basedir=None, end=None, loc=ssec_loc, site="rig", description=""): """ Make NetCDF files. All times are ignored and only the date information is used. One file for each date is written to the @@ -270,17 +255,13 @@ def fill_from_msg_files(nc, url): hk_data = _get_message_hk_data(nc, messages) # use middle stamp to avoid point from previous day at the beginning (if present) - midnight = mytime.datetime_to_epoch( - mytime.day_begin(messages[len(messages) / 2].stamp) - ) + midnight = mytime.datetime_to_epoch(mytime.day_begin(messages[len(messages) / 2].stamp)) offsets = times - midnight # use scalar array subtraction if offsets.dtype != int32: offsets = array(offsets, int32) var = nc.var("time") var[: len(offsets)] = offsets - var.units = messages[len(messages) / 2].stamp.strftime( - "seconds since %Y-%m-%d 00:00:00 0:00" - ) + var.units = messages[len(messages) / 2].stamp.strftime("seconds since %Y-%m-%d 00:00:00 0:00") # time offsets from base_time base_secs = mytime.datetime_to_epoch(mytime.day_begin(messages[0].stamp)) diff --git a/aossceilo/tests/test_nc.py b/aossceilo/tests/test_nc.py index 13d8dfbe0296be1feb323f5e2b99fa4e0b1847e3..a2e30d66e2846fbf348c2577af9ab626f7a36ce2 100644 --- a/aossceilo/tests/test_nc.py +++ b/aossceilo/tests/test_nc.py @@ -71,23 +71,12 @@ def test_create_nc(tmp_path): elif not var.shape: assert hasattr(var, "units") assert hasattr(var, "_FillValue") - if ( - var[:].dtype != "S1" - and var.name not in expect_fill - and var.name[:2] != "qc" - ): + if var[:].dtype != "S1" and var.name not in expect_fill and var.name[:2] != "qc": pass assert (var[:]) is not numpy.ma.masked.all() - dt = datetime.datetime.fromtimestamp( - nc_file.variables["base_time"][:], tz=datetime.timezone.utc - ) - midnight = datetime.datetime( - dt.year, dt.month, dt.day, tzinfo=datetime.timezone.utc - ) - assert ( - nc_file.variables["time_offset"][:] - == nc_file.variables["time"][:] + (midnight - dt).total_seconds() - ).all() + dt = datetime.datetime.fromtimestamp(nc_file.variables["base_time"][:], tz=datetime.timezone.utc) + midnight = datetime.datetime(dt.year, dt.month, dt.day, tzinfo=datetime.timezone.utc) + assert (nc_file.variables["time_offset"][:] == nc_file.variables["time"][:] + (midnight - dt).total_seconds()).all() os.remove(filename) @@ -104,34 +93,14 @@ def test_regen(tmp_path): for f in ascii_test_files: name = os.path.basename(f) date = _get_date(name) - directory = ( - tmp_path - / "cache" - / "aoss" - / "ceilo" - / "level_00" - / "version_00" - / date.strftime("%Y/%m/%d") - ) + directory = tmp_path / "cache" / "aoss" / "ceilo" / "level_00" / "version_00" / date.strftime("%Y/%m/%d") os.makedirs(directory) shutil.copyfile(f, directory / name) - os.makedirs( - tmp_path - / "cache" - / "aoss" - / "ceilo" - / "level_b1" - / "version_00" - / date.strftime("%Y/%m/%d") - ) + os.makedirs(tmp_path / "cache" / "aoss" / "ceilo" / "level_b1" / "version_00" / date.strftime("%Y/%m/%d")) os.environ["DATA_ROOT"] = str(tmp_path) p = Popen( - ( - os.path.join( - os.path.dirname(__file__), "../../scripts/regen_ceilo_level_b1.sh" - ), - ), + (os.path.join(os.path.dirname(__file__), "../../scripts/regen_ceilo_level_b1.sh"),), stdout=PIPE, stderr=PIPE, ) diff --git a/aossceilo/tidy.py b/aossceilo/tidy.py index 6c92592533bd74264ebc0aceec70cce87452fd22..53f8a61c58ad193f5584811baf8539c34cb3ba74 100644 --- a/aossceilo/tidy.py +++ b/aossceilo/tidy.py @@ -173,9 +173,7 @@ def unload_incoming(incoming_dir, praw_dir, cache_dir, latest_dir): LOG.warning("no files found in %s" % incoming_dir) return - raw_changes = [ - c.rename_incoming(fn, site=site, description=description) for fn in new_files - ] + raw_changes = [c.rename_incoming(fn, site=site, description=description) for fn in new_files] (praw_dirs, cache_dirs, renames, removes) = [list(x) for x in zip(*raw_changes)] raw_manager.daily_manage_raw( incoming_dir,