From f5348fda55c42c742a675dcc53076da623d9fbc6 Mon Sep 17 00:00:00 2001
From: David Hoese <david.hoese@ssec.wisc.edu>
Date: Wed, 2 Jul 2025 20:52:05 -0500
Subject: [PATCH] Update pre-commit
---
.pre-commit-config.yaml | 29 ++++++++++--------------
aossceilo/CONFIG.py | 24 +++++---------------
aossceilo/ingest.py | 13 ++++-------
aossceilo/level_b1/nc.py | 27 +++++------------------
aossceilo/message.py | 8 ++-----
aossceilo/nc.py | 35 +++++++----------------------
aossceilo/tests/test_nc.py | 45 ++++++--------------------------------
aossceilo/tidy.py | 4 +---
8 files changed, 45 insertions(+), 140 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index ccec9c9..f41623d 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,43 +1,36 @@
exclude: '^$'
fail_fast: false
repos:
- - repo: https://github.com/psf/black
- rev: 24.4.2
+ - repo: https://gitlab.com/vojko.pribudic.foss/pre-commit-update
+ rev: v0.8.0
hooks:
- - id: black
- language_version: python3
- exclude: versioneer.py
- args:
- - --target-version=py38
- - repo: https://github.com/pycqa/isort
- rev: 5.13.2
- hooks:
- - id: isort
- language_version: python3
+ - id: pre-commit-update
+ args: [ --dry-run ]
- repo: https://github.com/astral-sh/ruff-pre-commit
- # Ruff version.
- rev: 'v0.5.5'
+ rev: 'v0.12.1'
hooks:
- id: ruff
+ args: ["--fix"]
+ - id: ruff-format
- repo: https://github.com/pre-commit/pre-commit-hooks
- rev: v4.6.0
+ rev: v5.0.0
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
- id: check-yaml
args: [--unsafe]
- repo: https://github.com/PyCQA/bandit
- rev: '1.7.9' # Update me!
+ rev: '1.8.5' # Update me!
hooks:
- id: bandit
args: [--ini, .bandit]
- repo: https://github.com/pre-commit/mirrors-mypy
- rev: 'v1.11.0' # Use the sha / tag you want to point at
+ rev: 'v1.16.1' # Use the sha / tag you want to point at
hooks:
- id: mypy
additional_dependencies:
- types-docutils
- - types-pkg-resources
+ - types-setuptools
- types-PyYAML
- types-requests
ci:
diff --git a/aossceilo/CONFIG.py b/aossceilo/CONFIG.py
index 86ecb6a..02ad901 100644
--- a/aossceilo/CONFIG.py
+++ b/aossceilo/CONFIG.py
@@ -16,18 +16,14 @@ from datetime import datetime, timedelta
from metobscommon.util import CONFIG as c
-CEILO_INCOMING_DIR = os.environ.get(
- "CEILO_INCOMING_DIR", "/beach/incoming/Instrument_Data/METOBS/RIG/Ceilo/raw"
-)
+CEILO_INCOMING_DIR = os.environ.get("CEILO_INCOMING_DIR", "/beach/incoming/Instrument_Data/METOBS/RIG/Ceilo/raw")
CEILO_PRAW_DIR = os.environ.get("CEILO_PRAW_DIR", "/beach/raw/aoss/ceilo")
CEILO_CACHE_DIR = os.environ.get("CEILO_CACHE_DIR", "/beach/cache/aoss/ceilo")
CEILO_LATEST_DIR = os.environ.get("CEILO_LATEST_DIR", "/beach/cache/aoss/ceilo")
CEILO_DIR_FORMAT = os.environ.get("CEILO_DIR_FORMAT", "%Y/%m")
CEILO_ASCII_LOC = os.environ.get("CEILO_ASCII_LOC", "/beach/cache/aoss/ceilo")
CEILO_NC_LOC = os.environ.get("CEILO_NC_LOC", "/beach/cache/aoss/ceilo")
-CEILO_IMG_LOC = os.environ.get(
- "CEILO_IMG_LOC", "http://metobs.ssec.wisc.edu/pub/cache/aoss/ceilo"
-)
+CEILO_IMG_LOC = os.environ.get("CEILO_IMG_LOC", "http://metobs.ssec.wisc.edu/pub/cache/aoss/ceilo")
inst = "ceilo"
RE_DIGITS = re.compile(r"\d+")
@@ -45,9 +41,7 @@ def get_praw_dir(when=None):
def get_sraw_dir(when=None):
"""Return raw directory for specified date and data_type."""
- raise NotImplementedError(
- "This function is not used anymore, there should only be one primary storage location"
- )
+ raise NotImplementedError("This function is not used anymore, there should only be one primary storage location")
# when = when or datetime.now()
# return os.path.join( CEILO_SRAW_DIR, when.strftime( CEILO_DIR_FORMAT ) )
@@ -114,16 +108,12 @@ def get_img_filename(begin, end, ptype=1, tag="", site="rig", description=""):
def get_quicklook_filename(begin, end, ptype=1, site="rig", description=""):
"""Generate filename for a quicklook."""
- return get_img_filename(
- begin, end, ptype, tag="", site=site, description=description
- )
+ return get_img_filename(begin, end, ptype, tag="", site=site, description=description)
def get_thumbnail_filename(begin, end, ptype=1, site="rig", description=""):
"""Generate filename for a quicklook thumbnail."""
- return get_img_filename(
- begin, end, ptype, tag="tn", site=site, description=description
- )
+ return get_img_filename(begin, end, ptype, tag="tn", site=site, description=description)
def get_img_url(begin, end, ptype=1, tag="", site="rig", description=""):
@@ -132,9 +122,7 @@ def get_img_url(begin, end, ptype=1, tag="", site="rig", description=""):
CEILO_IMG_LOC,
"img",
begin.strftime(CEILO_DIR_FORMAT),
- get_img_filename(
- begin, end, ptype, tag=tag, site=site, description=description
- ),
+ get_img_filename(begin, end, ptype, tag=tag, site=site, description=description),
)
diff --git a/aossceilo/ingest.py b/aossceilo/ingest.py
index fd4e90a..e028a30 100755
--- a/aossceilo/ingest.py
+++ b/aossceilo/ingest.py
@@ -7,6 +7,7 @@ of each message. No validation of message data is done.
The output should match the legacy output written by the older Java software.
"""
+
import logging
import os
import re
@@ -66,9 +67,7 @@ def init_ceilo(portdev):
When this completes the instrument should be in autosend mode and generating messages.
"""
- port = serial.Serial(
- port=portdev, baudrate=2400, bytesize=7, parity="E", stopbits=1, timeout=1
- )
+ port = serial.Serial(port=portdev, baudrate=2400, bytesize=7, parity="E", stopbits=1, timeout=1)
init_commands = (
"\r\r\r",
"OPEN\r\n",
@@ -88,9 +87,7 @@ def init_ceilo(portdev):
port.close()
- return serial.Serial(
- port=portdev, baudrate=2400, bytesize=7, parity="E", stopbits=1, timeout=7.5
- )
+ return serial.Serial(port=portdev, baudrate=2400, bytesize=7, parity="E", stopbits=1, timeout=7.5)
def read_cfg(cfgfile):
@@ -117,9 +114,7 @@ def main():
"error": logging.ERROR,
}
parser.add_argument("-v", dest="loglvl", choices=levels.keys(), default="info")
- parser.add_argument(
- "--log-dir", help="Base directory where log files will be written"
- )
+ parser.add_argument("--log-dir", help="Base directory where log files will be written")
parser.add_argument("-o", dest="outdir", default=".")
parser.add_argument(
"-f",
diff --git a/aossceilo/level_b1/nc.py b/aossceilo/level_b1/nc.py
index f38e8e3..1379a31 100644
--- a/aossceilo/level_b1/nc.py
+++ b/aossceilo/level_b1/nc.py
@@ -70,9 +70,7 @@ ATTR_TYPE_MAP = {
def _get_value(var, value):
- return (
- var._FillValue if numpy.isnan(value) and hasattr(var, "_FillValue") else value
- )
+ return var._FillValue if numpy.isnan(value) and hasattr(var, "_FillValue") else value
def create_nc(input_files, out_files):
@@ -80,12 +78,7 @@ def create_nc(input_files, out_files):
messages = message.load_messages(map(os.path.realpath, input_files))
# raise FileNotFoundError(f"{messages} No messages were found in the input files")
- ncml = fromstring(
- importlib.resources.files(__name__)
- .joinpath("ceilo.ncml")
- .open("r", encoding="utf-8")
- .read()
- )
+ ncml = fromstring(importlib.resources.files(__name__).joinpath("ceilo.ncml").open("r", encoding="utf-8").read())
for f in out_files:
_create_one_nc(messages, ncml, f)
@@ -93,13 +86,9 @@ def create_nc(input_files, out_files):
def _create_one_nc(messages, ncml, out_file):
- now = datetime.datetime.strptime(
- os.path.basename(out_file).split(".")[1], "%Y-%m-%d"
- )
+ now = datetime.datetime.strptime(os.path.basename(out_file).split(".")[1], "%Y-%m-%d")
# get bounds of messages
- mask = (messages >= _compareMessage(now)) & (
- messages <= _compareMessage(now + datetime.timedelta(days=1))
- )
+ mask = (messages >= _compareMessage(now)) & (messages <= _compareMessage(now + datetime.timedelta(days=1)))
if not mask.any():
LOG.info(f"No files found for date range {now}")
@@ -129,9 +118,7 @@ def _create_one_nc(messages, ncml, out_file):
)
var[:] = base
- times = numpy.array(
- tuple(numpy.int64(timegm(m.stamp.timetuple())) for m in messages[mask])
- )
+ times = numpy.array(tuple(numpy.int64(timegm(m.stamp.timetuple())) for m in messages[mask]))
met_data = get_message_met_data(nc, messages[mask])
hk_data = get_message_hk_data(nc, messages[mask])
@@ -367,9 +354,7 @@ def main():
parser = argparse.ArgumentParser(
description="Script for creating and manipulating Viasala CT25K Ceilometer NetCDF files."
)
- parser.add_argument(
- "input", type=str, nargs="+", help="Input level 1 ASCII filepaths"
- )
+ parser.add_argument("input", type=str, nargs="+", help="Input level 1 ASCII filepaths")
parser.add_argument(
"-o",
"--output-files",
diff --git a/aossceilo/message.py b/aossceilo/message.py
index 9560acb..6f30aa9 100644
--- a/aossceilo/message.py
+++ b/aossceilo/message.py
@@ -144,9 +144,7 @@ class Message2(object):
@raises MessageError: If this instance cannot be created due to an error
parsing.
"""
- assert len(lines) == self.NUM_LINES, (
- "A Message2 must contain %s lines" % self.NUM_LINES
- )
+ assert len(lines) == self.NUM_LINES, "A Message2 must contain %s lines" % self.NUM_LINES
self._epoch = timegm(stamp.utctimetuple())
@@ -185,9 +183,7 @@ class Message2(object):
meas_params = lines[2].split()
if len(meas_params) < 10:
- LOG.warn(
- "Invalid measurement parameters for message with time %s", self.epoch
- )
+ LOG.warn("Invalid measurement parameters for message with time %s", self.epoch)
self._scale = _int(meas_params[0])
self._measurement_mode = _str(meas_params[1])
self._laser_pulse_energy = _float(meas_params[2])
diff --git a/aossceilo/nc.py b/aossceilo/nc.py
index 7565d6d..947440d 100644
--- a/aossceilo/nc.py
+++ b/aossceilo/nc.py
@@ -100,12 +100,8 @@ def _get_message_met_data(nc, messages):
backscatter=array([zeros(256) for idx in range(len(messages))]),
)
for i in range(len(messages)):
- data["alt_highest_signal"][i] = _get_value(
- nc.var("alt_highest_signal"), messages[i].alt_highest_signal
- )
- data["vertical_visibility"][i] = _get_value(
- nc.var("vertical_visibility"), messages[i].vertical_visibility
- )
+ data["alt_highest_signal"][i] = _get_value(nc.var("alt_highest_signal"), messages[i].alt_highest_signal)
+ data["vertical_visibility"][i] = _get_value(nc.var("vertical_visibility"), messages[i].vertical_visibility)
data["first_cbh"][i] = _get_value(nc.var("first_cbh"), messages[i].first_cbh)
data["second_cbh"][i] = _get_value(nc.var("second_cbh"), messages[i].second_cbh)
data["third_cbh"][i] = _get_value(nc.var("third_cbh"), messages[i].third_cbh)
@@ -143,9 +139,7 @@ def _get_message_hk_data(nc, messages):
data = {}
str_len = nc.dim("string_len").inq()[1]
data["detection_status"] = array([m.detection_status for m in messages], int32)
- data["range"] = array(
- range(15, 256 * 30, 30)
- ) # center of range buckets. Buckets are 30m
+ data["range"] = array(range(15, 256 * 30, 30)) # center of range buckets. Buckets are 30m
# so start at 15 and count 256 buckets.
for var_name in str_names:
data[var_name] = array(zeros((len(messages), str_len)))
@@ -154,9 +148,7 @@ def _get_message_hk_data(nc, messages):
for i in range(len(messages)):
for var_name in var_names:
- data[var_name][i] = _get_value(
- nc.var(var_name), getattr(messages[i], var_name)
- )
+ data[var_name][i] = _get_value(nc.var(var_name), getattr(messages[i], var_name))
for var_name in str_names:
s = getattr(messages[i], var_name)
data[var_name][i][:] = map(ord, list(s.ljust(str_len)))
@@ -177,12 +169,7 @@ def create(filename, lat, lon):
#
# TODO: eliminate the use of deprecated missing_value
#
- ncml = (
- importlib.resources.files(__name__)
- .joinpath("ceilo.ncml")
- .open("r", encoding="utf-8")
- .read()
- )
+ ncml = importlib.resources.files(__name__).joinpath("ceilo.ncml").open("r", encoding="utf-8").read()
nc = create_nc_from_ncml(filename, ncml)
#
@@ -194,9 +181,7 @@ def create(filename, lat, lon):
return nc
-def make_ceilo_files(
- begin, basedir=None, end=None, loc=ssec_loc, site="rig", description=""
-):
+def make_ceilo_files(begin, basedir=None, end=None, loc=ssec_loc, site="rig", description=""):
"""
Make NetCDF files. All times are ignored and only the date
information is used. One file for each date is written to the
@@ -270,17 +255,13 @@ def fill_from_msg_files(nc, url):
hk_data = _get_message_hk_data(nc, messages)
# use middle stamp to avoid point from previous day at the beginning (if present)
- midnight = mytime.datetime_to_epoch(
- mytime.day_begin(messages[len(messages) / 2].stamp)
- )
+ midnight = mytime.datetime_to_epoch(mytime.day_begin(messages[len(messages) / 2].stamp))
offsets = times - midnight # use scalar array subtraction
if offsets.dtype != int32:
offsets = array(offsets, int32)
var = nc.var("time")
var[: len(offsets)] = offsets
- var.units = messages[len(messages) / 2].stamp.strftime(
- "seconds since %Y-%m-%d 00:00:00 0:00"
- )
+ var.units = messages[len(messages) / 2].stamp.strftime("seconds since %Y-%m-%d 00:00:00 0:00")
# time offsets from base_time
base_secs = mytime.datetime_to_epoch(mytime.day_begin(messages[0].stamp))
diff --git a/aossceilo/tests/test_nc.py b/aossceilo/tests/test_nc.py
index 13d8dfb..a2e30d6 100644
--- a/aossceilo/tests/test_nc.py
+++ b/aossceilo/tests/test_nc.py
@@ -71,23 +71,12 @@ def test_create_nc(tmp_path):
elif not var.shape:
assert hasattr(var, "units")
assert hasattr(var, "_FillValue")
- if (
- var[:].dtype != "S1"
- and var.name not in expect_fill
- and var.name[:2] != "qc"
- ):
+ if var[:].dtype != "S1" and var.name not in expect_fill and var.name[:2] != "qc":
pass
assert (var[:]) is not numpy.ma.masked.all()
- dt = datetime.datetime.fromtimestamp(
- nc_file.variables["base_time"][:], tz=datetime.timezone.utc
- )
- midnight = datetime.datetime(
- dt.year, dt.month, dt.day, tzinfo=datetime.timezone.utc
- )
- assert (
- nc_file.variables["time_offset"][:]
- == nc_file.variables["time"][:] + (midnight - dt).total_seconds()
- ).all()
+ dt = datetime.datetime.fromtimestamp(nc_file.variables["base_time"][:], tz=datetime.timezone.utc)
+ midnight = datetime.datetime(dt.year, dt.month, dt.day, tzinfo=datetime.timezone.utc)
+ assert (nc_file.variables["time_offset"][:] == nc_file.variables["time"][:] + (midnight - dt).total_seconds()).all()
os.remove(filename)
@@ -104,34 +93,14 @@ def test_regen(tmp_path):
for f in ascii_test_files:
name = os.path.basename(f)
date = _get_date(name)
- directory = (
- tmp_path
- / "cache"
- / "aoss"
- / "ceilo"
- / "level_00"
- / "version_00"
- / date.strftime("%Y/%m/%d")
- )
+ directory = tmp_path / "cache" / "aoss" / "ceilo" / "level_00" / "version_00" / date.strftime("%Y/%m/%d")
os.makedirs(directory)
shutil.copyfile(f, directory / name)
- os.makedirs(
- tmp_path
- / "cache"
- / "aoss"
- / "ceilo"
- / "level_b1"
- / "version_00"
- / date.strftime("%Y/%m/%d")
- )
+ os.makedirs(tmp_path / "cache" / "aoss" / "ceilo" / "level_b1" / "version_00" / date.strftime("%Y/%m/%d"))
os.environ["DATA_ROOT"] = str(tmp_path)
p = Popen(
- (
- os.path.join(
- os.path.dirname(__file__), "../../scripts/regen_ceilo_level_b1.sh"
- ),
- ),
+ (os.path.join(os.path.dirname(__file__), "../../scripts/regen_ceilo_level_b1.sh"),),
stdout=PIPE,
stderr=PIPE,
)
diff --git a/aossceilo/tidy.py b/aossceilo/tidy.py
index 6c92592..53f8a61 100644
--- a/aossceilo/tidy.py
+++ b/aossceilo/tidy.py
@@ -173,9 +173,7 @@ def unload_incoming(incoming_dir, praw_dir, cache_dir, latest_dir):
LOG.warning("no files found in %s" % incoming_dir)
return
- raw_changes = [
- c.rename_incoming(fn, site=site, description=description) for fn in new_files
- ]
+ raw_changes = [c.rename_incoming(fn, site=site, description=description) for fn in new_files]
(praw_dirs, cache_dirs, renames, removes) = [list(x) for x in zip(*raw_changes)]
raw_manager.daily_manage_raw(
incoming_dir,
--
GitLab