Newer
Older
"""Insert RIG Tower data from a level 0 (raw) data file into a MetObs database.
All error messages will go to stderr, all other logging will go to stdout. So
to make sure errors are logged:
"%(prog)s 2> error.log"
"""
import logging
import logging.handlers
import sys
from pathlib import Path
from aosstower.level_00.parser import read_frames
logging.addLevelName(9, "TRACE")
ERR = logging.getLogger(__name__ + ".error")
_symbol_record_map = (
(1, "station"),
(2, "year"),
(3, "day"),
(4, "hour_minute"),
(5, "seconds"),
(6, "box_pressure"),
(7, "parosci_air_temp_period"),
(8, "parosci_pressure_period"),
(9, "parosci_air_temp"),
(10, "pressure"),
(11, "parosci_calc_sig"),
(12, "box_rh"),
(13, "box_air_temp"),
(14, "temp2"),
(15, "temp3"),
(16, "temp4"),
(17, "wind_speed"),
(18, "wind_direction"),
(19, "rh_shield_freq"),
(20, "relative_humidity"),
(21, "air_temp_6_3"),
(22, "dewpoint"),
(23, "rtd_shield_freq"),
(24, "air_temp"),
(25, "solar_flux"),
(26, "precipitation"),
(27, "accumulated_precipitation"),
(28, "altimeter"),
def configure_logging(level, logfn=None):
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(logging.Formatter("%(asctime)s: %(message)s"))
LOG.addHandler(handler)
LOG.setLevel(level)
handler = logging.StreamHandler(sys.stderr)
handler.setFormatter(logging.Formatter("FAIL: %(message)s"))
ERR.addHandler(handler)
ERR.setLevel(logging.INFO)
if logfn:
maxsize = 20 * 1024**3 # 20 MB
rotating_file = logging.handlers.RotatingFileHandler(logfn, mode="a", maxBytes=maxsize, backupCount=5)
rotating_file.setFormatter(logging.Formatter("%(asctime)s: %(message)s"))
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
"--loglvl",
dest="loglvl",
default="warn",
help="logging level",
choices=(["trace", "debug", "info", "warn", "error"]),
)
parser.add_argument("--logfn", help="Log to rotating file")
parser.add_argument(
"--debug",
action="store_true",
help="Don't submit records to the database, print them to stdout",
)
parser.add_argument(
"-s",
"--station",
dest="station",
default="RIG Tower",
help="Name of station to use to determine symbols",
)
parser.add_argument(
"--ldmp",
action="store_true",
help="Treat `src` file as a station name and read records from LoggerNet LDMP server (port: 1024)",
)
parser.add_argument("--tables", nargs="*", default=["1"], help="LoggerNet LDMP tables to read in")
parser.add_argument("dburl", help="Database to insert into")
parser.add_argument("src", help="Level 0 raw data file")
lvl = logging.getLevelName(args.loglvl.upper())
if isinstance(lvl, str):
lvl = logging.INFO
configure_logging(lvl, args.logfn)
LOG.info("Initializing DB with %s", args.dburl)
legacy_db.init_model(args.dburl)
station = legacy_db.Station.query.filter_by(name=args.station).first()
legacy_db.Session.commit()
if not station:
parser.exit(1, "Station not found: %s" % args.station)
if args.ldmp:
from aosstower.level_00.parser import LDMPGenerator
record_gen = LDMPGenerator(args.src, args.tables, symbol_names=[x[1] for x in _symbol_record_map])
src = Path(args.src).open()
record_gen = read_frames(src, tail=args.tail)
for _idx, record in enumerate(record_gen):
data = (record["stamp"], record)
LOG.info("Sending record: %s", record["stamp"].isoformat(" "))
failures = legacy_db.insert_records([data], station=station)
if failures:
ERR.info("(%s) %s", failures[0][0], record)
if __name__ == "__main__":
sys.exit(main())