legacy_db_insert.py

"""Insert RIG Tower data from a level 0 (raw) data file into a MetObs database.

All error messages will go to stderr, all other logging will go to stdout. So
to make sure errors are logged:

    "%(prog)s 2> error.log"

"""
import logging
import logging.handlers
import sys
from pathlib import Path

from metobscommon import legacy_db

from aosstower.level_00.parser import read_frames

logging.addLevelName(9, "TRACE")
LOG = logging.getLogger(__name__)
ERR = logging.getLogger(__name__ + ".error")


_symbol_record_map = (
    (1, "station"),
    (2, "year"),
    (3, "day"),
    (4, "hour_minute"),
    (5, "seconds"),
    (6, "box_pressure"),
    (7, "parosci_air_temp_period"),
    (8, "parosci_pressure_period"),
    (9, "parosci_air_temp"),
    (10, "pressure"),
    (11, "parosci_calc_sig"),
    (12, "box_rh"),
    (13, "box_air_temp"),
    (14, "temp2"),
    (15, "temp3"),
    (16, "temp4"),
    (17, "wind_speed"),
    (18, "wind_direction"),
    (19, "rh_shield_freq"),
    (20, "relative_humidity"),
    (21, "air_temp_6_3"),
    (22, "dewpoint"),
    (23, "rtd_shield_freq"),
    (24, "air_temp"),
    (25, "solar_flux"),
    (26, "precipitation"),
    (27, "accumulated_precipitation"),
    (28, "altimeter"),
)


def configure_logging(level, logfn=None):
    handler = logging.StreamHandler(sys.stdout)
    handler.setFormatter(logging.Formatter("%(asctime)s: %(message)s"))
    LOG.addHandler(handler)
    LOG.setLevel(level)

    handler = logging.StreamHandler(sys.stderr)
    handler.setFormatter(logging.Formatter("FAIL: %(message)s"))
    ERR.addHandler(handler)
    ERR.setLevel(logging.INFO)

    if logfn:
        maxsize = 20 * 1024**3  # 20 MB
        rotating_file = logging.handlers.RotatingFileHandler(logfn, mode="a", maxBytes=maxsize, backupCount=5)
        rotating_file.setFormatter(logging.Formatter("%(asctime)s: %(message)s"))
        LOG.addHandler(rotating_file)


def main():
    import argparse

    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument(
        "--loglvl",
        dest="loglvl",
        default="warn",
        help="logging level",
        choices=(["trace", "debug", "info", "warn", "error"]),
    )
    parser.add_argument("--logfn", help="Log to rotating file")
    parser.add_argument(
        "--debug",
        action="store_true",
        help="Don't submit records to the database, print them to stdout",
    )
    parser.add_argument(
        "-s",
        "--station",
        dest="station",
        default="RIG Tower",
        help="Name of station to use to determine symbols",
    )
    parser.add_argument(
        "--ldmp",
        action="store_true",
        help="Treat `src` file as a station name and read records from LoggerNet LDMP server (port: 1024)",
    )
    parser.add_argument("--tables", nargs="*", default=["1"], help="LoggerNet LDMP tables to read in")
    parser.add_argument("dburl", help="Database to insert into")
    parser.add_argument("src", help="Level 0 raw data file")

    args = parser.parse_args()
    lvl = logging.getLevelName(args.loglvl.upper())
    if isinstance(lvl, str):
        lvl = logging.INFO

    configure_logging(lvl, args.logfn)

    LOG.info("Initializing DB with %s", args.dburl)
    legacy_db.init_model(args.dburl)

    station = legacy_db.Station.query.filter_by(name=args.station).first()
    legacy_db.Session.commit()
    if not station:
        parser.exit(1, "Station not found: %s" % args.station)

    if args.ldmp:
        from aosstower.level_00.parser import LDMPGenerator

        record_gen = LDMPGenerator(args.src, args.tables, symbol_names=[x[1] for x in _symbol_record_map])
    else:
        src = Path(args.src).open()
        record_gen = read_frames(src, tail=args.tail)

    for _idx, record in enumerate(record_gen):
        data = (record["stamp"], record)
        LOG.info("Sending record: %s", record["stamp"].isoformat(" "))
        if args.debug:
            # we just want the above message
            continue
        failures = legacy_db.insert_records([data], station=station)
        if failures:
            ERR.info("(%s) %s", failures[0][0], record)


if __name__ == "__main__":
    sys.exit(main())