diff --git a/aosstower/level_00/influxdb.py b/aosstower/level_00/influxdb.py index 8ebfe21e33d54140c260c0120ff37eb2625a3e21..865b047305a5a0085381509d984f7ffd7f10eeb6 100644 --- a/aosstower/level_00/influxdb.py +++ b/aosstower/level_00/influxdb.py @@ -51,7 +51,7 @@ SYMBOLS = list(SYMBOL_CONVERSIONS.values()) def main(): import argparse parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('--logfn', help='Log to rotating file') + parser.add_argument('--logfn', help='Log to rotating file (Not Implemented)') parser.add_argument('--debug', action='store_true', help='Don\'t submit records to the database, print them to stdout') parser.add_argument('-t', '--tail', action='store_true', diff --git a/aosstower/level_00/legacy_db_insert.py b/aosstower/level_00/legacy_db_insert.py new file mode 100644 index 0000000000000000000000000000000000000000..1b0882d8af77ade732448d9e4a4b1462e08091dc --- /dev/null +++ b/aosstower/level_00/legacy_db_insert.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +"""Insert RIG Tower data from a level 0 (raw) data file into a MetObs database. + +All error messages will go to stderr, all other logging will go to stdout. So +to make sure errors are logged: + + "%(prog)s 2> error.log" + +""" +import logging +import logging.handlers +import sys +from metobscommon import legacy_db +from aosstower.level_00.parser import read_frames + +logging.addLevelName(9, 'TRACE') +LOG = logging.getLogger(__name__) +ERR = logging.getLogger(__name__ + '.error') + + +def configure_logging(level, logfn=None): + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(logging.Formatter("%(asctime)s: %(message)s")) + LOG.addHandler(handler) + LOG.setLevel(level) + + handler = logging.StreamHandler(sys.stderr) + handler.setFormatter(logging.Formatter("FAIL: %(message)s")) + ERR.addHandler(handler) + ERR.setLevel(logging.INFO) + + if logfn: + maxsize = (20 * 1024 ** 3) # 20 MB + rotating_file = logging.handlers.RotatingFileHandler( + logfn, mode='a', maxBytes=maxsize, + backupCount=5) + rotating_file.setFormatter(logging.Formatter('%(asctime)s: %(message)s')) + LOG.addHandler(rotating_file) + + +def main(*args): + import argparse + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('--loglvl', dest='loglvl', default='warn', + help='logging level', + choices=(['trace', 'debug', 'info', 'warn', 'error'])) + parser.add_argument('--logfn', help='Log to rotating file') + parser.add_argument('-s', '--station', dest='station', default='RIG Tower', + help='Name of station to use to determine symbols') + parser.add_argument('--ldmp', action='store_true', + help='Treat `src` file as a station name and read records from LoggerNet LDMP server (port: 1024)') + parser.add_argument('--tables', nargs='*', default=['1'], + help="LoggerNet LDMP tables to read in") + parser.add_argument('dburl', help='Database to insert into') + parser.add_argument('src', help='Level 0 raw data file') + + args = parser.parse_args() + lvl = logging._levelNames.get(args.loglvl.upper(), logging.INFO) + + configure_logging(lvl, args.logfn) + + LOG.info("Initializing DB with %s", args.dburl) + legacy_db.init_model(args.dburl) + + LOG.info("source: %s", args.src) + LOG.info("tail: %s", args.tail) + + station = legacy_db.Station.query.filter_by(name=args.station).first() + legacy_db.Session.commit() + if not station: + parser.exit(1, "Station not found: %s" % args.station) + + if args.ldmp: + from aosstower.level_00.parser import ldmp_generator + record_gen = ldmp_generator(args.src, args.tables) + else: + src = open(args.src, "r") + record_gen = read_frames(src, tail=args.tail) + + for record in record_gen: + data = (record['stamp'], record) + failures = legacy_db.insert_records([data], station=station) + if failures: + ERR.info("(%s) %s", failures[0][0], record)