From 4215322a9df84f3203dc7e98fd1533c18d9872ba Mon Sep 17 00:00:00 2001
From: davidh-ssec <david.hoese@ssec.wisc.edu>
Date: Tue, 19 Dec 2017 09:07:22 -0600
Subject: [PATCH] Add legacy db insert script
---
aosstower/level_00/influxdb.py | 2 +-
aosstower/level_00/legacy_db_insert.py | 85 ++++++++++++++++++++++++++
2 files changed, 86 insertions(+), 1 deletion(-)
create mode 100644 aosstower/level_00/legacy_db_insert.py
diff --git a/aosstower/level_00/influxdb.py b/aosstower/level_00/influxdb.py
index 8ebfe21..865b047 100644
--- a/aosstower/level_00/influxdb.py
+++ b/aosstower/level_00/influxdb.py
@@ -51,7 +51,7 @@ SYMBOLS = list(SYMBOL_CONVERSIONS.values())
def main():
import argparse
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
- parser.add_argument('--logfn', help='Log to rotating file')
+ parser.add_argument('--logfn', help='Log to rotating file (Not Implemented)')
parser.add_argument('--debug', action='store_true',
help='Don\'t submit records to the database, print them to stdout')
parser.add_argument('-t', '--tail', action='store_true',
diff --git a/aosstower/level_00/legacy_db_insert.py b/aosstower/level_00/legacy_db_insert.py
new file mode 100644
index 0000000..1b0882d
--- /dev/null
+++ b/aosstower/level_00/legacy_db_insert.py
@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+"""Insert RIG Tower data from a level 0 (raw) data file into a MetObs database.
+
+All error messages will go to stderr, all other logging will go to stdout. So
+to make sure errors are logged:
+
+ "%(prog)s 2> error.log"
+
+"""
+import logging
+import logging.handlers
+import sys
+from metobscommon import legacy_db
+from aosstower.level_00.parser import read_frames
+
+logging.addLevelName(9, 'TRACE')
+LOG = logging.getLogger(__name__)
+ERR = logging.getLogger(__name__ + '.error')
+
+
+def configure_logging(level, logfn=None):
+ handler = logging.StreamHandler(sys.stdout)
+ handler.setFormatter(logging.Formatter("%(asctime)s: %(message)s"))
+ LOG.addHandler(handler)
+ LOG.setLevel(level)
+
+ handler = logging.StreamHandler(sys.stderr)
+ handler.setFormatter(logging.Formatter("FAIL: %(message)s"))
+ ERR.addHandler(handler)
+ ERR.setLevel(logging.INFO)
+
+ if logfn:
+ maxsize = (20 * 1024 ** 3) # 20 MB
+ rotating_file = logging.handlers.RotatingFileHandler(
+ logfn, mode='a', maxBytes=maxsize,
+ backupCount=5)
+ rotating_file.setFormatter(logging.Formatter('%(asctime)s: %(message)s'))
+ LOG.addHandler(rotating_file)
+
+
+def main(*args):
+ import argparse
+ parser = argparse.ArgumentParser(description=__doc__,
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+ parser.add_argument('--loglvl', dest='loglvl', default='warn',
+ help='logging level',
+ choices=(['trace', 'debug', 'info', 'warn', 'error']))
+ parser.add_argument('--logfn', help='Log to rotating file')
+ parser.add_argument('-s', '--station', dest='station', default='RIG Tower',
+ help='Name of station to use to determine symbols')
+ parser.add_argument('--ldmp', action='store_true',
+ help='Treat `src` file as a station name and read records from LoggerNet LDMP server (port: 1024)')
+ parser.add_argument('--tables', nargs='*', default=['1'],
+ help="LoggerNet LDMP tables to read in")
+ parser.add_argument('dburl', help='Database to insert into')
+ parser.add_argument('src', help='Level 0 raw data file')
+
+ args = parser.parse_args()
+ lvl = logging._levelNames.get(args.loglvl.upper(), logging.INFO)
+
+ configure_logging(lvl, args.logfn)
+
+ LOG.info("Initializing DB with %s", args.dburl)
+ legacy_db.init_model(args.dburl)
+
+ LOG.info("source: %s", args.src)
+ LOG.info("tail: %s", args.tail)
+
+ station = legacy_db.Station.query.filter_by(name=args.station).first()
+ legacy_db.Session.commit()
+ if not station:
+ parser.exit(1, "Station not found: %s" % args.station)
+
+ if args.ldmp:
+ from aosstower.level_00.parser import ldmp_generator
+ record_gen = ldmp_generator(args.src, args.tables)
+ else:
+ src = open(args.src, "r")
+ record_gen = read_frames(src, tail=args.tail)
+
+ for record in record_gen:
+ data = (record['stamp'], record)
+ failures = legacy_db.insert_records([data], station=station)
+ if failures:
+ ERR.info("(%s) %s", failures[0][0], record)
--
GitLab