Add legacy db insert script

aosstower/level_00/influxdb.py

@@ -51,7 +51,7 @@ SYMBOLS = list(SYMBOL_CONVERSIONS.values())
 def main():
     import argparse
     parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-    parser.add_argument('--logfn', help='Log to rotating file')
+    parser.add_argument('--logfn', help='Log to rotating file (Not Implemented)')
     parser.add_argument('--debug', action='store_true',
                         help='Don\'t submit records to the database, print them to stdout')
     parser.add_argument('-t', '--tail', action='store_true',

aosstower/level_00/legacy_db_insert.py

+#!/usr/bin/env python
+"""Insert RIG Tower data from a level 0 (raw) data file into a MetObs database.
+
+All error messages will go to stderr, all other logging will go to stdout. So
+to make sure errors are logged:
+
+    "%(prog)s 2> error.log"
+
+"""
+import logging
+import logging.handlers
+import sys
+from metobscommon import legacy_db
+from aosstower.level_00.parser import read_frames
+
+logging.addLevelName(9, 'TRACE')
+LOG = logging.getLogger(__name__)
+ERR = logging.getLogger(__name__ + '.error')
+
+
+def configure_logging(level, logfn=None):
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setFormatter(logging.Formatter("%(asctime)s: %(message)s"))
+    LOG.addHandler(handler)
+    LOG.setLevel(level)
+
+    handler = logging.StreamHandler(sys.stderr)
+    handler.setFormatter(logging.Formatter("FAIL: %(message)s"))
+    ERR.addHandler(handler)
+    ERR.setLevel(logging.INFO)
+
+    if logfn:
+        maxsize = (20 * 1024 ** 3)  # 20 MB
+        rotating_file = logging.handlers.RotatingFileHandler(
+            logfn, mode='a', maxBytes=maxsize,
+            backupCount=5)
+        rotating_file.setFormatter(logging.Formatter('%(asctime)s: %(message)s'))
+        LOG.addHandler(rotating_file)
+
+
+def main(*args):
+    import argparse
+    parser = argparse.ArgumentParser(description=__doc__,
+                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument('--loglvl', dest='loglvl', default='warn',
+                        help='logging level',
+                        choices=(['trace', 'debug', 'info', 'warn', 'error']))
+    parser.add_argument('--logfn', help='Log to rotating file')
+    parser.add_argument('-s', '--station', dest='station', default='RIG Tower',
+                        help='Name of station to use to determine symbols')
+    parser.add_argument('--ldmp', action='store_true',
+                        help='Treat `src` file as a station name and read records from LoggerNet LDMP server (port: 1024)')
+    parser.add_argument('--tables', nargs='*', default=['1'],
+                        help="LoggerNet LDMP tables to read in")
+    parser.add_argument('dburl', help='Database to insert into')
+    parser.add_argument('src', help='Level 0 raw data file')
+
+    args = parser.parse_args()
+    lvl = logging._levelNames.get(args.loglvl.upper(), logging.INFO)
+
+    configure_logging(lvl, args.logfn)
+
+    LOG.info("Initializing DB with %s", args.dburl)
+    legacy_db.init_model(args.dburl)
+
+    LOG.info("source: %s", args.src)
+    LOG.info("tail: %s", args.tail)
+
+    station = legacy_db.Station.query.filter_by(name=args.station).first()
+    legacy_db.Session.commit()
+    if not station:
+        parser.exit(1, "Station not found: %s" % args.station)
+
+    if args.ldmp:
+        from aosstower.level_00.parser import ldmp_generator
+        record_gen = ldmp_generator(args.src, args.tables)
+    else:
+        src = open(args.src, "r")
+        record_gen = read_frames(src, tail=args.tail)
+
+    for record in record_gen:
+        data = (record['stamp'], record)
+        failures = legacy_db.insert_records([data], station=station)
+        if failures:
+            ERR.info("(%s) %s", failures[0][0], record)