#!/usr/bin/env python """Insert RIG Tower data from a level 0 (raw) data file into a MetObs database. All error messages will go to stderr, all other logging will go to stdout. So to make sure errors are logged: "%(prog)s 2> error.log" """ import logging import logging.handlers import sys from metobscommon import legacy_db from aosstower.level_00.parser import read_frames logging.addLevelName(9, 'TRACE') LOG = logging.getLogger(__name__) ERR = logging.getLogger(__name__ + '.error') _symbol_record_map = ( (1, 'station'), (2, 'year'), (3, 'day'), (4, 'hour_minute'), (5, 'seconds'), (6, 'box_pressure'), (7, 'parosci_air_temp_period'), (8, 'parosci_pressure_period'), (9, 'parosci_air_temp'), (10, 'pressure'), (11, 'parosci_calc_sig'), (12, 'box_rh'), (13, 'box_air_temp'), (14, 'temp2'), (15, 'temp3'), (16, 'temp4'), (17, 'wind_speed'), (18, 'wind_direction'), (19, 'rh_shield_freq'), (20, 'relative_humidity'), (21, 'air_temp_6_3'), (22, 'dewpoint'), (23, 'rtd_shield_freq'), (24, 'air_temp'), (25, 'solar_flux'), (26, 'precipitation'), (27, 'accumulated_precipitation'), (28, 'altimeter'), ) def configure_logging(level, logfn=None): handler = logging.StreamHandler(sys.stdout) handler.setFormatter(logging.Formatter("%(asctime)s: %(message)s")) LOG.addHandler(handler) LOG.setLevel(level) handler = logging.StreamHandler(sys.stderr) handler.setFormatter(logging.Formatter("FAIL: %(message)s")) ERR.addHandler(handler) ERR.setLevel(logging.INFO) if logfn: maxsize = (20 * 1024 ** 3) # 20 MB rotating_file = logging.handlers.RotatingFileHandler( logfn, mode='a', maxBytes=maxsize, backupCount=5) rotating_file.setFormatter(logging.Formatter('%(asctime)s: %(message)s')) LOG.addHandler(rotating_file) def main(): import argparse parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--loglvl', dest='loglvl', default='warn', help='logging level', choices=(['trace', 'debug', 'info', 'warn', 'error'])) parser.add_argument('--logfn', help='Log to rotating file') parser.add_argument('--debug', action='store_true', help='Don\'t submit records to the database, print them to stdout') parser.add_argument('-s', '--station', dest='station', default='RIG Tower', help='Name of station to use to determine symbols') parser.add_argument('--ldmp', action='store_true', help='Treat `src` file as a station name and read records from LoggerNet LDMP server (port: 1024)') parser.add_argument('--tables', nargs='*', default=['1'], help="LoggerNet LDMP tables to read in") parser.add_argument('dburl', help='Database to insert into') parser.add_argument('src', help='Level 0 raw data file') args = parser.parse_args() lvl = logging.getLevelName(args.loglvl.upper()) if isinstance(lvl, str): lvl = logging.INFO configure_logging(lvl, args.logfn) LOG.info("Initializing DB with %s", args.dburl) legacy_db.init_model(args.dburl) station = legacy_db.Station.query.filter_by(name=args.station).first() legacy_db.Session.commit() if not station: parser.exit(1, "Station not found: %s" % args.station) if args.ldmp: from aosstower.level_00.parser import ldmp_generator record_gen = ldmp_generator(args.src, args.tables, symbol_names=[x[1] for x in _symbol_record_map]) else: src = open(args.src, "r") record_gen = read_frames(src, tail=args.tail) for idx, record in enumerate(record_gen): data = (record['stamp'], record) LOG.info("Sending record: %s", record['stamp'].isoformat(' ')) if args.debug: # we just want the above message continue failures = legacy_db.insert_records([data], station=station) if failures: ERR.info("(%s) %s", failures[0][0], record) if __name__ == "__main__": sys.exit(main())