make_database.py

#!/usr/bin/env python
import os
import sys
import logging
from datetime import datetime

from metobscommon.model import RrdModel, ModelError
from aosstower.record import Record, LineParseError
from aosstower.model import initialize_rrd, VARS

LOG = logging


if __name__ == '__main__':

    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('-o', '--outdb', dest='outdb', default='out.db', 
            help='Destination RRD database. Must not exist')
    argdt = lambda v: datetime.strptime(v, '%Y-%m-%d')
    parser.add_argument('-s', '--db-start', type=argdt, default=datetime.now(),
            help='Reference start date for database (YYYY-MM-DD)')
    parser.add_argument('-d', '--db-days', type=int, default=366,
            help='Size of DB in days')
    parser.add_argument('-i', dest='files', type=argparse.FileType('r'),
            help="File containing list of time sorted input data files")

    args = parser.parse_args()

    logging.basicConfig(level=logging.INFO)

    if os.path.exists(args.outdb):
        os.remove(args.outdb)
    LOG.info("initializing database at start=%s for %d days",
             args.db_start, args.db_days)
    initialize_rrd(args.outdb, args.db_start, days=args.db_days)
    rrd = RrdModel(args.outdb)

    if args.files is None:
        LOG.info("files list not provided, reading from stdin")
        LOG.info("Enter time ordered data files, one per line, ^D when done")
        args.files = sys.stdin

    for each in args.files.readlines():
        fpath = each.strip()
        if not os.path.exists(fpath):
            LOG.warn("%s does not exist", fpath)
            continue

        LOG.info("adding %s", fpath)
        for line in open(fpath).readlines():
            if not line.strip():
                continue
            try:
                record = Record.create(line)
            except LineParseError as err:
                LOG.error(str(err))
                continue

            try:
                rrd.add_record(record['stamp'], record)
            except ModelError:
                LOG.exception("Could not insert: %s", record)