make_database.py

#!/usr/bin/env python
import os
import sys
import logging
from datetime import datetime

from metobscommon.model import RrdModel, ModelError
from aosstower.level_00.parser import read_records
from aosstower.level_00.rrd import initialize_rrd

LOG = logging


if __name__ == '__main__':

    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('-o', '--outdb', dest='outdb', default='out.db', 
            help='Destination RRD database. Must not exist')
    argdt = lambda v: datetime.strptime(v, '%Y-%m-%d')
    parser.add_argument('-s', '--db-start', type=argdt, default=datetime.now(),
            help='Reference start date for database (YYYY-MM-DD)')
    parser.add_argument('-d', '--db-days', type=int, default=366,
            help='Size of DB in days')
    parser.add_argument('-i', dest='files', type=argparse.FileType('r'),
            help="File containing list of time sorted input data files")

    args = parser.parse_args()

    logging.basicConfig(level=logging.INFO)

    if os.path.exists(args.outdb):
        os.remove(args.outdb)
    LOG.info("initializing database at start=%s for %d days",
             args.db_start, args.db_days)
    initialize_rrd(args.outdb, args.db_start, days=args.db_days)
    rrd = RrdModel(args.outdb)

    if args.files is None:
        LOG.info("files list not provided, reading from stdin")
        LOG.info("Enter time ordered data files, one per line, ^D when done")
        args.files = sys.stdin


    for each in args.files.readlines():
        fpath = each.strip()
        if not os.path.exists(fpath):
            LOG.warn("%s does not exist", fpath)
            continue

        LOG.info("adding %s", fpath)
        for record in read_records(fpath):
            try:
                rrd.add_record(record['stamp'], record)
            except ModelError as err:
                LOG.error("Insert failed: %s", err)