Start of script to make yearly dbs

scripts/make_database.py

+#!/usr/bin/env python
+import os
+import sys
+import glob
+import logging
+from datetime import datetime
+
+from metobs.data import wind_vector_components
+from aosstower.record import RecordV1, LineParseError
+from aosstower.model import RrdModel, ModelError
+
+LOG = logging
+
+if __name__ == '__main__':
+
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-o', '--outdb', dest='outdb', default='out.db', 
+            help='Destination RRD database. Must not exist')
+    argdt = lambda v: datetime.strptime(v, '%Y-%m-%d')
+    parser.add_argument('-s', '--db-start', type=argdt, default=datetime.now(),
+            help='Reference start date for database (YYYY-MM-DD)')
+    parser.add_argument('-d', '--db-days', type=int, default=365,
+            help='Size of DB in days')
+    parser.add_argument('files', nargs='?', type=argparse.FileType('r'),
+            default=sys.stdin, 
+            help="List of time sorted input data files")
+
+    args = parser.parse_args()
+
+    logging.basicConfig(level=logging.INFO)
+
+    assert not os.path.exists(args.outdb)
+    rrd = RrdModel(args.outdb)
+    rrd.initialize(args.db_start, days=args.db_days)
+    LOG.info("initilized %s", args.outdb)
+
+    for each in args.files.readlines():
+        fpath = each.strip()
+        if not os.path.exists(fpath):
+            LOG.warn("%s does not exist", fpath)
+            continue
+
+        LOG.info("adding %s", fpath)
+        for line in open(fpath).readlines():
+            if not line.strip():
+                continue
+            try:
+                record = RecordV1(line)
+            except LineParseError as err:
+                LOG.error(str(err))
+                continue
+
+            windspd = float(record['wind_speed'])
+            winddir = float(record['wind_dir'])
+            u_e, u_n, spd = wind_vector_components(windspd, winddir)
+            record['winddir_east'] = u_e
+            record['winddir_north'] = u_n
+            record['wind_speed'] = spd
+
+            try:
+                rrd.add_record(record)
+            except ModelError:
+                LOG.exception("Could not insert: %s", record)
+