From 09633f6cb0651b13828b4672d93b158674b520ab Mon Sep 17 00:00:00 2001
From: Bruce Flynn <brucef@ssec.wisc.edu>
Date: Mon, 23 Jun 2014 14:43:39 -0500
Subject: [PATCH] Moved some code to MetObsCommon
---
aosstower/model.py | 148 ---------------------------------------
scripts/make_database.py | 9 +--
setup.py | 9 +--
3 files changed, 6 insertions(+), 160 deletions(-)
diff --git a/aosstower/model.py b/aosstower/model.py
index 6224354..22caa55 100644
--- a/aosstower/model.py
+++ b/aosstower/model.py
@@ -1,46 +1,9 @@
import os
-import re
-import sys
from datetime import datetime, timedelta
import rrdtool
-import numpy as np
-from zope.interface import implementer
from metobs.data import wind_vector_degrees, to_unix_timestamp
-from metobscommon import interface
-
-
-class ModelError(Exception):
- """Base class for model errors.
- """
-
-
-class WrapErrors(object):
- """Class wrapper to catch exceptions and properly re-raise them such that
- the only exceptions to propagate are `ModelError`s. Essentially, this
- prevents anyone from having to import rrdtool lib.
- """
-
- def __init__(self, *exceptions):
- self.exceptions = exceptions
-
- def __call__(self, cls):
- def _wrap(fcn):
- def wrapped(*args, **kwargs):
- try:
- return fcn(*args, **kwargs)
- except self.exceptions as err:
- traceback = sys.exc_info()[2]
- raise ModelError, str(err), traceback
- wrapped.__doc__ = fcn.__doc__
- return wrapped
- for name in dir(cls):
- value = getattr(cls, name)
- if not name.startswith('_') and hasattr(value, '__call__'):
- setattr(cls, name, _wrap(value))
-
- return cls
def initialize(filepath, start=None, days=365, data_interval=5):
@@ -74,114 +37,3 @@ def initialize(filepath, start=None, days=365, data_interval=5):
'RRA:AVERAGE:0.5:{:d}:105120'.format(300/data_interval),
# 30 minute
'RRA:AVERAGE:0.5:{:d}:17520'.format(1800/data_interval))
-
-
-@WrapErrors(rrdtool.error)
-@implementer(interface.Model)
-class RrdModel(object):
- """Model for storing the Level0 uncalibrated data for non-scientific
- purposes, such as web-widgets.
- """
-
- def __init__(self, filepath):
- self._filepath = filepath
- self._averages = tuple()
- self._datasets = None
-
- @property
- def datasets(self):
- """Get dataset names available in the database.
- """
- if self._datasets is None:
- datasets = set()
- info = rrdtool.info(self._filepath)
- for key in info.keys():
- match = re.match('^ds\[(.*)\]', key)
- if not match:
- continue
- datasets.add(match.groups()[0])
- self._datasets = tuple(sorted(datasets))
- return self._datasets
-
- def averaging_intervals(self):
- """Lazy load averaging intervals from database.
- """
- if not self._averages:
- averages = set()
- info = rrdtool.info(self._filepath)
- for key in info.keys():
- if key.startswith('rra') and key.endswith('pdp_per_row'):
- averages.add(int(info[key] * info['step']))
- self._averages = tuple(sorted(averages))
- return self._averages
-
- def _format_data(self, stamp, data):
- """Format data for insert into RRD returning a template string and data
- line appropriate for arguments to rrdupdate.
- """
- validkeys = set(self.datasets).intersection(data.keys())
- if not validkeys:
- raise ModelError("No valid data keys provided", data)
- tmpl = ':'.join(validkeys)
- values = ':'.join([str(data[k]) for k in validkeys])
- values = '{:d}@{}'.format(to_unix_timestamp(stamp), values)
- return tmpl, values
-
- def add_record(self, stamp, record):
- """Add a single record to the database, where a record is a dict like
- object with keys for each dataset. Additional keys are ignored.
- """
- # Normalize to data interval
- utime = to_unix_timestamp(stamp)
- data_interval = min(self.averaging_intervals())
- stamp = datetime.utcfromtimestamp(utime - utime % data_interval)
-
- tmpl, data = self._format_data(stamp, dict(record))
- rrdtool.update(self._filepath, '--template=%s' % tmpl, data)
-
- def get_slice(self, start, end, names=None, average=5):
- """Get a slice of data from the database.
-
- :param start: Start time as datetime
- :param end: Inclusive end time as datetime
- :param names: Names to query for, defaults to all available, see ``datasets``
- :param average: Averaging interval supported by the database, see ``averaging_intervals``.
- """
- if average not in self.averaging_intervals():
- raise ValueError("Invalid average:%d", average)
- names = names or self.datasets[:]
-
- if isinstance(start, datetime):
- start = to_unix_timestamp(start)
- if isinstance(end, datetime):
- end = to_unix_timestamp(end)
-
- # normalize request times to averaging interval
- start -= start % average
- end -= end % average
-
- # we always get all the data, no matter what was requested
- range, columns, rawdata = rrdtool.fetch(self._filepath,
- 'AVERAGE',
- '-r {:d}'.format(average),
- '-s {:d}'.format(start),
- '-e {:d}'.format(end))
-
- src_data = np.array(rawdata)
- # NaN filled matrix of shape big enough for the request names
- dst_data = np.zeros((src_data.shape[0], len(names))) * float('nan')
-
- # get only the columns we're interested in
- for dst_idx, name in enumerate(names):
- if name in columns:
- dst_data[:, dst_idx] = src_data[:, columns.index(name)]
-
- # recompose the wind direction if asked for
- elif name == 'wind_dir':
- east = src_data[:, self.datasets.index('winddir_east')].astype(np.float64)
- north = src_data[:, self.datasets.index('winddir_north')].astype(np.float64)
- dst_data[:, dst_idx] = wind_vector_degrees(east, north)
-
- # generate column of times for the req average interval
- times = np.array([np.arange(start, end + average, average)])
- return np.concatenate((times.T, dst_data), axis=1)
diff --git a/scripts/make_database.py b/scripts/make_database.py
index eea1b98..0bcb84c 100755
--- a/scripts/make_database.py
+++ b/scripts/make_database.py
@@ -6,8 +6,9 @@ import logging
from datetime import datetime
from metobs.data import wind_vector_components
+from metobscommon.model import RrdModel
from aosstower.record import Record, LineParseError
-from aosstower import model as m
+from aosstower.model import initialize
LOG = logging
@@ -24,15 +25,15 @@ if __name__ == '__main__':
parser.add_argument('-d', '--db-days', type=int, default=365,
help='Size of DB in days')
parser.add_argument('-i', dest='files', type=argparse.FileType('r'),
- help="List of time sorted input data files")
+ help="File containing list of time sorted input data files")
args = parser.parse_args()
logging.basicConfig(level=logging.INFO)
assert not os.path.exists(args.outdb)
- m.initialize(args.outdb, args.db_start, days=args.db_days)
- rrd = m.RrdModel(args.outdb)
+ initialize(args.outdb, args.db_start, days=args.db_days)
+ rrd = RrdModel(args.outdb)
LOG.info("initilized %s", args.outdb)
if args.files is None:
diff --git a/setup.py b/setup.py
index 970b1d5..54ee24a 100644
--- a/setup.py
+++ b/setup.py
@@ -1,9 +1,4 @@
-try:
- from setuptools import setup, find_packages
-except ImportError:
- from ez_setup import use_setuptools
- use_setuptools()
- from setuptools import setup, find_packages
+from setuptools import setup, find_packages
setup(
name='AossTower',
@@ -11,9 +6,7 @@ setup(
description='UW AOSS Rooftop Instrument Group Met Tower',
url='http://metobs.ssec.wisc.edu',
install_requires=[
- 'python-rrdtool',
'numpy',
- 'metobs.data>=0.4a',
'MetObsCommon>=0.1dev'
],
dependency_links=['http://larch.ssec.wisc.edu/cgi-bin/repos.cgi'],
--
GitLab