Commit 2725c7c0 authored by Bruce Flynn's avatar Bruce Flynn

first adjustment to structure

parent 28f15496
......@@ -79,23 +79,21 @@ def validate_symbols(form):
SINGLE_STATION_MAX_RANGE = timedelta(days=5 * 365)
MULTI_STATION_MAX_RANGE = timedelta(days=1 * 365)
def check_data_limits_policy(start, end, stations):
if len(stations) == 0:
return
req_range = (end - start)
if len(stations) == 1 and req_range > SINGLE_STATION_MAX_RANGE:
raise HTTPBadRequest((
'Maximum time range allowed is {:d} days for single station query, '
'the current request is for {:d} days.')
.format(SINGLE_STATION_MAX_RANGE.days, req_range.days)
)
raise HTTPBadRequest(
('Maximum time range allowed is {:d} days for single station query, '
'the current request is for {:d} days.')
.format(SINGLE_STATION_MAX_RANGE.days, req_range.days))
elif len(stations) > 1 and req_range > MULTI_STATION_MAX_RANGE:
raise HTTPBadRequest((
'Maximum time range allowed is {:d} days for multi station query'
'the current request is for {:d} days.')
.format(SINGLE_STATION_MAX_RANGE.days, req_range.days)
)
raise HTTPBadRequest(
('Maximum time range allowed is {:d} days for multi station query'
'the current request is for {:d} days.')
.format(SINGLE_STATION_MAX_RANGE.days, req_range.days))
@view_config(route_name='data_form_csv')
......@@ -187,7 +185,6 @@ def csv_response(stations, sensors, form, rows):
fields = ['%s.%s' % (k, v)
for k in stations
for v in sensors]
LOG.debug('got shape %s for %s', rows.shape, fields)
def yielder():
yield '# AMRC AWS (http://amrc.ssec.wisc.edu/aws)\n'
......
......@@ -161,7 +161,17 @@ def _get_slice(type_, stations, symbols, start, end, avg):
"""
Get a slice of data for all station/symbol combinations. Data will be
returned for all combinations whether present or not and filled with NaN
where not avialable.
where not available.
Shape of the data returned will be::
<stamp 1>, <field 1 station 1>, ..., <field n station 1> --
<stamp 1>, <field 1 station 2>, ..., <field n station 2> | Record 1 (all same timestamp)
<stamp 1>, <field 1 station x>, ..., <field n station x> --
...
<stamp t>, <field 1 station 1>, ..., <field n station 1> --
<stamp t>, <field 1 station 2>, ..., <field n station 2> | Record R
<stamp t>, <field 1 station x>, ..., <field n station x> --
:param type_: A supported data type, e.g., rdr, or q1h
:param stations: A list of station names
......@@ -192,12 +202,14 @@ def _get_slice(type_, stations, symbols, start, end, avg):
AND data.station = series.station
ORDER BY series.stamp, data.station
""".format(data_table=table_name, stations=[str(s) for s in stations]))
'''
LOG.debug(_compile_query(select.bindparams(
start=start, end=end,
interval=timedelta(seconds=avg),
stations=tuple(stations))))
'''
rows = _fetchall(
select, start=start, end=end,
interval=timedelta(seconds=avg),
......@@ -258,7 +270,7 @@ def create_schema():
Column('_wind_vector_east', Float))
data_t.create()
data_q1h_t = Table(
data_t = Table(
'data_q1h', metadata,
Column('stamp', DateTime, primary_key=True),
Column('station', String, primary_key=True),
......@@ -268,7 +280,7 @@ def create_schema():
Column('pressure', Float),
Column('_wind_vector_north', Float),
Column('_wind_vector_east', Float))
data_q1h_t.create()
data_t.create()
station_dates_t = Table(
'station_dates', metadata,
......
......@@ -2,6 +2,8 @@ import logging
import numpy as np
import netCDF4
from .util import unixtime
LOG = logging.getLogger(__name__)
......@@ -16,7 +18,7 @@ schema = {
},
'dimensions': {
'station': None,
'obnum': None,
'time': None,
},
'variables': {
'station': {
......@@ -28,7 +30,7 @@ schema = {
},
},
'time': {
'dims': ('station', 'obnum'),
'dims': ('time',),
'type': np.int64,
'fill': -999,
'attrs': {
......@@ -42,18 +44,18 @@ schema = {
variables = {
'air_temp': {
'dims': ('station', 'obnum',),
'dims': ('station', 'time'),
'type': np.float32,
'fill': -999.0,
'attrs': {
'long_name': 'air temperature',
'standard_name': 'air_temperature',
'units': 'degC',
'coordinates': 'time'
'coordinates': 'station, time'
},
},
'pressure': {
'dims': ('station', 'obnum',),
'dims': ('station', 'time'),
'type': np.float32,
'fill': -999.0,
'attrs': {
......@@ -64,7 +66,7 @@ variables = {
},
},
'wind_spd': {
'dims': ('station', 'obnum',),
'dims': ('station', 'time'),
'type': np.float32,
'fill': -999.0,
'attrs': {
......@@ -75,7 +77,7 @@ variables = {
},
},
'wind_dir': {
'dims': ('station', 'obnum',),
'dims': ('station', 'time'),
'type': np.float32,
'fill': -999.0,
'attrs': {
......@@ -86,7 +88,7 @@ variables = {
},
},
'rh': {
'dims': ('station', 'obnum',),
'dims': ('station', 'time'),
'type': np.float32,
'fill': -999.0,
'attrs': {
......@@ -97,7 +99,7 @@ variables = {
},
},
'vtempdiff': {
'dims': ('station', 'obnum',),
'dims': ('station', 'time'),
'type': np.float32,
'fill': -999.0,
'attrs': {
......@@ -108,7 +110,7 @@ variables = {
}
def _fill_dataset(stations, symbols, data, dataset):
def _fill_dataset(stations, symbols, data, nc):
num_stations = len(stations)
# data[0:data.shape[0]:2,2]
for symidx, symbol in enumerate(symbols):
......@@ -120,53 +122,54 @@ def _fill_dataset(stations, symbols, data, dataset):
# to skip over the stamp colum
# symidx+1
arr[staidx,:] = data[staidx:data.shape[0]:num_stations,symidx+1].astype(float)
var = dataset.variables[symbol]
var = nc.variables[symbol]
# set to fill where currently NaN
arr[np.where(arr != arr)] = var._FillValue
var[:] = arr
# set station names
var = dataset.variables['station_name']
var = nc.variables['station']
var[:] = np.array(stations)
# set times
var = nc.variables['time']
var[:] = [unixtime(d) for d in data[0:data.shape[0]:2,0]]
def write_slice_to_netcdf(stations, symbols, data, dest, attrs=None):
attrs = attrs or {}
dataset = netCDF4.Dataset(dest, mode='w') #, diskless=True)
# global attrs
for name, value in schema['globals'].items():
setattr(dataset, name, value)
for name, value in attrs.items():
setattr(dataset, name, value)
with netCDF4.Dataset(dest, mode='w') as nc:
# global attrs
for name, value in schema['globals'].items():
setattr(nc, name, value)
# dimensions
for name, size in schema['dimensions'].items():
if size is not None:
dataset.createDimension(name, size=size)
elif name == 'station':
dataset.createDimension(name, len(stations))
elif name == 'obnum':
dataset.createDimension(name, data.shape[0]/len(stations))
else:
dataset.createDimension(name)
# schema variables
for name, dat in schema['variables'].items():
var = dataset.createVariable(
name, dat['type'], dat['dims'],
fill_value=dat.get('fill'), zlib=True)
for name, value in dat['attrs'].items():
setattr(var, name, value)
# data variables
for name, dat in variables.items():
if name not in symbols:
continue
var = dataset.createVariable(
name, dat['type'], dat['dims'],
fill_value=dat['fill'], zlib=True)
for name, value in dat['attrs'].items():
setattr(var, name, value)
for name, value in attrs.items():
setattr(nc, name, value)
_fill_dataset(stations, symbols, data, dataset)
# dimensions
for name, size in schema['dimensions'].items():
if size is not None:
nc.createDimension(name, size=size)
elif name == 'station':
nc.createDimension(name, len(stations))
elif name == 'time':
nc.createDimension(name, data.shape[0]/len(stations))
else:
nc.createDimension(name)
# schema variables
for name, dat in schema['variables'].items():
var = nc.createVariable(
name, dat['type'], dat['dims'],
fill_value=dat.get('fill'), zlib=True)
for aname, value in dat['attrs'].items():
setattr(var, aname, value)
# data variables
for name, dat in variables.items():
if name not in symbols:
continue
var = nc.createVariable(
name, dat['type'], dat['dims'],
fill_value=dat['fill'], zlib=True)
for name, value in dat['attrs'].items():
setattr(var, name, value)
dataset.close()
_fill_dataset(stations, symbols, data, nc)
......@@ -47,7 +47,7 @@ level = INFO
handlers = console
[logger_sa]
level = INFO
level = WARNING
handlers = console
qualname = sqlalchemy.engine
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment