Commit d977a497 authored by Bruce Flynn's avatar Bruce Flynn

merge in changes from amrc

Squashed commit of the following:

commit c5b3a0b78e9acba08b3b7837c6d900c5e971d1d7
Author: Bruce Flynn <brucef@ssec.wisc.edu>
Date:   Fri Jan 6 13:11:06 2017 -0600

    fix: broken tests

commit 3f020c44b0ea982730755eb532b4581562197264
Merge: 678f0141 4aaf5ffd
Author: Bruce Flynn <brucef@ssec.wisc.edu>
Date:   Fri Jan 6 13:03:09 2017 -0600

    Merge branch 'master' into from_armc

commit 678f0141
Author: Bruce Flynn <brucef@ssec.wisc.edu>
Date:   Fri Jan 6 18:56:07 2017 +0000

    not sure what this code is

commit 47f43547
Author: Bruce Flynn <brucef@ssec.wisc.edu>
Date:   Tue Oct 11 19:18:28 2016 +0000

    app: handle stations with no records

commit e7b6ab5b
Author: Bruce Flynn <brucef@ssec.wisc.edu>
Date:   Tue Oct 11 19:18:19 2016 +0000

    Correct package name
parent 4aaf5ffd
Pipeline #1370 failed with stage
in 1 minute and 21 seconds
......@@ -30,7 +30,7 @@ class DataForm(Schema):
symbols = Regex(r'^[a-zA-Z_0-9|\.]+$', not_empty=True)
start = Regex(r'^\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ$', not_empty=True)
end = Regex(r'^\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ$', not_empty=True)
avg = Int(min=10, max=1440, if_missing=10, if_empty=10)
avg = Int(min=10, max=1440, if_missing=60, if_empty=60)
def stamp_to_dt(val):
......@@ -73,7 +73,7 @@ def data_csv_view(request):
symbols = validate_symbols(form)
start = stamp_to_dt(form['start'])
end = stamp_to_dt(form['end'])
avg = int(form['avg']) * 60
avg = int(form['avg']) * 60 if form['avg'] else None
db = request.registry.settings['db']
get_slice = getattr(db, 'get_{}_slice'.format(form['type']))
......@@ -109,19 +109,29 @@ def data_netcdf_view(request):
symbols = validate_symbols(form)
start = stamp_to_dt(form['start'])
end = stamp_to_dt(form['end'])
avg = int(form['avg']) * 60
avg = int(form['avg']) * 60 if form['avg'] else None
# fetch data from database
db = request.registry.settings['db']
get_slice = getattr(db, 'get_{}_slice'.format(form['type']))
data = get_slice(stations, symbols, start, end, avg=avg)
if not data:
if not data.any():
raise HTTPNotFound('No data available matching your parameters')
# write rows to NetCDF
dest = mkstemp(suffix='.nc')[1]
nc.write_slice_to_netcdf(stations, symbols, data, dest)
attrs = {
'data_type': form['type'],
'start_time': form['start'],
'end_time': form['end'],
}
if form['avg']:
attrs.update({
'average_interval': form['avg'],
'average_units': 'minutes',
})
nc.write_slice_to_netcdf(stations, symbols, data, dest, attrs=attrs)
response = Response(app_iter=RemovingFileIter(open(dest)))
response.content_type = 'application/x-netcdf'
......@@ -133,8 +143,8 @@ def data_netcdf_view(request):
def stations_view(request):
db = request.registry.settings['db']
return [{'name': r['name'],
'min_available': util.unixtime(r['min']),
'max_available': util.unixtime(r['max'])}
'min_available': util.unixtime(r['min']) if r['min'] else None,
'max_available': util.unixtime(r['max']) if r['max'] else None}
for r in db.get_stations()]
......
......@@ -136,22 +136,21 @@ def insert_frames(frames):
return inserted
DEFAULT_AVG_SECS = 300
def _data_table_name(type_):
return 'data' if type_ == 'rdr' else 'data_{:s}'.format(type_)
def get_rdr_slice(stations, symbols, start, end, avg=DEFAULT_AVG_SECS):
def get_rdr_slice(stations, symbols, start, end, avg=None):
avg = max(avg, 600) if avg else 600
return _get_slice('rdr', stations, symbols, start, end, avg)
def get_q1h_slice(stations, symbols, start, end, avg=DEFAULT_AVG_SECS):
def get_q1h_slice(stations, symbols, start, end, avg=None):
avg = min(avg, 3600) if avg else 3600
return _get_slice('q1h', stations, symbols, start, end, avg)
def _get_slice(type_, stations, symbols, start, end, avg=300):
def _get_slice(type_, stations, symbols, start, end, avg):
"""
Get a slice of data for all station/symbol combinations. Data will be returned
for all combinations whether present or not and filled with NaN where not avialable.
......@@ -162,6 +161,7 @@ def _get_slice(type_, stations, symbols, start, end, avg=300):
'rh', 'pressure', 'wind_dir', 'wind_spd'.
:param start: Inclusive start datetime
:param end: Inclusive end datetime
:param avg: Averaging interval in seconds
"""
table_name = _data_table_name(type_)
select = sql.text("""
......
import logging
import numpy as np
import netCDF4
LOG = logging.getLogger(__name__)
NAME_STRLEN = 48
......@@ -100,14 +103,14 @@ variables = {
def _fill_dataset(stations, symbols, data, dataset):
num_stations = len(stations)
# data[0:data.shape[0]:2,2]
for symidx, symbol in enumerate(symbols):
# prefill with -999.9
arr = np.ones((len(stations), data.shape[0]/len(stations))) * -999.0
num_stations = len(stations)
for staidx, station in enumerate(stations):
# to get the rows for a particular station
# staidx:data.shape[0]:len(stations)
# staidx:data.shape[0]:num_stations
# to skip over the stamp colum
# symidx+1
arr[staidx,:] = data[staidx:data.shape[0]:num_stations,symidx+1].astype(float)
......@@ -115,15 +118,25 @@ def _fill_dataset(stations, symbols, data, dataset):
# set to fill where currently NaN
arr[np.where(arr != arr)] = var._FillValue
var[:] = arr
# set station names
var = dataset.variables['station_name']
var[:] = np.array(stations)
def write_slice_to_netcdf(stations, symbols, data, dest):
def write_slice_to_netcdf(stations, symbols, data, dest, attrs=None):
attrs = attrs or {}
dataset = netCDF4.Dataset(dest, mode='w') #, diskless=True)
# global attrs
for name, value in schema['globals'].items():
setattr(dataset, name, value)
for name, value in attrs.items():
setattr(dataset, name, value)
LOG.debug("data.shape: %s", data.shape)
LOG.debug(data)
# dimensions
for name, size in schema['dimensions'].items():
if size is not None:
......
......@@ -61,12 +61,12 @@ def test_get_rdr_slice(database):
('2016-01-01 00:30:00', :station_id, 3.0, 3.0, 3.0, 3.0, 0.0, 0.0)
"""), station_id=station_id)
avg = 300
avg = None
start = datetime(2016, 1, 1)
end = datetime(2016, 1, 1, 0, 59, 59)
data = db.get_rdr_slice(['STATION1'], ['air_temp'], start, end, avg)
# data should return 12 rows becasue there are 12 5min intervals in 1 hour
assert len(data) == 12
assert len(data) == 6
avg = 600
data = db.get_rdr_slice(['STATION1'], ['air_temp'], start, end, avg)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment