Commit a30225c4 authored by Bruce Flynn's avatar Bruce Flynn

fix numpy warnings

parent 0aef0649
......@@ -5,7 +5,6 @@ from datetime import datetime
from datetime import timedelta
import numpy as np
from metobs.data.calc import wind_vector_degrees
from psycopg2.extensions import adapt as sqlescape
from sqlalchemy import (
create_engine,
......@@ -21,9 +20,10 @@ from sqlalchemy import (
from sqlalchemy.pool import NullPool
from sqlalchemy.sql import compiler
from amrc_aws import config
from amrc_aws.station_aliases import stations
from amrc_aws.util import NaN
from . import config
from .station_aliases import stations
from .util import NaN
from .winds import wind_vector_degrees
LOG = logging.getLogger(__name__)
......@@ -146,15 +146,15 @@ def _data_table_name(type_):
def get_rdr_slice(stations, symbols, start, end, avg=None):
avg = max(avg, 600) if avg else 600
return _get_slice('rdr', stations, symbols, start, end, avg)
return _get_slice('rdr', stations, symbols, start, end, 600, avg)
def get_q1h_slice(stations, symbols, start, end, avg=None):
avg = min(avg, 3600) if avg else 3600
return _get_slice('q1h', stations, symbols, start, end, avg)
avg = max(avg, 3600) if avg else 3600
return _get_slice('q1h', stations, symbols, start, end, 3600, avg)
def _get_slice(type_, stations, symbols, start, end, avg):
def _get_slice(type_, stations, symbols, start, end, interval, avg):
"""
Get a slice of data for all station/symbol combinations. Data will be
returned for all combinations whether present or not and filled with NaN
......@@ -177,39 +177,51 @@ def _get_slice(type_, stations, symbols, start, end, avg):
'pressure', 'wind_dir', 'wind_spd'.
:param start: Inclusive start datetime
:param end: Inclusive end datetime
:param interval: Native data interval in seconds
:param avg: Averaging interval in seconds
"""
table_name = _data_table_name(type_)
select = sql.text("""
SELECT
series.stamp, air_temp, vtempdiff, rh, pressure, _wind_vector_east,
_wind_vector_north
SELECT
to_timestamp(extract(epoch from inner_stamp)::int - (extract(EPOCH from inner_stamp)::int % :avg)) as stamp,
avg(air_temp) as air_temp,
avg(vtempdiff) as vtempdiff,
avg(rh) as rh,
avg(pressure) as pressure,
avg(_wind_vector_east) as _wind_vector_east,
avg(_wind_vector_north) as _wind_vector_north
FROM (
SELECT name as station, stamp FROM
(SELECT generate_series(:start, :end, :interval) as stamp) stamps
LEFT OUTER JOIN
(SELECT unnest AS name FROM unnest(ARRAY{stations})) stations
ON True) AS series
LEFT OUTER JOIN (
SELECT * FROM {data_table} d
WHERE stamp >= :start AND stamp <= :end
AND station IN :stations
) AS data
ON series.stamp = data.stamp
AND data.station = series.station
ORDER BY series.stamp, data.station
SELECT
series.stamp as inner_stamp, air_temp, vtempdiff, rh, pressure, _wind_vector_east,
_wind_vector_north
FROM (
SELECT name as station, stamp FROM
(SELECT generate_series(:start, :end, :interval) as stamp) stamps
LEFT OUTER JOIN
(SELECT unnest AS name FROM unnest(ARRAY{stations})) stations ON True) AS series
LEFT OUTER JOIN (
SELECT * FROM {data_table} d
WHERE stamp >= :start AND stamp < :end AND station IN :stations
) AS data
ON series.stamp = data.stamp
AND data.station = series.station
ORDER BY series.stamp, data.station
) as y
GROUP BY stamp
ORDER BY stamp
""".format(data_table=table_name, stations=[str(s) for s in stations]))
'''
LOG.debug(_compile_query(select.bindparams(
start=start, end=end,
interval=timedelta(seconds=avg),
start=start, end=end - timedelta(seconds=1),
interval=timedelta(seconds=interval),
avg=avg,
stations=tuple(stations))))
'''
rows = _fetchall(
select, start=start, end=end,
interval=timedelta(seconds=avg),
select, start=start, end=end - timedelta(seconds=1),
interval=timedelta(seconds=interval),
avg=avg,
stations=tuple(stations))
arr = np.array(rows)
......
......@@ -6,16 +6,15 @@ The q1h format is assumed to have 2 header lines of the format::
Lat: 76.15S Lon: 168.40E Elev: 262m
"""
import re
import os
import re
from collections import OrderedDict
from datetime import datetime, timedelta
from metobs.data.calc import wind_vector_components
from amrc_aws import db
from amrc_aws.exc import ParseError
from amrc_aws.util import NaN, is_nan, printable
from . import db
from .exc import ParseError
from .util import NaN, is_nan, printable
from .winds import wind_vector_components
# RDR format NaN value
MISSING = 444.0
......@@ -156,6 +155,7 @@ def frames(filepath, frame_width=FRAME_WIDTH, error_callback=None):
'_type': 'q1h',
'_argosid': aid,
'_site': station_name,
'_stamp': get_frame_stamp(frame['year'], frame['month'], frame['day'], frame['hhmm'])
'_stamp': get_frame_stamp(frame['year'], frame['month'], frame['day'],
frame['hhmm'])
})
yield frame
......@@ -6,16 +6,15 @@ The RDR format is assumed to have 2 header lines of the format::
Lat : 67.01S Long : 142.66E Elev : 31 M
"""
import re
import os
import re
from collections import OrderedDict
from datetime import datetime, timedelta
from metobs.data.calc import wind_vector_components
from amrc_aws import db
from amrc_aws.exc import ParseError
from amrc_aws.util import NaN, is_nan, printable
from . import db
from .exc import ParseError
from .util import NaN, is_nan, printable
from .winds import wind_vector_components
# RDR format NaN value
MISSING = 444.0
......
"""
These functions were ripped from MetObsData (https://gitlab.ssec.wisc.edu/metobs/MetObsData)
to simply avoid the dependency.
"""
import numpy as np
def wind_vector_components(windspd, winddir):
"""
Decompose scalar or list/array polar wind direction and speed data
into the horizontal and vertical vector components and speed vector.
Inputs can be scalar or arrays.
"""
dir_rad = np.deg2rad(winddir)
spd_arr = np.array(windspd)
V_e = spd_arr * np.sin(dir_rad)
V_n = spd_arr * np.cos(dir_rad)
U_spd = np.sqrt(pow(V_e, 2) + pow(V_n, 2))
return V_e, V_n, U_spd
def wind_vector_degrees(vector_east, vector_north):
"""
Re-compose horizontal (east/west) and vertical (north/south) vector
components into wind direction in degrees.
Inputs can be scalar or arrays.
"""
rads = np.arctan2(vector_east, vector_north)
winddir = np.rad2deg(rads)
if isinstance(winddir, np.ndarray):
winddir[np.ma.less(np.ma.masked_invalid(winddir), 0)] += 360
elif winddir < 0:
winddir += 360
return winddir % 360
git+https://gitlab.ssec.wisc.edu/metobs/MetObsData.git@0.5
FormEncode==1.3.0
numpy==1.11.1
pyramid==1.7
......
......@@ -14,7 +14,6 @@ setup(
setup_requires=['setuptools_scm'],
install_requires=[
'numpy',
'metobs.data>=0.5',
'requests',
'psycopg2',
'sqlalchemy>=1.0',
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment