Commit 4aaf5ffd authored by Bruce Flynn's avatar Bruce Flynn

Fix up tests a bit

parent d6989d2f
......@@ -6,5 +6,5 @@ env
build
dist
*.sw?
.coverage
.coverage*
.cache
......@@ -5,8 +5,8 @@ AWS API Database Refresher
Refreshes database periodic data. This may include averaging of data and
calculation of statistics.
Currently, it's just updating the date range statistics so they do not need to
be recomputed every time which can be very expensive.
The current implementation upates the date range statistics as well as the list
of available station names.
"""
import argparse
import logging
......@@ -15,6 +15,7 @@ from amrc_aws import db
LOG = logging
def main():
parser = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
......@@ -30,6 +31,13 @@ def main():
LOG.info('Refreshing station dates')
db.refresh_station_dates()
db.engine.execute("""
INSERT INTO stations(name)
SELECT sitename FROM aws_sites
WHERE NOT EXISTS (
SELECT 1 FROM stations
WHERE stations.name == aws_sites.sitename)
""")
if __name__ == '__main__':
main()
#!/usr/bin/env python
"""
Update the ``stations`` table to include all stations listed in the ``aws_sites``
table maintained by SamB.
"""
import argparse
import logging
from amrc_aws import db
LOG = logging
def main():
parser = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
args = parser.parse_args()
db.init()
db.execute("""
INSERT INTO stations(name)
SELECT sitename FROM aws_sites
WHERE NOT EXISTS (
SELECT 1 FROM stations
WHERE stations.name == aws_sites.sitename)
""")
if __name__ == '__main__':
main()
......@@ -16,4 +16,6 @@ def is_nan(val):
def unixtime(dt):
if dt is None:
return None
return (dt - datetime(1970, 1, 1)).total_seconds()
......@@ -4,7 +4,7 @@
###
[app:main]
use = egg:amrc_aws#app
use = egg:AmrcAws#app
pyramid.reload_templates = true
pyramid.debug_authorization = false
......
......@@ -37,6 +37,5 @@ setup(
[console_scripts]
file_insert = amrc_aws.cli.file_insert:main
batch_insert = amrc_aws.cli.batch_insert:main
update_stations = amrc_aws.cli.update_stations
""",
)
......@@ -12,9 +12,7 @@ services:
db:
environment:
- POSTGRES_USER=docker
- POSTGRES_DB=docker
- POSTGRES_DB=__test
image: postgres:9.6
ports:
- 5432:5432
volumes:
- ./test_db_init.sh:/docker-entrypoint-initdb.d/test_db_init.sh
- 15432:5432
......@@ -52,7 +52,7 @@ Lat: 76.15S Lon: 168.40E Elev: 262m
def test_file_insert_q1h(database, q1h_file):
script = 'python -m amrc_aws.cli.file_insert -v {}'.format(q1h_file)
script = './env/bin/python -m amrc_aws.cli.file_insert -v {}'.format(q1h_file)
check_call(script, shell=True)
with db.connection() as conn:
......@@ -78,7 +78,7 @@ Lat : 82.32S Long : 75.99E Elev : 4027 M
def test_file_insert_rdr(database, rdr_file):
script = 'python -m amrc_aws.cli.file_insert -v {}'.format(rdr_file)
script = './env/bin/python -m amrc_aws.cli.file_insert -v {}'.format(rdr_file)
check_call(script, shell=True)
with db.connection() as conn:
......@@ -87,10 +87,10 @@ def test_file_insert_rdr(database, rdr_file):
def test_file_insert_skips_duplicates(database, rdr_file):
script = 'python -m amrc_aws.cli.file_insert -v {}'.format(rdr_file)
script = './env/bin/python -m amrc_aws.cli.file_insert -v {}'.format(rdr_file)
check_call(script, shell=True)
script = 'python -m amrc_aws.cli.file_insert -v {}'.format(rdr_file)
script = './env/bin/python -m amrc_aws.cli.file_insert -v {}'.format(rdr_file)
check_call(script, shell=True)
with db.connection() as conn:
......@@ -99,7 +99,7 @@ def test_file_insert_skips_duplicates(database, rdr_file):
def test_batch_insert(database, rdr_file, tmpdir):
script = 'python -m amrc_aws.cli.batch_insert -v -'
script = './env/bin/python -m amrc_aws.cli.batch_insert -v -'
fptr = tmpdir.join('files.txt')
fptr.write(rdr_file + '\n')
check_call(script, shell=True, stdin=open(fptr.strpath))
......
......@@ -21,7 +21,6 @@ def database():
with engine.connect() as conn:
conn.execute("DROP SCHEMA IF EXISTS {:s} CASCADE".format(schema))
conn.execute("CREATE SCHEMA {:s}".format(schema))
conn.execute("COMMIT")
engine.dispose()
db.init(sa.create_engine(os.environ['AWSAPI_DB_URL']))
......@@ -29,21 +28,27 @@ def database():
with db.connection() as conn:
conn.execute("INSERT INTO stations(name) VALUES ('STATION1')")
conn.execute("INSERT INTO station_dates(name, max, min) VALUES ('STATION1', now(), now())")
conn.execute("INSERT INTO stations(name) VALUES ('STATION2')")
return sa.create_engine(os.environ['AWSAPI_DB_URL'])
def test_station_names(database):
assert db.get_station_names() == ['STATION1']
assert db.get_station_names() == ['STATION1', 'STATION2']
def test_get_stations(database):
stations = db.get_stations()
assert len(stations) == 1
assert len(stations) == 2
assert isinstance(stations[0], dict)
assert stations[0].get('id')
assert stations[0].get('name') == 'STATION1'
# make sure None values for stations that don't have dates
assert stations[1].get('max') is None
assert stations[1].get('min') is None
def test_get_rdr_slice(database):
with db.connection() as conn:
......
#!/usr/bin/env bash
set -e
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<EOSQL
CREATE DATABASE __test;
GRANT ALL PRIVILEGES ON DATABASE __test TO docker;
EOSQL
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment