Commit fcff30b0 authored by Bruce Flynn's avatar Bruce Flynn

Require Python 3

parent 170d3e00
Pipeline #1512 passed with stage
in 1 minute and 55 seconds
......@@ -9,7 +9,7 @@ variables:
AWSAPI_DB_URL: postgresql://runner@postgres/__awsapi_test
test:
image: brucef/py2test
image: brucef/py3test
script:
- pip install git+https://gitlab.ssec.wisc.edu/metobs/MetObsData.git@0.5
- pip install -i https://sips.ssec.wisc.edu/eggs -e ".[testing]"
......
......@@ -2,7 +2,7 @@
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="jdk" jdkName="Python 3.6.0 (~/code/AmrcAws/env/bin/python3.6)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="TemplatesService">
......
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 2.7.13 (~/code/AmrcAws/env/bin/python)" project-jdk-type="Python SDK" />
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6.0 (~/code/AmrcAws/env/bin/python3.6)" project-jdk-type="Python SDK" />
</project>
\ No newline at end of file
AMRC Antarctic Weather Stations API
===================================
This package implements code to parse and insert AWS data (currently only in
the RDR format) into a PostgreSQL DB and serve it up via a HTTP GET API as
either CSV or NetCDF.
the RDR or Q1H formats) into a PostgreSQL DB and serve it up via a HTTP GET
API as either CSV or NetCDF.
Install
......@@ -29,7 +29,7 @@ following:
#. Install supervisord_ process watcher which is used to keep the processes
running in the case of error.
#. Create a few directories
gt
Starting AWS API
----------------
......
......@@ -25,7 +25,7 @@ def get_form(schema, request):
try:
return schema().to_python(request.params)
except Invalid as err:
raise HTTPBadRequest('Invalid request parameters: {:s}'
raise HTTPBadRequest('Invalid request parameters: {!r}'
.format(err))
......@@ -188,7 +188,7 @@ def csv_response(stations, sensors, form, rows):
# product the row for a timestamp
yield '{:%Y-%m-%dT%H:%M:%SZ},{}\n'.format(stamp, ','.join(vals))
return yielder()
return (l.encode('utf-8') for l in yielder())
class HeaderMiddleware(object):
......
......@@ -37,9 +37,9 @@ def main():
try:
inserted = db.insert_frames([frame])
if inserted == 0:
LOG.warn('No values inserted for line %d', idx + 1)
LOG.warning('No values inserted for line %d', idx + 1)
except db.IntegrityError:
LOG.warn('Line %d insert failed, possibly already exists', idx + 1)
LOG.warning('Line %d insert failed, possibly already exists', idx + 1)
if __name__ == '__main__':
......
#!/usr/bin/env python
"""
AWS API command to refresh station availability dates.
"""
import argparse
import logging
from amrc_aws import parsers, db, exc
LOG = logging
def main():
parser = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('-v', '--verbose', action='store_true')
args = parser.parse_args()
logging.basicConfig(
level=logging.DEBUG if args.verbose else logging.INFO,
format='[%(asctime)s %(levelname)s] %(message)s')
db.init()
LOG.info('refreshing ...')
db.refresh_station_dates()
......@@ -191,10 +191,12 @@ def _get_slice(type_, stations, symbols, start, end, avg):
AND data.station = series.station
ORDER BY series.stamp, data.station
""".format(data_table=table_name, stations=[str(s) for s in stations]))
'''
LOG.debug(_compile_query(select.bindparams(
start=start, end=end,
interval=timedelta(seconds=avg),
stations=tuple(stations))))
'''
rows = _fetchall(
select, start=start, end=end,
interval=timedelta(seconds=avg),
......@@ -283,8 +285,8 @@ def _compile_query(stmt):
comp.compile()
enc = dialect.encoding
params = {}
for k, v in comp.params.iteritems():
if isinstance(v, unicode):
for k, v in comp.params.items():
if isinstance(v, str):
v = v.encode(enc)
params[k] = sqlescape(v)
return (comp.string.encode(enc) % params).decode(enc)
return (comp.string % params)
import yaml
import config
from . import config
def load_alias_map(dbname):
......
......@@ -7,10 +7,8 @@ def printable(x):
"""
Filter any non printable chars and return str.
"""
if isinstance(x, str):
x = x.decode('ascii', errors='ignore').encode('ascii')
else:
x = x.encode('ascii', errors='ignore')
if isinstance(x, bytes):
x = x.decode('ascii', errors='ignore')
return ''.join(c for c in x if 31 < ord(c) < 127)
......
......@@ -2,14 +2,15 @@
set -e
VERSION=0.2
repo=https://gitlab.ssec.wisc.edu/brucef/AmrcAws
# If we don't have Conda on our path we must be installing on a system that
# does not have conda, so go ahead and install one
if ! which conda &> /dev/null
then
if [ ! -e Miniconda-latest-Linux-x86_64.sh ]
if [ ! -e Miniconda3-latest-Linux-x86_64.sh ]
then
wget https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh
wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
fi
bash Miniconda-latest-Linux-x86_64.sh -b -p ./env
export PATH=$PWD/env/bin:$PATH
......@@ -20,8 +21,6 @@ fi
conda install -y pip psycopg2 numpy netcdf4
repo=https://gitlab.ssec.wisc.edu/brucef/AmrcAws
pip install git+https://gitlab.ssec.wisc.edu/metobs/MetObsData.git@0.5
pip install git+${repo}.git@${VERSION}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment