Skip to content
Snippets Groups Projects
Unverified Commit d9564cc1 authored by David Hoese's avatar David Hoese
Browse files

Add quicklook images to file api and move error handlers

parent 9b6fc577
Branches
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
metobsapi/data_api.py
+ 68
− 62
View file @ d9564cc1
...@@ -4,11 +4,10 @@ from xml.dom.minidom import Document ...@@ -4,11 +4,10 @@ from xml.dom.minidom import Document
import numpy as np import numpy as np
import pandas as pd import pandas as pd
from flask import render_template, jsonify, Response from flask import render_template, Response
from flask_json import as_json_p from flask_json import as_json_p
from metobsapi.util import data_responses from metobsapi.util import data_responses
from metobsapi.util.error_handlers import ERROR_HANDLERS
from metobsapi.util.query_influx import build_queries, query from metobsapi.util.query_influx import build_queries, query
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
...@@ -140,6 +139,7 @@ def handle_csv(frame, epoch, sep=',', ...@@ -140,6 +139,7 @@ def handle_csv(frame, epoch, sep=',',
data_lines = [] data_lines = []
line_format = sep.join(["{time}", "{symbols}"]) line_format = sep.join(["{time}", "{symbols}"])
if frame is not None and not frame.empty:
for t, row in frame.iterrows(): for t, row in frame.iterrows():
line = line_format.format( line = line_format.format(
time=t, time=t,
...@@ -156,9 +156,9 @@ def handle_csv(frame, epoch, sep=',', ...@@ -156,9 +156,9 @@ def handle_csv(frame, epoch, sep=',',
status=status, status=status,
code=code, code=code,
message=message, message=message,
num_results=frame.shape[0], num_results=frame.shape[0] if frame is not None else 0,
epoch_str=epoch_str, epoch_str=epoch_str,
symbol_list=sep.join(frame.columns), symbol_list=sep.join(frame.columns) if frame is not None else '',
symbol_data="\n".join(data_lines), symbol_data="\n".join(data_lines),
) )
...@@ -168,22 +168,17 @@ def handle_csv(frame, epoch, sep=',', ...@@ -168,22 +168,17 @@ def handle_csv(frame, epoch, sep=',',
@as_json_p(optional=True) @as_json_p(optional=True)
def handle_json(frame, epoch, order='columns', def handle_json(frame, epoch, order='columns',
message='', code=200, status='success', **kwargs): message='', code=200, status='success', **kwargs):
package = {}
if frame is not None and not frame.empty:
# force conversion to float types so they can be json'd # force conversion to float types so they can be json'd
for column, data_type in zip(frame.columns, frame.dtypes.values): for column, data_type in zip(frame.columns, frame.dtypes.values):
if issubclass(data_type.type, np.integer): if issubclass(data_type.type, np.integer):
frame[column] = frame[column].astype(float) frame[column] = frame[column].astype(float)
# replace NaNs with None # replace NaNs with None
frame = frame.where(pd.notnull(frame), None) frame = frame.where(pd.notnull(frame), None)
output = {}
output['status'] = status
output['message'] = message
output['code'] = code
output['num_results'] = frame.shape[0]
package = {
'timestamps': frame.index.values,
}
package['timestamps'] = frame.index.values
if epoch: if epoch:
newStamps = [] newStamps = []
for stamp in package['timestamps']: for stamp in package['timestamps']:
...@@ -196,7 +191,21 @@ def handle_json(frame, epoch, order='columns', ...@@ -196,7 +191,21 @@ def handle_json(frame, epoch, order='columns',
package['symbols'] = frame.columns package['symbols'] = frame.columns
package['data'] = [frame.iloc[i].values for i in range(frame.shape[0])] package['data'] = [frame.iloc[i].values for i in range(frame.shape[0])]
# package['data'] = frame.values # package['data'] = frame.values
output['results'] = package else:
package['timestamps'] = []
if order == 'column':
package['data'] = {}
else:
package['data'] = []
package['symbols'] = []
output = {
'status': status,
'message': message,
'code': code,
'num_results': frame.shape[0] if frame is not None else 0,
'results': package,
}
return output, code return output, code
...@@ -209,8 +218,9 @@ def handle_xml(frame, epoch, sep=',', ...@@ -209,8 +218,9 @@ def handle_xml(frame, epoch, sep=',',
head.setAttribute('status', status) head.setAttribute('status', status)
head.setAttribute('code', str(code)) head.setAttribute('code', str(code))
head.setAttribute('message', message) head.setAttribute('message', message)
head.setAttribute('num_results', str(frame.shape[0])) head.setAttribute('num_results', str(frame.shape[0]) if frame is not None else str(0))
head.setAttribute('seperator', sep) head.setAttribute('seperator', sep)
data_elem = doc.createElement('data')
doc.appendChild(head) doc.appendChild(head)
columns_elem = doc.createElement('symbols') columns_elem = doc.createElement('symbols')
...@@ -224,6 +234,7 @@ def handle_xml(frame, epoch, sep=',', ...@@ -224,6 +234,7 @@ def handle_xml(frame, epoch, sep=',',
time_elem.setAttribute('format', data_responses.epoch_translation[epoch] + ' since epoch (1970-01-01 00:00:00)') time_elem.setAttribute('format', data_responses.epoch_translation[epoch] + ' since epoch (1970-01-01 00:00:00)')
columns_elem.appendChild(time_elem) columns_elem.appendChild(time_elem)
if frame is not None and not frame.empty:
for c in frame.columns: for c in frame.columns:
col_elem = doc.createElement('symbol') col_elem = doc.createElement('symbol')
col_elem.setAttribute('name', c) col_elem.setAttribute('name', c)
...@@ -232,9 +243,7 @@ def handle_xml(frame, epoch, sep=',', ...@@ -232,9 +243,7 @@ def handle_xml(frame, epoch, sep=',',
col_elem.setAttribute('site', parts[0]) col_elem.setAttribute('site', parts[0])
col_elem.setAttribute('inst', parts[1]) col_elem.setAttribute('inst', parts[1])
columns_elem.appendChild(col_elem) columns_elem.appendChild(col_elem)
head.appendChild(columns_elem)
data_elem = doc.createElement('data')
for idx, (t, row) in enumerate(frame.iterrows()): for idx, (t, row) in enumerate(frame.iterrows()):
row_elem = doc.createElement('row') row_elem = doc.createElement('row')
row_elem.setAttribute('id', str(idx)) row_elem.setAttribute('id', str(idx))
...@@ -243,19 +252,16 @@ def handle_xml(frame, epoch, sep=',', ...@@ -243,19 +252,16 @@ def handle_xml(frame, epoch, sep=',',
row_elem.appendChild(doc.createTextNode(sep)) row_elem.appendChild(doc.createTextNode(sep))
row_elem.appendChild(doc.createTextNode(str(point))) row_elem.appendChild(doc.createTextNode(str(point)))
data_elem.appendChild(row_elem) data_elem.appendChild(row_elem)
head.appendChild(columns_elem)
head.appendChild(data_elem) head.appendChild(data_elem)
# txt = doc.toprettyxml(indent=" ", encoding="utf-8")
txt = doc.toxml(encoding="utf-8") txt = doc.toxml(encoding="utf-8")
return Response(txt, mimetype='text/xml'), code return Response(txt, mimetype='text/xml'), code
def handle_error(fmt, error_str): def handle_error(fmt, error_str):
handler = ERROR_HANDLERS[fmt] handler = RESPONSE_HANDLERS[fmt]
err_code, err_msg = data_responses.ERROR_MESSAGES.get(error_str, (400, error_str)) err_code, err_msg = data_responses.ERROR_MESSAGES.get(error_str, (400, error_str))
res = handler(err_code, err_msg) res = handler(None, None, message=err_msg, code=err_code, status='error')
if fmt == 'json':
res = jsonify(**res)
return res, err_code return res, err_code
......
metobsapi/files_api.py
+ 26
− 23
View file @ d9564cc1
...@@ -4,11 +4,10 @@ from datetime import datetime ...@@ -4,11 +4,10 @@ from datetime import datetime
from datetime import timedelta as delta from datetime import timedelta as delta
import pandas as pd import pandas as pd
from flask import render_template, jsonify, Response from flask import render_template, Response
from flask_json import as_json_p from flask_json import as_json_p
from metobsapi.util import file_responses from metobsapi.util import file_responses
from metobsapi.util.error_handlers import ERROR_HANDLERS
from metobsapi.data_api import handle_date from metobsapi.data_api import handle_date
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
...@@ -92,18 +91,20 @@ def handle_csv(frame, message='', code=200, status='success'): ...@@ -92,18 +91,20 @@ def handle_csv(frame, message='', code=200, status='success'):
""" """
# Normalize the frame that was given so we only have expected information # Normalize the frame that was given so we only have expected information
if frame is not None and not frame.empty:
frame = frame[columns] frame = frame[columns]
data_lines = [] data_lines = []
if not frame.empty:
for row in frame.values: for row in frame.values:
data_lines.append(','.join(str(x) for x in row)) data_lines.append(','.join(str(x) for x in row))
else:
data_lines = []
rows = "\n".join(data_lines) rows = "\n".join(data_lines)
output = output.format( output = output.format(
status=status, status=status,
code=code, code=code,
message=message, message=message,
num_results=frame.shape[0], num_results=frame.shape[0] if frame is not None else 0,
column_list=",".join(columns), column_list=",".join(columns),
rows=rows, rows=rows,
) )
...@@ -140,8 +141,8 @@ EOF ...@@ -140,8 +141,8 @@ EOF
status=status, status=status,
code=code, code=code,
message=message, message=message,
num_results=frame.shape[0], num_results=frame.shape[0] if frame is not None else 0,
url_list='\n'.join(str(x) for x in frame['url']) url_list='\n'.join(str(x) for x in frame['url']) if frame is not None else '',
) )
return Response(output, mimetype='text/plain'), code return Response(output, mimetype='text/plain'), code
...@@ -168,7 +169,7 @@ bitsadmin /monitor ...@@ -168,7 +169,7 @@ bitsadmin /monitor
""" """
if not frame.empty: if frame is not None and not frame.empty:
urls = frame['url'] urls = frame['url']
directories = [] directories = []
commands = [] commands = []
...@@ -193,12 +194,14 @@ if not exist %cd%\\data\\{directory} ( ...@@ -193,12 +194,14 @@ if not exist %cd%\\data\\{directory} (
relpath=frame['relpath'][idx].replace('/', '\\') relpath=frame['relpath'][idx].replace('/', '\\')
) )
commands.append(url_str) commands.append(url_str)
else:
commands = []
output = output.format( output = output.format(
status=status, status=status,
code=code, code=code,
message=message, message=message,
num_results=frame.shape[0], num_results=frame.shape[0] if frame is not None else 0,
commands="\n".join(commands), commands="\n".join(commands),
) )
# windows line endings # windows line endings
...@@ -212,28 +215,19 @@ def handle_json(frame, message='', code=200, status='success'): ...@@ -212,28 +215,19 @@ def handle_json(frame, message='', code=200, status='success'):
output['status'] = status output['status'] = status
output['message'] = message output['message'] = message
output['code'] = code output['code'] = code
output['num_results'] = (len(list(frame.index))) output['num_results'] = (len(list(frame.index))) if frame is not None else 0
if not frame.empty: if frame is not None and not frame.empty:
body = [] body = []
for row in frame.values: for row in frame.values:
new_row = dict((k, row[idx]) for idx, k in enumerate(frame.columns)) new_row = dict((k, row[idx]) for idx, k in enumerate(frame.columns))
body.append(new_row) body.append(new_row)
output['data'] = body output['data'] = body
else:
output['data'] = []
return output, code return output, code
def handle_error(fmt, error_str, stream_id=None):
handler = ERROR_HANDLERS[fmt]
err_code, err_msg = file_responses.ERROR_MESSAGES[error_str]
if stream_id is not None:
err_msg += ": '{}'".format(stream_id)
res = handler(err_code, err_msg)
if fmt == 'json':
res = jsonify(**res)
return res, err_code
RESPONSE_HANDLERS = { RESPONSE_HANDLERS = {
'csv': handle_csv, 'csv': handle_csv,
'sh': handle_sh, 'sh': handle_sh,
...@@ -242,6 +236,15 @@ RESPONSE_HANDLERS = { ...@@ -242,6 +236,15 @@ RESPONSE_HANDLERS = {
} }
def handle_error(fmt, error_str, stream_id=None):
handler = RESPONSE_HANDLERS[fmt]
err_code, err_msg = file_responses.ERROR_MESSAGES[error_str]
if stream_id is not None:
err_msg += ": '{}'".format(stream_id)
res = handler(None, message=err_msg, code=err_code, status='error')
return res, err_code
def find_stream_files(fmt, begin_time, end_time, dates, streams): def find_stream_files(fmt, begin_time, end_time, dates, streams):
if fmt not in RESPONSE_HANDLERS: if fmt not in RESPONSE_HANDLERS:
return render_template('400.html', format=fmt), 400 return render_template('400.html', format=fmt), 400
......
metobsapi/tests/test_files_api.py
+ 2
− 2
View file @ d9564cc1
...@@ -122,7 +122,7 @@ class TestFilesAPI(unittest.TestCase): ...@@ -122,7 +122,7 @@ class TestFilesAPI(unittest.TestCase):
assert res['data'][0]['filename'] == fn assert res['data'][0]['filename'] == fn
def test_tower_multi_all_patterns(self): def test_tower_multi_all_patterns(self):
res = self.app.get('/api/files.json?streams=aoss.tower.*.l00.*:aoss.tower.nc-1d-1m.lb1.v00') res = self.app.get('/api/files.json?streams=aoss.tower.*.l00.*:aoss.tower.nc-daily.lb1.v00')
res = json.loads(str(res.data, encoding='utf-8')) res = json.loads(str(res.data, encoding='utf-8'))
fn = self._datetimes[0].strftime('rig_tower.%Y-%m-%d.ascii') fn = self._datetimes[0].strftime('rig_tower.%Y-%m-%d.ascii')
assert res['data'][0]['filename'] == fn assert res['data'][0]['filename'] == fn
...@@ -131,7 +131,7 @@ class TestFilesAPI(unittest.TestCase): ...@@ -131,7 +131,7 @@ class TestFilesAPI(unittest.TestCase):
def test_tower_dates(self): def test_tower_dates(self):
dates = tuple(dt.strftime('%Y-%m-%d') for dt in self._datetimes[::2]) dates = tuple(dt.strftime('%Y-%m-%d') for dt in self._datetimes[::2])
res = self.app.get('/api/files.json?streams=aoss.tower.nc-1d-1m.lb1.v00&dates={}:{}'.format(*dates)) res = self.app.get('/api/files.json?streams=aoss.tower.nc-daily.lb1.v00&dates={}:{}'.format(*dates))
res = json.loads(str(res.data, encoding='utf-8')) res = json.loads(str(res.data, encoding='utf-8'))
fn = self._datetimes[0].strftime('aoss_tower.%Y-%m-%d.nc') fn = self._datetimes[0].strftime('aoss_tower.%Y-%m-%d.nc')
assert res['data'][0]['filename'] == fn assert res['data'][0]['filename'] == fn
......
metobsapi/util/__init__.py
+ 2
− 0
View file @ d9564cc1
...@@ -25,6 +25,7 @@ def create_fake_archive(archive_info, root=FAKE_ARCHIVE_PATH, datetimes=None): ...@@ -25,6 +25,7 @@ def create_fake_archive(archive_info, root=FAKE_ARCHIVE_PATH, datetimes=None):
if datetimes is None: if datetimes is None:
datetimes = [datetime.utcnow()] datetimes = [datetime.utcnow()]
curr_dir = os.getcwd()
os.makedirs(root, exist_ok=True) os.makedirs(root, exist_ok=True)
os.chdir(root) os.chdir(root)
for site, inst_info in archive_info.items(): for site, inst_info in archive_info.items():
...@@ -65,4 +66,5 @@ def create_fake_archive(archive_info, root=FAKE_ARCHIVE_PATH, datetimes=None): ...@@ -65,4 +66,5 @@ def create_fake_archive(archive_info, root=FAKE_ARCHIVE_PATH, datetimes=None):
os.chdir('..') os.chdir('..')
os.chdir('..') os.chdir('..')
os.chdir('..') os.chdir('..')
os.chdir(curr_dir)
metobsapi/util/error_handlers.py deleted 100644 → 0
+ 0
− 50
View file @ 9b6fc577
from xml.dom.minidom import Document
def create_xml(code, message):
doc = Document()
header = 'metobs'
head = doc.createElement(header)
head.setAttribute('status', 'error')
head.setAttribute('code', code)
head.setAttribute('message', message)
head.setAttribute('num_results', '0')
doc.appendChild(head)
return doc.toprettyxml(indent=" ", encoding="utf-8")
def create_json(code, message):
json = {}
json['status'] = 'error'
json['code'] = code
json['num_results'] = 0
json['message'] = message
return json
def create_csv(code, message):
returnString = '# status: error<br>'
returnString += '# code: ' + str(code) + '<br>'
returnString += '# num_results: 0<br># message: ' + message
return returnString
def create_bat(code, message):
returnString = ':: status: error<br>'
returnString += ':: code: ' + str(code) + '<br>'
returnString += ':: num_results: 0<br># message: ' + message
return returnString
ERROR_HANDLERS = {
'csv': create_csv,
'xml': create_xml,
'json': create_json,
'bat': create_bat,
}
metobsapi/util/file_responses.py
+ 38
− 3
View file @ d9564cc1
...@@ -31,16 +31,51 @@ ARCHIVE_INFO = { ...@@ -31,16 +31,51 @@ ARCHIVE_INFO = {
'level_b1': { 'level_b1': {
'versions': ('version_00',), 'versions': ('version_00',),
'products': { 'products': {
'nc-1mo-1d': { 'nc-monthly': {
'frequency': ProductFrequency.MONTHLY_DIR, 'frequency': ProductFrequency.MONTHLY_DIR,
'pattern': 'aoss_tower.%Y-%m.nc', 'pattern': 'aoss_tower.%Y-%m.nc',
'display_name': 'Monthly NetCDF file (aoss_tower.YYYY-MM.nc)', 'display_name': 'Monthly NetCDF file (aoss_tower.YYYY-MM.nc)',
}, },
'nc-1d-1m': { 'nc-daily': {
'frequency': ProductFrequency.DAILY_DIR, 'frequency': ProductFrequency.DAILY_FILE,
'pattern': 'aoss_tower.%Y-%m-%d.nc', 'pattern': 'aoss_tower.%Y-%m-%d.nc',
'display_name': 'Daily NetCDF file (aoss_tower.YYYY-MM-DD.nc)', 'display_name': 'Daily NetCDF file (aoss_tower.YYYY-MM-DD.nc)',
}, },
'meteorogram-daily': {
'frequency': ProductFrequency.DAILY_FILE,
'pattern': 'aoss_tower.meteorogram.%Y-%m-%d*.png',
'display_name': 'Daily Meteorogram (aoss_tower.meteorogram.YYYY-MM-DD.png)',
},
'td-daily': {
'frequency': ProductFrequency.DAILY_FILE,
'pattern': 'aoss_tower.td.%Y-%m-%d*.png',
'display_name': 'Daily Air and Dewpoint Temperature (aoss_tower.td.YYYY-MM-DD.png)',
},
'pressure-daily': {
'frequency': ProductFrequency.DAILY_FILE,
'pattern': 'aoss_tower.pressure.%Y-%m-%d*.png',
'display_name': 'Daily Pressure (aoss_tower.pressure.YYYY-MM-DD.png)',
},
'wind-speed-daily': {
'frequency': ProductFrequency.DAILY_FILE,
'pattern': 'aoss_tower.wind_speed.%Y-%m-%d*.png',
'display_name': 'Daily Wind Speed (aoss_tower.wind_speed.YYYY-MM-DD.png)',
},
'wind-dir-daily': {
'frequency': ProductFrequency.DAILY_FILE,
'pattern': 'aoss_tower.wind_dir.%Y-%m-%d*.png',
'display_name': 'Daily Wind Direction (aoss_tower.wind_dir.YYYY-MM-DD.png)',
},
'accum-precip-daily': {
'frequency': ProductFrequency.DAILY_FILE,
'pattern': 'aoss_tower.accum_precip.%Y-%m-%d*.png',
'display_name': 'Daily Accumulated Precipitation (aoss_tower.accum_precip.YYYY-MM-DD.png)',
},
'solar-flux-daily': {
'frequency': ProductFrequency.DAILY_FILE,
'pattern': 'aoss_tower.solar_flux.%Y-%m-%d*.png',
'display_name': 'Daily Solar Flux (aoss_tower.solar_flux.YYYY-MM-DD.png)',
},
}, },
}, },
}, },
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please to comment