From 491152bb71012012f461d05df9834bfa8033c99f Mon Sep 17 00:00:00 2001
From: davidh-ssec <david.hoese@ssec.wisc.edu>
Date: Tue, 7 Mar 2017 21:55:01 -0600
Subject: [PATCH] Add basic XML format for data API
---
metobsapi/modifyData.py | 90 +++++++++++++-------------------
metobsapi/tests/test_data_api.py | 13 +++++
metobsapi/util/data_responses.py | 2 +
3 files changed, 52 insertions(+), 53 deletions(-)
diff --git a/metobsapi/modifyData.py b/metobsapi/modifyData.py
index 9cb16b3..3d9730a 100644
--- a/metobsapi/modifyData.py
+++ b/metobsapi/modifyData.py
@@ -2,7 +2,6 @@ import logging
from xml.dom.minidom import Document
from datetime import datetime, timedelta
from metobsapi.queryInflux import query
-from io import StringIO
import pandas as pd
from flask import render_template, jsonify, Response
from flask_json import as_json_p
@@ -110,10 +109,15 @@ def handle_csv(frame, symbols, epoch, sep=',', **kwargs):
symbols=sep.join(str(x) for x in row.values))
data_lines.append(line)
+ if not epoch:
+ epoch_str = '%Y-%m-%dT%H:%M:%SZ'
+ else:
+ epoch_str = data_responses.epoch_translation[epoch] + ' since epoch (1970-01-01 00:00:00)'
+
output = output.format(
message='',
num_results=frame.shape[0],
- epoch_str=data_responses.epoch_translation.get(epoch, 'YYYY-MM-DDTHH:MM:SSZ'),
+ epoch_str=epoch_str,
symbol_list=sep.join(symbols),
symbol_data="\n".join(data_lines),
)
@@ -152,67 +156,47 @@ def handle_json(frame, symbols, epoch, order='columns', **kwargs):
def handle_xml(frame, symbols, epoch, sep=',', **kwargs):
doc = Document()
header = 'metobs'
- timeStamps = list(frame.columns.values)
head = doc.createElement(header)
head.setAttribute('status', 'success')
head.setAttribute('code', '200')
head.setAttribute('message', '')
- head.setAttribute('num_results', str(len(timeStamps)))
-
+ head.setAttribute('num_results', str(frame.shape[0]))
head.setAttribute('seperator', sep)
doc.appendChild(head)
+ columns_elem = doc.createElement('symbols')
- stampElt = doc.createElement('timestamp')
-
+ time_elem = doc.createElement('symbol')
+ time_elem.setAttribute('name', 'time')
+ time_elem.setAttribute('short_name', 'time')
if not epoch:
- stampElt.setAttribute('format', '%Y-%m-%dT%H:%M:%SZ')
+ time_elem.setAttribute('format', '%Y-%m-%dT%H:%M:%SZ')
else:
- stampElt.setAttribute('format', data_responses.epoch_translation[epoch] + ' since epoch (1970-01-01 00:00:00)')
-
- dateStrings = StringIO()
-
- first = 0
-
- for dateString in timeStamps:
- if first == 0:
- dateStrings.write(dateString)
- first = 1
-
- else:
- dateStrings.write(sep + dateString)
-
- stamp_content = doc.createTextNode(dateStrings.getvalue())
- stampElt.appendChild(stamp_content)
- head.appendChild(stampElt)
-
- frame = frame.transpose()
-
- for symbol in symbols:
- first = 0
-
- dataStrings = StringIO()
-
- dataElt = doc.createElement('data')
- dataElt.setAttribute('symbol', symbol)
- dataElt.setAttribute('site', site)
- dataElt.setAttribute('inst', inst)
-
- dataList = frame[symbol]
- for data in dataList:
- if first == 0:
- dataStrings.write(str(data))
- first = 1
-
- else:
- dataStrings.write(sep + str(data))
-
- symbol_content = doc.createTextNode(dataStrings.getvalue())
- dataElt.appendChild(symbol_content)
- head.appendChild(dataElt)
-
- txt = doc.toprettyxml(indent=" ", encoding="utf-8")
+ time_elem.setAttribute('format', data_responses.epoch_translation[epoch] + ' since epoch (1970-01-01 00:00:00)')
+ columns_elem.appendChild(time_elem)
+
+ for c in frame.columns:
+ col_elem = doc.createElement('symbol')
+ col_elem.setAttribute('name', c)
+ parts = c.split('.')
+ col_elem.setAttribute('short_name', parts[2])
+ col_elem.setAttribute('site', parts[0])
+ col_elem.setAttribute('inst', parts[1])
+ columns_elem.appendChild(col_elem)
+ head.appendChild(columns_elem)
+
+ data_elem = doc.createElement('data')
+ for t, row in frame.iterrows():
+ row_elem = doc.createElement('row')
+ row_elem.appendChild(doc.createTextNode(str(t)))
+ for point in row:
+ row_elem.appendChild(doc.createTextNode(str(point)))
+ data_elem.appendChild(row_elem)
+ head.appendChild(data_elem)
+
+ # txt = doc.toprettyxml(indent=" ", encoding="utf-8")
+ txt = doc.toxml(encoding="utf-8")
return Response(txt, mimetype='text/xml')
@@ -297,7 +281,7 @@ def modify_data(fmt, begin, end, site, inst, symbols, interval,
try:
influx_symbols = handle_symbols(symbols)
except ValueError as e:
- return handle_error(fmt, str(e.message))
+ return handle_error(fmt, str(e))
result = query(site, inst, influx_symbols, begin, end, interval, epoch)
frame = handle_influxdb_result(result, influx_symbols, interval)
diff --git a/metobsapi/tests/test_data_api.py b/metobsapi/tests/test_data_api.py
index 3bc9d4d..648f5af 100644
--- a/metobsapi/tests/test_data_api.py
+++ b/metobsapi/tests/test_data_api.py
@@ -181,3 +181,16 @@ class TestDataAPI(unittest.TestCase):
# header, data, newline at end
self.assertEqual(len(res.split('\n')), 5 + 9 + 1)
self.assertIn("# code: 200", res)
+
+ @mock.patch('metobsapi.modifyData.query')
+ def test_one_symbol_xml(self, query_func):
+ from xml.dom.minidom import parseString
+ r = fake_data('1m', {('aoss', 'tower'): ['time', 'air_temp']}, 9)
+ query_func.return_value = r
+ # row should be the default
+ res = self.app.get('/api/data.xml?symbols=aoss.tower.air_temp&begin=-00:10:00')
+ res = parseString(res.data.decode())
+ # symbols: time and air_temp
+ self.assertEqual(len(res.childNodes[0].childNodes[0].childNodes), 2)
+ # data rows
+ self.assertEqual(len(res.childNodes[0].childNodes[1].childNodes), 9)
diff --git a/metobsapi/util/data_responses.py b/metobsapi/util/data_responses.py
index b42f48a..8b46a80 100644
--- a/metobsapi/util/data_responses.py
+++ b/metobsapi/util/data_responses.py
@@ -42,11 +42,13 @@ SYMBOL_TRANSLATIONS = {
'pressure': 'pressure',
'altimeter': 'altimeter',
'solar_flux': 'solar_flux',
+ 'wind_speed': 'wind_speed',
},
('mendota', 'buoy'): {
'air_temp': 'air_temp',
'dewpoint': 'dewpoint',
'rel_hum': 'rel_hum',
+ 'wind_speed': 'wind_speed',
'water_temp_1': 'water_temp_1',
'water_temp_2': 'water_temp_2',
'water_temp_3': 'water_temp_3',
--
GitLab