Add basic XML format for data API

metobsapi/modifyData.py

@@ -2,7 +2,6 @@ import logging
 from xml.dom.minidom import Document
 from datetime import datetime, timedelta
 from metobsapi.queryInflux import query
-from io import StringIO
 import pandas as pd
 from flask import render_template, jsonify, Response
 from flask_json import as_json_p
@@ -110,10 +109,15 @@ def handle_csv(frame, symbols, epoch, sep=',', **kwargs):
             symbols=sep.join(str(x) for x in row.values))
         data_lines.append(line)
 
+    if not epoch:
+        epoch_str = '%Y-%m-%dT%H:%M:%SZ'
+    else:
+        epoch_str = data_responses.epoch_translation[epoch] + ' since epoch (1970-01-01 00:00:00)'
+
     output = output.format(
         message='',
         num_results=frame.shape[0],
-        epoch_str=data_responses.epoch_translation.get(epoch, 'YYYY-MM-DDTHH:MM:SSZ'),
+        epoch_str=epoch_str,
         symbol_list=sep.join(symbols),
         symbol_data="\n".join(data_lines),
     )
@@ -152,67 +156,47 @@ def handle_json(frame, symbols, epoch, order='columns', **kwargs):
 def handle_xml(frame, symbols, epoch, sep=',', **kwargs):
     doc = Document()
     header = 'metobs'
-    timeStamps = list(frame.columns.values)
 
     head = doc.createElement(header)
     head.setAttribute('status', 'success')
     head.setAttribute('code', '200')
     head.setAttribute('message', '')
-    head.setAttribute('num_results', str(len(timeStamps)))
-
+    head.setAttribute('num_results', str(frame.shape[0]))
     head.setAttribute('seperator', sep)
 
     doc.appendChild(head)
+    columns_elem = doc.createElement('symbols')
 
-    stampElt = doc.createElement('timestamp')
-
+    time_elem = doc.createElement('symbol')
+    time_elem.setAttribute('name', 'time')
+    time_elem.setAttribute('short_name', 'time')
     if not epoch:
-        stampElt.setAttribute('format', '%Y-%m-%dT%H:%M:%SZ')
+        time_elem.setAttribute('format', '%Y-%m-%dT%H:%M:%SZ')
     else:
-        stampElt.setAttribute('format', data_responses.epoch_translation[epoch] + ' since epoch (1970-01-01 00:00:00)')
-
-    dateStrings = StringIO()
-
-    first = 0
-
-    for dateString in timeStamps:
-        if first == 0:
-            dateStrings.write(dateString)
-            first = 1
-
-        else:
-            dateStrings.write(sep + dateString)
-
-    stamp_content = doc.createTextNode(dateStrings.getvalue())
-    stampElt.appendChild(stamp_content)
-    head.appendChild(stampElt)
-
-    frame = frame.transpose()
-
-    for symbol in symbols:
-        first = 0
-
-        dataStrings = StringIO()
-
-        dataElt = doc.createElement('data')
-        dataElt.setAttribute('symbol', symbol)
-        dataElt.setAttribute('site', site)
-        dataElt.setAttribute('inst', inst)
-
-        dataList = frame[symbol]
-        for data in dataList:
-            if first == 0:
-                dataStrings.write(str(data))
-                first = 1
-
-            else:    
-                dataStrings.write(sep + str(data))
-
-        symbol_content = doc.createTextNode(dataStrings.getvalue())
-        dataElt.appendChild(symbol_content)
-        head.appendChild(dataElt)        
-
-    txt = doc.toprettyxml(indent="     ", encoding="utf-8")
+        time_elem.setAttribute('format', data_responses.epoch_translation[epoch] + ' since epoch (1970-01-01 00:00:00)')
+    columns_elem.appendChild(time_elem)
+
+    for c in frame.columns:
+        col_elem = doc.createElement('symbol')
+        col_elem.setAttribute('name', c)
+        parts = c.split('.')
+        col_elem.setAttribute('short_name', parts[2])
+        col_elem.setAttribute('site', parts[0])
+        col_elem.setAttribute('inst', parts[1])
+        columns_elem.appendChild(col_elem)
+    head.appendChild(columns_elem)
+
+    data_elem = doc.createElement('data')
+    for t, row in frame.iterrows():
+        row_elem = doc.createElement('row')
+        row_elem.appendChild(doc.createTextNode(str(t)))
+        for point in row:
+            row_elem.appendChild(doc.createTextNode(str(point)))
+        data_elem.appendChild(row_elem)
+    head.appendChild(data_elem)
+
+    # txt = doc.toprettyxml(indent="     ", encoding="utf-8")
+    txt = doc.toxml(encoding="utf-8")
     return Response(txt, mimetype='text/xml')
 
 
@@ -297,7 +281,7 @@ def modify_data(fmt, begin, end, site, inst, symbols, interval,
     try:
         influx_symbols = handle_symbols(symbols)
     except ValueError as e:
-        return handle_error(fmt, str(e.message))
+        return handle_error(fmt, str(e))
 
     result = query(site, inst, influx_symbols, begin, end, interval, epoch)
     frame = handle_influxdb_result(result, influx_symbols, interval)

metobsapi/tests/test_data_api.py

@@ -181,3 +181,16 @@ class TestDataAPI(unittest.TestCase):
         # header, data, newline at end
         self.assertEqual(len(res.split('\n')), 5 + 9 + 1)
         self.assertIn("# code: 200", res)
+
+    @mock.patch('metobsapi.modifyData.query')
+    def test_one_symbol_xml(self, query_func):
+        from xml.dom.minidom import parseString
+        r = fake_data('1m', {('aoss', 'tower'): ['time', 'air_temp']}, 9)
+        query_func.return_value = r
+        # row should be the default
+        res = self.app.get('/api/data.xml?symbols=aoss.tower.air_temp&begin=-00:10:00')
+        res = parseString(res.data.decode())
+        # symbols: time and air_temp
+        self.assertEqual(len(res.childNodes[0].childNodes[0].childNodes), 2)
+        # data rows
+        self.assertEqual(len(res.childNodes[0].childNodes[1].childNodes), 9)

metobsapi/util/data_responses.py

@@ -42,11 +42,13 @@ SYMBOL_TRANSLATIONS = {
         'pressure': 'pressure',
         'altimeter': 'altimeter',
         'solar_flux': 'solar_flux',
+        'wind_speed': 'wind_speed',
     },
     ('mendota', 'buoy'): {
         'air_temp': 'air_temp',
         'dewpoint': 'dewpoint',
         'rel_hum': 'rel_hum',
+        'wind_speed': 'wind_speed',
         'water_temp_1': 'water_temp_1',
         'water_temp_2': 'water_temp_2',
         'water_temp_3': 'water_temp_3',