Skip to content
Snippets Groups Projects
Verified Commit 537b0d51 authored by Owen Graham's avatar Owen Graham
Browse files

Move data spec definitions to new module

parent eaae6e50
Branches
No related tags found
No related merge requests found
"""Define `asccol` data specifications."""
import math
import asccol
def simple_time(s):
"""Convert an HHMM string to an (H, M) tuple."""
hhmm = int(s)
return divmod(hhmm, 100)
def filtered_int(s):
"""Like int, but ignore the special value 999."""
val = int(s)
return val if val != 999 else math.nan
def filtered_float(s):
"""Like float, but ignore the special values 444.0, 99.9, etc."""
val = float(s)
return val if val not in (444.0, 99.9, 999.9, 9999.9) else math.nan
ONE_HOUR = asccol.DataSpec(
leading=2, # 2 lines to delete
cols=(
('year', int, 4), # name, type (converter function), width
('jdate', int, 4),
('month', int, 3),
('day', int, 3),
('time', simple_time, 5),
('temp', filtered_float, 7),
('pressure', filtered_float, 7),
('wind_speed', filtered_float, 7),
('wind_dir', filtered_float, 7),
('rel_hum', filtered_float, 7),
('delta_t', filtered_float, 7),
),
)
SOUTH_POLE = asccol.DataSpec(
cols=(
('year', int, 5),
('month', int, 5),
('day', int, 5),
('time', simple_time, 5),
('wind_dir', filtered_int, 4),
('wind_speed', filtered_float, 6),
('visibility', filtered_int, 7),
('temp', filtered_float, 7),
('pressure', filtered_float, 7),
),
)
import datetime import datetime
from io import BytesIO from io import BytesIO
import json import json
import math
from types import SimpleNamespace from types import SimpleNamespace
from urllib.request import urlopen from urllib.request import urlopen
...@@ -13,6 +12,8 @@ import matplotlib.pyplot as plt ...@@ -13,6 +12,8 @@ import matplotlib.pyplot as plt
import numpy as np import numpy as np
from pyld import jsonld from pyld import jsonld
import data_spec
Measurement = make_dataclass('Measurement', ['url_name', 'field', 'title']) Measurement = make_dataclass('Measurement', ['url_name', 'field', 'title'])
measurements = {m.url_name: m for m in ( measurements = {m.url_name: m for m in (
Measurement('temperature', 1, 'Temperature (C)'), Measurement('temperature', 1, 'Temperature (C)'),
...@@ -23,55 +24,6 @@ measurements = {m.url_name: m for m in ( ...@@ -23,55 +24,6 @@ measurements = {m.url_name: m for m in (
ACCESS_URL = 'http://www.w3.org/ns/dcat#accessURL' ACCESS_URL = 'http://www.w3.org/ns/dcat#accessURL'
DISTRIBUTION = 'http://www.w3.org/ns/dcat#Distribution' DISTRIBUTION = 'http://www.w3.org/ns/dcat#Distribution'
def simple_time(s):
"""Convert an HHMM string to an (H, M) tuple."""
hhmm = int(s)
return divmod(hhmm, 100)
def filtered_int(s):
"""Like int, but ignore the special value 999."""
val = int(s)
return val if val != 999 else math.nan
def filtered_float(s):
"""Like float, but ignore the special values 444.0, 99.9, etc."""
val = float(s)
return val if val not in (444.0, 99.9, 999.9, 9999.9) else math.nan
ONE_HOUR_DATA = asccol.DataSpec(
leading=2, # 2 lines to delete
cols=(
('year', int, 4), # name, type (converter function), width
('jdate', int, 4),
('month', int, 3),
('day', int, 3),
('time', simple_time, 5),
('temp', filtered_float, 7),
('pressure', filtered_float, 7),
('wind_speed', filtered_float, 7),
('wind_dir', filtered_float, 7),
('rel_hum', filtered_float, 7),
('delta_t', filtered_float, 7),
),
)
SOUTH_POLE_DATA = asccol.DataSpec(
cols=(
('year', int, 5),
('month', int, 5),
('day', int, 5),
('time', simple_time, 5),
('wind_dir', filtered_int, 4),
('wind_speed', filtered_float, 6),
('visibility', filtered_int, 7),
('temp', filtered_float, 7),
('pressure', filtered_float, 7),
),
)
app = Flask(__name__) app = Flask(__name__)
app.jinja_env.trim_blocks = True app.jinja_env.trim_blocks = True
...@@ -350,13 +302,13 @@ def get_resources(link): ...@@ -350,13 +302,13 @@ def get_resources(link):
def read_data(station, year): def read_data(station, year):
"""Fetch data and convert it to a NumPy array.""" """Fetch data and convert it to a NumPy array."""
spec = (data_spec.SOUTH_POLE if station['id'] == 'south-pole'
else data_spec.ONE_HOUR)
data = [] data = []
resource_list = get_resources(get_link(station, year)) resource_list = get_resources(get_link(station, year))
for url in resource_list: for url in resource_list:
with urlopen(url) as f: with urlopen(url) as f:
lines = map(bytes.decode, f) lines = map(bytes.decode, f)
spec = (SOUTH_POLE_DATA if station['id'] == 'south-pole'
else ONE_HOUR_DATA)
for row in asccol.parse_data(lines, spec): for row in asccol.parse_data(lines, spec):
date = datetime.datetime(row.year, row.month, row.day, date = datetime.datetime(row.year, row.month, row.day,
*row.time) *row.time)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment