From 537b0d51172fdeed26322e73b0bcf2d937f90927 Mon Sep 17 00:00:00 2001
From: Owen Graham <ohgraham1@madisoncollege.edu>
Date: Tue, 14 Jun 2022 14:30:53 -0500
Subject: [PATCH] Move data spec definitions to new module
---
data_spec.py | 54 +++++++++++++++++++++++++++++++++++++++++++++++++
visualizer.py | 56 ++++-----------------------------------------------
2 files changed, 58 insertions(+), 52 deletions(-)
create mode 100644 data_spec.py
diff --git a/data_spec.py b/data_spec.py
new file mode 100644
index 0000000..9367c5c
--- /dev/null
+++ b/data_spec.py
@@ -0,0 +1,54 @@
+"""Define `asccol` data specifications."""
+
+import math
+
+import asccol
+
+
+def simple_time(s):
+ """Convert an HHMM string to an (H, M) tuple."""
+ hhmm = int(s)
+ return divmod(hhmm, 100)
+
+
+def filtered_int(s):
+ """Like int, but ignore the special value 999."""
+ val = int(s)
+ return val if val != 999 else math.nan
+
+
+def filtered_float(s):
+ """Like float, but ignore the special values 444.0, 99.9, etc."""
+ val = float(s)
+ return val if val not in (444.0, 99.9, 999.9, 9999.9) else math.nan
+
+
+ONE_HOUR = asccol.DataSpec(
+ leading=2, # 2 lines to delete
+ cols=(
+ ('year', int, 4), # name, type (converter function), width
+ ('jdate', int, 4),
+ ('month', int, 3),
+ ('day', int, 3),
+ ('time', simple_time, 5),
+ ('temp', filtered_float, 7),
+ ('pressure', filtered_float, 7),
+ ('wind_speed', filtered_float, 7),
+ ('wind_dir', filtered_float, 7),
+ ('rel_hum', filtered_float, 7),
+ ('delta_t', filtered_float, 7),
+ ),
+)
+SOUTH_POLE = asccol.DataSpec(
+ cols=(
+ ('year', int, 5),
+ ('month', int, 5),
+ ('day', int, 5),
+ ('time', simple_time, 5),
+ ('wind_dir', filtered_int, 4),
+ ('wind_speed', filtered_float, 6),
+ ('visibility', filtered_int, 7),
+ ('temp', filtered_float, 7),
+ ('pressure', filtered_float, 7),
+ ),
+)
diff --git a/visualizer.py b/visualizer.py
index dd588e8..4ca3061 100644
--- a/visualizer.py
+++ b/visualizer.py
@@ -1,7 +1,6 @@
import datetime
from io import BytesIO
import json
-import math
from types import SimpleNamespace
from urllib.request import urlopen
@@ -13,6 +12,8 @@ import matplotlib.pyplot as plt
import numpy as np
from pyld import jsonld
+import data_spec
+
Measurement = make_dataclass('Measurement', ['url_name', 'field', 'title'])
measurements = {m.url_name: m for m in (
Measurement('temperature', 1, 'Temperature (C)'),
@@ -23,55 +24,6 @@ measurements = {m.url_name: m for m in (
ACCESS_URL = 'http://www.w3.org/ns/dcat#accessURL'
DISTRIBUTION = 'http://www.w3.org/ns/dcat#Distribution'
-
-def simple_time(s):
- """Convert an HHMM string to an (H, M) tuple."""
- hhmm = int(s)
- return divmod(hhmm, 100)
-
-
-def filtered_int(s):
- """Like int, but ignore the special value 999."""
- val = int(s)
- return val if val != 999 else math.nan
-
-
-def filtered_float(s):
- """Like float, but ignore the special values 444.0, 99.9, etc."""
- val = float(s)
- return val if val not in (444.0, 99.9, 999.9, 9999.9) else math.nan
-
-
-ONE_HOUR_DATA = asccol.DataSpec(
- leading=2, # 2 lines to delete
- cols=(
- ('year', int, 4), # name, type (converter function), width
- ('jdate', int, 4),
- ('month', int, 3),
- ('day', int, 3),
- ('time', simple_time, 5),
- ('temp', filtered_float, 7),
- ('pressure', filtered_float, 7),
- ('wind_speed', filtered_float, 7),
- ('wind_dir', filtered_float, 7),
- ('rel_hum', filtered_float, 7),
- ('delta_t', filtered_float, 7),
- ),
-)
-SOUTH_POLE_DATA = asccol.DataSpec(
- cols=(
- ('year', int, 5),
- ('month', int, 5),
- ('day', int, 5),
- ('time', simple_time, 5),
- ('wind_dir', filtered_int, 4),
- ('wind_speed', filtered_float, 6),
- ('visibility', filtered_int, 7),
- ('temp', filtered_float, 7),
- ('pressure', filtered_float, 7),
- ),
-)
-
app = Flask(__name__)
app.jinja_env.trim_blocks = True
@@ -350,13 +302,13 @@ def get_resources(link):
def read_data(station, year):
"""Fetch data and convert it to a NumPy array."""
+ spec = (data_spec.SOUTH_POLE if station['id'] == 'south-pole'
+ else data_spec.ONE_HOUR)
data = []
resource_list = get_resources(get_link(station, year))
for url in resource_list:
with urlopen(url) as f:
lines = map(bytes.decode, f)
- spec = (SOUTH_POLE_DATA if station['id'] == 'south-pole'
- else ONE_HOUR_DATA)
for row in asccol.parse_data(lines, spec):
date = datetime.datetime(row.year, row.month, row.day,
*row.time)
--
GitLab