Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
V
visualizer-embed
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package registry
Container registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Owen Graham
visualizer-embed
Commits
537b0d51
Verified
Commit
537b0d51
authored
3 years ago
by
Owen Graham
Browse files
Options
Downloads
Patches
Plain Diff
Move data spec definitions to new module
parent
eaae6e50
Branches
Branches containing commit
No related tags found
No related merge requests found
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
data_spec.py
+54
-0
54 additions, 0 deletions
data_spec.py
visualizer.py
+4
-52
4 additions, 52 deletions
visualizer.py
with
58 additions
and
52 deletions
data_spec.py
0 → 100644
+
54
−
0
View file @
537b0d51
"""
Define `asccol` data specifications.
"""
import
math
import
asccol
def
simple_time
(
s
):
"""
Convert an HHMM string to an (H, M) tuple.
"""
hhmm
=
int
(
s
)
return
divmod
(
hhmm
,
100
)
def
filtered_int
(
s
):
"""
Like int, but ignore the special value 999.
"""
val
=
int
(
s
)
return
val
if
val
!=
999
else
math
.
nan
def
filtered_float
(
s
):
"""
Like float, but ignore the special values 444.0, 99.9, etc.
"""
val
=
float
(
s
)
return
val
if
val
not
in
(
444.0
,
99.9
,
999.9
,
9999.9
)
else
math
.
nan
ONE_HOUR
=
asccol
.
DataSpec
(
leading
=
2
,
# 2 lines to delete
cols
=
(
(
'
year
'
,
int
,
4
),
# name, type (converter function), width
(
'
jdate
'
,
int
,
4
),
(
'
month
'
,
int
,
3
),
(
'
day
'
,
int
,
3
),
(
'
time
'
,
simple_time
,
5
),
(
'
temp
'
,
filtered_float
,
7
),
(
'
pressure
'
,
filtered_float
,
7
),
(
'
wind_speed
'
,
filtered_float
,
7
),
(
'
wind_dir
'
,
filtered_float
,
7
),
(
'
rel_hum
'
,
filtered_float
,
7
),
(
'
delta_t
'
,
filtered_float
,
7
),
),
)
SOUTH_POLE
=
asccol
.
DataSpec
(
cols
=
(
(
'
year
'
,
int
,
5
),
(
'
month
'
,
int
,
5
),
(
'
day
'
,
int
,
5
),
(
'
time
'
,
simple_time
,
5
),
(
'
wind_dir
'
,
filtered_int
,
4
),
(
'
wind_speed
'
,
filtered_float
,
6
),
(
'
visibility
'
,
filtered_int
,
7
),
(
'
temp
'
,
filtered_float
,
7
),
(
'
pressure
'
,
filtered_float
,
7
),
),
)
This diff is collapsed.
Click to expand it.
visualizer.py
+
4
−
52
View file @
537b0d51
import
datetime
import
datetime
from
io
import
BytesIO
from
io
import
BytesIO
import
json
import
json
import
math
from
types
import
SimpleNamespace
from
types
import
SimpleNamespace
from
urllib.request
import
urlopen
from
urllib.request
import
urlopen
...
@@ -13,6 +12,8 @@ import matplotlib.pyplot as plt
...
@@ -13,6 +12,8 @@ import matplotlib.pyplot as plt
import
numpy
as
np
import
numpy
as
np
from
pyld
import
jsonld
from
pyld
import
jsonld
import
data_spec
Measurement
=
make_dataclass
(
'
Measurement
'
,
[
'
url_name
'
,
'
field
'
,
'
title
'
])
Measurement
=
make_dataclass
(
'
Measurement
'
,
[
'
url_name
'
,
'
field
'
,
'
title
'
])
measurements
=
{
m
.
url_name
:
m
for
m
in
(
measurements
=
{
m
.
url_name
:
m
for
m
in
(
Measurement
(
'
temperature
'
,
1
,
'
Temperature (C)
'
),
Measurement
(
'
temperature
'
,
1
,
'
Temperature (C)
'
),
...
@@ -23,55 +24,6 @@ measurements = {m.url_name: m for m in (
...
@@ -23,55 +24,6 @@ measurements = {m.url_name: m for m in (
ACCESS_URL
=
'
http://www.w3.org/ns/dcat#accessURL
'
ACCESS_URL
=
'
http://www.w3.org/ns/dcat#accessURL
'
DISTRIBUTION
=
'
http://www.w3.org/ns/dcat#Distribution
'
DISTRIBUTION
=
'
http://www.w3.org/ns/dcat#Distribution
'
def
simple_time
(
s
):
"""
Convert an HHMM string to an (H, M) tuple.
"""
hhmm
=
int
(
s
)
return
divmod
(
hhmm
,
100
)
def
filtered_int
(
s
):
"""
Like int, but ignore the special value 999.
"""
val
=
int
(
s
)
return
val
if
val
!=
999
else
math
.
nan
def
filtered_float
(
s
):
"""
Like float, but ignore the special values 444.0, 99.9, etc.
"""
val
=
float
(
s
)
return
val
if
val
not
in
(
444.0
,
99.9
,
999.9
,
9999.9
)
else
math
.
nan
ONE_HOUR_DATA
=
asccol
.
DataSpec
(
leading
=
2
,
# 2 lines to delete
cols
=
(
(
'
year
'
,
int
,
4
),
# name, type (converter function), width
(
'
jdate
'
,
int
,
4
),
(
'
month
'
,
int
,
3
),
(
'
day
'
,
int
,
3
),
(
'
time
'
,
simple_time
,
5
),
(
'
temp
'
,
filtered_float
,
7
),
(
'
pressure
'
,
filtered_float
,
7
),
(
'
wind_speed
'
,
filtered_float
,
7
),
(
'
wind_dir
'
,
filtered_float
,
7
),
(
'
rel_hum
'
,
filtered_float
,
7
),
(
'
delta_t
'
,
filtered_float
,
7
),
),
)
SOUTH_POLE_DATA
=
asccol
.
DataSpec
(
cols
=
(
(
'
year
'
,
int
,
5
),
(
'
month
'
,
int
,
5
),
(
'
day
'
,
int
,
5
),
(
'
time
'
,
simple_time
,
5
),
(
'
wind_dir
'
,
filtered_int
,
4
),
(
'
wind_speed
'
,
filtered_float
,
6
),
(
'
visibility
'
,
filtered_int
,
7
),
(
'
temp
'
,
filtered_float
,
7
),
(
'
pressure
'
,
filtered_float
,
7
),
),
)
app
=
Flask
(
__name__
)
app
=
Flask
(
__name__
)
app
.
jinja_env
.
trim_blocks
=
True
app
.
jinja_env
.
trim_blocks
=
True
...
@@ -350,13 +302,13 @@ def get_resources(link):
...
@@ -350,13 +302,13 @@ def get_resources(link):
def
read_data
(
station
,
year
):
def
read_data
(
station
,
year
):
"""
Fetch data and convert it to a NumPy array.
"""
"""
Fetch data and convert it to a NumPy array.
"""
spec
=
(
data_spec
.
SOUTH_POLE
if
station
[
'
id
'
]
==
'
south-pole
'
else
data_spec
.
ONE_HOUR
)
data
=
[]
data
=
[]
resource_list
=
get_resources
(
get_link
(
station
,
year
))
resource_list
=
get_resources
(
get_link
(
station
,
year
))
for
url
in
resource_list
:
for
url
in
resource_list
:
with
urlopen
(
url
)
as
f
:
with
urlopen
(
url
)
as
f
:
lines
=
map
(
bytes
.
decode
,
f
)
lines
=
map
(
bytes
.
decode
,
f
)
spec
=
(
SOUTH_POLE_DATA
if
station
[
'
id
'
]
==
'
south-pole
'
else
ONE_HOUR_DATA
)
for
row
in
asccol
.
parse_data
(
lines
,
spec
):
for
row
in
asccol
.
parse_data
(
lines
,
spec
):
date
=
datetime
.
datetime
(
row
.
year
,
row
.
month
,
row
.
day
,
date
=
datetime
.
datetime
(
row
.
year
,
row
.
month
,
row
.
day
,
*
row
.
time
)
*
row
.
time
)
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment