Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
A
AossTower
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Deploy
Model registry
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
MetObs
AossTower
Commits
a68618fc
Commit
a68618fc
authored
5 years ago
by
William Roberts
Browse files
Options
Downloads
Patches
Plain Diff
Add tests for influx.db and add urls to debug print information
parent
3100c32b
No related branches found
No related tags found
No related merge requests found
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
aosstower/level_00/influxdb.py
+15
-13
15 additions, 13 deletions
aosstower/level_00/influxdb.py
aosstower/tests/level_00/test_influxdb.py
+120
-0
120 additions, 0 deletions
aosstower/tests/level_00/test_influxdb.py
aosstower/tests/test_influxdb.py
+0
-23
0 additions, 23 deletions
aosstower/tests/test_influxdb.py
with
135 additions
and
36 deletions
aosstower/level_00/influxdb.py
+
15
−
13
View file @
a68618fc
...
...
@@ -107,15 +107,14 @@ class Updater(object):
# https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#offset-aliases
# 2 minute rolling average.
winds_frame_2m
=
frame
[[
'
wind_speed
'
,
'
wind_east
'
,
'
wind_north
'
]].
rolling
(
'
2T
'
,
closed
=
'
right
'
).
mean
()
frame
[
'
wind_speed_2m
'
]
=
winds_frame_2m
[
'
wind_speed
'
]
frame
[
'
wind_dir_2m
'
]
=
calc
.
wind_vector_degrees
(
winds_frame_2m
[
'
wind_east
'
],
winds_frame_2m
[
'
wind_north
'
])
# Makes 2 minute averages nans if given less than 2 minutes of data.
if
len
(
frame
[
frame
.
index
>
frame
.
index
[
-
1
]
-
timedelta
(
minutes
=
2
)])
<
120
/
self
.
data_interval
:
frame
[
'
wind_speed_2m
'
]
=
frame
[
'
wind_speed_2m
'
].
mask
(
frame
[
'
wind_speed_2m
'
]
>
-
1.
)
frame
[
'
wind_dir_2m
'
]
=
frame
[
'
wind_dir_2m
'
].
mask
(
frame
[
'
wind_dir_2m
'
]
>
-
1.
)
winds_frame_2m
=
winds_frame_2m
.
mask
(
winds_frame_2m
[
'
wind_speed
'
]
>
-
1
)
frame
[
'
wind_speed_2m
'
]
=
winds_frame_2m
[
'
wind_speed
'
]
frame
[
'
wind_dir_2m
'
]
=
calc
.
wind_vector_degrees
(
winds_frame_2m
[
'
wind_east
'
],
winds_frame_2m
[
'
wind_north
'
])
# 1 minute rolling peaks
wind_peak_1m
=
frame
[
'
wind_speed
'
].
rolling
(
window
=
'
1T
'
,
closed
=
'
right
'
).
max
()
# criteria for a fast wind to be considered a wind gust
# criteria for a fast wind to be considered a wind gust
. Note that it needs winds_frame_2m['wind_speed'].
gust_mask
=
(
winds_frame_2m
[
'
wind_speed
'
]
>=
calc
.
knots_to_mps
(
9.
))
&
\
(
wind_peak_1m
>=
winds_frame_2m
[
'
wind_speed
'
]
+
calc
.
knots_to_mps
(
5.
))
frame
[
'
gust_1m
'
]
=
wind_peak_1m
.
mask
(
~
gust_mask
)
...
...
@@ -132,7 +131,6 @@ def convert_to_influx_frame(record_gen, symbols, debug=False):
record
=
{
symbols
[
k
]
or
k
:
v
for
k
,
v
in
record
.
items
()
if
k
in
symbols
}
if
debug
:
print
(
idx
,
record
)
continue
yield
record
...
...
@@ -236,17 +234,21 @@ def main():
influx_gen
=
influxdb
.
grouper
(
influx_gen
,
args
.
bulk
)
updater
=
Updater
()
for
record
in
influx_gen
:
lines
=
influxdb
.
frame_records
(
record
,
**
station_tags
)
influxdb
.
insert
(
lines
,
host
=
args
.
host
,
port
=
args
.
port
,
dbname
=
args
.
dbname
)
if
not
args
.
debug
:
lines
=
influxdb
.
frame_records
(
record
,
**
station_tags
)
influxdb
.
insert
(
lines
,
host
=
args
.
host
,
port
=
args
.
port
,
dbname
=
args
.
dbname
)
# Record is in a list of size 1, but want just the record.
avg
=
updater
.
rolling_average
(
record
[
0
])
# Once every 5 minutes: 0 through 295 seconds inclusive in 5 second intervals.
if
wu_pw
and
args
.
ldmp
and
avg
is
not
None
:
if
avg
is
not
None
and
(
args
.
debug
or
wu_pw
and
args
.
ldmp
)
:
url
=
construct_url
(
get_url_data
(
avg
,
args
.
wu_id
,
wu_pw
))
resp
=
requests
.
post
(
url
)
if
resp
.
status_code
!=
200
:
warnings
.
warn
(
'
Data failed to upload to {0} with status code {1}: {2}
'
.
format
(
url
,
resp
.
status_code
,
resp
.
text
))
if
args
.
debug
:
print
(
url
)
elif
wu_pw
and
args
.
ldmp
:
resp
=
requests
.
post
(
url
)
if
resp
.
status_code
!=
200
:
warnings
.
warn
(
'
Data failed to upload to {0} with status code {1}: {2}
'
.
format
(
url
,
resp
.
status_code
,
resp
.
text
))
if
args
.
sleep_interval
:
time
.
sleep
(
args
.
sleep_interval
)
...
...
This diff is collapsed.
Click to expand it.
aosstower/tests/level_00/test_influxdb.py
0 → 100755
+
120
−
0
View file @
a68618fc
#!/usr/bin/env python
import
unittest
import
numpy
as
np
import
pandas
as
pd
import
datetime
from
aosstower.level_00.influxdb
import
Updater
,
construct_url
,
get_url_data
class
TestCase
:
def
__init__
(
self
,
input
,
expected_avg
,
expected_url
):
self
.
input
=
input
self
.
expected_avg
=
expected_avg
self
.
expected_url
=
expected_url
def
create_data
(
size
,
data_interval
=
datetime
.
timedelta
(
seconds
=
5
)):
return
[{
'
wind_speed
'
:
i
%
119
,
'
wind_dir
'
:
0
,
'
box_pressure
'
:
i
,
'
paro_air_temp_period
'
:
i
,
'
paro_pressure_period
'
:
i
,
'
paro_air_temp
'
:
i
,
'
pressure
'
:
i
,
'
paro_cal_sig
'
:
i
,
'
box_air_temp
'
:
i
,
'
air_temp_2
'
:
i
,
'
air_temp_3
'
:
i
,
'
air_temp_4
'
:
i
,
'
rh_shield_freq
'
:
i
,
'
rel_hum
'
:
i
,
'
air_temp_6_3m
'
:
i
,
'
dewpoint
'
:
i
,
'
rtd_shied_freq
'
:
i
,
'
air_temp
'
:
i
,
'
solar_flux
'
:
i
,
'
precip
'
:
i
,
'
accum_precip
'
:
-
99999
,
'
altimeter
'
:
i
,
'
timestamp
'
:
datetime
.
datetime
(
2019
,
1
,
1
,
0
,
3
,
33
)
+
data_interval
*
i
}
for
i
in
range
(
size
)]
def
_isnan
(
x
):
return
isinstance
(
x
,
float
)
and
np
.
isnan
(
x
)
class
TestInfluxdb
(
unittest
.
TestCase
):
def
setUp
(
self
):
self
.
updater
=
Updater
()
self
.
test_data
=
TestCase
(
create_data
(
209
),
[
{
'
wind_speed
'
:
17
,
'
wind_dir
'
:
0
,
'
box_pressure
'
:
17
,
'
paro_air_temp_period
'
:
17
,
'
paro_pressure_period
'
:
17
,
'
paro_air_temp
'
:
17
,
'
pressure
'
:
17
,
'
paro_cal_sig
'
:
17
,
'
box_air_temp
'
:
17
,
'
air_temp_2
'
:
17
,
'
air_temp_3
'
:
17
,
'
air_temp_4
'
:
17
,
'
rh_shield_freq
'
:
17
,
'
rel_hum
'
:
17
,
'
air_temp_6_3m
'
:
17
,
'
dewpoint
'
:
17
,
'
rtd_shied_freq
'
:
17
,
'
air_temp
'
:
17
,
'
solar_flux
'
:
17
,
'
precip
'
:
17
,
'
accum_precip
'
:
np
.
nan
,
'
altimeter
'
:
17
,
'
wind_east
'
:
0
,
'
wind_north
'
:
17
,
'
wind_speed_2m
'
:
np
.
nan
,
'
wind_dir_2m
'
:
np
.
nan
,
'
gust_1m
'
:
np
.
nan
,
'
gust_10m
'
:
np
.
nan
,
'
index
'
:
pd
.
Timestamp
(
'
2019-01-01 00:04:58
'
)},
{
'
wind_speed
'
:
77
,
'
wind_dir
'
:
0
,
'
box_pressure
'
:
77
,
'
paro_air_temp_period
'
:
77
,
'
paro_pressure_period
'
:
77
,
'
paro_air_temp
'
:
77
,
'
pressure
'
:
77
,
'
paro_cal_sig
'
:
77
,
'
box_air_temp
'
:
77
,
'
air_temp_2
'
:
77
,
'
air_temp_3
'
:
77
,
'
air_temp_4
'
:
77
,
'
rh_shield_freq
'
:
77
,
'
rel_hum
'
:
77
,
'
air_temp_6_3m
'
:
77
,
'
dewpoint
'
:
77
,
'
rtd_shied_freq
'
:
77
,
'
air_temp
'
:
77
,
'
solar_flux
'
:
77
,
'
precip
'
:
77
,
'
accum_precip
'
:
np
.
nan
,
'
altimeter
'
:
77
,
'
wind_east
'
:
0
,
'
wind_north
'
:
77
,
'
wind_speed_2m
'
:
65.5
,
'
wind_dir_2m
'
:
0
,
'
gust_1m
'
:
77
,
'
gust_10m
'
:
np
.
nan
,
'
index
'
:
pd
.
Timestamp
(
'
2019-01-01 00:09:58
'
)},
{
'
wind_speed
'
:
18
,
'
wind_dir
'
:
0
,
'
box_pressure
'
:
137
,
'
paro_air_temp_period
'
:
137
,
'
paro_pressure_period
'
:
137
,
'
paro_air_temp
'
:
137
,
'
pressure
'
:
137
,
'
paro_cal_sig
'
:
137
,
'
box_air_temp
'
:
137
,
'
air_temp_2
'
:
137
,
'
air_temp_3
'
:
137
,
'
air_temp_4
'
:
137
,
'
rh_shield_freq
'
:
137
,
'
rel_hum
'
:
137
,
'
air_temp_6_3m
'
:
137
,
'
dewpoint
'
:
137
,
'
rtd_shied_freq
'
:
137
,
'
air_temp
'
:
137
,
'
solar_flux
'
:
137
,
'
precip
'
:
137
,
'
accum_precip
'
:
np
.
nan
,
'
altimeter
'
:
137
,
'
wind_east
'
:
0
,
'
wind_north
'
:
18
,
'
wind_speed_2m
'
:
31.291666666666668
,
'
wind_dir_2m
'
:
0
,
'
gust_1m
'
:
np
.
nan
,
'
gust_10m
'
:
np
.
nan
,
'
index
'
:
pd
.
Timestamp
(
'
2019-01-01 00:14:58
'
)},
{
'
wind_speed
'
:
78
,
'
wind_dir
'
:
0
,
'
box_pressure
'
:
197
,
'
paro_air_temp_period
'
:
197
,
'
paro_pressure_period
'
:
197
,
'
paro_air_temp
'
:
197
,
'
pressure
'
:
197
,
'
paro_cal_sig
'
:
197
,
'
box_air_temp
'
:
197
,
'
air_temp_2
'
:
197
,
'
air_temp_3
'
:
197
,
'
air_temp_4
'
:
197
,
'
rh_shield_freq
'
:
197
,
'
rel_hum
'
:
197
,
'
air_temp_6_3m
'
:
197
,
'
dewpoint
'
:
197
,
'
rtd_shied_freq
'
:
197
,
'
air_temp
'
:
197
,
'
solar_flux
'
:
197
,
'
precip
'
:
197
,
'
accum_precip
'
:
np
.
nan
,
'
altimeter
'
:
197
,
'
wind_east
'
:
0
,
'
wind_north
'
:
78
,
'
wind_speed_2m
'
:
66.5
,
'
wind_dir_2m
'
:
0
,
'
gust_1m
'
:
78
,
'
gust_10m
'
:
118
,
'
index
'
:
pd
.
Timestamp
(
'
2019-01-01 00:19:58
'
)}],
[
'
http://weatherstation.wunderground.com/weatherstation/updateweatherstation.php?
'
'
ID=&PASSWORD=&dateutc=2019-01-01+00%3A04%3A58&winddir=0.0&winddir_avg2m=
'
'
&windspeedmph=38.02798&windspdmph_avg2m=&windgustmph=&windgustmph_10m=
'
'
&humidity=17.0&tempf=62.6&baromin=0.5020096628001095&dewptf=62.6
'
'
&solarradiation=17.0&rainin=17.0&dailyrainin=&softwaretype=SSEC-RIG
'
'
&action=updateraw
'
,
'
http://weatherstation.wunderground.com/weatherstation/updateweatherstation.php?
'
'
ID=&PASSWORD=&dateutc=2019-01-01+00%3A09%3A58&winddir=0.0&winddir_avg2m=0.0
'
'
&windspeedmph=172.24438&windspdmph_avg2m=146.51957000000002&windgustmph=172.24438
'
'
&windgustmph_10m=&humidity=77.0&tempf=170.6&baromin=2.2738084726828487
'
'
&dewptf=170.6&solarradiation=77.0&rainin=77.0&dailyrainin=&softwaretype=SSEC-RIG
'
'
&action=updateraw
'
,
'
http://weatherstation.wunderground.com/weatherstation/updateweatherstation.php?
'
'
ID=&PASSWORD=&dateutc=2019-01-01+00%3A14%3A58&winddir=0.0&winddir_avg2m=0.0
'
'
&windspeedmph=40.264920000000004&windspdmph_avg2m=69.99758083333334&windgustmph=
'
'
&windgustmph_10m=&humidity=137.0&tempf=278.6&baromin=4.045607282565588
'
'
&dewptf=278.6&solarradiation=137.0&rainin=137.0&dailyrainin=&softwaretype=SSEC-RIG
'
'
&action=updateraw
'
,
'
http://weatherstation.wunderground.com/weatherstation/updateweatherstation.php?
'
'
ID=&PASSWORD=&dateutc=2019-01-01+00%3A19%3A58&winddir=0.0&winddir_avg2m=0.0
'
'
&windspeedmph=174.48132&windspdmph_avg2m=148.75651000000002&windgustmph=174.48132
'
'
&windgustmph_10m=263.95892000000003&humidity=197.0&tempf=386.6
'
'
&baromin=5.817406092448327&dewptf=386.6&solarradiation=197.0&rainin=197.0
'
'
&dailyrainin=&softwaretype=SSEC-RIG&action=updateraw
'
])
def
test_updater
(
self
):
output
=
[]
for
record
in
self
.
test_data
.
input
:
avg
=
self
.
updater
.
rolling_average
(
record
)
if
avg
is
not
None
:
output
.
append
({
key
:
avg
[
key
][
-
1
]
for
key
in
avg
})
output
[
-
1
][
'
index
'
]
=
avg
.
index
[
-
1
]
self
.
assertGreaterEqual
(
len
(
self
.
test_data
.
expected_avg
),
len
(
output
))
self
.
assertEqual
(
len
(
self
.
test_data
.
expected_avg
[
len
(
output
)
-
1
]),
len
(
output
[
-
1
]))
for
key
in
output
[
-
1
]:
if
not
(
_isnan
(
output
[
-
1
][
key
])
and
_isnan
(
self
.
test_data
.
expected_avg
[
len
(
output
)
-
1
][
key
])):
self
.
assertEqual
(
self
.
test_data
.
expected_avg
[
len
(
output
)
-
1
][
key
],
output
[
-
1
][
key
])
self
.
assertEqual
(
len
(
self
.
test_data
.
expected_avg
),
len
(
output
))
def
test_construct_url
(
self
):
output
=
[]
for
record
in
self
.
test_data
.
input
:
avg
=
self
.
updater
.
rolling_average
(
record
)
if
avg
is
not
None
:
output
.
append
(
construct_url
(
get_url_data
(
avg
,
''
,
''
)))
self
.
assertGreaterEqual
(
len
(
self
.
test_data
.
expected_url
),
len
(
output
))
self
.
assertEqual
(
self
.
test_data
.
expected_url
[
len
(
output
)
-
1
],
output
[
-
1
])
self
.
assertEqual
(
len
(
self
.
test_data
.
expected_url
),
len
(
output
))
def
suite
():
"""
The test suite for influxdb.
"""
loader
=
unittest
.
TestLoader
()
mysuite
=
unittest
.
TestSuite
()
mysuite
.
addTest
(
loader
.
loadTestsFromTestCase
(
TestInfluxdb
))
return
mysuite
if
__name__
==
"
__main__
"
:
unittest
.
main
()
This diff is collapsed.
Click to expand it.
aosstower/tests/test_influxdb.py
deleted
100644 → 0
+
0
−
23
View file @
3100c32b
import
unittest
class
TestInfluxdb
(
unittest
.
TestCase
):
def
setUp
(
self
):
self
.
test_cases
=
[]
def
tearDown
(
self
):
return
def
suite
():
"""
The test suite for test_wind_functions.
"""
loader
=
unittest
.
TestLoader
()
mysuite
=
unittest
.
TestSuite
()
mysuite
.
addTest
(
loader
.
loadTestsFromTestCase
(
TestInfluxdb
))
return
mysuite
if
__name__
==
"
__main__
"
:
unittest
.
main
()
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment