Skip to content
Snippets Groups Projects
Unverified Commit 8961a3f1 authored by mnooj's avatar mnooj Committed by GitHub
Browse files

Add files via upload

parent 0b76fb9e
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
Procfile 0 → 100644
+ 1
− 0
View file @ 8961a3f1
web: gunicorn visualizer:app
\ No newline at end of file
requirements.txt 0 → 100644
+ 27
− 0
View file @ 8961a3f1
cachetools==5.0.0
certifi==2021.10.8
charset-normalizer==2.0.12
click==8.0.4
cycler==0.11.0
Flask==2.0.3
fonttools==4.29.1
frozendict==2.3.0
gunicorn==20.1.0
idna==3.3
itsdangerous==2.1.0
Jinja2==3.0.3
json5==0.9.6
kiwisolver==1.3.2
lxml==4.8.0
MarkupSafe==2.1.0
matplotlib==3.5.1
numpy==1.22.2
packaging==21.3
Pillow==9.0.1
PyLD==2.0.3
pyparsing==3.0.7
python-dateutil==2.8.2
requests==2.27.1
six==1.16.0
urllib3==1.26.8
Werkzeug==2.0.3
runtime.txt 0 → 100644
+ 1
− 0
View file @ 8961a3f1
python-3.10.2
\ No newline at end of file
static/logo.png 0 → 100644
+ 0
− 0
View file @ 8961a3f1
static/logo.png

209 KiB

templates/index.html 0 → 100644
+ 189
− 0
View file @ 8961a3f1
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<link rel='stylesheet' id='uwmadison-fonts-css' href='https://amrdc.ssec.wisc.edu/wp-test/wp-content/themes/amrdc-theme/dist/fonts/uw160/fonts.css?ver=1.0.0' type='text/css' media='all' />
<link rel='stylesheet' id='uwmadison-style-css' href='https://amrdc.ssec.wisc.edu/wp-test/wp-content/themes/amrdc-theme/dist/main.min.css?ver=0.7.0' type='text/css' media='all' />
<style type="text/css">
#header {
padding: 10px;
vertical-align: top;
}
#names_list,
#years_list,
#measurements,
#button,
#overlay_check {
display: inline-table;
padding: 10px;
zoom: 1;
}
h1 {
margin-right: 40%;
padding: 0px;
}
select {
width: 200px;
font-size: 16px;
border: 1px solid #CCC;
cursor: pointer;
background-color: #fff;
}
#repo_link {
padding-bottom: 30px;
}
</style>
</head>
<body>
<div id="header">
<div id="logo">
<a href="https://amrdcdata.ssec.wisc.edu"><img src="{{url_for('static', filename='logo.png')}}" width="75" height="75" align="left"/></a>
</div>
<div id="title">
<h1 align="right">Data Visualizer</h1>
</div>
</div>
<div id="names_list">
<select name="names" id="names" onchange="getYears()">
<option value="">Select Station</option>
</select>
</div>
<div id="years_list">
<select name="years" id="years">
<option value="">Select Year</option>
</select>
</div>
<div id="measurements">
<select name="meas" id="meas">
<option value="">Select Measurement</option>
<option value="1">Temperature</option>
<option value="2">Pressure</option>
<option value="3">Wind Speed</option>
</select>
</div>
<div id="button">
<input type="submit" value="Visualize" onclick="visualize()">
</div>
<div id="overlay_check">
<input type="checkbox" id="overlay_box" name="overlay_box" onclick="location.href = '/index2';">
<label for="overlay_box">Overlay second dataset</label>
</div>
<center>
<hr>
<div id="graph"></div>
<div id="divider"></div>
<div id="repo_link">
<a id="link_anchor"></a>
</div>
</center>
<script type="text/javascript">
fetch('/list')
.then(function (response) {
return response.json();
})
.then(function (data) {
appendData(data);
})
.catch(function (err) {
console.log('error: ' + err);
});
function appendData(data) {
var select = document.getElementById("names");
select.options.length = 0;
var option = document.createElement("option");
option.value = "";
option.text = "Select Station";
select.appendChild(option);
for (var i = 0; i < data.length; i++) {
var option = document.createElement("option");
option.value = data[i];
option.text = data[i];
select.appendChild(option);
}
}
function getYears() {
var name = document.getElementById("names").value;
data2 = {}
fetch("/years?name=" + name)
.then(function (response) {
return response.json();
})
.then(function (data2) {
appendYears(data2);
})
.catch(function (err) {
console.log('error: ' + err);
});
function appendYears(data2) {
var years = document.getElementById("years");
years.options.length = 0;
var option = document.createElement("option");
option.value = "";
option.text = "Select Year";
years.appendChild(option);
for (var i = 0; i < data2.length; i++) {
var option = document.createElement("option");
option.value = data2[i];
option.text = data2[i];
years.appendChild(option);
}
}
}
function visualize() {
var name = document.getElementById("names").value;
var year = document.getElementById("years").value;
var field = document.getElementById("meas").value;
var graph = document.getElementById("graph");
var repo_link = document.getElementById("repo_link");
var divider = document.getElementById("divider");
divider.innerHTML = "<hr>"
fetch("/plot?name=" + name + "&year=" + year + "&field=" + field)
.then(function (response) {
return response.text();
})
.then(function (response) {
graph.innerHTML = response;
})
.catch(function (err) {
console.log('error: ' + err);
});
fetch("/repo/" + name + "/" + year)
.then(function (response) {
return response.text();
})
.then(function (response) {
var a = document.getElementById('link_anchor');
a.innerHTML = "";
var link = document.createTextNode("Source: AMRDC Data Repository");
a.appendChild(link);
a.title = "AMRDC Data Repository";
a.href = response;
repo_link.appendChild(a);
})
.catch(function (err) {
console.log('error: ' + err);
});
}
</script>
</body>
</html>
\ No newline at end of file
templates/overlay.html 0 → 100644
+ 275
− 0
View file @ 8961a3f1
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<link rel='stylesheet' id='uwmadison-fonts-css' href='https://amrdc.ssec.wisc.edu/wp-test/wp-content/themes/amrdc-theme/dist/fonts/uw160/fonts.css?ver=1.0.0' type='text/css' media='all' />
<link rel='stylesheet' id='uwmadison-style-css' href='https://amrdc.ssec.wisc.edu/wp-test/wp-content/themes/amrdc-theme/dist/main.min.css?ver=0.7.0' type='text/css' media='all' />
<style type="text/css">
#header {
padding: 10px;
vertical-align: top;
}
#names_list,
#years_list,
#measurements,
#button,
#overlay_check {
display: inline-table;
padding: 10px;
zoom: 1;
}
h1 {
margin-right: 40%;
padding: 0px;
}
#names_list2,
#years_list2 {
display: inline-table;
padding: 0px 10px 10px 10px;
zoom: 1;
}
select {
width: 200px;
font-size: 16px;
border: 1px solid #CCC;
cursor: pointer;
background-color: #fff;
}
#repo_link2 {
padding-bottom: 30px;
}
</style>
</head>
<body>
<div id="header">
<div id="logo">
<a href="https://amrdcdata.ssec.wisc.edu"><img src="{{url_for('static', filename='logo.png')}}" width="75" height="75" align="left"/></a>
</div>
<div id="title">
<h1 align="right">Data Visualizer</h1>
</div>
</div>
<div id="names_list">
<select name="names" id="names" onchange="getYears()">
<option value="">Select Station</option>
</select>
</div>
<div id="years_list">
<select name="years" id="years">
<option value="">Select Year</option>
</select>
</div>
<div id="measurements">
<select name="meas" id="meas">
<option value="">Select Measurement</option>
<option value="1">Temperature</option>
<option value="2">Pressure</option>
<option value="3">Wind Speed</option>
</select>
</div>
<div id="button">
<input type="submit" value="Visualize" onclick="overlay()">
</div>
<div id="overlay_check">
<input type="checkbox" id="overlay_box" name="overlay_box" checked onclick="location.href = '/';">
<label for="overlay_box">Overlay second dataset</label>
</div>
<br>
<div id="names_list2">
<select name="names2" id="names2" onchange="getYears2()">
<option value="">Select Station</option>
</select>
</div>
<div id="years_list2">
<select name="years2" id="years2">
<option value="">Select Year</option>
</select>
</div>
<center>
<hr>
<div id="graph"></div>
<div id="divider"></div>
<div id="repo_link">
<a id="link_anchor"></a>
</div>
<div id="repo_link2">
<a id="link_anchor2"></a>
</div>
</center>
<script type="text/javascript">
fetch('/list')
.then(function (response) {
return response.json();
})
.then(function (data) {
appendData(data);
appendData2(data);
})
.catch(function (err) {
console.log('error: ' + err);
});
function appendData(data) {
var select = document.getElementById("names");
select.options.length = 0;
var option = document.createElement("option");
option.value = "";
option.text = "Select Station";
select.appendChild(option);
for (var i = 0; i < data.length; i++) {
var option = document.createElement("option");
option.value = data[i];
option.text = data[i];
select.appendChild(option);
}
}
function appendData2(data) {
var select = document.getElementById("names2");
select.options.length = 0;
var option = document.createElement("option");
option.value = "";
option.text = "Select Station";
select.appendChild(option);
for (var i = 0; i < data.length; i++) {
var option = document.createElement("option");
option.value = data[i];
option.text = data[i];
select.appendChild(option);
}
}
function getYears() {
var name = document.getElementById("names").value;
data2 = {}
fetch("/years?name=" + name)
.then(function (response) {
return response.json();
})
.then(function (data2) {
appendYears(data2);
})
.catch(function (err) {
console.log('error: ' + err);
});
function appendYears(data2) {
var years = document.getElementById("years");
years.options.length = 0;
var option = document.createElement("option");
option.value = "";
option.text = "Select Year";
years.appendChild(option);
for (var i = 0; i < data2.length; i++) {
var option = document.createElement("option");
option.value = data2[i];
option.text = data2[i];
years.appendChild(option);
}
}
}
function getYears2() {
var name = document.getElementById("names2").value;
data2 = {}
fetch("/years?name=" + name)
.then(function (response) {
return response.json();
})
.then(function (data2) {
appendYears(data2);
})
.catch(function (err) {
console.log('error: ' + err);
});
function appendYears(data2) {
var years = document.getElementById("years2");
years.options.length = 0;
var option = document.createElement("option");
option.value = "";
option.text = "Select Year";
years.appendChild(option);
for (var i = 0; i < data2.length; i++) {
var option = document.createElement("option");
option.value = data2[i];
option.text = data2[i];
years.appendChild(option);
}
}
}
function overlay() {
var name = document.getElementById("names").value;
var year = document.getElementById("years").value;
var field = document.getElementById("meas").value;
var name2 = document.getElementById("names2").value;
var year2 = document.getElementById("years2").value;
var graph = document.getElementById("graph");
var repo_link = document.getElementById("repo_link");
var repo_link2 = document.getElementById("repo_link2");
var divider = document.getElementById("divider");
divider.innerHTML = "<hr>"
fetch("/overlay?name=" + name + "&year=" + year + "&field=" + field + "&name2=" +
name2 + "&year2=" + year2)
.then(function (response) {
return response.text();
})
.then(function (response) {
graph.innerHTML = response;
})
.catch(function (err) {
console.log('error: ' + err);
});
fetch("/repo/" + name + "/" + year)
.then(function (response) {
return response.text();
})
.then(function (response) {
var a = document.getElementById('link_anchor');
a.innerHTML = "";
var link = document.createTextNode("Source #1: AMRDC Data Repository\n");
a.appendChild(link);
a.title = "AMRDC Data Repository";
a.href = response;
repo_link.appendChild(a);
})
.catch(function (err) {
console.log('error: ' + err);
});
fetch("/repo/" + name2 + "/" + year2)
.then(function (response) {
return response.text();
})
.then(function (response) {
var a = document.getElementById('link_anchor2');
a.innerHTML = "";
var link = document.createTextNode("Source #2: AMRDC Data Repository\n");
a.appendChild(link);
a.title = "AMRDC Data Repository";
a.href = response;
repo_link2.appendChild(a);
})
.catch(function (err) {
console.log('error: ' + err);
});
}
</script>
</body>
</html>
\ No newline at end of file
visualizer.py 0 → 100644
+ 266
− 0
View file @ 8961a3f1
# This program accesses AWS and South Pole station observational data via the AMRDC Data Repository
# and generates simple line charts based on temperature, pressure, and wind speed measurements.
# Programmed by Matthew G. Noojin for the Antarctic Meteorological Research and Data Center, 2022
from flask import Flask, jsonify, request, render_template
from urllib.request import urlopen
import datetime
import matplotlib.pyplot as plt
import matplotlib
import base64
from io import BytesIO
import numpy as np
from pyld import jsonld
import json
app = Flask(__name__)
RECORDS = "https://raw.githubusercontent.com/mnooj/amrdc_data_visualizer/main/amrdcrecords.txt"
plt.style.use('ggplot')
plt.rcParams['axes.xmargin'] = 0
matplotlib.use('Agg')
@app.route('/')
def home_page():
return render_template('index.html')
@app.route('/index2')
def overlay_index():
return render_template('overlay.html')
def get_record_list():
record_list = [
] # [ ["South Pole", ['1958', URL], ['1959, URL], ... ], ["Byrd", ...] ]
data = urlopen(RECORDS)
for line in data:
decoded_line = line.decode("utf-8")
decoded_line = decoded_line.rstrip('\n')
row = decoded_line.split(",")
for item in row:
if "\n" in item:
item.split("\n")
if len(row) == 1:
record_list.append([row[0]])
else:
record_list[-1].append([row[0], row[1]])
return record_list
## Returns a list of record_names + generates records_list of URLS organized by station name / year
@app.route("/list")
def get_record_names():
record_names = []
record_list = get_record_list()
for item in record_list:
record_names.append(item[0])
return jsonify(record_names)
## Returns a list of records by year for selected station
@app.route("/years", methods=['GET', 'POST'])
def get_record_years():
record_list = get_record_list()
name = request.args.get('name')
years = []
for row in record_list:
if row[0] == name:
for item in row[1:]:
years.append(item[0])
break
return jsonify(years)
## Returns a graph of selected data
@app.route("/plot", methods=['GET', 'POST'])
def plot():
selection = request.args.get('name')
year = request.args.get('year')
index = int(request.args.get('field'))
link = get_link(selection, year)
json_link = link + ".jsonld"
units = [None, "Temperature (C)", "Pressure (hPa)", "Wind Speed (m/s)"]
field = units[index]
data = readData(selection, year, json_link)
fig, plot = plt.subplots()
fig.set_figheight(6)
fig.set_figwidth(12)
plot.plot(data[:, 0], data[:, index])
avg = np.nanmean(data[:, index], dtype='float32')
plot.hlines(y=avg,
xmin=data[:, 0][0],
xmax=data[:, 0][-1],
linestyle='-',
alpha=0.7)
plot.set_ylabel(units[index])
plot.grid(True)
max = [data[0][0], data[0][index]]
min = [data[0][0], data[0][index]]
for row in data:
if max[1] <= row[index]:
max = [row[0], row[index]]
if min[1] >= row[index]:
min = [row[0], row[index]]
plot.set_title('Max ' + field + ': ' + str(max[1]) + ' - ' + str(max[0]) +
'. Min ' + field + ': ' + str(min[1]) + ' - ' + str(min[0]),
fontsize='small')
plt.suptitle(field + " measurements, " + selection + ' Station, ' +
str(data[0][0].year))
buf = BytesIO()
fig.savefig(buf, format="png")
data = base64.b64encode(buf.getbuffer()).decode("ascii")
return f"<img src='data:image/png;base64,{data}'/>"
@app.route("/overlay", methods=['GET', 'POST'])
def plot_overlay():
selection = request.args.get('name')
year = request.args.get('year')
index = int(request.args.get('field'))
link = get_link(selection, year)
json_link = link + ".jsonld"
selection2 = request.args.get('name2')
year2 = request.args.get('year2')
link2 = get_link(selection2, year2)
json_link2 = link2 + ".jsonld"
units = [None, "Temperature (C)", "Pressure (hPa)", "Wind Speed (m/s)"]
field = units[index]
data = readData(selection, year, json_link)
data2 = readData(selection2, year2, json_link2)
fig, plot = plt.subplots()
fig.set_figheight(6)
fig.set_figwidth(12)
temp_data = [] ## Should deal with Leap Year here
temp_data2 = []
for row in data:
if row[0].day == 29 and row[0].month == 2:
continue
else:
temp_data.append(row)
data = np.array(temp_data)
for row in data2:
if row[0].day == 29 and row[0].month == 2:
continue
else:
temp_data2.append(row)
data2 = np.array(temp_data2)
plot.plot(data[:, 0],
data[:, index],
label=(selection + ' ' + str(data[0][0].year)))
plot.plot(data[:, 0],
data2[:, index],
alpha=0.6,
label=(selection2 + ' ' + str(data2[0][0].year)))
avg1 = np.nanmean(data[:, index], dtype='float32')
avg2 = np.nanmean(data2[:, index], dtype='float32')
plot.hlines(y=avg1,
xmin=data[:, 0][0],
xmax=data[:, 0][-1],
alpha=0.7,
label=('Avg ' + selection + ' ' + str(data[0][0].year)))
plot.hlines(y=avg2,
xmin=data[:, 0][0],
xmax=data[:, 0][-1],
alpha=0.7,
label=('Avg ' + selection2 + ' ' + str(data2[0][0].year)))
plot.set_ylabel(units[index])
max = [data[0][0], data[0][index], selection]
min = [data[0][0], data[0][index], selection]
for row in data:
if max[1] <= row[index]:
max = [row[0], row[index], selection]
if min[1] >= row[index]:
min = [row[0], row[index], selection]
for row in data2:
if max[1] <= row[index]:
max = [row[0], row[index], selection2]
if min[1] >= row[index]:
min = [row[0], row[index], selection2]
plot.grid(True)
plot.set_title('Max ' + field + ': ' + str(max[1]) + ", " + max[2] +
' Station, ' + str(max[0]) + '. Min ' + field + ': ' +
str(min[1]) + ", " + min[2] + ' Station, ' + str(min[0]),
fontsize='small')
plot.legend()
plot.tick_params(labelbottom=False)
plt.suptitle(field + " measurements, " + selection + ' Station, ' +
str(data[0][0].year) + ' / ' + selection2 + ' Station, ' +
str(data2[0][0].year))
buf = BytesIO()
fig.savefig(buf, format="png")
data = base64.b64encode(buf.getbuffer()).decode("ascii")
return f"<img src='data:image/png;base64,{data}'/>"
@app.route("/repo/<string:selection>/<string:year>", methods=['GET', 'POST'])
def get_link(selection, year):
link = ""
record_list = get_record_list()
for item in record_list:
if item[0] == selection:
for record in item[1:]:
if record[0] == year:
link = record[1]
break
return link
def get_resources(station_yearly_record):
doc = jsonld.flatten(station_yearly_record)
doc = json.dumps(doc)
resource_list = []
data = doc.split('"')
for string in data:
if "download" in string:
resource_list.append(string)
pop_list = []
for item in reversed(resource_list):
if "10min" in item or "3hr" in item:
pop_list.append(resource_list.index(item))
for index in pop_list:
resource_list.pop(index)
return resource_list
def readData(name, year, json_link):
daily_data = []
resource_list = get_resources(json_link)
for url in resource_list:
data = urlopen(url)
if name == "South Pole Station":
for line in data:
decoded_line = line.decode("utf-8")
row = decoded_line.split()
time = int(row[3])
day = datetime.datetime(int(year), int(row[1]), int(row[2]),
int(time / 100), 00)
temp = float(row[7])
pressure = float(row[8])
windspeed = float(row[5])
daily_data.append([day, temp, pressure, windspeed])
else:
for line in range(2):
next(data)
for line in data:
decoded_line = line.decode("utf-8")
row = decoded_line.split()
time = int(row[4])
day = datetime.datetime(int(year), int(row[2]), int(row[3]),
int(time / 100), 00)
temp = float(row[5])
pressure = float(row[6])
windspeed = float(row[7])
daily_data.append([day, temp, pressure, windspeed])
for row in daily_data: ### Replace outliers with nan
if row[1] >= 99.9:
row[1] = np.nan
if row[2] >= 9999.9:
row[2] = np.nan
if row[3] >= 99.9:
row[3] = np.nan
daily_data = np.array(daily_data)
daily_data = daily_data[daily_data[:, 0].argsort()]
return daily_data
if __name__ == '__main__':
app.run()
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment