Skip to content
Snippets Groups Projects
Verified Commit c8ef6560 authored by David Hoese's avatar David Hoese
Browse files

Switch to pulling tile index information from postgres server

parent 22286e93
No related branches found
No related tags found
1 merge request!3Switch to pulling tile index information from postgres server
Pipeline #37008 failed with stages
in 1 minute and 26 seconds
......@@ -92,7 +92,7 @@ add_postgres_tables() {
docker exec -i ${PG_SERVER_NAME} psql -U ${POSTGRES_USER} <<EOF
CREATE TABLE IF NOT EXISTS g16_abi_radf_l1b_c01 (
gid SERIAL PRIMARY KEY,
start_time CHAR(19) NOT NULL UNIQUE,
start_time TIMESTAMP NOT NULL UNIQUE,
location VARCHAR(255) NOT NULL,
bbox_geometry geometry(POLYGON, 930916)
)
......@@ -145,6 +145,7 @@ EOF
start_test_container() {
debug "Starting test docker container (${image_url}:${image_tag})..."
docker run --rm -d --network ${NETWORK_NAME} --name test -p 8888:80 -v "$(pwd)":"/data" $@ ${image_url}:${image_tag}
# docker run --rm -it --network ${NETWORK_NAME} --name test -p 8888:80 -v "$(pwd)":"/data" $@ ${image_url}:${image_tag}
start_status=$?
# just wait a bit to let the server start
sleep 2
......@@ -181,6 +182,9 @@ kill_test_container() {
kill_postgres() {
debug "Killing postgres container..."
debug "-----------------------------------------"
docker logs ${PG_SERVER_NAME}
debug "-----------------------------------------"
docker kill ${PG_SERVER_NAME} >/dev/null
debug "Done killing postgres container."
}
......@@ -195,9 +199,12 @@ curl_layer_times() {
debug "Starting curl basic time request..."
# NOTE: The time doesn't actually exist and no image data is available. A blank image should be returned
# TODO: Verify that the expected time is returned
time_result=$(curl --fail -sS "http://localhost:8888/wms_times/g16/abi/radf/C01")
# time_result=$(curl --fail -sS "http://localhost:8888/wms_times/g16/abi/radf/C01")
time_result=$(curl "http://localhost:8888/wms_times/g16/abi/radf/C01")
time_status=$?
if [[ $time_status -ne 0 ]]; then
error "Requesting layer times failed"
error "$time_result"
return $time_status
fi
......
......@@ -23,23 +23,38 @@ MAP
# {{ product.replace('_', ' ').title() }} #
LAYER
NAME "{{ product }}_index"
NAME "{{ product }}_shapefile_index"
TYPE TILEINDEX
DATA "{{ layer_base_dir }}/{{ platform }}/abi/{{ sector }}/{{ product }}/{{ product }}"
END
LAYER
NAME "{{ product }}_postgres_index"
TYPE POLYGON
DATA "{{ postgis_geom_column }} from {{ platform }}_abi_{{ sector }}_l1b_{{ product.lower() }} using SRID={{ epsg_code }} using unique gid"
CONNECTIONTYPE postgis
CONNECTION "{{ postgis_connection_params }}"
METADATA
"wms_title" "{{ platform_long_name }} ABI {{ product.replace('_', ' ').title() }} Time Index"
"wms srs" "EPSG:{{ epsg_code }}"
"wms_extent" "-180 -90 180 90"
"wms_timeextent" "2017-01-01/2040-12-31"
"wms_timeformat" "YYYY-MM-DDTHH:MM:SS"
"wms_timeitem" "start_time" # column in postgres DB
"wms_timedefault" "2019-12-12T19:20:18"
"wms_enable_request" "*"
END
END
LAYER
NAME "{{ product }}"
TYPE RASTER
{% if postgis_connection_params %}
CONNECTIONTYPE postgis
CONNECTION "{{ postgis_connection_params }}"
TILEITEM "location"
TILEINDEX "{{ product }}_index"
#DATA "{{ postgis_geom_column }} from {{ postgis_table }} using SRID={{ epsg_code }} using unique id"
TILEINDEX "{{ product }}_postgres_index"
{% else %}
TILEITEM "location"
TILEINDEX "{{ product }}_index"
TILEINDEX "{{ product }}_shapefile_index"
{% endif %}
# Comment below to default to transparency
......@@ -53,10 +68,15 @@ MAP
PROCESSING "CLOSE_CONNECTION=DEFER"
METADATA
"wms_title" "{{ platform_long_name }} ABI {{ product.replace('_', ' ').title() }}"
"wms srs" "EPSG:{{ epsg_code }}"
"wms_extent" "-180 -90 180 90"
"wms_timeextent" "2017-01-01/2040-12-31"
"wms_timeformat" "YYYY-MM-DDTHH:MM:SS"
"wms_timeitem" "time" # time is a metadata item
{% if postgis_connection_params %}
"wms_timeitem" "start_time" # column in postgres DB
{% else %}
"wms_timeitem" "time" # field in shapefile
{% endif %}
"wms_timedefault" "2019-12-12T19:20:18"
"wms_enable_request" "*"
END
......
......@@ -74,15 +74,15 @@ order = order.upper()
if order not in ('ASC', 'DESC'):
bad_request("""'order' must be either 'ASC' or 'DESC'.""")
query_str = "SELECT start_time FROM {}"
query_str = """SELECT to_char(start_time, 'YYYY-MM-DD"T"HH24:MI:SS') FROM {}"""
if start_time is not None or end_time is not None:
query_str += " WHERE "
if start_time is not None:
query_str += "start_time >= '{}'".format(start_time.strftime('%Y-%m-%dT%H:%M:%S'))
query_str += "start_time >= timestamp '{}'".format(start_time.strftime('%Y-%m-%dT%H:%M:%S'))
if start_time is not None and end_time is not None:
query_str += " AND "
if end_time is not None:
query_str += "start_time <= '{}'".format(end_time.strftime('%Y-%m-%dT%H:%M:%S'))
query_str += "start_time <= timestamp '{}'".format(end_time.strftime('%Y-%m-%dT%H:%M:%S'))
query_str += " ORDER BY start_time {}".format(order)
if num_times != 0:
query_str += " LIMIT {}".format(num_times)
......
......@@ -15,7 +15,7 @@ sectors = "radm1 radm2 radc radf"
sectors = os.getenv("WMS_SECTORS", sectors).split(" ")
products = ["C{:02d}".format(x) for x in range(1, 17)] + ["true_color"]
products = os.getenv("WMS_PRODUCTS", " ".join(products)).split(" ")
platforms = "g16 g17"
platforms = "g16 g17 g18"
platforms = os.getenv("WMS_PLATFORMS", platforms).split(" ")
layer_base_dir = os.environ['LAYER_BASE_DIR']
......@@ -31,22 +31,38 @@ SECTOR_LONG_NAMES = {
PLATFORM_LONG_NAMES = {
'g16': 'GOES-16',
'g17': 'GOES-17',
'g18': 'GOES-18',
}
PLATFORM_EPSGS = {
'g16': 'EPSG:930916',
'g17': 'EPSG:930917',
'g18': 'EPSG:930918',
}
PLATFORM_POS_NAMES = {
'g16': 'GOES-EAST',
'g17': 'GOES-WEST',
'g18': 'GOES-WEST',
}
LAYER_TO_FORMAT = {
'C01': 'gray',
'true_color': 'rgba'
}
def _get_connect_str():
pw_filename = "__POSTGRES_PASSWORD_FILE__"
connect_str = "host=__POSTGRES_HOST__ " \
"port=__POSTGRES_PORT__ " \
"dbname=__POSTGRES_DBNAME__ " \
"user=__POSTGRES_USER__"
if not os.path.isfile(pw_filename) or '__' in connect_str:
print(pw_filename, os.path.isfile(pw_filename), connect_str)
raise ValueError("Backend has not been configured properly "
"and doesn't know how to communicate with the database. "
"Please contact site administrator.")
with open(pw_filename, 'r') as pw_file:
password = pw_file.read().strip()
connect_str += " password={}".format(password)
return connect_str
def main():
......@@ -58,6 +74,8 @@ def main():
help='Format string for each output mapfiles.')
args = parser.parse_args()
connect_params = _get_connect_str() if "__" not in "__POSTGRES_HOST__" else ""
for platform in platforms:
for sector in sectors:
tmpl_vars = {
......@@ -71,7 +89,8 @@ def main():
'epsg_code': PLATFORM_EPSGS[platform].replace('EPSG:', ''),
'PLATFORM_EPSGS': PLATFORM_EPSGS,
'PLATFORM_POS_NAMES': PLATFORM_POS_NAMES,
'postgis_connection_params': None, # TODO
'postgis_connection_params': connect_params,
'postgis_geom_column': 'bbox_geometry',
}
output_fn = args.output_pattern.format(**tmpl_vars)
......
#!/bin/bash -le
replace_pg_params() {
fn=$1
sed -i "s:__POSTGRES_HOST__:${POSTGRES_HOST}:g" $fn
sed -i "s:__POSTGRES_PORT__:${POSTGRES_PORT:-"5432"}:g" $fn
sed -i "s:__POSTGRES_DBNAME__:${POSTGRES_DBNAME:-"postgres"}:g" $fn
sed -i "s:__POSTGRES_USER__:${POSTGRES_USER:-"postgres"}:g" $fn
sed -i "s:__POSTGRES_PASSWORD_FILE__:${POSTGRES_PASSWORD_FILE:-"__POSTGRES_PASSWORD_FILE__"}:g" $fn
}
# Load environment variable options to overwrite in the config
export LAYER_BASE_DIR=${LAYER_BASE_DIR:-"/data/tiles"}
......@@ -7,20 +17,18 @@ export LAYER_BASE_DIR=${LAYER_BASE_DIR:-"/data/tiles"}
mf_tmpl="/work/abi_l1b_template.map"
export WMS_PLATFORMS=${WMS_PLATFORMS:-"g16 g17"}
export WMS_SECTORS=${WMS_SECTORS:-"radm1 radm2 radc radf"}
python3 /work/render.py $mf_tmpl "/work/mapfiles/{platform}_abi_{sector}_l1b.map"
sed -i "s:__LAYER_BASE_DIR__:$LAYER_BASE_DIR:g" /etc/apache2/sites-available/cspp_geo.conf
if [[ ${POSTGRES_HOST} != "" ]]; then
sed -i "s:wms_times_postgres:wms_times:g" /etc/apache2/sites-available/cspp_geo.conf
sed -i "s:__POSTGRES_HOST__:${POSTGRES_HOST}:g" /usr/lib/cgi-bin/layer_times_postgres.py
sed -i "s:__POSTGRES_PORT__:${POSTGRES_PORT:-"5432"}:g" /usr/lib/cgi-bin/layer_times_postgres.py
sed -i "s:__POSTGRES_DBNAME__:${POSTGRES_DBNAME:-"postgres"}:g" /usr/lib/cgi-bin/layer_times_postgres.py
sed -i "s:__POSTGRES_USER__:${POSTGRES_USER:-"postgres"}:g" /usr/lib/cgi-bin/layer_times_postgres.py
sed -i "s:__POSTGRES_PASSWORD_FILE__:${POSTGRES_PASSWORD_FILE:-"__POSTGRES_PASSWORD_FILE__"}:g" /usr/lib/cgi-bin/layer_times_postgres.py
replace_pg_params /usr/lib/cgi-bin/layer_times_postgres.py
replace_pg_params /work/render.py
else
sed -i "s:wms_times_shapes:wms_times:g" /etc/apache2/sites-available/cspp_geo.conf
fi
python3 /work/render.py $mf_tmpl "/work/mapfiles/{platform}_abi_{sector}_l1b.map"
/usr/sbin/apache2ctl configtest
/usr/sbin/apache2ctl -DFOREGROUND
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment