Skip to content
Snippets Groups Projects
Commit 495d1393 authored by David Hoese's avatar David Hoese
Browse files

Merge branch 'feature-pg-index' into 'master'

Switch to pulling tile index information from postgres server

See merge request !3
parents 666a3011 210a7dce
No related branches found
No related tags found
1 merge request!3Switch to pulling tile index information from postgres server
Pipeline #37086 passed with stages
in 1 minute and 45 seconds
......@@ -93,7 +93,7 @@ add_postgres_tables() {
docker exec -i ${PG_SERVER_NAME} psql -U ${POSTGRES_USER} <<EOF
CREATE TABLE IF NOT EXISTS g16_abi_radf_l1b_c01 (
gid SERIAL PRIMARY KEY,
start_time CHAR(19) NOT NULL UNIQUE,
start_time TIMESTAMP WITHOUT TIME ZONE NOT NULL UNIQUE,
location VARCHAR(255) NOT NULL,
bbox_geometry geometry(POLYGON, 930916)
)
......@@ -186,6 +186,9 @@ kill_test_container() {
kill_postgres() {
debug "Killing postgres container..."
debug "-----------------------------------------"
docker logs ${PG_SERVER_NAME}
debug "-----------------------------------------"
docker kill ${PG_SERVER_NAME} >/dev/null
debug "Done killing postgres container."
}
......@@ -201,6 +204,8 @@ curl_layer_times() {
time_result=$(curl --fail -sS "http://localhost:8888/wms_times/g16/abi/radf/C01")
time_status=$?
if [[ $time_status -ne 0 ]]; then
error "Requesting layer times failed"
error "$time_result"
return $time_status
fi
......@@ -218,14 +223,12 @@ curl_empty_tile() {
debug "Starting curl basic mapfile request for non-existent tile..."
# NOTE: The time doesn't actually exist and no image data is available. A blank image should be returned
curl --fail -sS -o "empty_tile.png" "http://localhost:8888/wms/g16/abi/radf/l1b?VERSION=1.1.1&REQUEST=GetMap&SERVICE=WMS&STYLES=&BBOX=-2500000%2c-2500000%2c2500000%2c2500000&WIDTH=256&HEIGHT=256&FORMAT=rgba&SRS=EPSG%3a930916&LAYERS=C01&TIME=2022-04-20T16:00:12Z" >/dev/null
cp empty_tile.png /tmp/empty_tile_test.png
check_image_content "empty_tile.png" 1
}
curl_valid_tile() {
debug "Starting curl basic mapfile request for valid tile..."
curl --fail -sS -o "valid_tile.png" "http://localhost:8888/wms/g16/abi/radf/l1b?VERSION=1.1.1&REQUEST=GetMap&SERVICE=WMS&STYLES=&BBOX=-2500000%2c-2500000%2c2500000%2c2500000&WIDTH=256&HEIGHT=256&FORMAT=rgba&SRS=EPSG%3a930916&LAYERS=C01&TIME=${C01_ISOTIME}Z" >/dev/null
cp valid_tile.png /tmp/valid_tile.png
check_image_content "valid_tile.png" 0
}
......@@ -285,7 +288,6 @@ run_basic_postgres_tests() {
start_postgres
start_pg_test_container
gtiff_fn="/data/$(create_fake_geotiff)"
add_shapefile_content "${gtiff_fn}"
add_postgres_projections
add_postgres_tables
add_postgres_content "${gtiff_fn}"
......
......@@ -23,23 +23,38 @@ MAP
# {{ product.replace('_', ' ').title() }} #
LAYER
NAME "{{ product }}_index"
NAME "{{ product }}_shapefile_index"
TYPE TILEINDEX
DATA "{{ layer_base_dir }}/{{ platform }}/abi/{{ sector }}/{{ product }}/{{ product }}"
END
LAYER
NAME "{{ product }}_postgres_index"
TYPE POLYGON
DATA "{{ postgis_geom_column }} from {{ platform }}_abi_{{ sector }}_l1b_{{ product.lower() }} using SRID={{ epsg_code }} using unique gid"
CONNECTIONTYPE postgis
CONNECTION "{{ postgis_connection_params }}"
METADATA
"wms_title" "{{ platform_long_name }} ABI {{ product.replace('_', ' ').title() }} Time Index"
"wms srs" "EPSG:{{ epsg_code }}"
"wms_extent" "-180 -90 180 90"
"wms_timeextent" "2017-01-01/2040-12-31"
"wms_timeformat" "YYYY-MM-DDTHH:MM:SS"
"wms_timeitem" "start_time" # column in postgres DB
"wms_timedefault" "2019-12-12T19:20:18"
"wms_enable_request" "*"
END
END
LAYER
NAME "{{ product }}"
TYPE RASTER
{% if postgis_connection_params %}
CONNECTIONTYPE postgis
CONNECTION "{{ postgis_connection_params }}"
TILEITEM "location"
TILEINDEX "{{ product }}_index"
#DATA "{{ postgis_geom_column }} from {{ postgis_table }} using SRID={{ epsg_code }} using unique id"
TILEINDEX "{{ product }}_postgres_index"
{% else %}
TILEITEM "location"
TILEINDEX "{{ product }}_index"
TILEINDEX "{{ product }}_shapefile_index"
{% endif %}
# Comment below to default to transparency
......@@ -53,10 +68,15 @@ MAP
PROCESSING "CLOSE_CONNECTION=DEFER"
METADATA
"wms_title" "{{ platform_long_name }} ABI {{ product.replace('_', ' ').title() }}"
"wms srs" "EPSG:{{ epsg_code }}"
"wms_extent" "-180 -90 180 90"
"wms_timeextent" "2017-01-01/2040-12-31"
"wms_timeformat" "YYYY-MM-DDTHH:MM:SS"
"wms_timeitem" "time" # time is a metadata item
{% if postgis_connection_params %}
"wms_timeitem" "start_time" # column in postgres DB
{% else %}
"wms_timeitem" "time" # field in shapefile
{% endif %}
"wms_timedefault" "2019-12-12T19:20:18"
"wms_enable_request" "*"
END
......
......@@ -74,15 +74,15 @@ order = order.upper()
if order not in ('ASC', 'DESC'):
bad_request("""'order' must be either 'ASC' or 'DESC'.""")
query_str = "SELECT start_time FROM {}"
query_str = """SELECT to_char(start_time, 'YYYY-MM-DD"T"HH24:MI:SS') FROM {}"""
if start_time is not None or end_time is not None:
query_str += " WHERE "
if start_time is not None:
query_str += "start_time >= '{}'".format(start_time.strftime('%Y-%m-%dT%H:%M:%S'))
query_str += "start_time >= timestamp '{}'".format(start_time.strftime('%Y-%m-%dT%H:%M:%S'))
if start_time is not None and end_time is not None:
query_str += " AND "
if end_time is not None:
query_str += "start_time <= '{}'".format(end_time.strftime('%Y-%m-%dT%H:%M:%S'))
query_str += "start_time <= timestamp '{}'".format(end_time.strftime('%Y-%m-%dT%H:%M:%S'))
query_str += " ORDER BY start_time {}".format(order)
if num_times != 0:
query_str += " LIMIT {}".format(num_times)
......
......@@ -15,7 +15,7 @@ sectors = "radm1 radm2 radc radf"
sectors = os.getenv("WMS_SECTORS", sectors).split(" ")
products = ["C{:02d}".format(x) for x in range(1, 17)] + ["true_color"]
products = os.getenv("WMS_PRODUCTS", " ".join(products)).split(" ")
platforms = "g16 g17"
platforms = "g16 g17 g18"
platforms = os.getenv("WMS_PLATFORMS", platforms).split(" ")
layer_base_dir = os.environ['LAYER_BASE_DIR']
......@@ -31,22 +31,38 @@ SECTOR_LONG_NAMES = {
PLATFORM_LONG_NAMES = {
'g16': 'GOES-16',
'g17': 'GOES-17',
'g18': 'GOES-18',
}
PLATFORM_EPSGS = {
'g16': 'EPSG:930916',
'g17': 'EPSG:930917',
'g18': 'EPSG:930918',
}
PLATFORM_POS_NAMES = {
'g16': 'GOES-EAST',
'g17': 'GOES-WEST',
'g18': 'GOES-WEST',
}
LAYER_TO_FORMAT = {
'C01': 'gray',
'true_color': 'rgba'
}
def _get_connect_str():
pw_filename = "__POSTGRES_PASSWORD_FILE__"
connect_str = "host=__POSTGRES_HOST__ " \
"port=__POSTGRES_PORT__ " \
"dbname=__POSTGRES_DBNAME__ " \
"user=__POSTGRES_USER__"
if not os.path.isfile(pw_filename) or '__' in connect_str:
print(pw_filename, os.path.isfile(pw_filename), connect_str)
raise ValueError("Backend has not been configured properly "
"and doesn't know how to communicate with the database. "
"Please contact site administrator.")
with open(pw_filename, 'r') as pw_file:
password = pw_file.read().strip()
connect_str += " password={}".format(password)
return connect_str
def main():
......@@ -58,6 +74,8 @@ def main():
help='Format string for each output mapfiles.')
args = parser.parse_args()
connect_params = _get_connect_str() if "__" not in "__POSTGRES_HOST__" else ""
for platform in platforms:
for sector in sectors:
tmpl_vars = {
......@@ -71,7 +89,8 @@ def main():
'epsg_code': PLATFORM_EPSGS[platform].replace('EPSG:', ''),
'PLATFORM_EPSGS': PLATFORM_EPSGS,
'PLATFORM_POS_NAMES': PLATFORM_POS_NAMES,
'postgis_connection_params': None, # TODO
'postgis_connection_params': connect_params,
'postgis_geom_column': 'bbox_geometry',
}
output_fn = args.output_pattern.format(**tmpl_vars)
......
#!/bin/bash -le
replace_pg_params() {
fn=$1
sed -i "s:__POSTGRES_HOST__:${POSTGRES_HOST}:g" $fn
sed -i "s:__POSTGRES_PORT__:${POSTGRES_PORT:-"5432"}:g" $fn
sed -i "s:__POSTGRES_DBNAME__:${POSTGRES_DBNAME:-"postgres"}:g" $fn
sed -i "s:__POSTGRES_USER__:${POSTGRES_USER:-"postgres"}:g" $fn
sed -i "s:__POSTGRES_PASSWORD_FILE__:${POSTGRES_PASSWORD_FILE:-"__POSTGRES_PASSWORD_FILE__"}:g" $fn
}
# Load environment variable options to overwrite in the config
export LAYER_BASE_DIR=${LAYER_BASE_DIR:-"/data/tiles"}
......@@ -7,20 +17,18 @@ export LAYER_BASE_DIR=${LAYER_BASE_DIR:-"/data/tiles"}
mf_tmpl="/work/abi_l1b_template.map"
export WMS_PLATFORMS=${WMS_PLATFORMS:-"g16 g17"}
export WMS_SECTORS=${WMS_SECTORS:-"radm1 radm2 radc radf"}
python3 /work/render.py $mf_tmpl "/work/mapfiles/{platform}_abi_{sector}_l1b.map"
sed -i "s:__LAYER_BASE_DIR__:$LAYER_BASE_DIR:g" /etc/apache2/sites-available/cspp_geo.conf
if [[ ${POSTGRES_HOST} != "" ]]; then
sed -i "s:wms_times_postgres:wms_times:g" /etc/apache2/sites-available/cspp_geo.conf
sed -i "s:__POSTGRES_HOST__:${POSTGRES_HOST}:g" /usr/lib/cgi-bin/layer_times_postgres.py
sed -i "s:__POSTGRES_PORT__:${POSTGRES_PORT:-"5432"}:g" /usr/lib/cgi-bin/layer_times_postgres.py
sed -i "s:__POSTGRES_DBNAME__:${POSTGRES_DBNAME:-"postgres"}:g" /usr/lib/cgi-bin/layer_times_postgres.py
sed -i "s:__POSTGRES_USER__:${POSTGRES_USER:-"postgres"}:g" /usr/lib/cgi-bin/layer_times_postgres.py
sed -i "s:__POSTGRES_PASSWORD_FILE__:${POSTGRES_PASSWORD_FILE:-"__POSTGRES_PASSWORD_FILE__"}:g" /usr/lib/cgi-bin/layer_times_postgres.py
replace_pg_params /usr/lib/cgi-bin/layer_times_postgres.py
replace_pg_params /work/render.py
else
sed -i "s:wms_times_shapes:wms_times:g" /etc/apache2/sites-available/cspp_geo.conf
fi
python3 /work/render.py $mf_tmpl "/work/mapfiles/{platform}_abi_{sector}_l1b.map"
/usr/sbin/apache2ctl configtest
/usr/sbin/apache2ctl -DFOREGROUND
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment