Merge branch 'feature-pg-index' into 'master'

Switch to pulling tile index information from postgres server

See merge request !3

ci/test_mapserver_image.sh

@@ -93,7 +93,7 @@ add_postgres_tables() {
     docker exec -i ${PG_SERVER_NAME} psql -U ${POSTGRES_USER} <<EOF
 CREATE TABLE IF NOT EXISTS g16_abi_radf_l1b_c01 (
     gid SERIAL PRIMARY KEY,
-    start_time CHAR(19) NOT NULL UNIQUE,
+    start_time TIMESTAMP WITHOUT TIME ZONE NOT NULL UNIQUE,
     location VARCHAR(255) NOT NULL,
     bbox_geometry geometry(POLYGON, 930916)
     )
@@ -186,6 +186,9 @@ kill_test_container() {
 
 kill_postgres() {
     debug "Killing postgres container..."
+    debug "-----------------------------------------"
+    docker logs ${PG_SERVER_NAME}
+    debug "-----------------------------------------"
     docker kill ${PG_SERVER_NAME} >/dev/null
     debug "Done killing postgres container."
 }
@@ -201,6 +204,8 @@ curl_layer_times() {
     time_result=$(curl --fail -sS "http://localhost:8888/wms_times/g16/abi/radf/C01")
     time_status=$?
     if [[ $time_status -ne 0 ]]; then
+        error "Requesting layer times failed"
+        error "$time_result"
         return $time_status
     fi
 
@@ -218,14 +223,12 @@ curl_empty_tile() {
     debug "Starting curl basic mapfile request for non-existent tile..."
     # NOTE: The time doesn't actually exist and no image data is available. A blank image should be returned
     curl --fail -sS -o "empty_tile.png" "http://localhost:8888/wms/g16/abi/radf/l1b?VERSION=1.1.1&REQUEST=GetMap&SERVICE=WMS&STYLES=&BBOX=-2500000%2c-2500000%2c2500000%2c2500000&WIDTH=256&HEIGHT=256&FORMAT=rgba&SRS=EPSG%3a930916&LAYERS=C01&TIME=2022-04-20T16:00:12Z" >/dev/null
-    cp empty_tile.png /tmp/empty_tile_test.png
     check_image_content "empty_tile.png" 1
 }
 
 curl_valid_tile() {
     debug "Starting curl basic mapfile request for valid tile..."
     curl --fail -sS -o "valid_tile.png" "http://localhost:8888/wms/g16/abi/radf/l1b?VERSION=1.1.1&REQUEST=GetMap&SERVICE=WMS&STYLES=&BBOX=-2500000%2c-2500000%2c2500000%2c2500000&WIDTH=256&HEIGHT=256&FORMAT=rgba&SRS=EPSG%3a930916&LAYERS=C01&TIME=${C01_ISOTIME}Z" >/dev/null
-    cp valid_tile.png /tmp/valid_tile.png
     check_image_content "valid_tile.png" 0
 }
 
@@ -285,7 +288,6 @@ run_basic_postgres_tests() {
     start_postgres
     start_pg_test_container
     gtiff_fn="/data/$(create_fake_geotiff)"
-    add_shapefile_content "${gtiff_fn}"
     add_postgres_projections
     add_postgres_tables
     add_postgres_content "${gtiff_fn}"

mapserver/abi_l1b_template.map

@@ -23,23 +23,38 @@ MAP
   # {{ product.replace('_', ' ').title() }} #
 
   LAYER
-    NAME "{{ product }}_index"
+    NAME "{{ product }}_shapefile_index"
     TYPE TILEINDEX
     DATA "{{ layer_base_dir }}/{{ platform }}/abi/{{ sector }}/{{ product }}/{{ product }}"
   END
 
+  LAYER
+    NAME "{{ product }}_postgres_index"
+    TYPE POLYGON
+    DATA "{{ postgis_geom_column }} from {{ platform }}_abi_{{ sector }}_l1b_{{ product.lower() }} using SRID={{ epsg_code }} using unique gid"
+    CONNECTIONTYPE postgis
+    CONNECTION "{{ postgis_connection_params }}"
+    METADATA
+      "wms_title" "{{ platform_long_name }} ABI {{ product.replace('_', ' ').title() }} Time Index"
+      "wms srs" "EPSG:{{ epsg_code }}"
+      "wms_extent" "-180 -90 180 90"
+      "wms_timeextent" "2017-01-01/2040-12-31"
+      "wms_timeformat" "YYYY-MM-DDTHH:MM:SS"
+      "wms_timeitem" "start_time" # column in postgres DB
+      "wms_timedefault" "2019-12-12T19:20:18"
+      "wms_enable_request" "*"
+    END
+  END
+
   LAYER
     NAME "{{ product }}"
     TYPE RASTER
     {% if postgis_connection_params %}
-    CONNECTIONTYPE postgis
-    CONNECTION "{{ postgis_connection_params }}"
     TILEITEM "location"
-    TILEINDEX "{{ product }}_index"
-    #DATA "{{ postgis_geom_column }} from {{ postgis_table }} using SRID={{ epsg_code }} using unique id"
+    TILEINDEX "{{ product }}_postgres_index"
     {% else %}
     TILEITEM "location"
-    TILEINDEX "{{ product }}_index"
+    TILEINDEX "{{ product }}_shapefile_index"
     {% endif %}
 
     # Comment below to default to transparency
@@ -53,10 +68,15 @@ MAP
     PROCESSING "CLOSE_CONNECTION=DEFER"
     METADATA
       "wms_title" "{{ platform_long_name }} ABI {{ product.replace('_', ' ').title() }}"
+      "wms srs" "EPSG:{{ epsg_code }}"
       "wms_extent" "-180 -90 180 90"
       "wms_timeextent" "2017-01-01/2040-12-31"
       "wms_timeformat" "YYYY-MM-DDTHH:MM:SS"
-      "wms_timeitem" "time" # time is a metadata item
+      {% if postgis_connection_params %}
+      "wms_timeitem" "start_time" # column in postgres DB
+      {% else %}
+      "wms_timeitem" "time" # field in shapefile
+      {% endif %}
       "wms_timedefault" "2019-12-12T19:20:18"
       "wms_enable_request" "*"
     END

mapserver/cgi-bin/layer_times_postgres.py

@@ -74,15 +74,15 @@ order = order.upper()
 if order not in ('ASC', 'DESC'):
     bad_request("""'order' must be either 'ASC' or 'DESC'.""")
 
-query_str = "SELECT start_time FROM {}"
+query_str = """SELECT to_char(start_time, 'YYYY-MM-DD"T"HH24:MI:SS') FROM {}"""
 if start_time is not None or end_time is not None:
     query_str += " WHERE "
 if start_time is not None:
-    query_str += "start_time >= '{}'".format(start_time.strftime('%Y-%m-%dT%H:%M:%S'))
+    query_str += "start_time >= timestamp '{}'".format(start_time.strftime('%Y-%m-%dT%H:%M:%S'))
 if start_time is not None and end_time is not None:
     query_str += " AND "
 if end_time is not None:
-    query_str += "start_time <= '{}'".format(end_time.strftime('%Y-%m-%dT%H:%M:%S'))
+    query_str += "start_time <= timestamp '{}'".format(end_time.strftime('%Y-%m-%dT%H:%M:%S'))
 query_str += " ORDER BY start_time {}".format(order)
 if num_times != 0:
     query_str += " LIMIT {}".format(num_times)

mapserver/render.py

@@ -15,7 +15,7 @@ sectors = "radm1 radm2 radc radf"
 sectors = os.getenv("WMS_SECTORS", sectors).split(" ")
 products = ["C{:02d}".format(x) for x in range(1, 17)] + ["true_color"]
 products = os.getenv("WMS_PRODUCTS", " ".join(products)).split(" ")
-platforms = "g16 g17"
+platforms = "g16 g17 g18"
 platforms = os.getenv("WMS_PLATFORMS", platforms).split(" ")
 
 layer_base_dir = os.environ['LAYER_BASE_DIR']
@@ -31,22 +31,38 @@ SECTOR_LONG_NAMES = {
 PLATFORM_LONG_NAMES = {
     'g16': 'GOES-16',
     'g17': 'GOES-17',
+    'g18': 'GOES-18',
 }
 
 PLATFORM_EPSGS = {
     'g16': 'EPSG:930916',
     'g17': 'EPSG:930917',
+    'g18': 'EPSG:930918',
 }
 
 PLATFORM_POS_NAMES = {
     'g16': 'GOES-EAST',
     'g17': 'GOES-WEST',
+    'g18': 'GOES-WEST',
 }
 
-LAYER_TO_FORMAT = {
-    'C01': 'gray',
-    'true_color': 'rgba'
-}
+
+def _get_connect_str():
+    pw_filename = "__POSTGRES_PASSWORD_FILE__"
+    connect_str = "host=__POSTGRES_HOST__ " \
+                  "port=__POSTGRES_PORT__ " \
+                  "dbname=__POSTGRES_DBNAME__ " \
+                  "user=__POSTGRES_USER__"
+    if not os.path.isfile(pw_filename) or '__' in connect_str:
+        print(pw_filename, os.path.isfile(pw_filename), connect_str)
+        raise ValueError("Backend has not been configured properly "
+                         "and doesn't know how to communicate with the database. "
+                         "Please contact site administrator.")
+
+    with open(pw_filename, 'r') as pw_file:
+        password = pw_file.read().strip()
+        connect_str += " password={}".format(password)
+    return connect_str
 
 
 def main():
@@ -58,6 +74,8 @@ def main():
                         help='Format string for each output mapfiles.')
     args = parser.parse_args()
 
+    connect_params = _get_connect_str() if "__" not in "__POSTGRES_HOST__" else ""
+
     for platform in platforms:
         for sector in sectors:
             tmpl_vars = {
@@ -71,7 +89,8 @@ def main():
                 'epsg_code': PLATFORM_EPSGS[platform].replace('EPSG:', ''),
                 'PLATFORM_EPSGS': PLATFORM_EPSGS,
                 'PLATFORM_POS_NAMES': PLATFORM_POS_NAMES,
-                'postgis_connection_params': None,  # TODO
+                'postgis_connection_params': connect_params,
+                'postgis_geom_column': 'bbox_geometry',
             }
             output_fn = args.output_pattern.format(**tmpl_vars)
 

mapserver/run.sh

 #!/bin/bash -le
 
+replace_pg_params() {
+    fn=$1
+    sed -i "s:__POSTGRES_HOST__:${POSTGRES_HOST}:g" $fn
+    sed -i "s:__POSTGRES_PORT__:${POSTGRES_PORT:-"5432"}:g" $fn
+    sed -i "s:__POSTGRES_DBNAME__:${POSTGRES_DBNAME:-"postgres"}:g" $fn
+    sed -i "s:__POSTGRES_USER__:${POSTGRES_USER:-"postgres"}:g" $fn
+    sed -i "s:__POSTGRES_PASSWORD_FILE__:${POSTGRES_PASSWORD_FILE:-"__POSTGRES_PASSWORD_FILE__"}:g" $fn
+}
+
+
 # Load environment variable options to overwrite in the config
 export LAYER_BASE_DIR=${LAYER_BASE_DIR:-"/data/tiles"}
 
@@ -7,20 +17,18 @@ export LAYER_BASE_DIR=${LAYER_BASE_DIR:-"/data/tiles"}
 mf_tmpl="/work/abi_l1b_template.map"
 export WMS_PLATFORMS=${WMS_PLATFORMS:-"g16 g17"}
 export WMS_SECTORS=${WMS_SECTORS:-"radm1 radm2 radc radf"}
-python3 /work/render.py $mf_tmpl "/work/mapfiles/{platform}_abi_{sector}_l1b.map"
 sed -i "s:__LAYER_BASE_DIR__:$LAYER_BASE_DIR:g" /etc/apache2/sites-available/cspp_geo.conf
 
 if [[ ${POSTGRES_HOST} != "" ]]; then
     sed -i "s:wms_times_postgres:wms_times:g" /etc/apache2/sites-available/cspp_geo.conf
-    sed -i "s:__POSTGRES_HOST__:${POSTGRES_HOST}:g" /usr/lib/cgi-bin/layer_times_postgres.py
-    sed -i "s:__POSTGRES_PORT__:${POSTGRES_PORT:-"5432"}:g" /usr/lib/cgi-bin/layer_times_postgres.py
-    sed -i "s:__POSTGRES_DBNAME__:${POSTGRES_DBNAME:-"postgres"}:g" /usr/lib/cgi-bin/layer_times_postgres.py
-    sed -i "s:__POSTGRES_USER__:${POSTGRES_USER:-"postgres"}:g" /usr/lib/cgi-bin/layer_times_postgres.py
-    sed -i "s:__POSTGRES_PASSWORD_FILE__:${POSTGRES_PASSWORD_FILE:-"__POSTGRES_PASSWORD_FILE__"}:g" /usr/lib/cgi-bin/layer_times_postgres.py
+    replace_pg_params /usr/lib/cgi-bin/layer_times_postgres.py
+    replace_pg_params /work/render.py
 else
     sed -i "s:wms_times_shapes:wms_times:g" /etc/apache2/sites-available/cspp_geo.conf
 fi
 
+python3 /work/render.py $mf_tmpl "/work/mapfiles/{platform}_abi_{sector}_l1b.map"
+
 /usr/sbin/apache2ctl configtest
 
 /usr/sbin/apache2ctl -DFOREGROUND