test_mapserver_image.sh

#!/usr/bin/env bash

debug() {
    echo >&2 "DEBUG: $@"
}

error() {
    echo >&2 "ERROR: $@"
}

if [[ $# -eq 1 ]]; then
    image=$1
elif [[ $# -eq 2 ]]; then
    # deprecated: use single URL:tag from now on
    image="$1:$2"
else
    error "Usage: ./test_mapserver_image.sh <image_url:image_tag>"
    exit 1
fi

MINIO_SERVER_NAME="test-minio-server"
export AWS_ACCESS_KEY_ID="minioadmin"
export AWS_SECRET_ACCESS_KEY="minioadmin"
S3_PORT=9000
PG_PORT=5432
PG_SERVER_NAME="postgres_db"
POSTGRES_USER="postgres"
POSTGRES_PASSWORD="1234"
NETWORK_NAME=`basename "$0"`
NETWORK_NAME=${NETWORK_NAME/.sh/}
C01_GTIFF_NAME="GOES-16_ABI_RadF_C01_20220302_194032_GOES-West.tif"
C01_ISOTIME="2022-03-02T19:40:32"
BASE_WORK_DIR=${BASE_WORK_DIR:-${TMPDIR:-/tmp}}
base_tmp_dir="__TOBECREATED__"

setup_test() {
    base_tmp_dir=$(mktemp -d ${BASE_WORK_DIR}/mapserver-tests-XXXXXXX)
    cd "${base_tmp_dir}"
    chmod 777 .
    echo "Temporary directory: ${base_tmp_dir}"

    docker network create --opt com.docker.network.driver.mtu=1400 ${NETWORK_NAME}
}

teardown_test() {
    kill_test_container || debug "Could not kill test container"
    kill_postgres || debug "Could not kill postgres container"
    kill_minio || debug "Could not kill minio container"
    if [ -d $base_tmp_dir ]; then
        docker run --rm --name tmpremover -v ${base_tmp_dir}:/dst ${image} bash -c "chmod -R 777 /dst"
        rm -rf $base_tmp_dir || debug "Could not remove temporary directory: ${base_tmp_dir}"
    fi
    docker network rm ${NETWORK_NAME} > /dev/null || debug "Could not remove docker network"
}

graceful_exit() {
    debug "FAIL"
    debug "Graceful exit"
    teardown_test
}

start_test_container() {
    debug "Starting test docker container (${image})..."
    docker run --rm -d --network ${NETWORK_NAME} --name test -p 8888:80 $@ ${image}
    start_status=$?
    # just wait a bit to let the server start
    sleep 2
    debug "Container started."
    debug "Installing rasterio into test container for geotiff creation"
    docker exec -i test python3 -m pip install --break-system-packages --root-user-action ignore rasterio fiona shapely
    return $start_status
}

start_shapefile_test_container() {
    start_test_container -v "$(pwd)":"/data"
}

start_pg_test_container() {
    mkdir pg_secrets
    echo "${POSTGRES_PASSWORD}" > pg_secrets/fake_file
    start_test_container -v "$(pwd)":"/data" -e POSTGRES_HOST=${PG_SERVER_NAME} -e POSTGRES_PORT=${PG_PORT} -e POSTGRES_PASSWORD_FILE="/data/pg_secrets/fake_file" "$@"
}

start_s3_pg_test_container() {
    start_pg_test_container -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} -e AWS_S3_ENDPOINT="${MINIO_SERVER_NAME}" -e AWS_S3_ENDPOINT_PORT="9000"
}

start_postgres() {
    debug "Starting Postgres database..."
    docker run --rm -d --network ${NETWORK_NAME} --name ${PG_SERVER_NAME} -e POSTGRES_PASSWORD="${POSTGRES_PASSWORD}" postgis/postgis
    create_status=$?
    debug "Sleeping for 5 seconds for DB to start up..."
    sleep 5
    return $create_status
}

start_minio() {
    base_dir=$1
    base_minio_dir="$1/minio_base"
    mkdir -p "${base_minio_dir}"
    docker run -d --rm --network ${NETWORK_NAME} -p 9000:9000 -p 9001:9001 --name ${MINIO_SERVER_NAME} --user $(id -u):$(id -g) -v ${base_minio_dir}:/data minio/minio server /data --console-address ":9001"
    create_status=$?
    debug "Sleeping for 5 seconds for MinIO to start up..."
    sleep 5
    return $create_status
}

kill_test_container() {
    debug "Killing docker container..."
    debug "-----------------------------------------"
    docker exec -i test bash -c "if [ -f /tmp/ms_error.txt ]; then cat /tmp/ms_error.txt; else echo 'No mapserver log file'; fi"
    debug "-----"
    docker logs test
    debug "-----------------------------------------"
    docker kill test >/dev/null
    debug "Done killing docker container."
}

kill_postgres() {
    debug "Killing postgres container..."
    debug "-----------------------------------------"
    docker logs ${PG_SERVER_NAME}
    debug "-----------------------------------------"
    docker kill ${PG_SERVER_NAME} >/dev/null
    debug "Done killing postgres container."
}

kill_minio() {
    debug "Killing MinIO container..."
    debug "-----------------------------------------"
    # To see HTTP trace run the following in a separate terminal once test MinIO container is started:
    # docker run --rm -it --network test_mapserver_image --entrypoint=/bin/sh minio/mc -c "mc alias set test http://test-minio-server:9000 minioadmin minioadmin --api s3v4 --path off; mc admin trace test"
    docker logs ${MINIO_SERVER_NAME}
    debug "-----------------------------------------"
    docker kill ${MINIO_SERVER_NAME} >/dev/null
    debug "Done killing minio container."
}

add_shapefile_content() {
    debug "Creating fake shapefile directory for C01"
    gtiff_location=$1
    docker exec test bash -c "mkdir -p /data/tiles/g16/abi/radf/C01; chmod -R a+rwX /data/tiles"
    debug "Creating fake shapefile file for C01"
    docker exec -i test python3 <<EOF
import fiona
from shapely.geometry import mapping, box
from glob import glob

s_file = fiona.open('/data/tiles/g16/abi/radf/C01/C01.shp',
                    'w',
                    driver='ESRI Shapefile',
                    schema={'geometry': 'Polygon', 'properties': {'location': 'str', 'time': 'str:19'}})
bbox = mapping(box(-25000000, -25000000, 25000000, 25000000))
s_file.write({
    "geometry": bbox,
    "properties": {
        "location": "${gtiff_location}",
        "time": "${C01_ISOTIME}",
    },
})
EOF
}

add_postgres_projections() {
    # copied from tile gen
    debug "Creating PostGIS projections"
    docker exec -i ${PG_SERVER_NAME} psql -U ${POSTGRES_USER} <<EOF
INSERT into spatial_ref_sys (srid, auth_name, auth_srid, srtext, proj4text)
values (930916, 'EPSG', 4269, 'PROJCRS["GOES-16 ABI Fixed Grid",BASEGEOGCRS["GOES-16 ABI Fixed Grid",DATUM["North American Datum 1983",ELLIPSOID["GRS 1980",6378137,298.257222101,LENGTHUNIT["metre",1]],ID["EPSG",6269]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8901]]],CONVERSION["unknown",METHOD["Geostationary Satellite (Sweep X)"],PARAMETER["Longitude of natural origin",-75,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8802]],PARAMETER["Satellite Height",35786023,LENGTHUNIT["metre",1,ID["EPSG",9001]]],PARAMETER["False easting",0,LENGTHUNIT["metre",1],ID["EPSG",8806]],PARAMETER["False northing",0,LENGTHUNIT["metre",1],ID["EPSG",8807]]],CS[Cartesian,2],AXIS["(E)",east,ORDER[1],LENGTHUNIT["metre",1,ID["EPSG",9001]]],AXIS["(N)",north,ORDER[2],LENGTHUNIT["metre",1,ID["EPSG",9001]]]]', '+proj=geos +sweep=x +lon_0=-75 +h=35786023 +x_0=0 +y_0=0 +datum=NAD83 +units=m +no_defs +type=crs') ON CONFLICT (srid) DO NOTHING
EOF
}

add_postgres_tables() {
    debug "Creating PostGIS table"
    docker exec -i ${PG_SERVER_NAME} psql -U ${POSTGRES_USER} <<EOF
CREATE TABLE IF NOT EXISTS g16_abi_radf_l1b_c01 (
    gid SERIAL PRIMARY KEY,
    start_time TIMESTAMP WITHOUT TIME ZONE NOT NULL UNIQUE,
    location VARCHAR(255) NOT NULL,
    bbox_geometry geometry(POLYGON, 930916)
    )
EOF
}

add_postgres_content() {
    debug "Creating values for fake DB row for C01"
    gtiff_location=$1
    insert_values=$(docker exec -i test python3 <<EOF
from shapely.geometry import box
from shapely import wkb

dt_str = "${C01_ISOTIME}"
location = "${gtiff_location}"
bbox = box(-25000000, -25000000, 25000000, 25000000)
bbox_wkb = wkb.dumps(bbox, hex=True, srid=930916)
values = (dt_str, location, bbox_wkb)
print(repr(values))
EOF
)
    debug "Inserting fake DB row for C01"
    docker exec -i ${PG_SERVER_NAME} psql -U ${POSTGRES_USER} <<EOF
INSERT into g16_abi_radf_l1b_c01 (start_time, location, bbox_geometry)
    VALUES ${insert_values} ON CONFLICT (start_time) DO NOTHING
    RETURNING (start_time)
EOF
}

create_fake_geotiff() {
    bucket_name="g16-abi-radf-l1b-c01"
    docker exec -i test bash -c "mkdir -p /data/${bucket_name}; chmod -R a+rwX /data/${bucket_name}"
    gtiff_fn="${bucket_name}/${C01_GTIFF_NAME}"
    docker exec -i test python3 <<EOF
import rasterio
from rasterio.transform import Affine
import numpy as np

transform = Affine.translation(-2500 * 1000 - 500, 2500 * 1000 - 500) * Affine.scale(1000, 1000)
with rasterio.open("/data/${gtiff_fn}", "w", driver='GTiff', height=5000, width=5000, dtype=np.uint8, count=1,
        crs="+proj=geos +sweep=x +lon_0=-75.0 +h=35786023 +x_0=0 +y_0=0 +ellps=GRS80 +units=m +no_defs", transform=transform) as gtiff_file:
    data = np.repeat(np.linspace(0, 255, 5000, dtype=np.uint8)[None, :], 5000, axis=0)
    gtiff_file.write(data, 1)
EOF
    creation_status=$?
    echo ${gtiff_fn}
    return $creation_status
}

create_fake_s3_geotiff() {
  fn=$(create_fake_geotiff)
  gtiff_fn=$(basename $fn)
  bucket_name=$(basename $(dirname $fn))

  aws s3api create-bucket --endpoint-url "http://localhost:9000" --bucket "${bucket_name}" 1>&2 || error "Can't create MinIO S3 bucket"
  aws s3 cp --endpoint-url "http://localhost:9000" "${fn}" s3://${bucket_name}/ 1>&2 || error "Can't copy geotiff to S3 bucket"
  echo "${bucket_name}/${gtiff_fn}"
}

curl_index() {
    debug "Starting curl basic request..."
    curl --fail -sS --max-time 5 "http://localhost:8888/" >/dev/null
}

curl_empty_tile() {
    debug "Starting curl basic mapfile request for non-existent tile..."
    # NOTE: The time doesn't actually exist and no image data is available. A blank image should be returned
#    docker exec -it test bash
    curl --fail -sS -o "empty_tile.png" "http://localhost:8888/wms/g16/abi/radf/l1b?VERSION=1.1.1&REQUEST=GetMap&SERVICE=WMS&STYLES=&BBOX=-2500000%2c-2500000%2c2500000%2c2500000&WIDTH=256&HEIGHT=256&FORMAT=rgba&SRS=EPSG%3a930916&LAYERS=C01&TIME=2022-04-20T16:00:12Z" >/dev/null
    check_image_content "empty_tile.png" 1
}

curl_valid_tile() {
    debug "Starting curl basic mapfile request for valid tile..."
#     docker exec -it test bash
    curl --fail -sS -o "valid_tile.png" "http://localhost:8888/wms/g16/abi/radf/l1b?VERSION=1.1.1&REQUEST=GetMap&SERVICE=WMS&STYLES=&BBOX=-2500000%2c-2500000%2c2500000%2c2500000&WIDTH=256&HEIGHT=256&FORMAT=rgba&SRS=EPSG%3a930916&LAYERS=C01&TIME=${C01_ISOTIME}Z" >/dev/null
    check_image_content "valid_tile.png" 0
}

check_image_content() {
    img_filename=$1
    all_zero=$2
    # if it is an XML file then it wasn't a successful download
    grep_found_xml=1
    grep "<?xml" $img_filename || grep_found_xml=0
    if [[ $grep_found_xml -ne 0 ]]; then
        error "Image being checked is XML file"
        cat $img_filename
        return 1
    fi
    docker exec -i test python3 -m pip install --break-system-packages --root-user-action ignore pillow >/dev/null
    incontainer_file="/tmp/$(basename $img_filename)"
    docker cp $img_filename test:${incontainer_file}
    docker exec -i test python3 <<EOF
import sys
import io
import os

import numpy as np
from PIL import Image

img = Image.open("${incontainer_file}")
img_arr = np.asarray(img)
assert img_arr.shape == (256, 256, 4)
if ${all_zero} == 1:
    print("\tChecking RGB data is all 0s")
    assert (img_arr[:, :, :3] == 0).all()
else:
    print("\tChecking RGB data is not all 0s")
    assert (img_arr[:, :, :3] != 0).any()
EOF
}


run_basic_shapefile_tests() {
    setup_test
    debug "Starting shapefile tests..."
    start_shapefile_test_container
    gtiff_fn="/data/$(create_fake_geotiff)"
    add_shapefile_content "${gtiff_fn}"

    curl_index
    curl_empty_tile
    curl_valid_tile
    teardown_test
    debug "SUCCESS: Shapefile test completed successfully"
}

run_basic_postgres_tests() {
    setup_test
    debug "Starting postgres tests..."
    start_postgres
    start_pg_test_container
    gtiff_fn="/data/$(create_fake_geotiff)"
    add_postgres_projections
    add_postgres_tables
    add_postgres_content "${gtiff_fn}"

    curl_index
    curl_empty_tile
    curl_valid_tile
    debug "SUCCESS: Postgres test completed successfully"
    teardown_test
}

run_s3_postgres_tests() {
    setup_test
    debug "Starting S3 Postgres tests..."
    start_postgres
    start_minio "${PWD}"
    start_s3_pg_test_container
    gtiff_fn="/vsis3/$(create_fake_s3_geotiff)"
    add_postgres_projections
    add_postgres_tables
    add_postgres_content "${gtiff_fn}"

    curl_index
    curl_empty_tile
    curl_valid_tile
    debug "SUCCESS: S3 Postgres test completed successfully"
    teardown_test
}


trap graceful_exit EXIT

set -e
run_basic_shapefile_tests
echo "#######"
run_basic_postgres_tests
echo "#######"
run_s3_postgres_tests
echo "#######"

trap - EXIT  # tests should have cleared this already, otherwise produces extra output
debug "SUCCESS"