Newer
Older
if [[ $# -eq 1 ]]; then
image=$1
elif [[ $# -eq 2 ]]; then
# deprecated: use single URL:tag from now on
image="$1:$2"
else
error "Usage: ./test_mapserver_image.sh <image_url:image_tag>"
MINIO_SERVER_NAME="test-minio-server"
export AWS_ACCESS_KEY_ID="minioadmin"
export AWS_SECRET_ACCESS_KEY="minioadmin"
S3_PORT=9000
PG_PORT=5432
PG_SERVER_NAME="postgres_db"
POSTGRES_USER="postgres"
POSTGRES_PASSWORD="1234"
NETWORK_NAME=`basename "$0"`
NETWORK_NAME=${NETWORK_NAME/.sh/}
C01_GTIFF_NAME="GOES-16_ABI_RadF_C01_20220302_194032_GOES-West.tif"
C01_ISOTIME="2022-03-02T19:40:32"
base_tmp_dir="__TOBECREATED__"
setup_test() {
base_tmp_dir=$(mktemp -d ${BASE_WORK_DIR}/mapserver-tests-XXXXXXX)
echo "Temporary directory: ${base_tmp_dir}"
docker network create --opt com.docker.network.driver.mtu=1400 ${NETWORK_NAME}
}
teardown_test() {
kill_test_container || debug "Could not kill test container"
kill_postgres || debug "Could not kill postgres container"
kill_minio || debug "Could not kill minio container"
if [ -d $base_tmp_dir ]; then
docker run --rm --name tmpremover -v ${base_tmp_dir}:/dst ${image} bash -c "chmod -R 777 /dst"
rm -rf $base_tmp_dir || debug "Could not remove temporary directory: ${base_tmp_dir}"
docker network rm ${NETWORK_NAME} > /dev/null || debug "Could not remove docker network"
}
graceful_exit() {
debug "FAIL"
debug "Graceful exit"
teardown_test
}
debug "Starting test docker container (${image})..."
docker run --rm -d --network ${NETWORK_NAME} --name test -p 8888:80 $@ ${image}
start_status=$?
# just wait a bit to let the server start
sleep 2
debug "Container started."
debug "Installing rasterio into test container for geotiff creation"
docker exec -i test python3 -m pip install --break-system-packages --root-user-action ignore rasterio fiona shapely
return $start_status
}
start_shapefile_test_container() {
start_test_container -v "$(pwd)":"/data"
}
start_pg_test_container() {
mkdir pg_secrets
echo "${POSTGRES_PASSWORD}" > pg_secrets/fake_file
start_test_container -v "$(pwd)":"/data" -e POSTGRES_HOST=${PG_SERVER_NAME} -e POSTGRES_PORT=${PG_PORT} -e POSTGRES_PASSWORD_FILE="/data/pg_secrets/fake_file" "$@"
start_s3_pg_test_container() {
start_pg_test_container -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} -e AWS_S3_ENDPOINT="${MINIO_SERVER_NAME}" -e AWS_S3_ENDPOINT_PORT="9000"
start_postgres() {
debug "Starting Postgres database..."
docker run --rm -d --network ${NETWORK_NAME} --name ${PG_SERVER_NAME} -e POSTGRES_PASSWORD="${POSTGRES_PASSWORD}" postgis/postgis
debug "Sleeping for 5 seconds for DB to start up..."
sleep 5
return $create_status
}
start_minio() {
base_dir=$1
base_minio_dir="$1/minio_base"
mkdir -p "${base_minio_dir}"
docker run -d --rm --network ${NETWORK_NAME} -p 9000:9000 -p 9001:9001 --name ${MINIO_SERVER_NAME} --user $(id -u):$(id -g) -v ${base_minio_dir}:/data minio/minio server /data --console-address ":9001"
create_status=$?
debug "Sleeping for 5 seconds for MinIO to start up..."
sleep 5
return $create_status
}
kill_test_container() {
debug "Killing docker container..."
debug "-----------------------------------------"
docker exec -i test bash -c "if [ -f /tmp/ms_error.txt ]; then cat /tmp/ms_error.txt; else echo 'No mapserver log file'; fi"
debug "-----"
docker logs test
debug "-----------------------------------------"
docker kill test >/dev/null
debug "Done killing docker container."
}
kill_postgres() {
debug "Killing postgres container..."
debug "-----------------------------------------"
docker logs ${PG_SERVER_NAME}
debug "-----------------------------------------"
docker kill ${PG_SERVER_NAME} >/dev/null
debug "Done killing postgres container."
}
kill_minio() {
debug "Killing MinIO container..."
debug "-----------------------------------------"
# To see HTTP trace run the following in a separate terminal once test MinIO container is started:
# docker run --rm -it --network test_mapserver_image --entrypoint=/bin/sh minio/mc -c "mc alias set test http://test-minio-server:9000 minioadmin minioadmin --api s3v4 --path off; mc admin trace test"
docker logs ${MINIO_SERVER_NAME}
debug "-----------------------------------------"
docker kill ${MINIO_SERVER_NAME} >/dev/null
debug "Done killing minio container."
}
add_shapefile_content() {
debug "Creating fake shapefile directory for C01"
gtiff_location=$1
docker exec test bash -c "mkdir -p /data/tiles/g16/abi/radf/C01; chmod -R a+rwX /data/tiles"
docker exec -i test python3 <<EOF
import fiona
from shapely.geometry import mapping, box
from glob import glob
s_file = fiona.open('/data/tiles/g16/abi/radf/C01/C01.shp',
'w',
driver='ESRI Shapefile',
schema={'geometry': 'Polygon', 'properties': {'location': 'str', 'time': 'str:19'}})
bbox = mapping(box(-25000000, -25000000, 25000000, 25000000))
s_file.write({
"geometry": bbox,
"properties": {
"location": "${gtiff_location}",
"time": "${C01_ISOTIME}",
},
})
EOF
add_postgres_projections() {
# copied from tile gen
debug "Creating PostGIS projections"
docker exec -i ${PG_SERVER_NAME} psql -U ${POSTGRES_USER} <<EOF
INSERT into spatial_ref_sys (srid, auth_name, auth_srid, srtext, proj4text)
values (930916, 'EPSG', 4269, 'PROJCRS["GOES-16 ABI Fixed Grid",BASEGEOGCRS["GOES-16 ABI Fixed Grid",DATUM["North American Datum 1983",ELLIPSOID["GRS 1980",6378137,298.257222101,LENGTHUNIT["metre",1]],ID["EPSG",6269]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8901]]],CONVERSION["unknown",METHOD["Geostationary Satellite (Sweep X)"],PARAMETER["Longitude of natural origin",-75,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8802]],PARAMETER["Satellite Height",35786023,LENGTHUNIT["metre",1,ID["EPSG",9001]]],PARAMETER["False easting",0,LENGTHUNIT["metre",1],ID["EPSG",8806]],PARAMETER["False northing",0,LENGTHUNIT["metre",1],ID["EPSG",8807]]],CS[Cartesian,2],AXIS["(E)",east,ORDER[1],LENGTHUNIT["metre",1,ID["EPSG",9001]]],AXIS["(N)",north,ORDER[2],LENGTHUNIT["metre",1,ID["EPSG",9001]]]]', '+proj=geos +sweep=x +lon_0=-75 +h=35786023 +x_0=0 +y_0=0 +datum=NAD83 +units=m +no_defs +type=crs') ON CONFLICT (srid) DO NOTHING
EOF
}
add_postgres_tables() {
debug "Creating PostGIS table"
docker exec -i ${PG_SERVER_NAME} psql -U ${POSTGRES_USER} <<EOF
CREATE TABLE IF NOT EXISTS g16_abi_radf_l1b_c01 (
gid SERIAL PRIMARY KEY,
start_time TIMESTAMP WITHOUT TIME ZONE NOT NULL UNIQUE,
location VARCHAR(255) NOT NULL,
bbox_geometry geometry(POLYGON, 930916)
)
EOF
}
add_postgres_content() {
debug "Creating values for fake DB row for C01"
gtiff_location=$1
insert_values=$(docker exec -i test python3 <<EOF
from shapely.geometry import box
from shapely import wkb
dt_str = "${C01_ISOTIME}"
location = "${gtiff_location}"
bbox = box(-25000000, -25000000, 25000000, 25000000)
bbox_wkb = wkb.dumps(bbox, hex=True, srid=930916)
values = (dt_str, location, bbox_wkb)
print(repr(values))
EOF
)
debug "Inserting fake DB row for C01"
docker exec -i ${PG_SERVER_NAME} psql -U ${POSTGRES_USER} <<EOF
INSERT into g16_abi_radf_l1b_c01 (start_time, location, bbox_geometry)
VALUES ${insert_values} ON CONFLICT (start_time) DO NOTHING
RETURNING (start_time)
create_fake_geotiff() {
bucket_name="g16-abi-radf-l1b-c01"
docker exec -i test bash -c "mkdir -p /data/${bucket_name}; chmod -R a+rwX /data/${bucket_name}"
gtiff_fn="${bucket_name}/${C01_GTIFF_NAME}"
docker exec -i test python3 <<EOF
import rasterio
from rasterio.transform import Affine
import numpy as np
transform = Affine.translation(-2500 * 1000 - 500, 2500 * 1000 - 500) * Affine.scale(1000, 1000)
with rasterio.open("/data/${gtiff_fn}", "w", driver='GTiff', height=5000, width=5000, dtype=np.uint8, count=1,
crs="+proj=geos +sweep=x +lon_0=-75.0 +h=35786023 +x_0=0 +y_0=0 +ellps=GRS80 +units=m +no_defs", transform=transform) as gtiff_file:
data = np.repeat(np.linspace(0, 255, 5000, dtype=np.uint8)[None, :], 5000, axis=0)
gtiff_file.write(data, 1)
EOF
creation_status=$?
echo ${gtiff_fn}
return $creation_status
}
create_fake_s3_geotiff() {
fn=$(create_fake_geotiff)
gtiff_fn=$(basename $fn)
bucket_name=$(basename $(dirname $fn))
aws s3api create-bucket --endpoint-url "http://localhost:9000" --bucket "${bucket_name}" 1>&2 || error "Can't create MinIO S3 bucket"
aws s3 cp --endpoint-url "http://localhost:9000" "${fn}" s3://${bucket_name}/ 1>&2 || error "Can't copy geotiff to S3 bucket"
echo "${bucket_name}/${gtiff_fn}"
}
curl_index() {
debug "Starting curl basic request..."
curl --fail -sS --max-time 5 "http://localhost:8888/" >/dev/null
}
debug "Starting curl basic mapfile request for non-existent tile..."
# NOTE: The time doesn't actually exist and no image data is available. A blank image should be returned
# docker exec -it test bash
curl --fail -sS -o "empty_tile.png" "http://localhost:8888/wms/g16/abi/radf/l1b?VERSION=1.1.1&REQUEST=GetMap&SERVICE=WMS&STYLES=&BBOX=-2500000%2c-2500000%2c2500000%2c2500000&WIDTH=256&HEIGHT=256&FORMAT=rgba&SRS=EPSG%3a930916&LAYERS=C01&TIME=2022-04-20T16:00:12Z" >/dev/null
check_image_content "empty_tile.png" 1
}
curl_valid_tile() {
debug "Starting curl basic mapfile request for valid tile..."
# docker exec -it test bash
curl --fail -sS -o "valid_tile.png" "http://localhost:8888/wms/g16/abi/radf/l1b?VERSION=1.1.1&REQUEST=GetMap&SERVICE=WMS&STYLES=&BBOX=-2500000%2c-2500000%2c2500000%2c2500000&WIDTH=256&HEIGHT=256&FORMAT=rgba&SRS=EPSG%3a930916&LAYERS=C01&TIME=${C01_ISOTIME}Z" >/dev/null
check_image_content "valid_tile.png" 0
check_image_content() {
img_filename=$1
all_zero=$2
# if it is an XML file then it wasn't a successful download
grep_found_xml=1
grep "<?xml" $img_filename || grep_found_xml=0
if [[ $grep_found_xml -ne 0 ]]; then
error "Image being checked is XML file"
cat $img_filename
return 1
fi
docker exec -i test python3 -m pip install --break-system-packages --root-user-action ignore pillow >/dev/null
incontainer_file="/tmp/$(basename $img_filename)"
docker cp $img_filename test:${incontainer_file}
docker exec -i test python3 <<EOF
import sys
import io
import os
import numpy as np
from PIL import Image
img = Image.open("${incontainer_file}")
img_arr = np.asarray(img)
assert img_arr.shape == (256, 256, 4)
if ${all_zero} == 1:
print("\tChecking RGB data is all 0s")
assert (img_arr[:, :, :3] == 0).all()
else:
print("\tChecking RGB data is not all 0s")
assert (img_arr[:, :, :3] != 0).any()
gtiff_fn="/data/$(create_fake_geotiff)"
add_shapefile_content "${gtiff_fn}"
curl_index
curl_empty_tile
curl_valid_tile
teardown_test
debug "SUCCESS: Shapefile test completed successfully"
start_postgres
start_pg_test_container
gtiff_fn="/data/$(create_fake_geotiff)"
add_postgres_projections
add_postgres_content "${gtiff_fn}"
curl_index
curl_empty_tile
curl_valid_tile
debug "SUCCESS: Postgres test completed successfully"
teardown_test
run_s3_postgres_tests() {
setup_test
debug "Starting S3 Postgres tests..."
start_postgres
start_minio "${PWD}"
start_s3_pg_test_container
gtiff_fn="/vsis3/$(create_fake_s3_geotiff)"
add_postgres_projections
add_postgres_tables
add_postgres_content "${gtiff_fn}"
curl_index
curl_empty_tile
curl_valid_tile
debug "SUCCESS: S3 Postgres test completed successfully"
teardown_test
}
trap graceful_exit EXIT
set -e
run_basic_shapefile_tests
run_basic_postgres_tests
run_s3_postgres_tests
echo "#######"
trap - EXIT # tests should have cleared this already, otherwise produces extra output
debug "SUCCESS"