Skip to content
Snippets Groups Projects
Verified Commit 2e59a096 authored by David Hoese's avatar David Hoese
Browse files

Add initial proof of concept of S3 geotiff loading

parent 0c46f8bf
No related branches found
No related tags found
1 merge request!5Add initial attempt at S3 geotiff reading
Pipeline #38124 failed with stages
in 12 minutes and 7 seconds
......@@ -42,6 +42,7 @@ setup_test() {
teardown_test() {
kill_test_container || echo "Could not kill test container"
kill_postgres || echo "Could not kill postgres container"
kill_minio || echo "Could not kill minio container"
if [ -d $base_tmp_dir ]; then
rm -rf $base_tmp_dir
fi
......@@ -71,24 +72,37 @@ start_shapefile_test_container() {
start_pg_test_container() {
mkdir pg_secrets
echo "${POSTGRES_PASSWORD}" > pg_secrets/fake_file
start_test_container -v "$(pwd)":"/data" -e POSTGRES_HOST=${PG_SERVER_NAME} -e POSTGRES_PORT=${PG_PORT} -e POSTGRES_PASSWORD_FILE="/data/pg_secrets/fake_file"
start_test_container -v "$(pwd)":"/data" -e POSTGRES_HOST=${PG_SERVER_NAME} -e POSTGRES_PORT=${PG_PORT} -e POSTGRES_PASSWORD_FILE="/data/pg_secrets/fake_file" "$@"
}
# start_minio() {
# base_dir=$1
# docker run -d --rm --name ${MINIO_SERVER_NAME} --user ${UID}:${UID} -p ${S3_PORT}:9000 -p 9001:9001 -v ${base_dir}:/data minio/minio server /data --console-address ":9001"
# }
start_s3_pg_test_container() {
start_pg_test_container -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} -e AWS_S3_ENDPOINT="${MINIO_SERVER_NAME}:9000"
}
start_postgres() {
debug "Starting Postgres database..."
docker run --rm -d --network ${NETWORK_NAME} --name ${PG_SERVER_NAME} -e POSTGRES_PASSWORD="${POSTGRES_PASSWORD}" postgis/postgis
create_status=$?
debug "Sleeping for 5 seconds for DB to start up..."
sleep 5
return $create_status
}
start_minio() {
base_dir=$1
docker run -d --rm --network ${NETWORK_NAME} -p 9000:9000 -p 9001:9001 --name ${MINIO_SERVER_NAME} --user ${UID}:${UID} -v ${base_dir}:/data minio/minio server /data --console-address ":9001"
create_status=$?
debug "Sleeping for 5 seconds for MinIO to start up..."
docker run --rm -d -v /tmp:/tmp --network ${NETWORK_NAME} --entrypoint=/bin/sh minio/mc "mc alias set test http://test-minio-server:9000 ${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY} --api s3v4 --path off; mc admin trace test >> /tmp/minio_trace.log" &
sleep 5
return $create_status
}
kill_test_container() {
debug "Killing docker container..."
debug "-----------------------------------------"
docker exec -i test bash -c "if [ -f /tmp/ms_error.txt ]; then cat /tmp/ms_error.txt; else echo 'No mapserver log file'; fi"
debug "-----"
docker logs test
debug "-----------------------------------------"
docker kill test >/dev/null
......@@ -104,6 +118,17 @@ kill_postgres() {
debug "Done killing postgres container."
}
kill_minio() {
debug "Killing MinIO container..."
debug "-----------------------------------------"
# To see HTTP trace run the following in a separate terminal once test MinIO container is started:
# docker run --rm -it --network test_mapserver_image --entrypoint=/bin/sh minio/mc -c "mc alias set test http://test-minio-server:9000 minioadmin minioadmin --api s3v4 --path off; mc admin trace test"
docker logs ${MINIO_SERVER_NAME}
debug "-----------------------------------------"
docker kill ${MINIO_SERVER_NAME} >/dev/null
debug "Done killing minio container."
}
add_shapefile_content() {
debug "Creating fake shapefile directory for C01"
gtiff_location=$1
......@@ -192,6 +217,7 @@ EOF
echo ${gtiff_fn}
return $creation_status
}
curl_index() {
debug "Starting curl basic request..."
curl --fail -sS --max-time 5 "http://localhost:8888/" >/dev/null
......@@ -227,6 +253,7 @@ curl_empty_tile() {
curl_valid_tile() {
debug "Starting curl basic mapfile request for valid tile..."
# docker exec -it test bash
curl --fail -sS -o "valid_tile.png" "http://localhost:8888/wms/g16/abi/radf/l1b?VERSION=1.1.1&REQUEST=GetMap&SERVICE=WMS&STYLES=&BBOX=-2500000%2c-2500000%2c2500000%2c2500000&WIDTH=256&HEIGHT=256&FORMAT=rgba&SRS=EPSG%3a930916&LAYERS=C01&TIME=${C01_ISOTIME}Z" >/dev/null
check_image_content "valid_tile.png" 0
}
......@@ -299,6 +326,25 @@ run_basic_postgres_tests() {
teardown_test
}
run_s3_postgres_tests() {
setup_test
debug "Starting S3 Postgres tests..."
start_postgres
start_minio "${PWD}"
start_s3_pg_test_container
gtiff_fn="/vsis3/$(create_fake_geotiff)"
add_postgres_projections
add_postgres_tables
add_postgres_content "${gtiff_fn}"
curl_index
curl_layer_times "[\"${C01_ISOTIME}\"]"
curl_empty_tile
curl_valid_tile
debug "SUCCESS: S3 Postgres test completed successfully"
teardown_test
}
trap graceful_exit EXIT
......@@ -307,6 +353,8 @@ run_basic_shapefile_tests
echo "#######"
run_basic_postgres_tests
echo "#######"
run_s3_postgres_tests
echo "#######"
trap - EXIT # tests should have cleared this already, otherwise produces extra output
debug "SUCCESS"
......@@ -4,6 +4,16 @@ MAP
IMAGETYPE rgb
SIZE 256 256
EXTENT -5434894.885 -5434894.885 5434894.885 5434894.885
CONFIG "GDAL_DISABLE_READDIR_ON_OPEN" "TRUE"
CONFIG "CPL_VSIL_CURL_USE_CACHE" "TRUE"
CONFIG "CPL_VSIL_CURL_CACHE_SIZE" "128000000"
CONFIG "CPL_VSIL_CURL_USE_HEAD" "FALSE"
CONFIG "AWS_ACCESS_KEY_ID" "{{ env.get('AWS_ACCESS_KEY_ID', '') }}"
CONFIG "AWS_SECRET_ACCESS_KEY" "{{ env.get('AWS_SECRET_ACCESS_KEY', '') }}"
CONFIG "AWS_S3_ENDPOINT" "{{ env.get('AWS_S3_ENDPOINT') }}"
CONFIG "AWS_HTTPS" "{{ env.get('AWS_HTTPS', 'FALSE') }}"
CONFIG "AWS_VIRTUAL_HOSTING" "{{ env.get('AWS_VIRTUAL_HOSTING', 'FALSE') }}"
CONFIG "CPL_VSIL_CURL_ALLOWED_EXTENSIONS" ".tif"
PROJECTION
"{{ projection }}"
......@@ -34,6 +44,7 @@ MAP
DATA "{{ postgis_geom_column }} from {{ platform }}_abi_{{ sector }}_l1b_{{ product.lower() }} using SRID={{ epsg_code }} using unique gid"
CONNECTIONTYPE postgis
CONNECTION "{{ postgis_connection_params }}"
PROCESSING "CLOSE_CONNECTION=DEFER"
METADATA
"wms_title" "{{ platform_long_name }} ABI {{ product.replace('_', ' ').title() }} Time Index"
"wms srs" "EPSG:{{ epsg_code }}"
......@@ -59,10 +70,6 @@ MAP
# Comment below to default to transparency
#OFFSITE 0 0 0
# Uncomment if projection is different than entire mapfile
#PROJECTION
# "{{ projection }}"
#END
PROCESSING "RESAMPLE=AVERAGE"
# Used to keep FastCGI connections open longer
PROCESSING "CLOSE_CONNECTION=DEFER"
......
......@@ -96,7 +96,7 @@ def main():
with open(args.template_fn, 'r') as tmpl_file, open(output_fn, 'w') as mapfile:
template = jinja2.Template(tmpl_file.read())
mapfile.write(template.render(tmpl_vars))
mapfile.write(template.render(tmpl_vars, env=os.environ))
if __name__ == "__main__":
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment