Add initial proof of concept of S3 geotiff loading

2e59a096 · David Hoese · 0c46f8bf · 2e59a096 · 2e59a096 · 2e59a096
Verified Commit 2e59a096 authored 2 years ago by David Hoese
--- a/ci/test_mapserver_image.sh
+++ b/ci/test_mapserver_image.sh
@@ -42,6 +42,7 @@ setup_test() {
 teardown_test() {
    kill_test_container || echo "Could not kill test container"
    kill_postgres || echo "Could not kill postgres container"
+    kill_minio || echo "Could not kill minio container"
    if [ -d $base_tmp_dir ]; then
        rm -rf $base_tmp_dir
    fi
@@ -71,24 +72,37 @@ start_shapefile_test_container() {
 start_pg_test_container() {
    mkdir pg_secrets
    echo "${POSTGRES_PASSWORD}" > pg_secrets/fake_file
-    start_test_container -v "$(pwd)":"/data" -e POSTGRES_HOST=${PG_SERVER_NAME} -e POSTGRES_PORT=${PG_PORT} -e POSTGRES_PASSWORD_FILE="/data/pg_secrets/fake_file"
+    start_test_container -v "$(pwd)":"/data" -e POSTGRES_HOST=${PG_SERVER_NAME} -e POSTGRES_PORT=${PG_PORT} -e POSTGRES_PASSWORD_FILE="/data/pg_secrets/fake_file" "$@"
 }

-# start_minio() {
-#     base_dir=$1
-#     docker run -d --rm --name ${MINIO_SERVER_NAME} --user ${UID}:${UID} -p ${S3_PORT}:9000 -p 9001:9001 -v ${base_dir}:/data minio/minio server /data --console-address ":9001"
-# }
+start_s3_pg_test_container() {
+    start_pg_test_container -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} -e AWS_S3_ENDPOINT="${MINIO_SERVER_NAME}:9000"
+}

 start_postgres() {
    debug "Starting Postgres database..."
    docker run --rm -d --network ${NETWORK_NAME} --name ${PG_SERVER_NAME} -e POSTGRES_PASSWORD="${POSTGRES_PASSWORD}" postgis/postgis
+    create_status=$?
    debug "Sleeping for 5 seconds for DB to start up..."
    sleep 5
+    return $create_status
+}
+
+start_minio() {
+    base_dir=$1
+    docker run -d --rm --network ${NETWORK_NAME} -p 9000:9000 -p 9001:9001 --name ${MINIO_SERVER_NAME} --user ${UID}:${UID} -v ${base_dir}:/data minio/minio server /data --console-address ":9001"
+    create_status=$?
+    debug "Sleeping for 5 seconds for MinIO to start up..."
+    docker run --rm -d -v /tmp:/tmp --network ${NETWORK_NAME} --entrypoint=/bin/sh minio/mc "mc alias set test http://test-minio-server:9000 ${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY} --api s3v4 --path off; mc admin trace test >> /tmp/minio_trace.log" &
+    sleep 5
+    return $create_status
 }

 kill_test_container() {
    debug "Killing docker container..."
    debug "-----------------------------------------"
+    docker exec -i test bash -c "if [ -f /tmp/ms_error.txt ]; then cat /tmp/ms_error.txt; else echo 'No mapserver log file'; fi"
+    debug "-----"
    docker logs test
    debug "-----------------------------------------"
    docker kill test >/dev/null
@@ -104,6 +118,17 @@ kill_postgres() {
    debug "Done killing postgres container."
 }

+kill_minio() {
+    debug "Killing MinIO container..."
+    debug "-----------------------------------------"
+    # To see HTTP trace run the following in a separate terminal once test MinIO container is started:
+    # docker run --rm -it --network test_mapserver_image --entrypoint=/bin/sh minio/mc -c "mc alias set test http://test-minio-server:9000 minioadmin minioadmin --api s3v4 --path off; mc admin trace test"
+    docker logs ${MINIO_SERVER_NAME}
+    debug "-----------------------------------------"
+    docker kill ${MINIO_SERVER_NAME} >/dev/null
+    debug "Done killing minio container."
+}
+
 add_shapefile_content() {
    debug "Creating fake shapefile directory for C01"
    gtiff_location=$1
@@ -192,6 +217,7 @@ EOF
    echo ${gtiff_fn}
    return $creation_status
 }
+
 curl_index() {
    debug "Starting curl basic request..."
    curl --fail -sS --max-time 5 "http://localhost:8888/" >/dev/null
@@ -227,6 +253,7 @@ curl_empty_tile() {

 curl_valid_tile() {
    debug "Starting curl basic mapfile request for valid tile..."
+#     docker exec -it test bash
    curl --fail -sS -o "valid_tile.png" "http://localhost:8888/wms/g16/abi/radf/l1b?VERSION=1.1.1&REQUEST=GetMap&SERVICE=WMS&STYLES=&BBOX=-2500000%2c-2500000%2c2500000%2c2500000&WIDTH=256&HEIGHT=256&FORMAT=rgba&SRS=EPSG%3a930916&LAYERS=C01&TIME=${C01_ISOTIME}Z" >/dev/null
    check_image_content "valid_tile.png" 0
 }
@@ -299,6 +326,25 @@ run_basic_postgres_tests() {
    teardown_test
 }

+run_s3_postgres_tests() {
+    setup_test
+    debug "Starting S3 Postgres tests..."
+    start_postgres
+    start_minio "${PWD}"
+    start_s3_pg_test_container
+    gtiff_fn="/vsis3/$(create_fake_geotiff)"
+    add_postgres_projections
+    add_postgres_tables
+    add_postgres_content "${gtiff_fn}"
+
+    curl_index
+    curl_layer_times "[\"${C01_ISOTIME}\"]"
+    curl_empty_tile
+    curl_valid_tile
+    debug "SUCCESS: S3 Postgres test completed successfully"
+    teardown_test
+}
+

 trap graceful_exit EXIT

@@ -307,6 +353,8 @@ run_basic_shapefile_tests
 echo "#######"
 run_basic_postgres_tests
 echo "#######"
+run_s3_postgres_tests
+echo "#######"

 trap - EXIT  # tests should have cleared this already, otherwise produces extra output
 debug "SUCCESS"
--- a/mapserver/abi_l1b_template.map
+++ b/mapserver/abi_l1b_template.map
@@ -4,6 +4,16 @@ MAP
  IMAGETYPE rgb
  SIZE 256 256
  EXTENT -5434894.885 -5434894.885 5434894.885 5434894.885
+  CONFIG "GDAL_DISABLE_READDIR_ON_OPEN" "TRUE"
+  CONFIG "CPL_VSIL_CURL_USE_CACHE" "TRUE"
+  CONFIG "CPL_VSIL_CURL_CACHE_SIZE" "128000000"
+  CONFIG "CPL_VSIL_CURL_USE_HEAD" "FALSE"
+  CONFIG "AWS_ACCESS_KEY_ID" "{{ env.get('AWS_ACCESS_KEY_ID', '') }}"
+  CONFIG "AWS_SECRET_ACCESS_KEY" "{{ env.get('AWS_SECRET_ACCESS_KEY', '') }}"
+  CONFIG "AWS_S3_ENDPOINT" "{{ env.get('AWS_S3_ENDPOINT') }}"
+  CONFIG "AWS_HTTPS" "{{ env.get('AWS_HTTPS', 'FALSE') }}"
+  CONFIG "AWS_VIRTUAL_HOSTING" "{{ env.get('AWS_VIRTUAL_HOSTING', 'FALSE') }}"
+  CONFIG "CPL_VSIL_CURL_ALLOWED_EXTENSIONS" ".tif"

  PROJECTION
    "{{ projection }}"
@@ -34,6 +44,7 @@ MAP
    DATA "{{ postgis_geom_column }} from {{ platform }}_abi_{{ sector }}_l1b_{{ product.lower() }} using SRID={{ epsg_code }} using unique gid"
    CONNECTIONTYPE postgis
    CONNECTION "{{ postgis_connection_params }}"
+    PROCESSING "CLOSE_CONNECTION=DEFER"
    METADATA
      "wms_title" "{{ platform_long_name }} ABI {{ product.replace('_', ' ').title() }} Time Index"
      "wms srs" "EPSG:{{ epsg_code }}"
@@ -59,10 +70,6 @@ MAP

    # Comment below to default to transparency
    #OFFSITE 0 0 0
-    # Uncomment if projection is different than entire mapfile
-    #PROJECTION
-    #  "{{ projection }}"
-    #END
    PROCESSING "RESAMPLE=AVERAGE"
    # Used to keep FastCGI connections open longer
    PROCESSING "CLOSE_CONNECTION=DEFER"

--- a/mapserver/render.py
+++ b/mapserver/render.py
@@ -96,7 +96,7 @@ def main():

            with open(args.template_fn, 'r') as tmpl_file, open(output_fn, 'w') as mapfile:
                template = jinja2.Template(tmpl_file.read())
-                mapfile.write(template.render(tmpl_vars))
+                mapfile.write(template.render(tmpl_vars, env=os.environ))


 if __name__ == "__main__":