Add initial attempt at S3 geotiff reading

This adds the initial configuration changes and testing for loading geotiff data from an S3 bucket. It currently uses MinIO as a temporary S3 server. It includes a lot of new configuration/environment variable options that maybe should have been added earlier for performance. Some are specific to curl/s3 loading and authentication.

TODO:

• Test performance differences between local geotiff and local S3 server. This won't be a fair comparison as S3 will be remote, but it should reveal any major problems (I hope).
• Allow more of the configuration values to be specified from environment variables.
• Update the helm chart to load these environment variables from the values.yaml
• Think of a transition plan for geosphere-test (and geosphere) for switching to S3. Either transition all geotiffs (manual copy/upload) or all future geotiffs or wait to enable it at a later date.

This is step one in getting this to work. The other steps include:

1. tile-gen needs to glob/search the S3 bucket for all geotiffs instead of expecting them locally
2. geo2grid needs to push geotiffs to S3. It probably shouldn't do local and S3 as it will slow processing...but maybe if tested we can check how long this really would be and do it on geosphere-test to verify operations.

ci/test_mapserver_image.sh

@@ -90,10 +90,11 @@ start_postgres() {
 
 start_minio() {
     base_dir=$1
-    docker run -d --rm --network ${NETWORK_NAME} -p 9000:9000 -p 9001:9001 --name ${MINIO_SERVER_NAME} --user ${UID}:${UID} -v ${base_dir}:/data minio/minio server /data --console-address ":9001"
+    base_minio_dir="$1/minio_base"
+    mkdir -p "${base_minio_dir}"
+    docker run -d --rm --network ${NETWORK_NAME} -p 9000:9000 -p 9001:9001 --name ${MINIO_SERVER_NAME} --user ${UID}:${UID} -v ${base_minio_dir}:/data minio/minio server /data --console-address ":9001"
     create_status=$?
     debug "Sleeping for 5 seconds for MinIO to start up..."
-    docker run --rm -d -v /tmp:/tmp --network ${NETWORK_NAME} --entrypoint=/bin/sh minio/mc "mc alias set test http://test-minio-server:9000 ${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY} --api s3v4 --path off; mc admin trace test >> /tmp/minio_trace.log" &
     sleep 5
     return $create_status
 }
@@ -199,7 +200,7 @@ EOF
 }
 
 create_fake_geotiff() {
-    bucket_name="g16_abi_radf_l1b_C01"
+    bucket_name="g16-abi-radf-l1b-c01"
     docker exec -i test bash -c "mkdir -p /data/${bucket_name}; chmod -R a+rwX /data/${bucket_name}"
     gtiff_fn="${bucket_name}/${C01_GTIFF_NAME}"
     docker exec -i test python3 <<EOF
@@ -218,6 +219,16 @@ EOF
     return $creation_status
 }
 
+create_fake_s3_geotiff() {
+  fn=$(create_fake_geotiff)
+  gtiff_fn=$(basename $fn)
+  bucket_name=$(basename $(dirname $fn))
+
+  aws s3api create-bucket --endpoint-url "http://localhost:9000" --bucket "${bucket_name}" 1>&2
+  aws s3 cp --endpoint-url "http://localhost:9000" "${fn}" s3://${bucket_name}/ 1>&2
+  echo "${bucket_name}/${gtiff_fn}"
+}
+
 curl_index() {
     debug "Starting curl basic request..."
     curl --fail -sS --max-time 5 "http://localhost:8888/" >/dev/null
@@ -332,7 +343,7 @@ run_s3_postgres_tests() {
     start_postgres
     start_minio "${PWD}"
     start_s3_pg_test_container
-    gtiff_fn="/vsis3/$(create_fake_geotiff)"
+    gtiff_fn="/vsis3/$(create_fake_s3_geotiff)"
     add_postgres_projections
     add_postgres_tables
     add_postgres_content "${gtiff_fn}"