Skip to content
Snippets Groups Projects
Verified Commit 0b3d9aba authored by David Hoese's avatar David Hoese
Browse files

Fix testing for newer versions of MinIO

Instead of assuming existing data, use aws CLI to copy geotiff to MinIO
parent 2e59a096
No related branches found
No related tags found
1 merge request!5Add initial attempt at S3 geotiff reading
Pipeline #38190 failed with stages
in 6 seconds
......@@ -90,10 +90,11 @@ start_postgres() {
start_minio() {
base_dir=$1
docker run -d --rm --network ${NETWORK_NAME} -p 9000:9000 -p 9001:9001 --name ${MINIO_SERVER_NAME} --user ${UID}:${UID} -v ${base_dir}:/data minio/minio server /data --console-address ":9001"
base_minio_dir="$1/minio_base"
mkdir -p "${base_minio_dir}"
docker run -d --rm --network ${NETWORK_NAME} -p 9000:9000 -p 9001:9001 --name ${MINIO_SERVER_NAME} --user ${UID}:${UID} -v ${base_minio_dir}:/data minio/minio server /data --console-address ":9001"
create_status=$?
debug "Sleeping for 5 seconds for MinIO to start up..."
docker run --rm -d -v /tmp:/tmp --network ${NETWORK_NAME} --entrypoint=/bin/sh minio/mc "mc alias set test http://test-minio-server:9000 ${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY} --api s3v4 --path off; mc admin trace test >> /tmp/minio_trace.log" &
sleep 5
return $create_status
}
......@@ -199,7 +200,7 @@ EOF
}
create_fake_geotiff() {
bucket_name="g16_abi_radf_l1b_C01"
bucket_name="g16-abi-radf-l1b-c01"
docker exec -i test bash -c "mkdir -p /data/${bucket_name}; chmod -R a+rwX /data/${bucket_name}"
gtiff_fn="${bucket_name}/${C01_GTIFF_NAME}"
docker exec -i test python3 <<EOF
......@@ -218,6 +219,16 @@ EOF
return $creation_status
}
create_fake_s3_geotiff() {
fn=$(create_fake_geotiff)
gtiff_fn=$(basename $fn)
bucket_name=$(basename $(dirname $fn))
aws s3api create-bucket --endpoint-url "http://localhost:9000" --bucket "${bucket_name}" 1>&2
aws s3 cp --endpoint-url "http://localhost:9000" "${fn}" s3://${bucket_name}/ 1>&2
echo "${bucket_name}/${gtiff_fn}"
}
curl_index() {
debug "Starting curl basic request..."
curl --fail -sS --max-time 5 "http://localhost:8888/" >/dev/null
......@@ -332,7 +343,7 @@ run_s3_postgres_tests() {
start_postgres
start_minio "${PWD}"
start_s3_pg_test_container
gtiff_fn="/vsis3/$(create_fake_geotiff)"
gtiff_fn="/vsis3/$(create_fake_s3_geotiff)"
add_postgres_projections
add_postgres_tables
add_postgres_content "${gtiff_fn}"
......
......@@ -4,7 +4,16 @@ MAP
IMAGETYPE rgb
SIZE 256 256
EXTENT -5434894.885 -5434894.885 5434894.885 5434894.885
# See https://developmentseed.org/titiler/advanced/performance_tuning/
# CONFIG "GDAL_INGESTED_BYTES_AT_OPEN" "75000" # investigate all geotiffs
CONFIG "GDAL_DISABLE_READDIR_ON_OPEN" "TRUE"
CONFIG "GDAL_HTTP_MERGE_CONSECUTIVE_RANGES" "YES"
CONFIG "GDAL_CACHEMAX" "200"
CONFIG "VSI_CACHE" "TRUE"
# CONFIG "VSI_CACHE_SIZE" "25000000"
CONFIG GDAL_HTTP_MULTIPLEX "YES"
CONFIG GDAL_HTTP_VERSION "2"
CONFIG "PROJ_NETWORK" "OFF"
CONFIG "CPL_VSIL_CURL_USE_CACHE" "TRUE"
CONFIG "CPL_VSIL_CURL_CACHE_SIZE" "128000000"
CONFIG "CPL_VSIL_CURL_USE_HEAD" "FALSE"
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment