diff --git a/ci_geosphere/values-postgis.yaml b/ci_geosphere/values-postgis.yaml new file mode 100644 index 0000000000000000000000000000000000000000..688dd527935a40511b791047957aeb43306e5eae --- /dev/null +++ b/ci_geosphere/values-postgis.yaml @@ -0,0 +1,7 @@ +metrics: + enabled: true + serviceMonitor: + enabled: true + namespace: monitoring + additionalLabels: + release: "prometheus-operator" \ No newline at end of file diff --git a/ci_tests/_tests_common.sh b/ci_tests/_tests_common.sh index a1dede3f2ef6ead5fb87e51a6bab1804262290da..cd11400d3f2fbfca1cc4556e08ed95b64875818d 100644 --- a/ci_tests/_tests_common.sh +++ b/ci_tests/_tests_common.sh @@ -14,6 +14,11 @@ install_rabbitmq_chart() { ./helpers/deploy_rabbitmq.sh "${TEST_VALUES_DIR}" "${RELEASE_PREFIX}" } +install_postgis_chart() { + TEST_VALUES_DIR=$1 + ./helpers/deploy_postgis.sh "${TEST_VALUES_DIR}" "${RELEASE_PREFIX}" +} + install_postgres_chart() { TEST_VALUES_DIR=$1 } @@ -38,6 +43,7 @@ install_tile_gen_chart() { TEST_VALUES_DIR=$1 geotiff_pvc=$(get_pvc_name ${GEO2GRID_RELEASE_NAME}) sed -i "s/geosphere-rabbit-rabbitmq/${RABBITMQ_SERVICE_NAME}/g" "${TEST_VALUES_DIR}/values-tile-gen-g16-radm1.yaml" + sed -i "s/geosphere-postgis-postgresql/${POSTGIS_SERVICE_NAME}/g" "${TEST_VALUES_DIR}/values-tile-gen-g16-radm1.yaml" sed -i "s/cspp-geo-geo2grid/${GEO2GRID_RELEASE_NAME}/g" "${TEST_VALUES_DIR}/values-tile-gen-g16-radm1.yaml" ./helpers/deploy_tile_gen.sh "${TEST_VALUES_DIR}" "${RELEASE_PREFIX}" "-g16-radm1" } @@ -53,6 +59,7 @@ install_mapserver_chart() { install_mapcache_chart() { TEST_VALUES_DIR=$1 sed -i "s/geosphere-rabbit-rabbitmq/${RABBITMQ_SERVICE_NAME}/g" "${TEST_VALUES_DIR}/values-mapcache.yaml" + sed -i "s/geosphere-postgis-postgresql/${POSTGIS_SERVICE_NAME}/g" "${TEST_VALUES_DIR}/values-mapcache.yaml" sed -i "s/geosphere-mapserver/${MAPSERVER_RELEASE_NAME}/g" "${TEST_VALUES_DIR}/values-mapcache.yaml" ./helpers/deploy_mapcache.sh "${TEST_VALUES_DIR}" "${RELEASE_PREFIX}" } diff --git a/ci_tests/basic_postgres/values-client.yaml b/ci_tests/basic_postgres/values-client.yaml new file mode 100644 index 0000000000000000000000000000000000000000..558e745ac17ab813111157a92deace71b25858d8 --- /dev/null +++ b/ci_tests/basic_postgres/values-client.yaml @@ -0,0 +1,2 @@ +infoServer: "geosphere-mapserver" +tileServer: "geosphere-mapcache" diff --git a/ci_tests/basic_postgres/values-geo2grid-g16-radm1.yaml b/ci_tests/basic_postgres/values-geo2grid-g16-radm1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..605be5b0dea3ca96b40fca89bed5bc97bb43fc65 --- /dev/null +++ b/ci_tests/basic_postgres/values-geo2grid-g16-radm1.yaml @@ -0,0 +1,18 @@ +rabbitIn: + host: "geosphere-rabbit-rabbitmq" + username: "user" + passwordSecret: "geosphere-rabbit-rabbitmq" + topic: "data.goes.g16.abi.radm1.l1b.netcdf.all.complete" +rabbitOut: + host: "geosphere-rabbit-rabbitmq" + username: "user" + passwordSecret: "geosphere-rabbit-rabbitmq" +source: +# s3Endpoint: "http://geosphere-minio:9000" + existingClaim: "cspp-geo-grb" +destination: + persistence: + enabled: true + storageClass: longhorn + cleanup: + enabled: false diff --git a/ci_tests/basic_postgres/values-geosphere-rabbit.yaml b/ci_tests/basic_postgres/values-geosphere-rabbit.yaml new file mode 100644 index 0000000000000000000000000000000000000000..65d1a3d1d0a4447624597358b6cff3d35d6fc769 --- /dev/null +++ b/ci_tests/basic_postgres/values-geosphere-rabbit.yaml @@ -0,0 +1,4 @@ +# persisting the secret only causes issues for failed tests +persistence: + enabled: false +rbacEnabled: false \ No newline at end of file diff --git a/ci_tests/basic_postgres/values-grb-g16.yaml b/ci_tests/basic_postgres/values-grb-g16.yaml new file mode 100644 index 0000000000000000000000000000000000000000..20be65e84fdea66dc10dc7e17578b3b9598e834f --- /dev/null +++ b/ci_tests/basic_postgres/values-grb-g16.yaml @@ -0,0 +1,17 @@ +apidFilter: "GOES-16-ABI-Only.xml" +uploadDst: "/dst" +rabbitOut: + host: "geosphere-rabbit-rabbitmq" + username: "user" + passwordSecret: "geosphere-rabbit-rabbitmq" +caduSender: + enabled: true + existingClaim: "geosphere-ci-tests-cadu" + env: + - name: CADU_SLEEP + value: "60" +persistence: + enabled: true + storageClass: longhorn + cleanup: + enabled: false \ No newline at end of file diff --git a/ci_tests/basic_postgres/values-mapcache.yaml b/ci_tests/basic_postgres/values-mapcache.yaml new file mode 100644 index 0000000000000000000000000000000000000000..68c089a5653d72b9122ab6e0fda75cb3128d5173 --- /dev/null +++ b/ci_tests/basic_postgres/values-mapcache.yaml @@ -0,0 +1,25 @@ +rabbitIn: + host: "geosphere-rabbit-rabbitmq" + username: "user" + passwordSecret: "geosphere-rabbit-rabbitmq" + topic: "data.goes.g16.abi.*.l1b.tiles.*.complete" +wms: + host: "geosphere-mapserver" +cache: + persistence: + enabled: true + storageClass: "local-path" + size: 1Ti + cleanup: + # every 6 hours + schedule: "0 */6 * * *" + age: "+2" +seed: + images: true + overlays: true +database: + postgresHost: "geosphere-postgis-postgresql" + postgresPort: 5432 + postgresDatabaseName: "postgres" + postgresUser: "postgres" + postgresPasswordSecret: "geosphere-postgis-postgresql" diff --git a/ci_tests/basic_postgres/values-mapserver.yaml b/ci_tests/basic_postgres/values-mapserver.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f24f3336d23bf9c94366c7fc3de57ddcff2ca162 --- /dev/null +++ b/ci_tests/basic_postgres/values-mapserver.yaml @@ -0,0 +1,5 @@ +imageSource: + # s3Endpoint: "http://geosphere-minio:9000" + existingClaim: "cspp-geo-geo2grid" +tileSource: + existingClaim: "geosphere-tile-gen-shapefiles" diff --git a/ci_tests/basic_postgres/values-postgis.yaml b/ci_tests/basic_postgres/values-postgis.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/ci_tests/basic_postgres/values-tile-gen-g16-radm1.yaml b/ci_tests/basic_postgres/values-tile-gen-g16-radm1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b639064d7d046bc5a47eb64c646053d8978462c --- /dev/null +++ b/ci_tests/basic_postgres/values-tile-gen-g16-radm1.yaml @@ -0,0 +1,23 @@ +rabbitIn: + host: "geosphere-rabbit-rabbitmq" + username: "user" + passwordSecret: "geosphere-rabbit-rabbitmq" + topic: "data.goes.g16.abi.radm1.l1b.geotiff.all.complete" +rabbitOut: + host: "geosphere-rabbit-rabbitmq" + username: "user" + passwordSecret: "geosphere-rabbit-rabbitmq" +source: + existingClaim: "cspp-geo-geo2grid" +destination: + persistence: + enabled: true + storageClass: "longhorn" + cleanup: + enabled: false +database: + postgresHost: "geosphere-postgis-postgresql" + postgresPort: 5432 + postgresDatabaseName: "postgres" + postgresUser: "postgres" + postgresPasswordSecret: "geosphere-postgis-postgresql" diff --git a/ci_tests/get_release_names.sh b/ci_tests/get_release_names.sh index 7970afc838941b5ee2dfd37ab4d96090ed8fb8cb..66bc010d349041a2ec9d22a2a0cc2f1733eb4f17 100755 --- a/ci_tests/get_release_names.sh +++ b/ci_tests/get_release_names.sh @@ -4,6 +4,8 @@ export RELEASE_PREFIX="${CI_JOB_NAME//_/-}-${CI_JOB_ID//_/-}-" export GRB_RELEASE_NAME="${RELEASE_PREFIX}cspp-geo-grb" export RABBITMQ_RELEASE_NAME="${RELEASE_PREFIX}geosphere-rabbit" export RABBITMQ_SERVICE_NAME="${RABBITMQ_RELEASE_NAME}-rabbitmq" +export POSTGIS_RELEASE_NAME="${RELEASE_PREFIX}geosphere-postgis" +export POSTGIS_SERVICE_NAME="${POSTGIS_RELEASE_NAME}-postgresql" export GEO2GRID_RELEASE_NAME="${RELEASE_PREFIX}cspp-geo-geo2grid-g16-radm1" export TILEGEN_RELEASE_NAME="${RELEASE_PREFIX}geosphere-tile-gen-g16-radm1" export MAPSERVER_RELEASE_NAME="${RELEASE_PREFIX}geosphere-mapserver" diff --git a/ci_tests/run_basic_postgres_test.sh b/ci_tests/run_basic_postgres_test.sh index 20d602bdd3703e32d2239a764682f2913fe9c10c..c955a8c0734a75bc4e9dde38ef3dccf5dbc1d33f 100755 --- a/ci_tests/run_basic_postgres_test.sh +++ b/ci_tests/run_basic_postgres_test.sh @@ -1,2 +1,53 @@ #!/usr/bin/env bash +install_basic_postgres_charts() { + install_rabbitmq_chart $1 + install_postgis_chart $1 + install_grb_chart $1 + install_geo2grid_chart $1 + install_tile_gen_chart $1 + install_mapserver_chart $1 + install_mapcache_chart $1 + install_client_chart $1 + debug "$(helm -n $ns list)" +} + +set -e +source ci_tests/_tests_common.sh +ns=$(./helpers/get_namespace.sh) +source ci_tests/get_release_names.sh +install_basic_postgres_charts "ci_tests/basic_postgres" + +# give kubernetes a bit to create the resources +debug "Waiting for Kubernetes to deploy and schedule components..." +sleep 60 +debug "Done waiting" + +debug "Getting GRB pod name" +GRB_POD_NAME=$(get_pod_name "${GRB_RELEASE_NAME}") +debug "GRB pod name: ${GRB_POD_NAME}" +GEO2GRID_POD_NAME=$(get_pod_name "${GEO2GRID_RELEASE_NAME}") +debug "Geo2Grid pod name: ${GEO2GRID_POD_NAME}" +TILEGEN_POD_NAME=$(get_pod_name "${TILEGEN_RELEASE_NAME}") +debug "Tile Generation pod name: ${TILEGEN_POD_NAME}" +MAPSERVER_POD_NAME=$(get_pod_name "${MAPSERVER_RELEASE_NAME}") +debug "MapServer pod name: ${MAPSERVER_POD_NAME}" +MAPCACHE_POD_NAME=$(get_pod_name "${MAPCACHE_RELEASE_NAME}") +debug "MapCache pod name: ${MAPCACHE_POD_NAME}" +CLIENT_POD_NAME=$(get_pod_name "${CLIENT_RELEASE_NAME}") +debug "Client pod name: ${CLIENT_POD_NAME}" + +wait_for_pod ${GRB_POD_NAME} || error "GRB pod was never ready" +run_grb_checks || error "GRB checks failed" +wait_for_pod ${GEO2GRID_POD_NAME} || error "Geo2Grid pod was never ready" +run_geo2grid_checks || error "Geo2Grid checks failed" +wait_for_pod ${TILEGEN_POD_NAME} || error "Tile Gen pod was never ready" +run_tile_gen_checks || error "Tile Generation checks failed" +wait_for_pod ${MAPSERVER_POD_NAME} || error "MapServer pod was never ready" +run_mapserver_checks || error "MapServer checks failed" +wait_for_pod ${MAPCACHE_POD_NAME} || error "MapCache pod was never ready" +run_mapcache_checks || error "MapCache checks failed" +wait_for_pod ${CLIENT_POD_NAME} || error "Client pod was never ready" +run_client_checks || error "Client checks failed" + + diff --git a/helpers/deploy_postgis.sh b/helpers/deploy_postgis.sh new file mode 100755 index 0000000000000000000000000000000000000000..110f85039b80457b664979eed7dda85114b729a6 --- /dev/null +++ b/helpers/deploy_postgis.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +set -e + +if [ $# -ne 1 ] && [ $# -ne 2 ]; then + echo "Usage: ./helpers/deploy_postgis.sh <values_base_dir> [<release_prefix>]" + exit 1 +fi + +# ci_geosphere-test +values_base="$1" + +# Prefix Example: testing-YYYYMMDD- +if [[ $# -eq 1 ]]; then + release_prefix="" +else + release_prefix="$2" +fi + + +ns=$(./helpers/get_namespace.sh) +# get password from any previous installation +# if we don't do this the password will get out of sync +# where password is accessed from the kubectl secret: +# kubectl get secret --namespace geosphere <secret-name> -o jsonpath="{.data.postgresql-password}" | base64 --decode +release_name="${release_prefix}geosphere-postgis" +sec_info=$(kubectl get secret --namespace $ns "${release_name}-postgresql" || echo "") +echo $sec_info +auth_sec="${release_name}-postgresql" +if [[ "$sec_info" != "" ]]; then + pw=$(kubectl get secret --namespace $ns $auth_sec -o jsonpath="{.data.postgresql-password}" | base64 -d); + EXTRA_ARGS="--set postgresqlPassword=$pw"; +fi +echo "Adding extra arguments: $EXTRA_ARGS" +helm repo add bitnami "https://charts.bitnami.com/bitnami" + +helm upgrade -v 2 --install -f ${values_base}/values-postgis.yaml $EXTRA_ARGS --namespace $ns "${release_name}" bitnami/postgres +# do a little waiting for the postgres pod to be ready so future stages +# don't fail to communicate with it +echo "Start waiting for postgres: $(date +%Y%m%dT%H%M%S)" +kubectl wait -n $ns --selector=app.kubernetes.io/instance=${release_name} --timeout 120s --for=condition=Ready +status=$? +echo "Done waiting for postgis: $(date +%Y%m%dT%H%M%S) : $status" +exit $status