Add first attempt at postgis ci tests

ci_geosphere/values-postgis.yaml

+metrics:
+  enabled: true
+  serviceMonitor:
+    enabled: true
+    namespace: monitoring
+    additionalLabels:
+      release: "prometheus-operator"
\ No newline at end of file

ci_tests/_tests_common.sh

@@ -14,6 +14,11 @@ install_rabbitmq_chart() {
     ./helpers/deploy_rabbitmq.sh "${TEST_VALUES_DIR}" "${RELEASE_PREFIX}"
 }
 
+install_postgis_chart() {
+    TEST_VALUES_DIR=$1
+    ./helpers/deploy_postgis.sh "${TEST_VALUES_DIR}" "${RELEASE_PREFIX}"
+}
+
 install_postgres_chart() {
     TEST_VALUES_DIR=$1
 }
@@ -38,6 +43,7 @@ install_tile_gen_chart() {
     TEST_VALUES_DIR=$1
     geotiff_pvc=$(get_pvc_name ${GEO2GRID_RELEASE_NAME})
     sed -i "s/geosphere-rabbit-rabbitmq/${RABBITMQ_SERVICE_NAME}/g" "${TEST_VALUES_DIR}/values-tile-gen-g16-radm1.yaml"
+    sed -i "s/geosphere-postgis-postgresql/${POSTGIS_SERVICE_NAME}/g" "${TEST_VALUES_DIR}/values-tile-gen-g16-radm1.yaml"
     sed -i "s/cspp-geo-geo2grid/${GEO2GRID_RELEASE_NAME}/g" "${TEST_VALUES_DIR}/values-tile-gen-g16-radm1.yaml"
     ./helpers/deploy_tile_gen.sh "${TEST_VALUES_DIR}" "${RELEASE_PREFIX}" "-g16-radm1"
 }
@@ -53,6 +59,7 @@ install_mapserver_chart() {
 install_mapcache_chart() {
     TEST_VALUES_DIR=$1
     sed -i "s/geosphere-rabbit-rabbitmq/${RABBITMQ_SERVICE_NAME}/g" "${TEST_VALUES_DIR}/values-mapcache.yaml"
+    sed -i "s/geosphere-postgis-postgresql/${POSTGIS_SERVICE_NAME}/g" "${TEST_VALUES_DIR}/values-mapcache.yaml"
     sed -i "s/geosphere-mapserver/${MAPSERVER_RELEASE_NAME}/g" "${TEST_VALUES_DIR}/values-mapcache.yaml"
     ./helpers/deploy_mapcache.sh "${TEST_VALUES_DIR}" "${RELEASE_PREFIX}"
 }

ci_tests/basic_postgres/values-client.yaml

+infoServer: "geosphere-mapserver"
+tileServer: "geosphere-mapcache"

ci_tests/basic_postgres/values-geo2grid-g16-radm1.yaml

+rabbitIn:
+  host: "geosphere-rabbit-rabbitmq"
+  username: "user"
+  passwordSecret: "geosphere-rabbit-rabbitmq"
+  topic: "data.goes.g16.abi.radm1.l1b.netcdf.all.complete"
+rabbitOut:
+  host: "geosphere-rabbit-rabbitmq"
+  username: "user"
+  passwordSecret: "geosphere-rabbit-rabbitmq"
+source:
+#  s3Endpoint: "http://geosphere-minio:9000"
+  existingClaim: "cspp-geo-grb"
+destination:
+  persistence:
+    enabled: true
+    storageClass: longhorn
+    cleanup:
+      enabled: false

ci_tests/basic_postgres/values-geosphere-rabbit.yaml

+# persisting the secret only causes issues for failed tests
+persistence:
+  enabled: false
+rbacEnabled: false
\ No newline at end of file

ci_tests/basic_postgres/values-grb-g16.yaml

+apidFilter: "GOES-16-ABI-Only.xml"
+uploadDst: "/dst"
+rabbitOut:
+  host: "geosphere-rabbit-rabbitmq"
+  username: "user"
+  passwordSecret: "geosphere-rabbit-rabbitmq"
+caduSender:
+  enabled: true
+  existingClaim: "geosphere-ci-tests-cadu"
+  env:
+    - name: CADU_SLEEP
+      value: "60"
+persistence:
+  enabled: true
+  storageClass: longhorn
+  cleanup:
+    enabled: false
\ No newline at end of file

ci_tests/basic_postgres/values-mapcache.yaml

+rabbitIn:
+  host: "geosphere-rabbit-rabbitmq"
+  username: "user"
+  passwordSecret: "geosphere-rabbit-rabbitmq"
+  topic: "data.goes.g16.abi.*.l1b.tiles.*.complete"
+wms:
+  host: "geosphere-mapserver"
+cache:
+  persistence:
+    enabled: true
+    storageClass: "local-path"
+    size: 1Ti
+    cleanup:
+      # every 6 hours
+      schedule: "0 */6 * * *"
+      age: "+2"
+seed:
+  images: true
+  overlays: true
+database:
+  postgresHost: "geosphere-postgis-postgresql"
+  postgresPort: 5432
+  postgresDatabaseName: "postgres"
+  postgresUser: "postgres"
+  postgresPasswordSecret: "geosphere-postgis-postgresql"

ci_tests/basic_postgres/values-mapserver.yaml

+imageSource:
+  #  s3Endpoint: "http://geosphere-minio:9000"
+  existingClaim: "cspp-geo-geo2grid"
+tileSource:
+  existingClaim: "geosphere-tile-gen-shapefiles"

ci_tests/basic_postgres/values-tile-gen-g16-radm1.yaml

+rabbitIn:
+  host: "geosphere-rabbit-rabbitmq"
+  username: "user"
+  passwordSecret: "geosphere-rabbit-rabbitmq"
+  topic: "data.goes.g16.abi.radm1.l1b.geotiff.all.complete"
+rabbitOut:
+  host: "geosphere-rabbit-rabbitmq"
+  username: "user"
+  passwordSecret: "geosphere-rabbit-rabbitmq"
+source:
+  existingClaim: "cspp-geo-geo2grid"
+destination:
+  persistence:
+    enabled: true
+    storageClass: "longhorn"
+    cleanup:
+      enabled: false
+database:
+  postgresHost: "geosphere-postgis-postgresql"
+  postgresPort: 5432
+  postgresDatabaseName: "postgres"
+  postgresUser: "postgres"
+  postgresPasswordSecret: "geosphere-postgis-postgresql"

ci_tests/get_release_names.sh

@@ -4,6 +4,8 @@ export RELEASE_PREFIX="${CI_JOB_NAME//_/-}-${CI_JOB_ID//_/-}-"
 export GRB_RELEASE_NAME="${RELEASE_PREFIX}cspp-geo-grb"
 export RABBITMQ_RELEASE_NAME="${RELEASE_PREFIX}geosphere-rabbit"
 export RABBITMQ_SERVICE_NAME="${RABBITMQ_RELEASE_NAME}-rabbitmq"
+export POSTGIS_RELEASE_NAME="${RELEASE_PREFIX}geosphere-postgis"
+export POSTGIS_SERVICE_NAME="${POSTGIS_RELEASE_NAME}-postgresql"
 export GEO2GRID_RELEASE_NAME="${RELEASE_PREFIX}cspp-geo-geo2grid-g16-radm1"
 export TILEGEN_RELEASE_NAME="${RELEASE_PREFIX}geosphere-tile-gen-g16-radm1"
 export MAPSERVER_RELEASE_NAME="${RELEASE_PREFIX}geosphere-mapserver"

ci_tests/run_basic_postgres_test.sh

 #!/usr/bin/env bash
 
+install_basic_postgres_charts() {
+    install_rabbitmq_chart $1
+    install_postgis_chart $1
+    install_grb_chart $1
+    install_geo2grid_chart $1
+    install_tile_gen_chart $1
+    install_mapserver_chart $1
+    install_mapcache_chart $1
+    install_client_chart $1
+    debug "$(helm -n $ns list)"
+}
+
+set -e
+source ci_tests/_tests_common.sh
+ns=$(./helpers/get_namespace.sh)
+source ci_tests/get_release_names.sh
+install_basic_postgres_charts "ci_tests/basic_postgres"
+
+# give kubernetes a bit to create the resources
+debug "Waiting for Kubernetes to deploy and schedule components..."
+sleep 60
+debug "Done waiting"
+
+debug "Getting GRB pod name"
+GRB_POD_NAME=$(get_pod_name "${GRB_RELEASE_NAME}")
+debug "GRB pod name: ${GRB_POD_NAME}"
+GEO2GRID_POD_NAME=$(get_pod_name "${GEO2GRID_RELEASE_NAME}")
+debug "Geo2Grid pod name: ${GEO2GRID_POD_NAME}"
+TILEGEN_POD_NAME=$(get_pod_name "${TILEGEN_RELEASE_NAME}")
+debug "Tile Generation pod name: ${TILEGEN_POD_NAME}"
+MAPSERVER_POD_NAME=$(get_pod_name "${MAPSERVER_RELEASE_NAME}")
+debug "MapServer pod name: ${MAPSERVER_POD_NAME}"
+MAPCACHE_POD_NAME=$(get_pod_name "${MAPCACHE_RELEASE_NAME}")
+debug "MapCache pod name: ${MAPCACHE_POD_NAME}"
+CLIENT_POD_NAME=$(get_pod_name "${CLIENT_RELEASE_NAME}")
+debug "Client pod name: ${CLIENT_POD_NAME}"
+
+wait_for_pod ${GRB_POD_NAME} || error "GRB pod was never ready"
+run_grb_checks || error "GRB checks failed"
+wait_for_pod ${GEO2GRID_POD_NAME} || error "Geo2Grid pod was never ready"
+run_geo2grid_checks || error "Geo2Grid checks failed"
+wait_for_pod ${TILEGEN_POD_NAME} || error "Tile Gen pod was never ready"
+run_tile_gen_checks || error "Tile Generation checks failed"
+wait_for_pod ${MAPSERVER_POD_NAME} || error "MapServer pod was never ready"
+run_mapserver_checks || error "MapServer checks failed"
+wait_for_pod ${MAPCACHE_POD_NAME} || error "MapCache pod was never ready"
+run_mapcache_checks || error "MapCache checks failed"
+wait_for_pod ${CLIENT_POD_NAME} || error "Client pod was never ready"
+run_client_checks || error "Client checks failed"
+
+

helpers/deploy_postgis.sh

+#!/usr/bin/env bash
+
+set -e
+
+if [ $# -ne 1 ] && [ $# -ne 2 ]; then
+    echo "Usage: ./helpers/deploy_postgis.sh <values_base_dir> [<release_prefix>]"
+    exit 1
+fi
+
+# ci_geosphere-test
+values_base="$1"
+
+# Prefix Example: testing-YYYYMMDD-
+if [[ $# -eq 1 ]]; then
+    release_prefix=""
+else
+    release_prefix="$2"
+fi
+
+
+ns=$(./helpers/get_namespace.sh)
+# get password from any previous installation
+# if we don't do this the password will get out of sync
+# where password is accessed from the kubectl secret:
+#   kubectl get secret --namespace geosphere <secret-name> -o jsonpath="{.data.postgresql-password}" | base64 --decode
+release_name="${release_prefix}geosphere-postgis"
+sec_info=$(kubectl get secret --namespace $ns "${release_name}-postgresql" || echo "")
+echo $sec_info
+auth_sec="${release_name}-postgresql"
+if [[ "$sec_info" != "" ]]; then
+    pw=$(kubectl get secret --namespace $ns $auth_sec -o jsonpath="{.data.postgresql-password}" | base64 -d);
+    EXTRA_ARGS="--set postgresqlPassword=$pw";
+fi
+echo "Adding extra arguments: $EXTRA_ARGS"
+helm repo add bitnami "https://charts.bitnami.com/bitnami"
+
+helm upgrade -v 2 --install -f ${values_base}/values-postgis.yaml $EXTRA_ARGS --namespace $ns "${release_name}" bitnami/postgres
+# do a little waiting for the postgres pod to be ready so future stages
+# don't fail to communicate with it
+echo "Start waiting for postgres: $(date +%Y%m%dT%H%M%S)"
+kubectl wait -n $ns --selector=app.kubernetes.io/instance=${release_name} --timeout 120s --for=condition=Ready
+status=$?
+echo "Done waiting for postgis: $(date +%Y%m%dT%H%M%S) : $status"
+exit $status