diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 582426fec7fe9579041f03f04f8f3b1c1a17aff5..ee2c71068f6f1d8bd13118bff0aef63aa6e6c6ee 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -157,17 +157,7 @@ get_chart_client_test: extends: .helm_based_job stage: deploy WMS script: - - ns=$(./helpers/get_namespace.sh) - - cd geosphere-mapserver/chart - - source geosphere-mapserver/cibuild.env - # copy secret kubeconfig to the mounted (pwd) directory - - cp $kubekorner_k3s_config . - - kubeconfig=$(basename $kubekorner_k3s_config) - - echo "Deploying version $docker_tag to cluster namespace $ns" - # copy extra values files to the local directory (where helm has access via docker mount) - - cp ../../${VALUES_DIR}/values-mapserver.yaml . - # namespace names are the same as domain names - - helm upgrade -v 2 --install --kubeconfig $kubeconfig --namespace $ns -f values-mapserver.yaml geosphere-mapserver geosphere-mapserver/ + - ./helpers/deploy_mapserver.sh "${VALUES_DIR}" dependencies: - get_chart_mapserver @@ -177,17 +167,17 @@ get_chart_client_test: extends: .helm_based_job stage: deploy WMTS script: - - ns=$(./helpers/get_namespace.sh) - - cd geosphere-mapcache/chart - - source geosphere-mapcache/cibuild.env - # copy secret kubeconfig to the mounted (pwd) directory - - cp $kubekorner_k3s_config . - - kubeconfig=$(basename $kubekorner_k3s_config) - - echo "Deploying version $docker_tag to cluster namespace $ns" - # copy extra values files to the local directory (where helm has access via docker mount) - - cp ../../${VALUES_DIR}/values-mapcache.yaml . - # namespace names are the same as domain names - - helm upgrade -v 2 --install --kubeconfig $kubeconfig --namespace $ns -f values-mapcache.yaml geosphere-mapcache geosphere-mapcache/ + - ./helpers/deploy_mapcache.sh "${VALUES_DIR}" dependencies: - get_chart_mapcache +.deploy_client: + variables: + VALUES_DIR: "" + extends: .helm_based_job + stage: deploy Client + script: + - ./helpers/deploy_client.sh "${VALUES_DIR}" + dependencies: + - get_chart_client + diff --git a/ci_geosphere-test/gitlab-ci.yaml b/ci_geosphere-test/gitlab-ci.yaml index 3b60262a895ea434170672052238ae40682dd423..8c30f981c6d233028f6d3d4485ff350f67fe36ad 100644 --- a/ci_geosphere-test/gitlab-ci.yaml +++ b/ci_geosphere-test/gitlab-ci.yaml @@ -130,24 +130,12 @@ gstest deploy client: environment: name: geosphere-test url: http://geosphere-test.ssec.wisc.edu - extends: .helm_based_job - stage: deploy Client - script: - - ns="geosphere-test" - - cd geosphere-client/chart - - source geosphere-client/cibuild.env - # copy secret kubeconfig to the mounted (pwd) directory - - cp $kubekorner_k3s_config . - - kubeconfig=$(basename $kubekorner_k3s_config) - - echo "Deploying version $docker_tag to cluster namespace $ns" - # copy extra values files to the local directory (where helm has access via docker mount) - - cp ../../ci_geosphere-test/values-client.yaml . - # namespace names are the same as domain names - - helm upgrade -v 2 --install --kubeconfig $kubeconfig --namespace $ns -f values-client.yaml geosphere-client geosphere-client/ + variables: + VALUES_DIR: "ci_geosphere-test" + extends: .deploy_client dependencies: - get_chart_client_test rules: - if: $CI_COMMIT_TAG !~ /$GEOSPHERE_TEST_TAG_REGEX/ when: never - when: on_success - diff --git a/ci_geosphere-test/values-mapserver.yaml b/ci_geosphere-test/values-mapserver.yaml deleted file mode 100644 index 971734a5a1bf2273126b788608effa879169ca10..0000000000000000000000000000000000000000 --- a/ci_geosphere-test/values-mapserver.yaml +++ /dev/null @@ -1,16 +0,0 @@ -imageSource: - # s3Endpoint: "http://geosphere-minio:9000" - existingClaim: "cspp-geo-geo2grid" -tileSource: - existingClaim: "geosphere-tile-gen-shapefiles" -ingress: - enabled: true - annotations: - ingress.kubernetes.io/ssl-redirect: "true" - hosts: - - host: geosphere-test.ssec.wisc.edu - paths: ["/wms", "/wms_times"] - tls: - - hosts: - - "geosphere-test.ssec.wisc.edu" - secretName: "geosphere-tls-certs" diff --git a/ci_geosphere-test/values-tile-gen-g16-radc.yaml b/ci_geosphere-test/values-tile-gen-g16-radc.yaml deleted file mode 100644 index 7e9dc56494e83136877e3869c30397fe0a283e42..0000000000000000000000000000000000000000 --- a/ci_geosphere-test/values-tile-gen-g16-radc.yaml +++ /dev/null @@ -1,18 +0,0 @@ -rabbitIn: - # use the rabbitmq from the production environment (geosphere) - host: "geosphere-rabbit-rabbitmq.geosphere" - username: "user" - passwordSecret: "geosphere-rabbit-rabbitmq-production" - topic: "data.goes.g16.abi.radc.l1b.geotiff.all.complete" -rabbitOut: - host: "geosphere-rabbit-rabbitmq" - username: "user" - passwordSecret: "geosphere-rabbit-rabbitmq" -source: - existingClaim: "cspp-geo-geo2grid" -destination: - # s3Endpoint: "http://geosphere-minio:9000" - persistence: - enabled: true - storageClass: "longhorn" - existingClaim: "geosphere-tile-gen-shapefiles" diff --git a/ci_geosphere-test/values-tile-gen-g16-radf.yaml b/ci_geosphere-test/values-tile-gen-g16-radf.yaml deleted file mode 100644 index 86975c2f64f707b8e6322f2d1670f5374c2a81a5..0000000000000000000000000000000000000000 --- a/ci_geosphere-test/values-tile-gen-g16-radf.yaml +++ /dev/null @@ -1,19 +0,0 @@ -rabbitIn: - # use the rabbitmq from the production environment (geosphere) - host: "geosphere-rabbit-rabbitmq.geosphere" - username: "user" - passwordSecret: "geosphere-rabbit-rabbitmq-production" - topic: "data.goes.g16.abi.radf.l1b.geotiff.all.complete" -rabbitOut: - host: "geosphere-rabbit-rabbitmq" - username: "user" - passwordSecret: "geosphere-rabbit-rabbitmq" -source: - existingClaim: "cspp-geo-geo2grid" -destination: - # s3Endpoint: "http://geosphere-minio:9000" - persistence: - enabled: true - storageClass: "longhorn" - existingClaim: "geosphere-tile-gen-shapefiles" - diff --git a/ci_geosphere-test/values-tile-gen-g16-radm2.yaml b/ci_geosphere-test/values-tile-gen-g16-radm2.yaml deleted file mode 100644 index c4548c4aebf4f166d80f70f46c6975bb8e7010f2..0000000000000000000000000000000000000000 --- a/ci_geosphere-test/values-tile-gen-g16-radm2.yaml +++ /dev/null @@ -1,18 +0,0 @@ -rabbitIn: - # use the rabbitmq from the production environment (geosphere) - host: "geosphere-rabbit-rabbitmq.geosphere" - username: "user" - passwordSecret: "geosphere-rabbit-rabbitmq-production" - topic: "data.goes.g16.abi.radm2.l1b.geotiff.all.complete" -rabbitOut: - host: "geosphere-rabbit-rabbitmq" - username: "user" - passwordSecret: "geosphere-rabbit-rabbitmq" -source: - existingClaim: "cspp-geo-geo2grid" -destination: - # s3Endpoint: "http://geosphere-minio:9000" - persistence: - enabled: true - storageClass: "longhorn" - existingClaim: "geosphere-tile-gen-shapefiles" diff --git a/ci_geosphere/gitlab-ci.yaml b/ci_geosphere/gitlab-ci.yaml index 8230ba221fc833ac00b24583bab6e3ea106951bd..6aa8f9265773db7aeeaaa6690791d2e8355c4c99 100644 --- a/ci_geosphere/gitlab-ci.yaml +++ b/ci_geosphere/gitlab-ci.yaml @@ -193,22 +193,9 @@ gs deploy client: environment: name: geosphere url: http://geosphere.ssec.wisc.edu - extends: .helm_based_job - stage: deploy Client - script: - - ns="geosphere" - - cd geosphere-client/chart - - source geosphere-client/cibuild.env - # copy secret kubeconfig to the mounted (pwd) directory - - cp $kubekorner_k3s_config . - - kubeconfig=$(basename $kubekorner_k3s_config) - - echo "Deploying version $docker_tag to cluster namespace $ns" - # copy extra values files to the local directory (where helm has access via docker mount) - - cp ../../ci_geosphere/values-client.yaml . - # namespace names are the same as domain names - - helm upgrade -v 2 --install --kubeconfig $kubeconfig --namespace $ns -f values-client.yaml geosphere-client geosphere-client/ - dependencies: - - get_chart_client + variables: + VALUES_DIR: "ci_geosphere" + extends: .deploy_client rules: - if: $CI_COMMIT_TAG !~ /$GEOSPHERE_TAG_REGEX/ when: never diff --git a/ci_tests/get_release_names.sh b/ci_tests/get_release_names.sh index 3e39bf300f6cfad894de357c9284b2f7c194c69b..b75d6fc298f9ec56d7280a630e6107ebff1d7bac 100755 --- a/ci_tests/get_release_names.sh +++ b/ci_tests/get_release_names.sh @@ -2,7 +2,8 @@ export RELEASE_PREFIX="${CI_JOB_NAME//_/-}-${CI_JOB_ID//_/-}-" export GRB_RELEASE_NAME="${RELEASE_PREFIX}cspp-geo-grb" -export RABBITMQ_RELEASE_NAME="geosphere-rabbit" +export RABBITMQ_RELEASE_NAME="${RELEASE_PREFIX}geosphere-rabbit" +export RABBITMQ_SERVICE_NAME="${RABBITMQ_RELEASE_NAME}-rabbitmq" export GEO2GRID_RELEASE_NAME="${RELEASE_PREFIX}cspp-geo-geo2grid-g16-radm1" export TILEGEN_RELEASE_NAME="${RELEASE_PREFIX}geosphere-tilegen-g16-radm1" export MAPSERVER_RELEASE_NAME="${RELEASE_PREFIX}geosphere-mapserver" diff --git a/ci_tests/gitlab-ci.yaml b/ci_tests/gitlab-ci.yaml index 2457abf08ec291e2eccb979852e1369ff01b9535..b726c5a8d7b7b5db00cca324ae1cb45ea98001d2 100644 --- a/ci_tests/gitlab-ci.yaml +++ b/ci_tests/gitlab-ci.yaml @@ -7,65 +7,6 @@ # Create TLS certificate secret: # kubectl create -n geosphere secret tls geosphere-tls-certs --cert=geosphere_ssec_wisc_edu.crt --key=geosphere_ssec_wisc_edu.key -#gs create geotiff storage: -# extends: .helm_based_job -# stage: create storage -# script: -# - ns=$(./helpers/get_namespace.sh) -# # copy secret kubeconfig to the mounted (pwd) directory -# - cp $kubekorner_k3s_config . -# - kubeconfig=$(basename $kubekorner_k3s_config) -# - ./helpers/create_pvc.sh "$ns" "ci_geosphere/geotiff-pvc.yaml" "cspp-geo-geo2grid" "$kubeconfig" -# # this job doesn't actually need any artifacts from previous jobs -# dependencies: [] -# rules: -# - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX -# when: never -# - if: $CREATE_STORAGE -# -#gs create shapefile storage: -# extends: .helm_based_job -# stage: create storage -# script: -# - ns=$(./helpers/get_namespace.sh) -# # copy secret kubeconfig to the mounted (pwd) directory -# - cp $kubekorner_k3s_config . -# - kubeconfig=$(basename $kubekorner_k3s_config) -# - ./helpers/create_pvc.sh "$ns" "ci_geosphere/shapefiles-pvc.yaml" "geosphere-tile-gen-shapefiles" "$kubeconfig" -# # this job doesn't actually need any artifacts from previous jobs -# dependencies: [] -# rules: -# - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX -# when: never -# # this will always be true for tags -# - changes: -# - ci_geosphere/shapefiles-pvc.yaml -# - if: $CREATE_STORAGE -# -#gs deploy rabbit: -# environment: -# name: geosphere -# url: http://geosphere.ssec.wisc.edu -# extends: .helm_based_job -# stage: deploy rabbit -# script: -# - ./helpers/deploy_rabbitmq.sh ci_geosphere -# - cp ${kubekorner_k3s_config} . -# - kubeconfig=$(basename ${kubekorner_k3s_config}) -# - |- -# kubectl get secret --kubeconfig "${kubeconfig}" geosphere-rabbit-rabbitmq --namespace=geosphere -oyaml | grep -v '^\s*namespace:\s' | grep -v "[Hh]elm" | grep -v "[tT]ime" | grep -v "selfLink" | grep -v "uid" | grep -v "resourceVersion" | sed 's/ name: .*/ name: geosphere-rabbit-rabbitmq-production/' | kubectl_stdin apply --kubeconfig "${kubeconfig}" --namespace=geosphere-test -f - -# # this job doesn't actually need any artifacts from previous jobs -# dependencies: [] -# rules: -# - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX -# when: never -# # no need to build if another project triggered us -# - if: $CI_PIPELINE_SOURCE == "pipeline" -# when: never -# - changes: -# - ci_geosphere/values-geosphere-rabbit.yaml -# - if: $DEPLOY_RABBIT -# test_basic_grb: extends: .helm_based_job stage: test @@ -82,132 +23,3 @@ test_basic_grb: - get_chart_client rules: - when: on_success -# -#gs deploy geo2grid g16 radf: -# extends: .deploy_geo2grid -# variables: -# VALUES_DIR: "ci_geosphere" -# DEPLOY_SUFFIX: "-g16-radf" -# rules: -# - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX -# when: never -# - when: on_success -# -#gs deploy geo2grid g16 radc: -# extends: .deploy_geo2grid -# variables: -# VALUES_DIR: "ci_geosphere" -# DEPLOY_SUFFIX: "-g16-radc" -# rules: -# - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX -# when: never -# - when: on_success -# -#gs deploy geo2grid g16 radm1: -# extends: .deploy_geo2grid -# variables: -# VALUES_DIR: "ci_geosphere" -# DEPLOY_SUFFIX: "-g16-radm1" -# rules: -# - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX -# when: never -# - when: on_success -# -#gs deploy geo2grid g16 radm2: -# extends: .deploy_geo2grid -# variables: -# VALUES_DIR: "ci_geosphere" -# DEPLOY_SUFFIX: "-g16-radm2" -# rules: -# - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX -# when: never -# - when: on_success -# -#gs deploy tile gen g16 radf: -# extends: .deploy_tile_gen -# variables: -# VALUES_DIR: "ci_geosphere" -# DEPLOY_SUFFIX: "-g16-radf" -# rules: -# - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX -# when: never -# - when: on_success -# -#gs deploy tile gen g16 radc: -# extends: .deploy_tile_gen -# variables: -# VALUES_DIR: "ci_geosphere" -# DEPLOY_SUFFIX: "-g16-radc" -# rules: -# - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX -# when: never -# - when: on_success -# -#gs deploy_tile_gen_g16_radm1: -# extends: .deploy_tile_gen -# variables: -# VALUES_DIR: "ci_geosphere" -# DEPLOY_SUFFIX: "-g16-radm1" -# rules: -# - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX -# when: never -# - when: on_success -# -#gs deploy_tile_gen_g16_radm2: -# extends: .deploy_tile_gen -# variables: -# VALUES_DIR: "ci_geosphere" -# DEPLOY_SUFFIX: "-g16-radm2" -# rules: -# - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX -# when: never -# - when: on_success -# -#gs deploy mapserver: -# environment: -# name: geosphere -# url: http://geosphere.ssec.wisc.edu -# variables: -# VALUES_DIR: "ci_geosphere" -# extends: .deploy_mapserver -# rules: -# - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX -# when: never -# - when: on_success -# -#gs deploy mapcache: -# environment: -# name: geosphere -# url: http://geosphere.ssec.wisc.edu -# variables: -# VALUES_DIR: "ci_geosphere" -# extends: .deploy_mapcache -# rules: -# - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX -# when: never -# - when: on_success -# -#gs deploy client: -# environment: -# name: geosphere -# url: http://geosphere.ssec.wisc.edu -# extends: .helm_based_job -# stage: deploy Client -# script: -# - ns="geosphere" -# - cd geosphere-client/chart -# - source geosphere-client/cibuild.env -# # copy secret kubeconfig to the mounted (pwd) directory -# - cp $kubekorner_k3s_config . -# - kubeconfig=$(basename $kubekorner_k3s_config) -# - echo "Deploying version $docker_tag to cluster namespace $ns" -# # copy extra values files to the local directory (where helm has access via docker mount) -# - cp ../../ci_geosphere/values-client.yaml . -# # namespace names are the same as domain names -# - helm upgrade -v 2 --install --kubeconfig $kubeconfig --namespace $ns -f values-client.yaml geosphere-client geosphere-client/ -# dependencies: -# - get_chart_client -# rules: -# - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX -# when: never -# - when: on_success diff --git a/ci_tests/run_basic_grb_test.sh b/ci_tests/run_basic_grb_test.sh index 5f218228f9924855b84d079d5194bc5a5a9b119a..b6fe0890c82bf2fcb4c049cb738b337c15b6cafc 100755 --- a/ci_tests/run_basic_grb_test.sh +++ b/ci_tests/run_basic_grb_test.sh @@ -11,17 +11,40 @@ error() { install_charts() { # use the production rabbitmq config - ./helpers/deploy_rabbitmq.sh "ci_tests" + ./helpers/deploy_rabbitmq.sh "ci_tests" "${RELEASE_PREFIX}" # GRB pushd geosphere-grb/chart + sed -i "s/geosphere-rabbit-rabbitmq/${RABBITMQ_SERVICE_NAME}/g" "ci_tests/values-grb-g16.yaml" helm upgrade -v 2 --install --namespace $ns -f ../../ci_tests/values-grb-g16.yaml ${GRB_RELEASE_NAME} cspp-geo-grb/ popd # Geo2Grid grb_pvc=$(get_pvc_name ${GRB_RELEASE_NAME}) + sed -i "s/geosphere-rabbit-rabbitmq/${RABBITMQ_SERVICE_NAME}/g" "ci_tests/values-geo2grid-g16-radm1.yaml" sed -i "s/cspp-geo-grb/${GRB_RELEASE_NAME}/g" "ci_tests/values-geo2grid-g16-radm1.yaml" ./helpers/deploy_geo2grid.sh "ci_tests" "${RELEASE_PREFIX}" "-g16-radm1" + + # Tile Gen + geotiff_pvc=$(get_pvc_name ${GEO2GRID_RELEASE_NAME}) + sed -i "s/geosphere-rabbit-rabbitmq/${RABBITMQ_SERVICE_NAME}/g" "ci_tests/values-tile-gen-g16-radm1.yaml" + sed -i "s/cspp-geo-geo2grid/${GEO2GRID_RELEASE_NAME}/g" "ci_tests/values-tile-gen-g16-radm1.yaml" + ./helpers/deploy_tile_gen.sh "ci_tests" "${RELEASE_PREFIX}" "-g16-radm1" + + # MapServer + sed -i "s/cspp-geo-geo2grid/${GEO2GRID_RELEASE_NAME}/g" "ci_tests/values-mapserver.yaml" + sed -i "s/geosphere-tile-gen-shapefiles/${TILEGEN_RELEASE_NAME}/g" "ci_tests/values-mapserver.yaml" + ./helpers/deploy_mapserver.sh "ci_test" + + # MapCache + sed -i "s/geosphere-rabbit-rabbitmq/${RABBITMQ_SERVICE_NAME}/g" "ci_tests/values-mapcache.yaml" + sed -i "s/geosphere-mapserver/${MAPSERVER_RELEASE_NAME}/g" "ci_tests/values-mapcache.yaml" + ./helpers/deploy_mapcache.sh "ci_test" + + # Client + sed -i "s/geosphere-mapserver/${MAPSERVER_RELEASE_NAME}/g" "ci_tests/values-client.yaml" + sed -i "s/geosphere-mapcache/${MAPCACHE_RELEASE_NAME}/g" "ci_tests/values-client.yaml" + ./helpers/deploy_client.sh "ci_test" } get_pod_name() { @@ -46,7 +69,7 @@ run_grb_checks() { num_files=$(kubectl exec -n $ns ${GRB_POD_NAME} -c cspp-geo-grb-notifier -- ls -1 /dst | wc -l || echo 0) let count=count+1 if [[ $num_files -gt 16 ]]; then - debug "Got at least 16 files" + debug "Got at least 16 files ($num_files)" break fi if [[ $count -gt 10 ]]; then @@ -62,15 +85,32 @@ run_geo2grid_checks() { num_files=$(kubectl exec -n $ns ${GEO2GRID_POD_NAME} -- ls -1 /dst/geotiffs/g16/abi/radm1/ | wc -l || echo 0) let count=count+1 if [[ $num_files -gt 16 ]]; then - debug "Got at least 16 files" + debug "Got at least 16 files ($num_files)" break fi if [[ $count -gt 10 ]]; then - error "Took too long for Geo2Grid to produce files" + error "Took too long for Geo2Grid to produce files ($num_files so far)" fi sleep 10 done } + +run_tile_gen_checks() { + return 0 +} + +run_mapserver_checks() { + return 0 +} + +run_mapcache_checks() { + return 0 +} + +run_client_checks() { + # just try to load the webpage at all + curl -s "http://${CLIENT_RELEASE_NAME}/" +} set -e ns=$(./helpers/get_namespace.sh) @@ -85,9 +125,25 @@ GRB_POD_NAME=$(get_pod_name "${GRB_RELEASE_NAME}") debug "GRB pod name: ${GRB_POD_NAME}" GEO2GRID_POD_NAME=$(get_pod_name "${GEO2GRID_RELEASE_NAME}") debug "Geo2Grid pod name: ${GEO2GRID_POD_NAME}" +TILEGEN_POD_NAME=$(get_pod_name "${TILEGEN_RELEASE_NAME}") +debug "Tile Generation pod name: ${TILEGEN_POD_NAME}" +MAPSERVER_POD_NAME=$(get_pod_name "${MAPSERVER_RELEASE_NAME}") +debug "MapServer pod name: ${MAPSERVER_POD_NAME}" +MAPCACHE_POD_NAME=$(get_pod_name "${MAPCACHE_RELEASE_NAME}") +debug "MapCache pod name: ${MAPCACHE_POD_NAME}" +CLIENT_POD_NAME=$(get_pod_name "${CLIENT_RELEASE_NAME}") +debug "Client pod name: ${CLIENT_POD_NAME}" wait_for_pod ${GRB_POD_NAME} || error "GRB pod was never ready" run_grb_checks || error "GRB checks failed" wait_for_pod ${GEO2GRID_POD_NAME} || error "Geo2Grid pod was never ready" run_geo2grid_checks || error "Geo2Grid checks failed" +wait_for_pod ${TILEGEN_POD_NAME} || error "Tile Gen pod was never ready" +run_tile_gen_checks || error "Tile Generation checks failed" +wait_for_pod ${MAPSERVER_POD_NAME} || error "MapServer pod was never ready" +run_mapserver_checks || error "MapServer checks failed" +wait_for_pod ${MAPCACHE_POD_NAME} || error "MapCache pod was never ready" +run_mapcache_checks || error "MapCache checks failed" +wait_for_pod ${CLIENT_POD_NAME} || error "Client pod was never ready" +run_client_checks || error "Client checks failed" diff --git a/ci_tests/values-client.yaml b/ci_tests/values-client.yaml new file mode 100644 index 0000000000000000000000000000000000000000..558e745ac17ab813111157a92deace71b25858d8 --- /dev/null +++ b/ci_tests/values-client.yaml @@ -0,0 +1,2 @@ +infoServer: "geosphere-mapserver" +tileServer: "geosphere-mapcache" diff --git a/ci_tests/values-mapcache.yaml b/ci_tests/values-mapcache.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ba26713979300370877732db76c1437c1e9c372 --- /dev/null +++ b/ci_tests/values-mapcache.yaml @@ -0,0 +1,19 @@ +rabbitIn: + host: "geosphere-rabbit-rabbitmq" + username: "user" + passwordSecret: "geosphere-rabbit-rabbitmq" + topic: "data.goes.g16.abi.*.l1b.tiles.*.complete" +wms: + host: "geosphere-mapserver" +cache: + persistence: + enabled: true + storageClass: "local-path" + size: 1Ti + cleanup: + # every 6 hours + schedule: "0 */6 * * *" + age: "+2" +seed: + images: true + overlays: true diff --git a/ci_tests/values-mapserver.yaml b/ci_tests/values-mapserver.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f24f3336d23bf9c94366c7fc3de57ddcff2ca162 --- /dev/null +++ b/ci_tests/values-mapserver.yaml @@ -0,0 +1,5 @@ +imageSource: + # s3Endpoint: "http://geosphere-minio:9000" + existingClaim: "cspp-geo-geo2grid" +tileSource: + existingClaim: "geosphere-tile-gen-shapefiles" diff --git a/ci_geosphere-test/values-tile-gen-g16-radm1.yaml b/ci_tests/values-tile-gen-g16-radm1.yaml similarity index 54% rename from ci_geosphere-test/values-tile-gen-g16-radm1.yaml rename to ci_tests/values-tile-gen-g16-radm1.yaml index f0aaa5b81716d71d93dbc6e6f018b41c86722f13..581446c88a470806c5988f3ed38c34363d921a8a 100644 --- a/ci_geosphere-test/values-tile-gen-g16-radm1.yaml +++ b/ci_tests/values-tile-gen-g16-radm1.yaml @@ -1,8 +1,7 @@ rabbitIn: - # use the rabbitmq from the production environment (geosphere) - host: "geosphere-rabbit-rabbitmq.geosphere" + host: "geosphere-rabbit-rabbitmq" username: "user" - passwordSecret: "geosphere-rabbit-rabbitmq-production" + passwordSecret: "geosphere-rabbit-rabbitmq" topic: "data.goes.g16.abi.radm1.l1b.geotiff.all.complete" rabbitOut: host: "geosphere-rabbit-rabbitmq" @@ -11,8 +10,8 @@ rabbitOut: source: existingClaim: "cspp-geo-geo2grid" destination: - # s3Endpoint: "http://geosphere-minio:9000" persistence: enabled: true storageClass: "longhorn" - existingClaim: "geosphere-tile-gen-shapefiles" + cleanup: + enabled: false diff --git a/helpers/deploy_client.sh b/helpers/deploy_client.sh new file mode 100755 index 0000000000000000000000000000000000000000..88489998daead5db00d9347cd0c80af37062e7d4 --- /dev/null +++ b/helpers/deploy_client.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -e + +if [ $# -ne 2 ] && [ $# -ne 3 ]; then + echo "Usage: ./helpers/deploy_client.sh <values_base_dir> [<release_prefix>]" + exit 1 +fi + +# ci_geosphere-test +values_base="$1" + +# Prefix Example: testing-YYYYMMDD- +if [[ $# -eq 1 ]]; then + release_prefix="" +else + release_prefix="$2" +fi + +ns=$(./helpers/get_namespace.sh) +cd geosphere-client/chart +source geosphere-client/cibuild.env +echo "Deploying version $docker_tag to cluster namespace $ns" +helm upgrade -v 2 --install --namespace $ns -f ../../${values_base}/values-client.yaml "${release_prefix}geosphere-client" geosphere-client/ diff --git a/helpers/deploy_mapcache.sh b/helpers/deploy_mapcache.sh new file mode 100755 index 0000000000000000000000000000000000000000..ae254b6a0dc91007d2302ec822ae117910ec1e47 --- /dev/null +++ b/helpers/deploy_mapcache.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -e + +if [ $# -ne 2 ] && [ $# -ne 3 ]; then + echo "Usage: ./helpers/deploy_mapcache.sh <values_base_dir> [<release_prefix>]" + exit 1 +fi + +# ci_geosphere-test +values_base="$1" + +# Prefix Example: testing-YYYYMMDD- +if [[ $# -eq 1 ]]; then + release_prefix="" +else + release_prefix="$2" +fi + +ns=$(./helpers/get_namespace.sh) +cd geosphere-mapcache/chart +source geosphere-mapcache/cibuild.env +echo "Deploying version $docker_tag to cluster namespace $ns" +helm upgrade -v 2 --install --namespace $ns -f ../../${values_base}/values-mapcache.yaml "${release_prefix}geosphere-mapcache" geosphere-mapcache/ diff --git a/helpers/deploy_mapserver.sh b/helpers/deploy_mapserver.sh new file mode 100755 index 0000000000000000000000000000000000000000..b57dc0a8a654e70edb0a492cc272a68f73b54897 --- /dev/null +++ b/helpers/deploy_mapserver.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -e + +if [ $# -ne 2 ] && [ $# -ne 3 ]; then + echo "Usage: ./helpers/deploy_mapserver.sh <values_base_dir> [<release_prefix>]" + exit 1 +fi + +# ci_geosphere-test +values_base="$1" + +# Prefix Example: testing-YYYYMMDD- +if [[ $# -eq 1 ]]; then + release_prefix="" +else + release_prefix="$2" +fi + +ns=$(./helpers/get_namespace.sh) +cd geosphere-mapserver/chart +source geosphere-mapserver/cibuild.env +echo "Deploying version $docker_tag to cluster namespace $ns" +helm upgrade -v 2 --install --namespace $ns -f ../../${values_base}/values-mapserver.yaml "${release_prefix}geosphere-mapserver" geosphere-mapserver/ diff --git a/helpers/deploy_rabbitmq.sh b/helpers/deploy_rabbitmq.sh index 1d681a18cbfa3b3c805082038800f49fede1c7ac..c6b4541ccc151dadbb70be3c92b463113c047590 100755 --- a/helpers/deploy_rabbitmq.sh +++ b/helpers/deploy_rabbitmq.sh @@ -10,6 +10,14 @@ fi # ci_geosphere-test values_base="$1" +# Prefix Example: testing-YYYYMMDD- +if [[ $# -eq 1 ]]; then + release_prefix="" +else + release_prefix="$2" +fi + + ns=$(./helpers/get_namespace.sh) # get password from any previous installation # if we don't do this the password will get out of sync @@ -32,11 +40,13 @@ fi echo "Adding extra arguments: $EXTRA_ARGS" # install third-party rabbitmq server helm repo add bitnami "https://charts.bitnami.com/bitnami" -helm upgrade -v 2 --install -f ${values_base}/values-geosphere-rabbit.yaml $EXTRA_ARGS --namespace $ns geosphere-rabbit bitnami/rabbitmq + +release_name="${release_prefix}geosphere-rabbit" +helm upgrade -v 2 --install -f ${values_base}/values-geosphere-rabbit.yaml $EXTRA_ARGS --namespace $ns "${release_name}" bitnami/rabbitmq # do a little waiting for the rabbitmq pod to be ready so future stages # don't fail to communicate with it echo "Start waiting for rabbitmq: $(date +%Y%m%dT%H%M%S)" -kubectl wait -n $ns pod/geosphere-rabbit-rabbitmq-0 --timeout 120s --for=condition=Ready +kubectl wait -n $ns pod/"${release_name}-rabbitmq-0" --timeout 120s --for=condition=Ready status=$? echo "Done waiting for rabbitmq: $(date +%Y%m%dT%H%M%S) : $status" exit $status diff --git a/helpers/deploy_tile_gen.sh b/helpers/deploy_tile_gen.sh index 554e4abbdf9bacba1fe149d2e10844a6a0fb6b76..3fabf07d0edd332c1ca33770a6dcd2c80bdd9e37 100755 --- a/helpers/deploy_tile_gen.sh +++ b/helpers/deploy_tile_gen.sh @@ -2,24 +2,26 @@ set -e -if [ $# -ne 2 ]; then - echo "Usage: ./helpers/deploy_tile_gen.sh <values_base_dir> <deploy_suffix>" +if [ $# -ne 2 ] && [ $# -ne 3 ]; then + echo "Usage: ./helpers/deploy_tile_gen.sh <values_base_dir> [<release_prefix>] <release_suffix>" exit 1 fi # ci_geosphere-test values_base="$1" -# Example: -g16-radf -deploy_suffix="$2" + +# Prefix Example: testing-YYYYMMDD- +# Suffix Example: -g16-radf +if [[ $# -eq 2 ]]; then + release_prefix="" + release_suffix="$2" +else + release_prefix="$2" + release_suffix="$3" +fi ns=$(./helpers/get_namespace.sh) cd geosphere-tile-gen/chart source geosphere-tile-gen/cibuild.env -# copy secret kubeconfig to the mounted (pwd) directory -cp $kubekorner_k3s_config . -kubeconfig=$(basename $kubekorner_k3s_config) echo "Deploying version $docker_tag to cluster namespace $ns" -# copy extra values files to the local directory (where helm has access via docker mount) -cp ../../${values_base}/values-tile-gen${deploy_suffix}.yaml . -# namespace names are the same as domain names -helm upgrade -v 2 --install --kubeconfig $kubeconfig --namespace $ns -f values-tile-gen${deploy_suffix}.yaml geosphere-tile-gen${deploy_suffix} geosphere-tile-gen/ +helm upgrade -v 2 --install --namespace $ns -f ../../${values_base}/values-tile-gen${release_suffix}.yaml "${release_prefix}geosphere-tile-gen${release_suffix}" geosphere-tile-gen/