diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c08fe7451937f29293449bcb77a838bc6b3983d5..ff8297987331537ee07b52f1ebb7ef95c90b5ed6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,7 @@ stages: - get chart - build prereqs + - test - create storage - deploy rabbit - deploy GRB @@ -16,6 +17,14 @@ variables: DOCKER_DRIVER: overlay2 HELM_EXPERIMENTAL_OCI: "1" CICHART_IMAGE: "$CI_REGISTRY_IMAGE/cichart:latest" + GEOSPHERE_TAG_REGEX: "r[0-9]+_[0-9]+" + GEOSPHERE_TEST_TAG_REGEX: "test-r[0-9]+_[0-9]+" + GCP_TAG_REGEX: "gcp-r[0-9]+_[0-9]+" + TESTS_BRANCH_REGEX: "master" + GEOSPHERE_NAMESPACE: "geosphere" + GEOSPHERE_TEST_NAMESPACE: "geosphere-test" + GCP_NAMESPACE: "default" + TESTS_NAMESPACE: "geosphere-ci-tests" include: @@ -23,6 +32,7 @@ include: - local: "/helpers/docker_base.yaml" - local: "/helpers/helm_base.yaml" - local: "/helpers/get_chart.yaml" + - local: "/ci_tests/gitlab-ci.yaml" - local: "/ci_geosphere/gitlab-ci.yaml" - local: "/ci_geosphere-test/gitlab-ci.yaml" # - local: "/ci_gcp/gitlab-ci.yaml" @@ -124,10 +134,6 @@ get_chart_client_test: - ./helpers/deploy_geo2grid.sh "${VALUES_DIR}" "${DEPLOY_SUFFIX}" dependencies: - get_chart_geo2grid - rules: - - if: '$kubekorner_k3s_config == null' - when: never - - when: on_success .deploy_tile_gen: # environment: @@ -142,10 +148,6 @@ get_chart_client_test: - ./helpers/deploy_tile_gen.sh "${VALUES_DIR}" "${DEPLOY_SUFFIX}" dependencies: - get_chart_tilegen - rules: - - if: '$kubekorner_k3s_config == null' - when: never - - when: on_success .deploy_mapserver: variables: @@ -166,10 +168,6 @@ get_chart_client_test: - helm upgrade -v 2 --install --kubeconfig $kubeconfig --namespace $ns -f values-mapserver.yaml geosphere-mapserver geosphere-mapserver/ dependencies: - get_chart_mapserver - rules: - - if: '$kubekorner_k3s_config == null' - when: never - - when: on_success .deploy_mapcache: variables: @@ -190,8 +188,4 @@ get_chart_client_test: - helm upgrade -v 2 --install --kubeconfig $kubeconfig --namespace $ns -f values-mapcache.yaml geosphere-mapcache geosphere-mapcache/ dependencies: - get_chart_mapcache - rules: - - if: '$kubekorner_k3s_config == null' - when: never - - when: on_success diff --git a/ci_geosphere-test/gitlab-ci.yaml b/ci_geosphere-test/gitlab-ci.yaml index 28133b2bc45439f667fe433caf21ab088c6fd9d3..052db5e5d42211622ac82dd5600cc322931cb309 100644 --- a/ci_geosphere-test/gitlab-ci.yaml +++ b/ci_geosphere-test/gitlab-ci.yaml @@ -43,9 +43,7 @@ gstest deploy rabbit: # this job doesn't actually need any artifacts from previous jobs dependencies: [] rules: - - if: '$CI_COMMIT_BRANCH != "master"' - when: never - - if: '$kubekorner_k3s_config == null' + - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TEST_TAG_REGEX when: never # no need to build if another project triggered us - if: $CI_PIPELINE_SOURCE == "pipeline" @@ -124,9 +122,7 @@ gstest deploy mapcache: VALUES_DIR: "ci_geosphere-test" extends: .deploy_mapcache rules: - - if: '$CI_COMMIT_BRANCH != "master"' - when: never - - if: '$kubekorner_k3s_config == null' + - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TEST_TAG_REGEX when: never - when: on_success @@ -151,9 +147,7 @@ gstest deploy client: dependencies: - get_chart_client_test rules: - - if: '$CI_COMMIT_BRANCH != "master"' - when: never - - if: '$kubekorner_k3s_config == null' + - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TEST_TAG_REGEX when: never - when: on_success diff --git a/ci_geosphere/gitlab-ci.yaml b/ci_geosphere/gitlab-ci.yaml index 26e7e63d09229602741fb08e83803f3ea0a9878b..1fdcd73ef9044000d0b04365f2a868b06da66909 100644 --- a/ci_geosphere/gitlab-ci.yaml +++ b/ci_geosphere/gitlab-ci.yaml @@ -19,13 +19,8 @@ gs create geotiff storage: # this job doesn't actually need any artifacts from previous jobs dependencies: [] rules: - - if: "$CI_COMMIT_TAG == null" + - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX when: never - - if: '$kubekorner_k3s_config == null' - when: never - # this will always be true for tags - - changes: - - ci_geosphere/geotiff-pvc.yaml - if: $CREATE_STORAGE gs create shapefile storage: @@ -40,9 +35,7 @@ gs create shapefile storage: # this job doesn't actually need any artifacts from previous jobs dependencies: [] rules: - - if: "$CI_COMMIT_TAG == null" - when: never - - if: '$kubekorner_k3s_config == null' + - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX when: never # this will always be true for tags - changes: @@ -64,9 +57,7 @@ gs deploy rabbit: # this job doesn't actually need any artifacts from previous jobs dependencies: [] rules: - - if: "$CI_COMMIT_TAG == null" - when: never - - if: '$kubekorner_k3s_config == null' + - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX when: never # no need to build if another project triggered us - if: $CI_PIPELINE_SOURCE == "pipeline" @@ -96,9 +87,7 @@ gs deploy g16 grb: dependencies: - get_chart_grb rules: - - if: "$CI_COMMIT_TAG == null" - when: never - - if: '$kubekorner_k3s_config == null' + - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX when: never - when: on_success @@ -108,9 +97,7 @@ gs deploy geo2grid g16 radf: VALUES_DIR: "ci_geosphere" DEPLOY_SUFFIX: "-g16-radf" rules: - - if: "$CI_COMMIT_TAG == null" - when: never - - if: '$kubekorner_k3s_config == null' + - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX when: never - when: on_success @@ -120,9 +107,7 @@ gs deploy geo2grid g16 radc: VALUES_DIR: "ci_geosphere" DEPLOY_SUFFIX: "-g16-radc" rules: - - if: "$CI_COMMIT_TAG == null" - when: never - - if: '$kubekorner_k3s_config == null' + - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX when: never - when: on_success @@ -132,9 +117,7 @@ gs deploy geo2grid g16 radm1: VALUES_DIR: "ci_geosphere" DEPLOY_SUFFIX: "-g16-radm1" rules: - - if: "$CI_COMMIT_TAG == null" - when: never - - if: '$kubekorner_k3s_config == null' + - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX when: never - when: on_success @@ -144,9 +127,7 @@ gs deploy geo2grid g16 radm2: VALUES_DIR: "ci_geosphere" DEPLOY_SUFFIX: "-g16-radm2" rules: - - if: "$CI_COMMIT_TAG == null" - when: never - - if: '$kubekorner_k3s_config == null' + - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX when: never - when: on_success @@ -156,9 +137,7 @@ gs deploy tile gen g16 radf: VALUES_DIR: "ci_geosphere" DEPLOY_SUFFIX: "-g16-radf" rules: - - if: "$CI_COMMIT_TAG == null" - when: never - - if: '$kubekorner_k3s_config == null' + - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX when: never - when: on_success @@ -168,9 +147,7 @@ gs deploy tile gen g16 radc: VALUES_DIR: "ci_geosphere" DEPLOY_SUFFIX: "-g16-radc" rules: - - if: "$CI_COMMIT_TAG == null" - when: never - - if: '$kubekorner_k3s_config == null' + - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX when: never - when: on_success @@ -180,9 +157,7 @@ gs deploy_tile_gen_g16_radm1: VALUES_DIR: "ci_geosphere" DEPLOY_SUFFIX: "-g16-radm1" rules: - - if: "$CI_COMMIT_TAG == null" - when: never - - if: '$kubekorner_k3s_config == null' + - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX when: never - when: on_success @@ -192,9 +167,7 @@ gs deploy_tile_gen_g16_radm2: VALUES_DIR: "ci_geosphere" DEPLOY_SUFFIX: "-g16-radm2" rules: - - if: "$CI_COMMIT_TAG == null" - when: never - - if: '$kubekorner_k3s_config == null' + - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX when: never - when: on_success @@ -206,9 +179,7 @@ gs deploy mapserver: VALUES_DIR: "ci_geosphere" extends: .deploy_mapserver rules: - - if: "$CI_COMMIT_TAG == null" - when: never - - if: '$kubekorner_k3s_config == null' + - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX when: never - when: on_success @@ -220,9 +191,7 @@ gs deploy mapcache: VALUES_DIR: "ci_geosphere" extends: .deploy_mapcache rules: - - if: "$CI_COMMIT_TAG == null" - when: never - - if: '$kubekorner_k3s_config == null' + - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX when: never - when: on_success @@ -247,8 +216,6 @@ gs deploy client: dependencies: - get_chart_client rules: - - if: "$CI_COMMIT_TAG == null" - when: never - - if: '$kubekorner_k3s_config == null' + - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX when: never - when: on_success diff --git a/ci_tests/gitlab-ci.yaml b/ci_tests/gitlab-ci.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5836fd650e17a1986af0e8aac20233bff9a870f6 --- /dev/null +++ b/ci_tests/gitlab-ci.yaml @@ -0,0 +1,221 @@ +# This file is included as part of the main repository .gitlab-ci.yml file +# NOTE: Since this is running from the root of the repository all referenced +# files must be relative to the root directory. +# Most jobs in this file are only meant to be run when a git tag is created + +# Prerequisites: +# Create TLS certificate secret: +# kubectl create -n geosphere secret tls geosphere-tls-certs --cert=geosphere_ssec_wisc_edu.crt --key=geosphere_ssec_wisc_edu.key + +#gs create geotiff storage: +# extends: .helm_based_job +# stage: create storage +# script: +# - ns=$(./helpers/get_namespace.sh) +# # copy secret kubeconfig to the mounted (pwd) directory +# - cp $kubekorner_k3s_config . +# - kubeconfig=$(basename $kubekorner_k3s_config) +# - ./helpers/create_pvc.sh "$ns" "ci_geosphere/geotiff-pvc.yaml" "cspp-geo-geo2grid" "$kubeconfig" +# # this job doesn't actually need any artifacts from previous jobs +# dependencies: [] +# rules: +# - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX +# when: never +# - if: $CREATE_STORAGE +# +#gs create shapefile storage: +# extends: .helm_based_job +# stage: create storage +# script: +# - ns=$(./helpers/get_namespace.sh) +# # copy secret kubeconfig to the mounted (pwd) directory +# - cp $kubekorner_k3s_config . +# - kubeconfig=$(basename $kubekorner_k3s_config) +# - ./helpers/create_pvc.sh "$ns" "ci_geosphere/shapefiles-pvc.yaml" "geosphere-tile-gen-shapefiles" "$kubeconfig" +# # this job doesn't actually need any artifacts from previous jobs +# dependencies: [] +# rules: +# - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX +# when: never +# # this will always be true for tags +# - changes: +# - ci_geosphere/shapefiles-pvc.yaml +# - if: $CREATE_STORAGE +# +#gs deploy rabbit: +# environment: +# name: geosphere +# url: http://geosphere.ssec.wisc.edu +# extends: .helm_based_job +# stage: deploy rabbit +# script: +# - ./helpers/deploy_rabbitmq.sh ci_geosphere +# - cp ${kubekorner_k3s_config} . +# - kubeconfig=$(basename ${kubekorner_k3s_config}) +# - |- +# kubectl get secret --kubeconfig "${kubeconfig}" geosphere-rabbit-rabbitmq --namespace=geosphere -oyaml | grep -v '^\s*namespace:\s' | grep -v "[Hh]elm" | grep -v "[tT]ime" | grep -v "selfLink" | grep -v "uid" | grep -v "resourceVersion" | sed 's/ name: .*/ name: geosphere-rabbit-rabbitmq-production/' | kubectl_stdin apply --kubeconfig "${kubeconfig}" --namespace=geosphere-test -f - +# # this job doesn't actually need any artifacts from previous jobs +# dependencies: [] +# rules: +# - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX +# when: never +# # no need to build if another project triggered us +# - if: $CI_PIPELINE_SOURCE == "pipeline" +# when: never +# - changes: +# - ci_geosphere/values-geosphere-rabbit.yaml +# - if: $DEPLOY_RABBIT +# +#gs deploy g16 grb: +## environment: +## name: geosphere +## url: http://geosphere.ssec.wisc.edu +# extends: .helm_based_job +# stage: deploy GRB +# script: +# - ns=$(./helpers/get_namespace.sh) +# - cd geosphere-grb/chart +# - source cspp-geo-grb/cibuild.env +# # copy secret kubeconfig to the mounted (pwd) directory +# - cp $kubekorner_k3s_config . +# - kubeconfig=$(basename $kubekorner_k3s_config) +# - echo "Deploying version $docker_tag to cluster namespace $ns" +# # copy extra values files to the local directory (where helm has access via docker mount) +# - cp ../../ci_geosphere/values-grb-g16.yaml . +# # namespace names are the same as domain names +# - helm upgrade -v 2 --install --kubeconfig $kubeconfig --namespace $ns --set persistence.enabled=true --set persistence.storageClass=longhorn -f values-grb-g16.yaml cspp-geo-grb cspp-geo-grb/ +# dependencies: +# - get_chart_grb +# rules: +# - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX +# when: never +# - when: on_success +# +#gs deploy geo2grid g16 radf: +# extends: .deploy_geo2grid +# variables: +# VALUES_DIR: "ci_geosphere" +# DEPLOY_SUFFIX: "-g16-radf" +# rules: +# - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX +# when: never +# - when: on_success +# +#gs deploy geo2grid g16 radc: +# extends: .deploy_geo2grid +# variables: +# VALUES_DIR: "ci_geosphere" +# DEPLOY_SUFFIX: "-g16-radc" +# rules: +# - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX +# when: never +# - when: on_success +# +#gs deploy geo2grid g16 radm1: +# extends: .deploy_geo2grid +# variables: +# VALUES_DIR: "ci_geosphere" +# DEPLOY_SUFFIX: "-g16-radm1" +# rules: +# - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX +# when: never +# - when: on_success +# +#gs deploy geo2grid g16 radm2: +# extends: .deploy_geo2grid +# variables: +# VALUES_DIR: "ci_geosphere" +# DEPLOY_SUFFIX: "-g16-radm2" +# rules: +# - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX +# when: never +# - when: on_success +# +#gs deploy tile gen g16 radf: +# extends: .deploy_tile_gen +# variables: +# VALUES_DIR: "ci_geosphere" +# DEPLOY_SUFFIX: "-g16-radf" +# rules: +# - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX +# when: never +# - when: on_success +# +#gs deploy tile gen g16 radc: +# extends: .deploy_tile_gen +# variables: +# VALUES_DIR: "ci_geosphere" +# DEPLOY_SUFFIX: "-g16-radc" +# rules: +# - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX +# when: never +# - when: on_success +# +#gs deploy_tile_gen_g16_radm1: +# extends: .deploy_tile_gen +# variables: +# VALUES_DIR: "ci_geosphere" +# DEPLOY_SUFFIX: "-g16-radm1" +# rules: +# - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX +# when: never +# - when: on_success +# +#gs deploy_tile_gen_g16_radm2: +# extends: .deploy_tile_gen +# variables: +# VALUES_DIR: "ci_geosphere" +# DEPLOY_SUFFIX: "-g16-radm2" +# rules: +# - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX +# when: never +# - when: on_success +# +#gs deploy mapserver: +# environment: +# name: geosphere +# url: http://geosphere.ssec.wisc.edu +# variables: +# VALUES_DIR: "ci_geosphere" +# extends: .deploy_mapserver +# rules: +# - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX +# when: never +# - when: on_success +# +#gs deploy mapcache: +# environment: +# name: geosphere +# url: http://geosphere.ssec.wisc.edu +# variables: +# VALUES_DIR: "ci_geosphere" +# extends: .deploy_mapcache +# rules: +# - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX +# when: never +# - when: on_success +# +#gs deploy client: +# environment: +# name: geosphere +# url: http://geosphere.ssec.wisc.edu +# extends: .helm_based_job +# stage: deploy Client +# script: +# - ns="geosphere" +# - cd geosphere-client/chart +# - source geosphere-client/cibuild.env +# # copy secret kubeconfig to the mounted (pwd) directory +# - cp $kubekorner_k3s_config . +# - kubeconfig=$(basename $kubekorner_k3s_config) +# - echo "Deploying version $docker_tag to cluster namespace $ns" +# # copy extra values files to the local directory (where helm has access via docker mount) +# - cp ../../ci_geosphere/values-client.yaml . +# # namespace names are the same as domain names +# - helm upgrade -v 2 --install --kubeconfig $kubeconfig --namespace $ns -f values-client.yaml geosphere-client geosphere-client/ +# dependencies: +# - get_chart_client +# rules: +# - if: $CI_COMMIT_TAG !~ $GEOSPHERE_TAG_REGEX +# when: never +# - when: on_success diff --git a/ci_tests/values-grb-g16.yaml b/ci_tests/values-grb-g16.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d27012396e64d7c7cd79d1f8752a46df83f2e9f --- /dev/null +++ b/ci_tests/values-grb-g16.yaml @@ -0,0 +1,9 @@ +apidFilter: "GOES-16-ABI-Only.xml" +uploadDst: "/dst" +#rabbitOut: +# host: "geosphere-rabbit-rabbitmq" +# username: "user" +# passwordSecret: "geosphere-rabbit-rabbitmq" +caduSender: + enabled: true + existingClaim: "geosphere-ci-tests-cadu" \ No newline at end of file diff --git a/helpers/get_namespace.sh b/helpers/get_namespace.sh index b897775995907bc5eec36347de1474a4391f14b3..696d2376b0b64e25128cc546e1c8cb291d0b05c5 100755 --- a/helpers/get_namespace.sh +++ b/helpers/get_namespace.sh @@ -1,10 +1,22 @@ #!/usr/bin/env bash +GEOSPHERE_NAMESPACE="${GEOSPHERE_NAMESPACE:-"geosphere"}" +GEOSPHERE_TEST_NAMESPACE="${GEOSPHERE_TEST_NAMESPACE:-"geosphere-test"}" +TESTS_NAMESPACE="${TESTS_NAMESPACE:-"geosphere-ci-tests"}" + if [[ -n "$CI_COMMIT_TAG" ]]; then - ns="geosphere"; -elif [[ "$CI_COMMIT_BRANCH" == "master" ]]; then - ns="geosphere-test"; -else + if [[ $CI_COMMIT_TAG =~ $GEOSPHERE_TAG_REGEX ]]; then + ns="${GEOSPHERE_NAMESPACE}"; + elif [[ $CI_COMMIT_TAG =~ $GEOSPHERE_TEST_TAG_REGEX ]]; then + ns="${GEOSPHERE_TEST_NAMESPACE}"; + elif [[ $CI_COMMIT_TAG =~ $GCP_TAG_REGEX ]]; then + ns="${GCP_NAMESPACE}"; + fi +elif [[ $CI_COMMIT_BRANCH =~ $TESTS_BRANCH_REGEX ]]; then + ns="${TESTS_NAMESPACE}"; +fi + +if [[ -z $ns ]]; then echo "ERROR: Kubernetes jobs don't run unless in master branch or tagged" exit 1 fi