From c829cb02ce9f56800b088bfc6b72a2676e1197b6 Mon Sep 17 00:00:00 2001 From: David Hoese <david.hoese@ssec.wisc.edu> Date: Fri, 24 Jul 2020 09:52:45 -0500 Subject: [PATCH] Add more complex PVC creation for GRB and Geo2Grid --- .gitlab-ci.yml | 30 +++++++++++++++++------- helpers/create_pvc.sh | 30 ++++++++++++++++++++++++ production/geotiff-pvc.yaml | 12 ++++++++++ production/values-geo2grid-g16-radf.yaml | 4 +++- 4 files changed, 66 insertions(+), 10 deletions(-) create mode 100755 helpers/create_pvc.sh create mode 100644 production/geotiff-pvc.yaml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 05087b1..9b768cf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,7 @@ stages: - get chart - build prereqs + - create storage - deploy rabbit - deploy GRB - deploy G2G @@ -55,6 +56,25 @@ build sidecar rabbit init: - if: $BUILD_PREREQS when: always +create geotiff storage: + extends: .helm_based_job + stage: create storage + script: + - if [ -n "$CI_COMMIT_TAG" ]; then + ns="geosphere"; + else + ns="geosphere-test"; + fi + # copy secret kubeconfig to the mounted (pwd) directory + - cp $kubekorner_k3s_config . + - kubeconfig=$(basename $kubekorner_k3s_config) + - ./helpers/create_pvc.sh $ns production/geotiff-pvc.yaml $kubeconfig + rules: + - changes: + - production/geotiff-pvc.yaml + when: always + - if: $CREATE_STORAGE + when: always .get_chart_tmpl: extends: .helm_based_job @@ -166,18 +186,10 @@ deploy_g16_grb: - cp $kubekorner_k3s_config . - kubeconfig=$(basename $kubekorner_k3s_config) - echo "Deploying version $docker_tag to cluster namespace $ns" - - pvc=$(kubectl --kubeconfig $kubeconfig get pvc --namespace $ns cspp-geo-grb -o jsonpath="{.metadata.name}" || echo "") - - if [ "$pvc" != "" ]; then - EXTRA_ARGS="--set persistence.existingClaim=$pvc"; - else - EXTRA_ARGS="--set persistence.enabled=true"; - fi - - EXTRA_ARGS="--set persistence.existingClaim=cspp-geo-grb-netcdf-pv" - - echo $EXTRA_ARGS # copy extra values files to the local directory (where helm has access via docker mount) - cp ../../production/values-grb-g16.yaml . # namespace names are the same as domain names - - helm upgrade -v 2 --install --kubeconfig $kubeconfig --namespace $ns $EXTRA_ARGS --set persistence.storageClass=longhorn -f values-grb-g16.yaml cspp-geo-grb cspp-geo-grb/ + - helm upgrade -v 2 --install --kubeconfig $kubeconfig --namespace $ns --set persistence.enabled --set persistence.storageClass=longhorn -f values-grb-g16.yaml cspp-geo-grb cspp-geo-grb/ dependencies: - get_chart_grb rules: diff --git a/helpers/create_pvc.sh b/helpers/create_pvc.sh new file mode 100755 index 0000000..9cb3868 --- /dev/null +++ b/helpers/create_pvc.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +if [[ $# -eq 2 ]]; then + ns=$1 + pvc=$2 + EXTRA_ARGS="" +else if [[ $# -eq 3 ]]; then + ns=$1 + pvc=$2 + kubeconfig=$3 + EXTRA_ARGS="--kubeconfig $kubeconfig" +else + echo "Usage: ./create_pvc.sh <kubernetes namespace> <PVC YAML definition> <kubernetes config file>" + exit 1 +fi + + +# If another project started this build then we know the provided definition +# didn't change. Let's make sure it exists on the cluster and if so, don't +# try to recreate it. +if [[ $CI_PIPELINE_SOURCE == "pipeline" ]]; then + pvc_exists=$(kubectl $EXTRA_ARGS get pvc --namespace $ns $pvc -o jsonpath="{.metadata.name}" || echo "") + if [[ "$pvc_exists" != "" ]]; then + # it exists + exit 0 + fi +fi + +kubectl $EXTRA_ARGS apply -f $pvc + diff --git a/production/geotiff-pvc.yaml b/production/geotiff-pvc.yaml new file mode 100644 index 0000000..d392ba4 --- /dev/null +++ b/production/geotiff-pvc.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: cspp-geo-geo2grid + labels: {} +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 2Ti + storageClassName: "longhorn" diff --git a/production/values-geo2grid-g16-radf.yaml b/production/values-geo2grid-g16-radf.yaml index e4029c9..e605074 100644 --- a/production/values-geo2grid-g16-radf.yaml +++ b/production/values-geo2grid-g16-radf.yaml @@ -8,8 +8,10 @@ rabbitOut: username: "user" passwordSecret: "geosphere-rabbit-rabbitmq" source: - s3Endpoint: "http://geosphere-minio:9000" +# s3Endpoint: "http://geosphere-minio:9000" + existingClaim: "cspp-geo-grb" destination: + existingClaim: "cspp-geo-geo2grid" # s3Endpoint: "http://geosphere-minio:9000" s3Secret: "geosphere-minio" s3AccessKey: "accesskey" -- GitLab