Add initial copy of PVCs and geo2grid configuration to GCP CI

ci_gcp/geotiff-pvc.yaml

+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: cspp-geo-geo2grid
+  labels: {}
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 1Ti
+  storageClassName: "standard"

ci_gcp/gitlab-ci.yaml

 # This file is included as part of the main repository .gitlab-ci.yml file
 
-test gcp connection:
-  environment:
-    name: gcp
-  stage: test
-  tags:
-    - docker
-    - kubernetes
-  image: google/cloud-sdk:latest
-  # don't need any artifacts for this to run
-  dependencies: []
-  before_script:
-    - gcloud auth activate-service-account "${GEOSPHERE_DEPLOY_GCP_SA_EMAIL}" --key-file="${GEOSPHERE_DEPLOY_GCP_SA_KEY}"
-    # If this produces a warning about not having permission make sure the SA
-    # has the project "Viewer" role. If it still produces a warning then try
-    # enabling this Cloud Resource Manager API. See admin/GCP_README.md
-    # for more information.
-    - gcloud config set project "${GEOSPHERE_DEPLOY_GCP_PROJECT_NAME}"
-  script:
-    - gcloud compute instances list
-    - if [ "${GEOSPHERE_DEPLOY_GCP_CLUSTER_NAME}" != "" ]; then
-        gcloud container clusters get-credentials "${GEOSPHERE_DEPLOY_GCP_CLUSTER_NAME}" --zone "${GEOSPHERE_DEPLOY_GCP_ZONE_NAME}";
-        kubectl get all;
-      fi
-
-
-gstest deploy grb:
+.gcloud_base:
   environment:
     name: gcp
   extends: .helm_based_job
   image: gitlab.ssec.wisc.edu:5555/cspp_geo/geosphere/gcloud-kubectl-helm/gcloud-kubectl-helm:6d3e308b
-  stage: deploy GRB
   variables:
     KUBECONFIG: "/root/.kube/config"
   before_script:
@@ -44,15 +18,30 @@ gstest deploy grb:
     - helm registry login -u ${CI_REGISTRY_USER} -p ${CI_JOB_TOKEN} ${CI_REGISTRY}
     - helm repo add stable https://kubernetes-charts.storage.googleapis.com
     - helm repo update
+
+test gcp connection:
+  extends: .gcloud_base
+  stage: test
+  # don't need any artifacts for this to run
+  dependencies: []
+  script:
+    - gcloud compute instances list
+    - if [ "${GEOSPHERE_DEPLOY_GCP_CLUSTER_NAME}" != "" ]; then
+        gcloud container clusters get-credentials "${GEOSPHERE_DEPLOY_GCP_CLUSTER_NAME}" --zone "${GEOSPHERE_DEPLOY_GCP_ZONE_NAME}";
+        kubectl get all;
+      fi
+
+gstest deploy grb:
+  extends: .gcloud_base
+  stage: deploy GRB
   script:
-    - ns="default"
+    - ns=$(./helpers/get_namespace.sh)
     - cd geosphere-grb/chart
     - source cspp-geo-grb/cibuild.env
     # copy private ssh key to the chart for inclusion in the secret
     - cp $GRB_PROXY_SSH_KEY cspp-geo-grb/secrets/grb_ssh_proxy_rsa
     - echo "Deploying version $docker_tag to cluster namespace $ns"
     - helm upgrade -v 2 --install --namespace $ns -f ../../ci_gcp/values-grb-g16.yaml cspp-geo-grb cspp-geo-grb/
-#    - helm upgrade -v 2 --install --kubeconfig $kubeconfig --namespace $ns --set persistence.enabled=true --set persistence.storageClass=longhorn -f values-grb-g16.yaml cspp-geo-grb cspp-geo-grb/
   dependencies:
     - get_chart_grb
   rules:
@@ -60,33 +49,75 @@ gstest deploy grb:
       when: never
     - when: on_success
 
+gs create geotiff storage:
+  extends: .gcloud_base
+  stage: create storage
+  script:
+    - ns=$(./helpers/get_namespace.sh)
+    - ./helpers/create_pvc.sh "$ns" "ci_${ns}/geotiff-pvc.yaml" "cspp-geo-geo2grid" "$KUBECONFIG"
+  # this job doesn't actually need any artifacts from previous jobs
+  dependencies: []
+  rules:
+    - if: '$CI_COMMIT_BRANCH != "gcp"'
+      when: never
+    - when: on_success
+    # this will always be true for tags
+    - changes:
+        - ci_geosphere/geotiff-pvc.yaml
+    - if: $CREATE_STORAGE
 
-gstest deploy client:
+gs create shapefile storage:
+  extends: .gcloud_base
+  stage: create storage
+  script:
+    - ns=$(./helpers/get_namespace.sh)
+    - ./helpers/create_pvc.sh "$ns" "ci_${ns}/shapefiles-pvc.yaml" "geosphere-tile-gen-shapefiles" "$KUBECONFIG"
+  # this job doesn't actually need any artifacts from previous jobs
+  dependencies: []
+  rules:
+    - if: "$CI_COMMIT_TAG == null"
+      when: never
+    - if: '$kubekorner_k3s_config == null'
+      when: never
+    # this will always be true for tags
+    - changes:
+        - ci_geosphere/shapefiles-pvc.yaml
+    - if: $CREATE_STORAGE
+
+gs deploy rabbit:
   environment:
-    name: gcp
+    name: geosphere
+    url: http://geosphere.ssec.wisc.edu
   extends: .helm_based_job
-  #image: google/cloud-sdk:latest
-  image: kiwigrid/gcloud-kubectl-helm
+  stage: deploy rabbit
+  script:
+    - ./helpers/deploy_rabbitmq.sh ci_geosphere
+    - cp ${kubekorner_k3s_config} .
+    - kubeconfig=$(basename ${kubekorner_k3s_config})
+    - |-
+      kubectl get secret --kubeconfig "${kubeconfig}" geosphere-rabbit-rabbitmq --namespace=geosphere -oyaml | grep -v '^\s*namespace:\s' | grep -v "[Hh]elm" | grep -v "[tT]ime" | grep -v "selfLink" | grep -v "uid" | grep -v "resourceVersion" | sed 's/ name: .*/ name: geosphere-rabbit-rabbitmq-production/' | kubectl_stdin apply --kubeconfig "${kubeconfig}" --namespace=geosphere-test -f -
+  # this job doesn't actually need any artifacts from previous jobs
+  dependencies: []
+  rules:
+    - if: "$CI_COMMIT_TAG == null"
+      when: never
+    - if: '$kubekorner_k3s_config == null'
+      when: never
+    # no need to build if another project triggered us
+    - if: $CI_PIPELINE_SOURCE == "pipeline"
+      when: never
+    - changes:
+        - ci_geosphere/values-geosphere-rabbit.yaml
+    - if: $DEPLOY_RABBIT
+
+gstest deploy client:
+  extends: .gcloud_base
   stage: deploy Client
-  variables:
-    KUBECONFIG: "/home/gkh/.kube/config"
-  before_script:
-    - gcloud --verbosity=debug auth activate-service-account "${GEOSPHERE_DEPLOY_GCP_SA_EMAIL}" --key-file="${GEOSPHERE_DEPLOY_GCP_SA_KEY}"
-    # If this produces a warning about not having permission make sure the SA
-    # has the project "Viewer" role. If it still produces a warning then try
-    # enabling this Cloud Resource Manager API. See admin/GCP_README.md
-    # for more information.
-    - gcloud --verbosity=debug config set project "${GEOSPHERE_DEPLOY_GCP_PROJECT_NAME}"
-    - gcloud --verbosity=debug container clusters get-credentials "${GEOSPHERE_DEPLOY_GCP_CLUSTER_NAME}" --zone "${GEOSPHERE_DEPLOY_GCP_ZONE_NAME}";
-    - helm registry login -u ${CI_REGISTRY_USER} -p ${CI_JOB_TOKEN} ${CI_REGISTRY}
-    - helm repo add stable https://kubernetes-charts.storage.googleapis.com
-    - helm repo update
   script:
-    - ns="default"
+    - ns=$(./helpers/get_namespace.sh)
     - cd geosphere-client/chart
+    - source geosphere-client/cibuild.env
     - echo "Deploying version $docker_tag to cluster namespace $ns"
-    - helm list --kubeconfig $HOME/.kube/config
-    # namespace names are the same as domain names
     - helm upgrade -v 2 --kubeconfig $HOME/.kube/config --install --namespace $ns -f ../../ci_gcp/values-client.yaml geosphere-client geosphere-client/
   dependencies:
     - get_chart_client_test

ci_gcp/shapefiles-pvc.yaml

+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: geosphere-tile-gen-shapefiles
+  labels: {}
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 2Gi
+  storageClassName: "standard"

ci_gcp/values-geo2grid-g16-radf.yaml

+rabbitIn:
+  host: "geosphere-rabbit-rabbitmq"
+  username: "user"
+  passwordSecret: "geosphere-rabbit-rabbitmq"
+  topic: "data.goes.g16.abi.radf.l1b.netcdf.all.complete"
+rabbitOut:
+  host: "geosphere-rabbit-rabbitmq"
+  username: "user"
+  passwordSecret: "geosphere-rabbit-rabbitmq"
+source:
+#  s3Endpoint: "http://geosphere-minio:9000"
+  existingClaim: "cspp-geo-grb"
+destination:
+  persistence:
+    existingClaim: "cspp-geo-geo2grid"
+    cleanup:
+      age: "+1"
+#  s3Endpoint: "http://geosphere-minio:9000"
+  s3Secret: "geosphere-minio"
+  s3AccessKey: "accesskey"
+  s3SecretKey: "secretkey"

ci_gcp/values-grb-g16.yaml

@@ -3,14 +3,13 @@ fanoutServer: "fanout1"
 sshProxy: "ash.ssec.wisc.edu"
 sshUser: "davidh"
 sshPrivateKeyFile: "secrets/grb_ssh_proxy_rsa"
-uploadDst: ""
 leftPort: 50060
 rightPort: 50070
-#uploadDst: "s3://g16-abi-l1b-netcdf"
-#s3Endpoint: "http://geosphere-minio:9000"
-#s3Secret: "geosphere-minio"
-#s3AccessKey: "accesskey"
-#s3SecretKey: "secretkey"
+uploadDst: "/dst"
+persistence:
+  enabled: true
+  size: 100Gi
+  storageClass: "standard"
 #rabbitOut:
 #  host: "geosphere-rabbit-rabbitmq"
 #  username: "user"

helpers/get_namespace.sh

@@ -4,6 +4,8 @@ if [[ -n "$CI_COMMIT_TAG" ]]; then
     ns="geosphere";
 elif [[ "$CI_COMMIT_BRANCH" == "master" ]]; then
     ns="geosphere-test";
+elif [[ "$CI_COMMIT_BRANCH" == "gcp" ]]; then
+    ns="default";
 else
     echo "ERROR: Kubernetes jobs don't run unless in master branch or tagged"
     exit 1