From 79be40ffe4095de42ec17a95b29a80e712c69b92 Mon Sep 17 00:00:00 2001
From: David Hoese <david.hoese@ssec.wisc.edu>
Date: Mon, 23 Nov 2020 08:58:30 -0600
Subject: [PATCH] Initial attempt at adding geo2grid chart to the ci tests

---
 ci_geosphere/gitlab-ci.yaml             |  8 +-------
 ci_tests/cleanup_basic_grb_test.sh      |  8 +++++++-
 ci_tests/get_release_names.sh           |  7 +++++++
 ci_tests/gitlab-ci.yaml                 |  5 +++++
 ci_tests/run_basic_grb_test.sh          | 23 +++++++++++++++++++++++
 ci_tests/values-geo2grid-g16-radm1.yaml | 15 +++++++++++++++
 helpers/deploy_geo2grid.sh              |  7 +------
 helpers/deploy_rabbitmq.sh              | 13 +++++--------
 8 files changed, 64 insertions(+), 22 deletions(-)
 create mode 100644 ci_tests/values-geo2grid-g16-radm1.yaml

diff --git a/ci_geosphere/gitlab-ci.yaml b/ci_geosphere/gitlab-ci.yaml
index 3d293a1..8230ba2 100644
--- a/ci_geosphere/gitlab-ci.yaml
+++ b/ci_geosphere/gitlab-ci.yaml
@@ -76,14 +76,8 @@ gs deploy g16 grb:
     - ns=$(./helpers/get_namespace.sh)
     - cd geosphere-grb/chart
     - source cspp-geo-grb/cibuild.env
-    # copy secret kubeconfig to the mounted (pwd) directory
-    - cp $kubekorner_k3s_config .
-    - kubeconfig=$(basename $kubekorner_k3s_config)
     - echo "Deploying version $docker_tag to cluster namespace $ns"
-    # copy extra values files to the local directory (where helm has access via docker mount)
-    - cp ../../ci_geosphere/values-grb-g16.yaml .
-    # namespace names are the same as domain names
-    - helm upgrade -v 2 --install --kubeconfig $kubeconfig --namespace $ns --set persistence.enabled=true --set persistence.storageClass=longhorn -f values-grb-g16.yaml cspp-geo-grb cspp-geo-grb/
+    - helm upgrade -v 2 --install --namespace $ns --set persistence.enabled=true --set persistence.storageClass=longhorn -f ../../ci_geosphere/values-grb-g16.yaml cspp-geo-grb cspp-geo-grb/
   dependencies:
     - get_chart_grb
   rules:
diff --git a/ci_tests/cleanup_basic_grb_test.sh b/ci_tests/cleanup_basic_grb_test.sh
index 2de195f..4078fde 100755
--- a/ci_tests/cleanup_basic_grb_test.sh
+++ b/ci_tests/cleanup_basic_grb_test.sh
@@ -23,4 +23,10 @@ check_and_uninstall() {
 
 ns=$(./helpers/get_namespace.sh)
 source ./ci_tests/get_release_names.sh
-check_and_uninstall "${ns}" "${GRB_RELEASE_NAME}"
\ No newline at end of file
+check_and_uninstall "${ns}" "${GRB_RELEASE_NAME}"
+check_and_uninstall "${ns}" "${RABBITMQ_RELEASE_NAME}"
+check_and_uninstall "${ns}" "${GEO2GRID_RELEASE_NAME}"
+check_and_uninstall "${ns}" "${TILEGEN_RELEASE_NAME}"
+check_and_uninstall "${ns}" "${MAPSERVER_RELEASE_NAME}"
+check_and_uninstall "${ns}" "${MAPCACHE_RELEASE_NAME}"
+check_and_uninstall "${ns}" "${CLIENT_RELEASE_NAME}"
diff --git a/ci_tests/get_release_names.sh b/ci_tests/get_release_names.sh
index 35c0c80..0e65bb8 100755
--- a/ci_tests/get_release_names.sh
+++ b/ci_tests/get_release_names.sh
@@ -2,3 +2,10 @@
 
 export RELEASE_PREFIX="${CI_JOB_NAME//_/-}-${CI_JOB_ID//_/-}"
 export GRB_RELEASE_NAME="${RELEASE_PREFIX}-cspp-geo-grb"
+export RABBITMQ_RELEASE_NAME="geosphere-rabbit"
+export GEO2GRID_RELEASE_NAME="${RELEASE_PREFIX}-cspp-geo-geo2grid"
+export TILEGEN_RELEASE_NAME="${RELEASE_PREFIX}-geosphere-tilegen"
+export MAPSERVER_RELEASE_NAME="${RELEASE_PREFIX}-geosphere-mapserver"
+export MAPCACHE_RELEASE_NAME="${RELEASE_PREFIX}-geosphere-mapcache"
+export CLIENT_RELEASE_NAME="${RELEASE_PREFIX}-geosphere-client"
+
diff --git a/ci_tests/gitlab-ci.yaml b/ci_tests/gitlab-ci.yaml
index 1fe1f8e..2457abf 100644
--- a/ci_tests/gitlab-ci.yaml
+++ b/ci_tests/gitlab-ci.yaml
@@ -75,6 +75,11 @@ test_basic_grb:
     - ./ci_tests/cleanup_basic_grb_test.sh
   dependencies:
     - get_chart_grb
+    - get_chart_geo2grid
+    - get_chart_tilegen
+    - get_chart_mapserver
+    - get_chart_mapcache
+    - get_chart_client
   rules:
     - when: on_success
 #
diff --git a/ci_tests/run_basic_grb_test.sh b/ci_tests/run_basic_grb_test.sh
index 360fab1..3971fc7 100755
--- a/ci_tests/run_basic_grb_test.sh
+++ b/ci_tests/run_basic_grb_test.sh
@@ -10,9 +10,16 @@ error() {
 }
 
 install_charts() {
+    # use the production rabbitmq config
+    ./helpers/deploy_rabbitmq.sh ci_geosphere
+
+    # GRB
     pushd geosphere-grb/chart
     helm upgrade -v 2 --install --namespace $ns -f ../../ci_tests/values-grb-g16.yaml ${GRB_RELEASE_NAME} cspp-geo-grb/
     popd
+
+    # Geo2Grid
+    ./helpers/deploy_geo2grid.sh "ci_tests" "-g16-radm1"
 }
 
 get_pod_name() {
@@ -36,6 +43,21 @@ run_grb_checks() {
     done
 }
 
+run_geo2grid_checks() {
+    count=0
+    while true; do
+      num_files=$(kubectl exec -n $ns ${GEO2GRID_POD_NAME} -c cspp-geo-grb-notifier -- ls -1 /dst/geotiffs/g16/abi/radm1/ | wc -l || echo 0)
+      let count=count+1
+      if [[ $num_files -gt 16 ]]; then
+          debug "Got at least 16 files"
+          break
+      fi
+      if [[ $count -gt 10 ]]; then
+          error "Took too long for Geo2Grid to produce files"
+      fi
+      sleep 10
+    done
+}
 set -e
 
 ns=$(./helpers/get_namespace.sh)
@@ -50,3 +72,4 @@ GRB_POD_NAME=$(get_pod_name "${GRB_RELEASE_NAME}")
 debug "GRB pod name: ${GRB_POD_NAME}"
 
 run_grb_checks || error "GRB checks failed"
+run_geo2grid_checks || error "Geo2Grid checks failed"
diff --git a/ci_tests/values-geo2grid-g16-radm1.yaml b/ci_tests/values-geo2grid-g16-radm1.yaml
new file mode 100644
index 0000000..9317c03
--- /dev/null
+++ b/ci_tests/values-geo2grid-g16-radm1.yaml
@@ -0,0 +1,15 @@
+rabbitIn:
+  host: "geosphere-rabbit-rabbitmq"
+  username: "user"
+  passwordSecret: "geosphere-rabbit-rabbitmq"
+  topic: "data.goes.g16.abi.radm1.l1b.netcdf.all.complete"
+rabbitOut:
+  host: "geosphere-rabbit-rabbitmq"
+  username: "user"
+  passwordSecret: "geosphere-rabbit-rabbitmq"
+source:
+#  s3Endpoint: "http://geosphere-minio:9000"
+  existingClaim: "cspp-geo-grb"
+destination:
+  persistence:
+    enabled: true
diff --git a/helpers/deploy_geo2grid.sh b/helpers/deploy_geo2grid.sh
index 0a65e9f..d3a74ad 100755
--- a/helpers/deploy_geo2grid.sh
+++ b/helpers/deploy_geo2grid.sh
@@ -15,11 +15,6 @@ deploy_suffix="$2"
 ns=$(./helpers/get_namespace.sh)
 cd geosphere-geo2grid/chart
 source cspp-geo-geo2grid/cibuild.env
-# copy secret kubeconfig to the mounted (pwd) directory
-cp $kubekorner_k3s_config .
-kubeconfig=$(basename $kubekorner_k3s_config)
 echo "Deploying version $docker_tag to cluster namespace $ns"
-# copy extra values files to the local directory (where helm has access via docker mount)
-cp ../../${values_base}/values-geo2grid${deploy_suffix}.yaml .
 # namespace names are the same as domain names
-helm upgrade -v 2 --install --kubeconfig $kubeconfig --namespace $ns -f values-geo2grid${deploy_suffix}.yaml cspp-geo-geo2grid${deploy_suffix} cspp-geo-geo2grid/
+helm upgrade -v 2 --install --namespace $ns -f ../../${values_base}/values-geo2grid${deploy_suffix}.yaml cspp-geo-geo2grid${deploy_suffix} cspp-geo-geo2grid/
diff --git a/helpers/deploy_rabbitmq.sh b/helpers/deploy_rabbitmq.sh
index 723a7a8..1d681a1 100755
--- a/helpers/deploy_rabbitmq.sh
+++ b/helpers/deploy_rabbitmq.sh
@@ -11,9 +11,6 @@ fi
 values_base="$1"
 
 ns=$(./helpers/get_namespace.sh)
-# copy secret kubeconfig to the mounted (pwd) directory
-cp $kubekorner_k3s_config .
-kubeconfig=$(basename $kubekorner_k3s_config)
 # get password from any previous installation
 # if we don't do this the password will get out of sync
 # WARNING: If you can't figure out why the password isn't working and you
@@ -24,22 +21,22 @@ kubeconfig=$(basename $kubekorner_k3s_config)
 #   rabbitmqctl authenticate_user user <password>
 # where password is accessed from the kubectl secret:
 #   kubectl get secret --namespace geosphere geosphere-rabbit-rabbitmq -o jsonpath="{.data.rabbitmq-password}" | base64 --decode
-sec_info=$(kubectl --kubeconfig $kubeconfig get secret --namespace $ns geosphere-rabbit-rabbitmq || echo "")
+sec_info=$(kubectl get secret --namespace $ns geosphere-rabbit-rabbitmq || echo "")
 echo $sec_info
 auth_sec="geosphere-rabbit-rabbitmq"
 if [[ "$sec_info" != "" ]]; then
-    pw=$(kubectl --kubeconfig $kubeconfig get secret --namespace $ns $auth_sec -o jsonpath="{.data.rabbitmq-password}" | base64 -d);
-    ec=$(kubectl --kubeconfig $kubeconfig get secret --namespace $ns $auth_sec -o jsonpath="{.data.rabbitmq-erlang-cookie}" | base64 -d);
+    pw=$(kubectl get secret --namespace $ns $auth_sec -o jsonpath="{.data.rabbitmq-password}" | base64 -d);
+    ec=$(kubectl get secret --namespace $ns $auth_sec -o jsonpath="{.data.rabbitmq-erlang-cookie}" | base64 -d);
     EXTRA_ARGS="--set auth.password=$pw --set auth.erlangCookie=$ec";
 fi
 echo "Adding extra arguments: $EXTRA_ARGS"
 # install third-party rabbitmq server
 helm repo add bitnami "https://charts.bitnami.com/bitnami"
-helm upgrade -v 2 --install --kubeconfig $kubeconfig -f ${values_base}/values-geosphere-rabbit.yaml $EXTRA_ARGS --namespace $ns geosphere-rabbit bitnami/rabbitmq
+helm upgrade -v 2 --install -f ${values_base}/values-geosphere-rabbit.yaml $EXTRA_ARGS --namespace $ns geosphere-rabbit bitnami/rabbitmq
 # do a little waiting for the rabbitmq pod to be ready so future stages
 # don't fail to communicate with it
 echo "Start waiting for rabbitmq: $(date +%Y%m%dT%H%M%S)"
-kubectl wait --kubeconfig $kubeconfig -n $ns pod/geosphere-rabbit-rabbitmq-0 --timeout 120s --for=condition=Ready
+kubectl wait -n $ns pod/geosphere-rabbit-rabbitmq-0 --timeout 120s --for=condition=Ready
 status=$?
 echo "Done waiting for rabbitmq: $(date +%Y%m%dT%H%M%S) : $status"
 exit $status
-- 
GitLab