First attempt at adding separate tile generation

Mapserver still needs updating

.gitlab-ci.yml

@@ -5,6 +5,7 @@ stages:
   - deploy rabbit
   - deploy GRB
   - deploy G2G
+  - deploy tile gen
   - deploy WMS
   - deploy WMTS
   - deploy Client
@@ -75,6 +76,24 @@ create geotiff storage:
     - if: $CREATE_STORAGE
       when: always
 
+create shapefile storage:
+  extends: .helm_based_job
+  stage: create storage
+  script:
+    - ns=$(./helpers/get_namespace.sh)
+    # copy secret kubeconfig to the mounted (pwd) directory
+    - cp $kubekorner_k3s_config .
+    - kubeconfig=$(basename $kubekorner_k3s_config)
+    - ./helpers/create_pvc.sh "$ns" "ci_geosphere-test/shapefiles-pvc.yaml" "geosphere-tile-gen-shapefiles" "$kubeconfig"
+  # this job doesn't actually need any artifacts from previous jobs
+  dependencies: []
+  rules:
+    - changes:
+        - ci_geosphere-test/shapefiles-pvc.yaml
+      when: always
+    - if: $CREATE_STORAGE
+      when: always
+
 .get_chart_tmpl:
   extends: .helm_based_job
   stage: get chart
@@ -123,6 +142,12 @@ get_chart_geo2grid:
     SUBCOMP_REPOS: "geosphere-geo2grid"
     SUBCOMP_CHART_DIR: "cspp-geo-geo2grid"
 
+get_chart_tilegen:
+  extends: .get_chart_tmpl
+  variables:
+    SUBCOMP_REPOS: "geosphere-tile-gen"
+    SUBCOMP_CHART_DIR: "geosphere-tile-gen"
+
 get_chart_mapserver:
   extends: .get_chart_tmpl
   variables:
@@ -240,6 +265,43 @@ deploy_geo2grid_g16_radf:
       when: never
     - when: on_success
 
+.deploy_tile_gen:
+  environment:
+    name: geosphere-test
+    url: http://geosphere-test.ssec.wisc.edu
+  variables:
+    DEPLOY_SUFFIX: ""
+  extends: .helm_based_job
+  stage: deploy tile gen
+  script:
+    - ./helpers/deploy_tile_gen.sh "$DEPLOY_SUFFIX"
+  dependencies:
+    - get_chart_tilegen
+  rules:
+    - if: '$kubekorner_k3s_config == ""'
+      when: never
+    - when: on_success
+
+deploy_tile_gen_g16_radf:
+  extends: .deploy_tile_gen
+  variables:
+    DEPLOY_SUFFIX: "-g16-radf"
+
+deploy_tile_gen_g16_radc:
+  extends: .deploy_tile_gen
+  variables:
+    DEPLOY_SUFFIX: "-g16-radc"
+
+#deploy_tile_gen_g16_radm1:
+#  extends: .deploy_tile_gen
+#  variables:
+#    DEPLOY_SUFFIX: "-g16-radm1"
+#
+#deploy_tile_gen_g16_radm2:
+#  extends: .deploy_tile_gen
+#  variables:
+#    DEPLOY_SUFFIX: "-g16-radm2"
+
 deploy_mapserver:
   environment:
     name: geosphere-test

ci_geosphere-test/shapefiles-pvc.yaml

+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: geosphere-tile-gen-shapefiles
+  labels: {}
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 2Gi
+  storageClassName: "longhorn"

ci_geosphere-test/values-mapserver.yaml

-rabbitIn:
-  host: "geosphere-rabbit-rabbitmq"
-  username: "user"
-  passwordSecret: "geosphere-rabbit-rabbitmq"
-  topic: "data.goes.g16.abi.*.l1b.geotiff.all.complete"
-rabbitOut:
-  host: "geosphere-rabbit-rabbitmq"
-  username: "user"
-  passwordSecret: "geosphere-rabbit-rabbitmq"
 source:
   #  s3Endpoint: "http://geosphere-minio:9000"
   existingClaim: "cspp-geo-geo2grid"

ci_geosphere-test/values-tile-gen-g16-radc.yaml

+rabbitIn:
+  host: "geosphere-rabbit-rabbitmq"
+  username: "user"
+  passwordSecret: "geosphere-rabbit-rabbitmq"
+  topic: "data.goes.g16.abi.radf.l1b.geotiff.all.complete"
+rabbitOut:
+  host: "geosphere-rabbit-rabbitmq"
+  username: "user"
+  passwordSecret: "geosphere-rabbit-rabbitmq"
+source:
+  #  s3Endpoint: "http://geosphere-minio:9000"
+  existingClaim: "geosphere-tile-gen-shapefiles"

ci_geosphere-test/values-tile-gen-g16-radf.yaml

+rabbitIn:
+  host: "geosphere-rabbit-rabbitmq"
+  username: "user"
+  passwordSecret: "geosphere-rabbit-rabbitmq"
+  topic: "data.goes.g16.abi.radf.l1b.geotiff.all.complete"
+rabbitOut:
+  host: "geosphere-rabbit-rabbitmq"
+  username: "user"
+  passwordSecret: "geosphere-rabbit-rabbitmq"
+source:
+  #  s3Endpoint: "http://geosphere-minio:9000"
+  existingClaim: "geosphere-tile-gen-shapefiles"

helpers/deploy_tile_gen.sh

+#!/usr/bin/env bash
+
+set -e
+
+if [ $# -ne 1 ]; then
+    echo "Usage: ./helpers/deploy_tile_gen.sh <deploy_suffix>"
+    exit 1
+fi
+
+# Example: -g16-radf
+deploy_suffx="$1"
+
+ns=$(./helpers/get_namespace.sh)
+cd geosphere-tile-gen/chart
+source geosphere-tile-gen/cibuild.env
+# copy secret kubeconfig to the mounted (pwd) directory
+cp $kubekorner_k3s_config .
+kubeconfig=$(basename $kubekorner_k3s_config)
+echo "Deploying version $docker_tag to cluster namespace $ns"
+# copy extra values files to the local directory (where helm has access via docker mount)
+cp ../../ci_geosphere-test/values-tile-gen${deploy_suffix}.yaml .
+# namespace names are the same as domain names
+helm upgrade -v 2 --install --kubeconfig $kubeconfig --namespace $ns -f values-tile-gen${deploy_suffx}.yaml geosphere-tile-gen${deploy_suffx} geosphere-tile-gen/