Add initial version of the mapserver chart

.gitlab-ci.yml

+stages:
+  - build image
+  - test image
+  - build chart
+  - deploy
+#  - registry_cleanup
+image: docker:19.03.1
+variables:
+  DOCKER_TLS_CERTDIR: ""
+  DOCKER_HOST: "tcp://localhost:2375"
+#  DOCKER_TLS_CERTDIR: "/certs"
+#  DOCKER_HOST: "tcp://localhost:2376"
+  DOCKER_DRIVER: overlay2
+  HELM_EXPERIMENTAL_OCI: "1"
+services:
+  - docker:19.03.1-dind
+
+include:
+  - project: 'cspp_geo/geosphere/geosphere-deploy'
+    file: '/helpers/docker_base.yaml'
+  - project: 'cspp_geo/geosphere/geosphere-deploy'
+    file: '/helpers/build_image.yaml'
+
+build_image_tile_gen:
+  extends: .build_image
+  variables:
+    IMAGE_NAME: "tile-gen"
+    IMAGE_DIR: "tile_gen"
+
+build_image_mapserver:
+  extends: .build_image
+  variables:
+    IMAGE_NAME: "geosphere-mapserver"
+    IMAGE_DIR: "mapserver"
+
+build_chart:
+  stage: build chart
+  tags:
+    - docker
+    - kubernetes
+  before_script:
+    - docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY}
+    - alias helm='docker run -t --rm -e HELM_EXPERIMENTAL_OCI="$HELM_EXPERIMENTAL_OCI" -v $(pwd):/apps -w /apps -v ~/.kube:/root/.kube -v ~/.helm:/root/.helm -v ~/.config/helm:/root/.config/helm -v ~/.cache/helm:/root/.cache/helm alpine/helm:3.2.3'
+    - helm registry login -u ${CI_REGISTRY_USER} -p ${CI_JOB_TOKEN} ${CI_REGISTRY}
+  script:
+    - image_url="${CI_REGISTRY_IMAGE}/charts/geosphere-mapserver"
+    - if [ -z "$CI_COMMIT_TAG" ]; then
+          docker_tag=$CI_COMMIT_SHORT_SHA;
+      else
+          docker_tag=$CI_COMMIT_TAG;
+      fi;
+    - chart_tag=$docker_tag
+    - cd chart
+    - |-
+      sed -i "s/^appVersion: .*\$/appVersion: ${docker_tag}/g" geosphere-mapserver/Chart.yaml
+    - helm lint geosphere-mapserver
+    - helm chart save geosphere-mapserver ${image_url}:${chart_tag}
+    - helm chart push ${image_url}:${chart_tag}
+  artifacts:
+    paths:
+      - chart/geosphere-mapserver
+    when: on_success
+    expire_in: 6 hours
+
+#deploy_application:
+#  stage: deploy
+#  trigger:
+#    project: cspp_geo/geosphere/geosphere-deploy
+#    strategy: depend
+
+
+#registry_cleanup:
+#  stage: registry_cleanup
+#  tags:
+#    - docker
+#  script:
+#    - ci/registry_cleanup.sh

README.md

+# GeoSphere - MapServer (WMS)
+
+This repository holds the configuration files necessary to build a docker
+image and helm chart for running a MapServer server in a containerized
+environment. The important parts of the repository are discussed below.
+
+* GitLab Project: https://gitlab.ssec.wisc.edu/cspp_geo/geosphere/geosphere-mapserver
+* Image Registry: https://gitlab.ssec.wisc.edu/cspp_geo/geosphere/geosphere-mapserver/container_registry
+* Chart Registry: https://gitlab.ssec.wisc.edu/cspp_geo/geosphere/geosphere-mapserver/container_registry
+* Issue Tracker: https://gitlab.ssec.wisc.edu/cspp_geo/geosphere/geosphere-mapserver/-/issues
+
+* `chart`: Configuration and definition of a helm chart that combines the
+  other parts of this repository into a Kubernetes installable helm chart.
+  See `chart/README.md` for more information.
+* `tile_gen`: Docker configuration for the tile generation sidecar image. See
+  the `README.md` file inside for more information.
+* `mapserver`: Docker configuration for the main MapServer image. See
+  the `README.md` file inside for more information.
+* `.gitlab-ci.yml`: Continuous Integration configuration to build and test the
+  docker images in this repository and build the helm chart. Images and charts
+  are uploaded to the registries (see links above). This configuration also
+  triggers deployment of the application by starting the CI processing in the
+  [geosphere-deploy](https://gitlab.ssec.wisc.edu/cspp_geo/geosphere/geosphere-deploy/)
+  repository. See that repository for more information about the operation
+  and test deployment of the GeoSphere application.

chart/geosphere-mapserver/.helmignore

+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

chart/geosphere-mapserver/Chart.yaml

+apiVersion: v2
+name: geosphere-mapserver
+description: A Helm chart for GeoSphere's MapServer
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application.
+appVersion: 1.16.0

chart/geosphere-mapserver/templates/NOTES.txt

+1. Get the application URL by running these commands:
+{{- if .Values.ingress.enabled }}
+{{- range $host := .Values.ingress.hosts }}
+  {{- range .paths }}
+  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ . }}
+  {{- end }}
+{{- end }}
+{{- else if contains "NodePort" .Values.service.type }}
+  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "geosphere-mapserver.fullname" . }})
+  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+  echo http://$NODE_IP:$NODE_PORT
+{{- else if contains "LoadBalancer" .Values.service.type }}
+     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+           You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "geosphere-mapserver.fullname" . }}'
+  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "geosphere-mapserver.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
+  echo http://$SERVICE_IP:{{ .Values.service.port }}
+{{- else if contains "ClusterIP" .Values.service.type }}
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "geosphere-mapserver.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+  echo "Visit http://127.0.0.1:8080 to use your application"
+  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:80
+{{- end }}

chart/geosphere-mapserver/templates/_helpers.tpl

+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "geosphere-mapserver.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "geosphere-mapserver.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "geosphere-mapserver.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Common labels
+*/}}
+{{- define "geosphere-mapserver.labels" -}}
+helm.sh/chart: {{ include "geosphere-mapserver.chart" . }}
+{{ include "geosphere-mapserver.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}
+
+{{/*
+Selector labels
+*/}}
+{{- define "geosphere-mapserver.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "geosphere-mapserver.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end -}}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "geosphere-mapserver.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create -}}
+    {{ default (include "geosphere-mapserver.fullname" .) .Values.serviceAccount.name }}
+{{- else -}}
+    {{ default "default" .Values.serviceAccount.name }}
+{{- end -}}
+{{- end -}}

chart/geosphere-mapserver/templates/deployment.yaml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "geosphere-mapserver.fullname" . }}
+  labels:
+    {{- include "geosphere-mapserver.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "geosphere-mapserver.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "geosphere-mapserver.selectorLabels" . | nindent 8 }}
+    spec:
+    {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+    volumes:
+      {{- if .Values.source.existingClaim }}
+      - name: src
+        persistentVolumeClaim:
+          claimName: "{{ .Values.source.existingClaim }}"
+      {{- else }}
+      # TODO: For testing only (should require s3 source)
+      - name: src
+        emptyDir: {}
+      {{- end }}
+      # put shapefiles in memory storage (small)
+      - name: dst
+        emptyDir:
+          medium: Memory
+      {{- end }}
+      containers:
+        - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.image.repository }}:{{ .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          env:
+            - name: WMS_PRODUCTS
+              value: "{{ join " " .Values.products }}"
+            - name: WMS_PLATFORMS
+              value: "{{ join " " .Values.platforms }}"
+            - name: WMS_SECTORS
+              value: "{{ join " " .Values.sectors }}"
+          ports:
+            - name: http
+              containerPort: 80
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /
+              port: http
+          readinessProbe:
+            httpGet:
+              path: /
+              port: http
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+        - name: {{ (printf "%s%s" .Chart.Name "-tile-gen") }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.tileGen.repository }}:{{ .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.tileGen.pullPolicy }}
+          env:
+            - name: G2G_PRODUCTS
+              value: "{{ join " " .Values.products }}"
+            - name: TILE_ARGS
+              value: "{{ .Values.tileArgs }}"
+            {{- with .Values.rabbitIn }}
+            {{- if .host }}
+            - name: RABBITMQ_HOST
+              value: "{{ .host }}"
+            - name: RABBITMQ_EXCHANGE_NAME
+              value: "{{ .exchange }}"
+            - name: RABBITMQ_USERNAME
+              value: "{{ .username }}"
+            {{- if .passwordSecret }}
+            - name: RABBITMQ_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .passwordSecret }}
+                  key: rabbitmq-password
+            {{- else }}
+            - name: RABBITMQ_PASSWORD
+              value: {{ .password }}
+            {{- end }}
+            - name: AMQPFIND_TOPIC
+              value: "{{ .topic }}"
+            - name: AMQPFIND_ARGS
+              value: "{{ .amqpFindArgs }}"
+            {{- end }}
+            {{- end }}
+            {{- with .Values.rabbitOut }}
+            {{- if .host }}
+            - name: RABBITMQ_OUT_HOST
+              value: "{{ .host }}"
+            - name: RABBITMQ_OUT_EXCHANGE_NAME
+              value: "{{ .exchange }}"
+            - name: RABBITMQ_OUT_USERNAME
+              value: "{{ .username }}"
+            {{- if .passwordSecret }}
+            - name: RABBITMQ_OUT_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .passwordSecret }}
+                  key: rabbitmq-password
+            {{- else }}
+            - name: RABBITMQ_OUT_PASSWORD
+              value: {{ .password }}
+            {{- end }}
+            - name: AMQPSEND_ARGS
+              value: "{{ .amqpSendArgs }}"
+            {{- end }}
+            {{- end }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+    {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+    {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}

chart/geosphere-mapserver/templates/ingress.yaml

+{{- if .Values.ingress.enabled -}}
+{{- $fullName := include "geosphere-mapserver.fullname" . -}}
+{{- $svcPort := .Values.service.port -}}
+{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
+apiVersion: networking.k8s.io/v1beta1
+{{- else -}}
+apiVersion: extensions/v1beta1
+{{- end }}
+kind: Ingress
+metadata:
+  name: {{ $fullName }}
+  labels:
+    {{- include "geosphere-mapserver.labels" . | nindent 4 }}
+  {{- with .Values.ingress.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+{{- if .Values.ingress.tls }}
+  tls:
+  {{- range .Values.ingress.tls }}
+    - hosts:
+      {{- range .hosts }}
+        - {{ . | quote }}
+      {{- end }}
+      secretName: {{ .secretName }}
+  {{- end }}
+{{- end }}
+  rules:
+  {{- range .Values.ingress.hosts }}
+    - host: {{ .host | quote }}
+      http:
+        paths:
+        {{- range .paths }}
+          - path: {{ . }}
+            backend:
+              serviceName: {{ $fullName }}
+              servicePort: {{ $svcPort }}
+        {{- end }}
+  {{- end }}
+{{- end }}

chart/geosphere-mapserver/templates/service.yaml

+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "geosphere-mapserver.fullname" . }}
+  labels:
+    {{- include "geosphere-mapserver.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    {{- include "geosphere-mapserver.selectorLabels" . | nindent 4 }}

chart/geosphere-mapserver/templates/tests/test-connection.yaml

+apiVersion: v1
+kind: Pod
+metadata:
+  name: "{{ include "geosphere-mapserver.fullname" . }}-test-connection"
+  labels:
+{{ include "geosphere-mapserver.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": test-success
+spec:
+  containers:
+    - name: wget
+      image: busybox
+      command: ['wget']
+      args:  ['{{ include "geosphere-mapserver.fullname" . }}:{{ .Values.service.port }}']
+  restartPolicy: Never

chart/geosphere-mapserver/values.yaml

+# Default values for geosphere-mapserver.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+replicaCount: 1
+
+image:
+  repository: gitlab.ssec.wisc.edu:5555/cspp_geo/geosphere/geosphere-mapserver/geosphere-mapserver
+  pullPolicy: IfNotPresent
+tileGen:
+  repository: gitlab.ssec.wisc.edu:5555/cspp_geo/geosphere/geosphere-mapserver/tile-gen
+  pullPolicy: IfNotPresent
+
+products:
+  - C01
+  - C02
+  - C03
+  - C04
+  - C05
+  - C06
+  - C07
+  - C08
+  - C09
+  - C10
+  - C11
+  - C12
+  - C13
+  - C14
+  - C15
+  - C16
+  - true_color_night
+platforms:
+  - "g16"
+sectors:
+  - "radf"
+  - "radc"
+  - "radm1"
+  - "radm2"
+
+source:
+  # not implemented:
+  s3Endpoint: ""
+  existingClaim: ""
+rabbitIn:
+  host: ""
+  exchange: "satellite"
+  username: guest
+  password: guest
+  passwordSecret: ""
+  topic: "data.goes.*.abi.*.l1b.geotiff.all.complete"
+  amqpFindArgs: ""
+rabbitOut:
+  host: ""
+  exchange: "satellite"
+  username: guest
+  password: guest
+  passwordSecret: ""
+  amqpSendArgs: ""
+tileArgs: ""
+
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""
+
+podSecurityContext: {}
+  # fsGroup: 2000
+
+securityContext: {}
+  # capabilities:
+  #   drop:
+  #   - ALL
+  # readOnlyRootFilesystem: true
+  # runAsNonRoot: true
+  # runAsUser: 1000
+
+service:
+  type: ClusterIP
+  port: 80
+
+ingress:
+  enabled: false
+  annotations: {}
+    # kubernetes.io/ingress.class: nginx
+    # kubernetes.io/tls-acme: "true"
+  hosts:
+    - host: chart-example.local
+      paths: []
+  tls: []
+  #  - secretName: chart-example-tls
+  #    hosts:
+  #      - chart-example.local
+
+resources: {}
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as Minikube. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # limits:
+  #   cpu: 100m
+  #   memory: 128Mi
+  # requests:
+  #   cpu: 100m
+  #   memory: 128Mi
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}

tile_gen/generate_tiles.py

@@ -84,6 +84,9 @@ def main():
                         help="Assume that all input geotiffs have the same "
                              "bounding box so no need to compute it for every "
                              "input.")
+    parser.add_argument('--link-or-copy', action='store_true',
+                        help="Copy (or hardlink if possible) geotiff files to "
+                             "be next to the created shapefiles.")
     parser.add_argument('out_dir',
                         help="Output path to save tile information to (ex. '/data/tiles/{product}')")
     parser.add_argument('input_files', nargs="+",
@@ -104,9 +107,11 @@ def main():
         if args.remap:
             # remap if needed
             prod_files = list(remap_tifs(prod_files, out_dir, args.remap_suffix))
-        else:
+        elif args.link_or_copy:
             # hardlink if needed
             prod_files = list(link_or_copy(prod_files, out_dir))
+        else:
+            out_dir = os.path.dirname(prod_files[0])
 
         # get all products in the current directory
         ext = os.path.splitext(prod_files[-1])[-1]

tile_gen/run.sh

@@ -15,8 +15,19 @@
 # Verify that the data mount is available
 test -d "/data"
 
-export AMQPFIND_ARGS=${AMQPFIND_ARGS:-"-H cspp-geo-rabbit -X satellite -u guest -p guest"}
-export AMQPSEND_ARGS=${AMQPSEND_ARGS:-"-D 15 -H cspp-geo-rabbit -X satellite -u guest -p guest"}
+export RABBITMQ_HOST=${RABBITMQ_HOST:-""}
+export RABBITMQ_EXCHANGE_NAME=${RABBITMQ_EXCHANGE_NAME:-"satellite"}
+export RABBITMQ_USERNAME=${RABBITMQ_USERNAME:-"guest"}
+export RABBITMQ_PASSWORD=${RABBITMQ_PASSWORD:-"guest"}
+export RABBITMQ_OUT_HOST=${RABBITMQ_OUT_HOST:-${RABBITMQ_HOST}}
+export RABBITMQ_OUT_EXCHANGE_NAME=${RABBITMQ_OUT_EXCHANGE_NAME:-${RABBITMQ_EXCHANGE_NAME}}
+export RABBITMQ_OUT_USERNAME=${RABBITMQ_OUT_USERNAME:-${RABBITMQ_USERNAME}}
+export RABBITMQ_OUT_PASSWORD=${RABBITMQ_OUT_PASSWORD:-${RABBITMQ_PASSWORD}}
+export AMQPFIND_ARGS=${AMQPFIND_ARGS:-""}
+export AMQPSEND_ARGS=${AMQPSEND_ARGS:-"-D 15"}
+# append provided args
+export AMQPFIND_ARGS="-H ${RABBITMQ_HOST} -X ${RABBITMQ_EXCHANGE_NAME} -u ${RABBITMQ_USERNAME} -p ${RABBITMQ_PASSWORD} ${AMQPSEND_ARGS}"
+export AMQPSEND_ARGS="-H ${RABBITMQ_OUT_HOST} -X ${RABBITMQ_OUT_EXCHANGE_NAME} -u ${RABBITMQ_OUT_USERNAME} -p ${RABBITMQ_OUT_PASSWORD} ${AMQPSEND_ARGS}"
 export AMQPFIND_TOPIC=${AMQPFIND_TOPIC:-'data.goes.*.abi.*.l1b.geotiff.all.complete'}
 export G2G_PRODUCTS=${G2G_PRODUCTS:-"C01 C02 C03 C04 C05 C06 C07 C08 C09 C10 C11 C12 C13 C14 C15 C16 true_color"}
 export TILE_ARGS=${TILE_ARGS:-""}