diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ee2c71068f6f1d8bd13118bff0aef63aa6e6c6ee..c8edb23c65b9160ab35e1cd4d27bb8b9bbcc3a24 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,7 @@ stages: - build prereqs - test - create storage + - test - deploy rabbit - deploy GRB - deploy G2G @@ -37,7 +38,7 @@ include: - local: "/ci_tests/gitlab-ci.yaml" - local: "/ci_geosphere/gitlab-ci.yaml" - local: "/ci_geosphere-test/gitlab-ci.yaml" -# - local: "/ci_gcp/gitlab-ci.yaml" + - local: "/ci_gcp/gitlab-ci.yaml" build ci: diff --git a/admin/GCP_README.md b/admin/GCP_README.md new file mode 100644 index 0000000000000000000000000000000000000000..a420fdb5d4f5de80ae5b954b7c57f61bd0c148c1 --- /dev/null +++ b/admin/GCP_README.md @@ -0,0 +1,84 @@ +# Google Cloud Platform - Google Kubernetes Engine Administration + +This document describes administration procedures for creating and using +a Kubernetes cluster on Google Cloud Platform. + +Disclaimer: This document is *NOT* a substitute for the Google Cloud Platform +documentation. Please read their documentation for updated and accurate +information. + +## Accounts and Projects + +Accounts related to SSEC work or other University of Wisconsin work must be +created by DoIT. See https://it.wisc.edu/services/google-cloud-platform/ for +more information. By working with DoIT you should get access through your +Google Suite `@wisc.edu` account to a GCP "project" that you've named. You can +then login to the GCP Console (https://console.cloud.google.com/) where you'll +have access to everything related to your work on GCP. + +NOTE: Once logged in, be careful not to create resources on GCP unless you +know what you are doing, even in tutorials. Otherwise you may end up +accidentally charging to your business account. + +## Service Accounts and Kubectl + +GCP allows you to create "Service Accounts" for controlling access to specific +parts of a GCP project. This is useful for automating access to your future +GKE cluster from GitLab (or other) Continuous Integration (CI) jobs. Note that +a GCP Service Account is different than a Kubernetes Service Account. + +To create a Service Account, first read through: + +https://cloud.google.com/iam/docs/service-accounts + +Next, go to your GCP Console, click the menu in the upper-left and go to +"Project Settings". From there click on "Service Accounts" and then +"+ CREATE SERVICE ACCOUNT". Follow the instructions to create your service +account. To give your SA permission to create resources on your Kubernetes +cluster give it at least the "roles/container.developer" role. See +https://cloud.google.com/iam/docs/understanding-roles#kubernetes-engine-roles +for more information. + +Once created you will likely want to create a key to authenticate to GCP +from your CI environment. Click the 3 dots to the right of your newly created +key in the list of Service Accounts and select "Create key". See +https://cloud.google.com/iam/docs/creating-managing-service-account-keys +for more information. + +Now that you have that key, you should be able to use the `gcloud` utility +(must be installed separately) in your CI jobs to: + +```bash +gcloud auth activate-service-account <sa-account-name>@<gcp-project-id>.iam.gserviceaccount.com \ + --key-file=/path/to/json/key/you/downloaded.json +``` + +Now you can get the kube cluster configuration information by doing: + +```bash +gcloud container clusters get-credentials <cluster-name> --zone <zone-name> +``` + +This will likely write information to `~/.kube/config` about your cluster. You +can then use `kubectl` as normal (unless there are multiple clusters +configured then you may need to use the `--cluster`). + +If your SA account includes the "Project/Viewer" role you can do: + +```bash +gcloud config set project <project-id> +``` + +If the above gives you a warning then double check the Project/Viewer role +has been added to the Service Account. Otherwise, try enabling this API: + +https://console.developers.google.com/apis/api/cloudresourcemanager.googleapis.com/overview?project=<project-id> + +And then try again. If that still doesn't work try this Stackoverflow thread +for more help: https://stackoverflow.com/a/59931415/433202 + +## Install Helm + +Follow the instructions in the Helm documentation in the Google web console: + +https://helm.sh/docs/intro/install/#from-apt-debianubuntu diff --git a/ci_gcp/geotiff-pvc.yaml b/ci_gcp/geotiff-pvc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa75f73107f296468974a6784f56b8eec9bfbd52 --- /dev/null +++ b/ci_gcp/geotiff-pvc.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: cspp-geo-geo2grid + labels: {} +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Ti + storageClassName: "standard" diff --git a/ci_gcp/gitlab-ci.yaml b/ci_gcp/gitlab-ci.yaml index 3e5a6b5bd7551393b18a7157fae8a1bf4964db69..464f192d75788ef423f1a58da8584b9f6d82fcad 100644 --- a/ci_gcp/gitlab-ci.yaml +++ b/ci_gcp/gitlab-ci.yaml @@ -1 +1,128 @@ # This file is included as part of the main repository .gitlab-ci.yml file + +.gcloud_base: + environment: + name: gcp + extends: .helm_based_job + image: gitlab.ssec.wisc.edu:5555/cspp_geo/geosphere/gcloud-kubectl-helm/gcloud-kubectl-helm:6d3e308b + variables: + KUBECONFIG: "/root/.kube/config" + before_script: + - gcloud --verbosity=debug auth activate-service-account "${GEOSPHERE_DEPLOY_GCP_SA_EMAIL}" --key-file="${GEOSPHERE_DEPLOY_GCP_SA_KEY}" + # If this produces a warning about not having permission make sure the SA + # has the project "Viewer" role. If it still produces a warning then try + # enabling this Cloud Resource Manager API. See admin/GCP_README.md + # for more information. + - gcloud --verbosity=debug config set project "${GEOSPHERE_DEPLOY_GCP_PROJECT_NAME}" + - gcloud --verbosity=debug container clusters get-credentials "${GEOSPHERE_DEPLOY_GCP_CLUSTER_NAME}" --zone "${GEOSPHERE_DEPLOY_GCP_ZONE_NAME}"; + - helm registry login -u ${CI_REGISTRY_USER} -p ${CI_JOB_TOKEN} ${CI_REGISTRY} + - helm repo add stable https://kubernetes-charts.storage.googleapis.com + - helm repo update + +test gcp connection: + extends: .gcloud_base + stage: test + # don't need any artifacts for this to run + dependencies: [] + script: + - gcloud compute instances list + - if [ "${GEOSPHERE_DEPLOY_GCP_CLUSTER_NAME}" != "" ]; then + gcloud container clusters get-credentials "${GEOSPHERE_DEPLOY_GCP_CLUSTER_NAME}" --zone "${GEOSPHERE_DEPLOY_GCP_ZONE_NAME}"; + kubectl get all; + fi + +gstest deploy grb: + extends: .gcloud_base + stage: deploy GRB + script: + - ns=$(./helpers/get_namespace.sh) + - cd geosphere-grb/chart + - source cspp-geo-grb/cibuild.env + # copy private ssh key to the chart for inclusion in the secret + - cp $GRB_PROXY_SSH_KEY cspp-geo-grb/secrets/grb_ssh_proxy_rsa + - echo "Deploying version $docker_tag to cluster namespace $ns" + - helm upgrade -v 2 --install --namespace $ns -f ../../ci_gcp/values-grb-g16.yaml cspp-geo-grb cspp-geo-grb/ + dependencies: + - get_chart_grb + rules: + - if: '$CI_COMMIT_BRANCH != "gcp"' + when: never + - when: on_success + +gs create geotiff storage: + extends: .gcloud_base + stage: create storage + script: + - ns=$(./helpers/get_namespace.sh) + - ./helpers/create_pvc.sh "$ns" "ci_${ns}/geotiff-pvc.yaml" "cspp-geo-geo2grid" "$KUBECONFIG" + # this job doesn't actually need any artifacts from previous jobs + dependencies: [] + rules: + - if: '$CI_COMMIT_BRANCH != "gcp"' + when: never + - when: on_success + # this will always be true for tags + - changes: + - ci_geosphere/geotiff-pvc.yaml + - if: $CREATE_STORAGE + +gs create shapefile storage: + extends: .gcloud_base + stage: create storage + script: + - ns=$(./helpers/get_namespace.sh) + - ./helpers/create_pvc.sh "$ns" "ci_${ns}/shapefiles-pvc.yaml" "geosphere-tile-gen-shapefiles" "$KUBECONFIG" + # this job doesn't actually need any artifacts from previous jobs + dependencies: [] + rules: + - if: "$CI_COMMIT_TAG == null" + when: never + - if: '$kubekorner_k3s_config == null' + when: never + # this will always be true for tags + - changes: + - ci_geosphere/shapefiles-pvc.yaml + - if: $CREATE_STORAGE + +gs deploy rabbit: + environment: + name: geosphere + url: http://geosphere.ssec.wisc.edu + extends: .helm_based_job + stage: deploy rabbit + script: + - ./helpers/deploy_rabbitmq.sh ci_geosphere + - cp ${kubekorner_k3s_config} . + - kubeconfig=$(basename ${kubekorner_k3s_config}) + - |- + kubectl get secret --kubeconfig "${kubeconfig}" geosphere-rabbit-rabbitmq --namespace=geosphere -oyaml | grep -v '^\s*namespace:\s' | grep -v "[Hh]elm" | grep -v "[tT]ime" | grep -v "selfLink" | grep -v "uid" | grep -v "resourceVersion" | sed 's/ name: .*/ name: geosphere-rabbit-rabbitmq-production/' | kubectl_stdin apply --kubeconfig "${kubeconfig}" --namespace=geosphere-test -f - + # this job doesn't actually need any artifacts from previous jobs + dependencies: [] + rules: + - if: "$CI_COMMIT_TAG == null" + when: never + - if: '$kubekorner_k3s_config == null' + when: never + # no need to build if another project triggered us + - if: $CI_PIPELINE_SOURCE == "pipeline" + when: never + - changes: + - ci_geosphere/values-geosphere-rabbit.yaml + - if: $DEPLOY_RABBIT + +gstest deploy client: + extends: .gcloud_base + stage: deploy Client + script: + - ns=$(./helpers/get_namespace.sh) + - cd geosphere-client/chart + - source geosphere-client/cibuild.env + - echo "Deploying version $docker_tag to cluster namespace $ns" + - helm upgrade -v 2 --kubeconfig $HOME/.kube/config --install --namespace $ns -f ../../ci_gcp/values-client.yaml geosphere-client geosphere-client/ + dependencies: + - get_chart_client_test + rules: + - if: '$CI_COMMIT_BRANCH != "gcp"' + when: never + - when: on_success + diff --git a/ci_gcp/shapefiles-pvc.yaml b/ci_gcp/shapefiles-pvc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fda3aeb09e948a30931edec9ec89f6b3ac99532a --- /dev/null +++ b/ci_gcp/shapefiles-pvc.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: geosphere-tile-gen-shapefiles + labels: {} +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 2Gi + storageClassName: "standard" diff --git a/ci_gcp/values-client.yaml b/ci_gcp/values-client.yaml new file mode 100644 index 0000000000000000000000000000000000000000..38c9dc6007a937d265646468723f572e8d866d33 --- /dev/null +++ b/ci_gcp/values-client.yaml @@ -0,0 +1,17 @@ +service: + type: LoadBalancer +ingress: + enabled: true +# annotations: +# nginx.ingress.kubernetes.io/ssl-redirect: "false" +# ingress.kubernetes.io/ssl-redirect: "true" + hosts: + - host: "" + paths: ["/"] +# tls: +# - hosts: +# - "geosphere.ssec.wisc.edu" +# secretName: "geosphere-tls-certs" + +infoServer: "http://geosphere.ssec.wisc.edu" +tileServer: "http://geosphere{1-4}.ssec.wisc.edu/mapcache/wmts" diff --git a/ci_gcp/values-geo2grid-g16-radf.yaml b/ci_gcp/values-geo2grid-g16-radf.yaml new file mode 100644 index 0000000000000000000000000000000000000000..20f67bbc265b4acef83f36fa4851d936dce25918 --- /dev/null +++ b/ci_gcp/values-geo2grid-g16-radf.yaml @@ -0,0 +1,21 @@ +rabbitIn: + host: "geosphere-rabbit-rabbitmq" + username: "user" + passwordSecret: "geosphere-rabbit-rabbitmq" + topic: "data.goes.g16.abi.radf.l1b.netcdf.all.complete" +rabbitOut: + host: "geosphere-rabbit-rabbitmq" + username: "user" + passwordSecret: "geosphere-rabbit-rabbitmq" +source: +# s3Endpoint: "http://geosphere-minio:9000" + existingClaim: "cspp-geo-grb" +destination: + persistence: + existingClaim: "cspp-geo-geo2grid" + cleanup: + age: "+1" +# s3Endpoint: "http://geosphere-minio:9000" + s3Secret: "geosphere-minio" + s3AccessKey: "accesskey" + s3SecretKey: "secretkey" diff --git a/ci_gcp/values-grb-g16.yaml b/ci_gcp/values-grb-g16.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ada79ef7d320147a8042dde52423524dbd98a59 --- /dev/null +++ b/ci_gcp/values-grb-g16.yaml @@ -0,0 +1,16 @@ +apidFilter: "GOES-16-ABI-Only.xml" +fanoutServer: "fanout1" +sshProxy: "ash.ssec.wisc.edu" +sshUser: "davidh" +sshPrivateKeyFile: "secrets/grb_ssh_proxy_rsa" +leftPort: 50060 +rightPort: 50070 +uploadDst: "/dst" +persistence: + enabled: true + size: 100Gi + storageClass: "standard" +#rabbitOut: +# host: "geosphere-rabbit-rabbitmq" +# username: "user" +# passwordSecret: "geosphere-rabbit-rabbitmq" \ No newline at end of file diff --git a/helpers/shell_aliases.sh b/helpers/shell_aliases.sh index 082d11f33a3ffaecec3d4d9d41862a1d94fe17e7..26349c1d1c5448c3a72676ea591d8a36e56c3c11 100644 --- a/helpers/shell_aliases.sh +++ b/helpers/shell_aliases.sh @@ -7,6 +7,17 @@ helm() { } export -f helm +helm_debug() { + docker run -t --rm -e HELM_EXPERIMENTAL_OCI="$HELM_EXPERIMENTAL_OCI" -e KUBECONFIG="/root/.kube/config" -v $(pwd):/apps -v /root/.kube:/root/.kube -v ~/.helm:/root/.helm -v ~/.config:/root/.config -v ~/.cache:/root/.cache --entrypoint="" alpine/helm:3.2.3 "$@" +} +export -f helm_debug + + +helm_for_gcp() { + docker run -t --rm -e HELM_EXPERIMENTAL_OCI="$HELM_EXPERIMENTAL_OCI" -e KUBECONFIG="/root/.kube/config" -v $(pwd):/apps -v ~/.kube:/root/.kube -v ~/.helm:/root/.helm -v ~/.config:/root/.config -v ~/.cache:/root/.cache alpine/helm:3.2.3 "$@" +} +export -f helm_for_gcp + kubectl() { docker run -a stdout --rm -v $(pwd):/apps -w /apps alpine/k8s:1.18.2 kubectl "$@" }