diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eba71e69106f8b58fb56e2aeef56ff5215c9bbc4..35085b0f33139ba2a0da383b276c987008ba1ced 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,7 +3,7 @@ stages: - build prereqs - test - create storage - - deploy rabbit + - deploy infrastructure - deploy GRB - deploy G2G - deploy tile gen diff --git a/admin/kubekorner_geosphere_prometheus_rules.yaml b/admin/kubekorner_geosphere_prometheus_rules.yaml index abafad6980f0a2174386004ff6b126fb14b9934e..225ade0749ee5b1b4d3c4a931117661a34e33305 100644 --- a/admin/kubekorner_geosphere_prometheus_rules.yaml +++ b/admin/kubekorner_geosphere_prometheus_rules.yaml @@ -220,14 +220,14 @@ spec: summary: "Host unusual disk read rate (instance {{ $labels.instance }})" description: "Disk is probably reading too much data (> 150 MB/s)\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" - alert: HostUnusualDiskWriteRate - expr: sum by (instance) (irate(node_disk_written_bytes_total[5m])) / 1024 / 1024 > 150 + expr: sum by (instance) (irate(node_disk_written_bytes_total[5m])) / 1024 / 1024 > 300 for: 5m labels: severity: warning ruleGroup: geosphere-node annotations: summary: "Host unusual disk write rate (instance {{ $labels.instance }})" - description: "Disk is probably writing too much data (> 150 MB/s)\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + description: "Disk is probably writing too much data (> 300 MB/s)\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" - alert: HostOutOfDiskSpace expr: (node_filesystem_avail_bytes{mountpoint="/"} * 100) / node_filesystem_size_bytes{mountpoint="/"} < 10 for: 5m diff --git a/ci_gcp/gitlab-ci.yaml b/ci_gcp/gitlab-ci.yaml index 85ab31f0e9a4a5d40bd6be514211906a74cb9268..c8e79c58aa91168bda37cf8834962e4861f108a4 100644 --- a/ci_gcp/gitlab-ci.yaml +++ b/ci_gcp/gitlab-ci.yaml @@ -84,7 +84,7 @@ gcp deploy rabbit: name: geosphere url: http://geosphere.ssec.wisc.edu extends: .helm_based_job - stage: deploy rabbit + stage: deploy infrastructure script: - ./helpers/deploy_rabbitmq.sh ci_geosphere # this job doesn't actually need any artifacts from previous jobs diff --git a/ci_geosphere-test/gitlab-ci.yaml b/ci_geosphere-test/gitlab-ci.yaml index 4e32c6119033272d3e936070d9f2ebadd86b8a85..872a41c086ca45273bef106d66aad7e6a131f0bc 100644 --- a/ci_geosphere-test/gitlab-ci.yaml +++ b/ci_geosphere-test/gitlab-ci.yaml @@ -37,7 +37,7 @@ gstest deploy rabbit: name: geosphere-test url: http://geosphere-test.ssec.wisc.edu extends: .helm_based_job - stage: deploy rabbit + stage: deploy infrastructure script: - ./helpers/deploy_rabbitmq.sh ci_geosphere-test # this job doesn't actually need any artifacts from previous jobs diff --git a/ci_geosphere-test/values-mapcache.yaml b/ci_geosphere-test/values-mapcache.yaml index b63894016f233139fcabeb88ffd9451832820685..15a3e0d0ccbde6c05a4e3206994ebb2f3b8f5409 100644 --- a/ci_geosphere-test/values-mapcache.yaml +++ b/ci_geosphere-test/values-mapcache.yaml @@ -13,6 +13,12 @@ cache: cleanup: # every 6 hours schedule: "0 */6 * * *" +database: + postgresHost: "geosphere-postgis-postgresql" + postgresPort: 5432 + postgresDatabaseName: "postgres" + postgresUser: "postgres" + postgresPasswordSecret: "geosphere-postgis-postgresql-production" seed: images: false overlays: false diff --git a/ci_geosphere/gitlab-ci.yaml b/ci_geosphere/gitlab-ci.yaml index c04a9e3e80e342e1e080211930bca5541da6c9d1..cf3676dc3e064a04f1819debd753f97b1b639dd7 100644 --- a/ci_geosphere/gitlab-ci.yaml +++ b/ci_geosphere/gitlab-ci.yaml @@ -12,10 +12,20 @@ gs create geotiff storage: stage: create storage script: - ns=$(./helpers/get_namespace.sh) - # copy secret kubeconfig to the mounted (pwd) directory - - cp $kubekorner_k3s_config . - - kubeconfig=$(basename $kubekorner_k3s_config) - - ./helpers/create_pvc.sh "$ns" "ci_geosphere/geotiff-pvc.yaml" "cspp-geo-geo2grid" "$kubeconfig" + - ./helpers/create_pvc.sh "$ns" "ci_geosphere/geotiff-pvc.yaml" "cspp-geo-geo2grid" + # this job doesn't actually need any artifacts from previous jobs + dependencies: [] + rules: + - if: $CI_COMMIT_TAG !~ /^r[0-9]+_[0-9]+/ + when: never + - if: $CREATE_STORAGE + +gs create postgres storage: + extends: .helm_based_job + stage: create storage + script: + - ns=$(./helpers/get_namespace.sh) + - ./helpers/create_pvc.sh "$ns" "ci_geosphere/postgres-pvc.yaml" "geosphere-postgis" # this job doesn't actually need any artifacts from previous jobs dependencies: [] rules: @@ -28,10 +38,7 @@ gs create shapefile storage: stage: create storage script: - ns=$(./helpers/get_namespace.sh) - # copy secret kubeconfig to the mounted (pwd) directory - - cp $kubekorner_k3s_config . - - kubeconfig=$(basename $kubekorner_k3s_config) - - ./helpers/create_pvc.sh "$ns" "ci_geosphere/shapefiles-pvc.yaml" "geosphere-tile-gen-shapefiles" "$kubeconfig" + - ./helpers/create_pvc.sh "$ns" "ci_geosphere/shapefiles-pvc.yaml" "geosphere-tile-gen-shapefiles" # this job doesn't actually need any artifacts from previous jobs dependencies: [] rules: @@ -47,13 +54,11 @@ gs deploy rabbit: name: geosphere url: http://geosphere.ssec.wisc.edu extends: .helm_based_job - stage: deploy rabbit + stage: deploy infrastructure script: - ./helpers/deploy_rabbitmq.sh ci_geosphere - - cp ${kubekorner_k3s_config} . - - kubeconfig=$(basename ${kubekorner_k3s_config}) - |- - kubectl get secret --kubeconfig "${kubeconfig}" geosphere-rabbit-rabbitmq --namespace=geosphere -oyaml | grep -v '^\s*namespace:\s' | grep -v "[Hh]elm" | grep -v "[tT]ime" | grep -v "selfLink" | grep -v "uid" | grep -v "resourceVersion" | sed 's/ name: .*/ name: geosphere-rabbit-rabbitmq-production/' | kubectl apply --kubeconfig "${kubeconfig}" --namespace=geosphere-test -f - + kubectl get secret geosphere-rabbit-rabbitmq --namespace=geosphere -oyaml | grep -v '^\s*namespace:\s' | grep -v "[Hh]elm" | grep -v "[tT]ime" | grep -v "selfLink" | grep -v "uid" | grep -v "resourceVersion" | sed 's/ name: .*/ name: geosphere-rabbit-rabbitmq-production/' | kubectl apply --namespace=geosphere-test -f - # this job doesn't actually need any artifacts from previous jobs dependencies: [] rules: @@ -66,6 +71,28 @@ gs deploy rabbit: - ci_geosphere/values-geosphere-rabbit.yaml - if: $DEPLOY_RABBIT +gs deploy postgres: + environment: + name: geosphere + url: http://geosphere.ssec.wisc.edu + extends: .helm_based_job + stage: deploy infrastructure + script: + - ./helpers/deploy_postgis.sh ci_geosphere + - |- + kubectl get secret geosphere-postgis-postgresql --namespace=geosphere -oyaml | grep -v '^\s*namespace:\s' | grep -v "[Hh]elm" | grep -v "[tT]ime" | grep -v "selfLink" | grep -v "uid" | grep -v "resourceVersion" | sed 's/ name: .*/ name: geosphere-postgis-postgresql-production/' | kubectl apply --namespace=geosphere-test -f - + # this job doesn't actually need any artifacts from previous jobs + dependencies: [] + rules: + - if: $CI_COMMIT_TAG !~ /^r[0-9]+_[0-9]+/ + when: never + # no need to build if another project triggered us + - if: $CI_PIPELINE_SOURCE == "pipeline" + when: never + - changes: + - ci_geosphere/values-postgis.yaml + - if: $DEPLOY_POSTGIS + gs deploy g16 grb: # environment: # name: geosphere diff --git a/ci_geosphere/postgres-pvc.yaml b/ci_geosphere/postgres-pvc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bdf13e894e1f944c6fb911dc54ae501fcc0a4347 --- /dev/null +++ b/ci_geosphere/postgres-pvc.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: geosphere-postgis + labels: {} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 8Gi + storageClassName: "longhorn" diff --git a/ci_geosphere/values-mapcache.yaml b/ci_geosphere/values-mapcache.yaml index ca8afa3bd8a964aa4f75d65364f8bf4545bc86a5..82827b0ecc6961a73834740d0e56e201a5a01ba3 100644 --- a/ci_geosphere/values-mapcache.yaml +++ b/ci_geosphere/values-mapcache.yaml @@ -14,6 +14,12 @@ cache: # every 6 hours schedule: "0 */6 * * *" age: "+2" +database: + postgresHost: "geosphere-postgis-postgresql" + postgresPort: 5432 + postgresDatabaseName: "postgres" + postgresUser: "postgres" + postgresPasswordSecret: "geosphere-postgis-postgresql" seed: images: true overlays: true diff --git a/ci_geosphere/values-postgis.yaml b/ci_geosphere/values-postgis.yaml index 688dd527935a40511b791047957aeb43306e5eae..73cd7cddb48af6cf6657b59609e6fed4cad0cd4b 100644 --- a/ci_geosphere/values-postgis.yaml +++ b/ci_geosphere/values-postgis.yaml @@ -1,3 +1,6 @@ +persistence: + enabled: true + existingClaim: "geosphere-postgis" metrics: enabled: true serviceMonitor: diff --git a/ci_geosphere/values-tile-gen-g16-radc.yaml b/ci_geosphere/values-tile-gen-g16-radc.yaml index f622126f8cd1e56a6e0ec2322a64fe446733ab86..0c0ef0c9ca23e52b91bc61fab577642d4b9fad03 100644 --- a/ci_geosphere/values-tile-gen-g16-radc.yaml +++ b/ci_geosphere/values-tile-gen-g16-radc.yaml @@ -15,3 +15,9 @@ destination: enabled: true storageClass: "longhorn" existingClaim: "geosphere-tile-gen-shapefiles" +database: + postgresHost: "geosphere-postgis-postgresql" + postgresPort: 5432 + postgresDatabaseName: "postgres" + postgresUser: "postgres" + postgresPasswordSecret: "geosphere-postgis-postgresql" diff --git a/ci_geosphere/values-tile-gen-g16-radf.yaml b/ci_geosphere/values-tile-gen-g16-radf.yaml index 7438d4c1ed108c43207e973aa387ede37b937d5e..81ca71026522a4ee5b6366b347b7e8a10202def9 100644 --- a/ci_geosphere/values-tile-gen-g16-radf.yaml +++ b/ci_geosphere/values-tile-gen-g16-radf.yaml @@ -15,4 +15,9 @@ destination: enabled: true storageClass: "longhorn" existingClaim: "geosphere-tile-gen-shapefiles" - +database: + postgresHost: "geosphere-postgis-postgresql" + postgresPort: 5432 + postgresDatabaseName: "postgres" + postgresUser: "postgres" + postgresPasswordSecret: "geosphere-postgis-postgresql" diff --git a/ci_geosphere/values-tile-gen-g16-radm1.yaml b/ci_geosphere/values-tile-gen-g16-radm1.yaml index 98c4408d9c7faf6ef5789bad180369cd9893f0d2..0235b39a268a7e93d0ba7dc457f5ea90c2027ba0 100644 --- a/ci_geosphere/values-tile-gen-g16-radm1.yaml +++ b/ci_geosphere/values-tile-gen-g16-radm1.yaml @@ -15,3 +15,9 @@ destination: enabled: true storageClass: "longhorn" existingClaim: "geosphere-tile-gen-shapefiles" +database: + postgresHost: "geosphere-postgis-postgresql" + postgresPort: 5432 + postgresDatabaseName: "postgres" + postgresUser: "postgres" + postgresPasswordSecret: "geosphere-postgis-postgresql" diff --git a/ci_geosphere/values-tile-gen-g16-radm2.yaml b/ci_geosphere/values-tile-gen-g16-radm2.yaml index b9c8c46db33ea8943392bb209d25fd100a72a047..81f8e205ca307b8b95846ee6c5853fe58805d471 100644 --- a/ci_geosphere/values-tile-gen-g16-radm2.yaml +++ b/ci_geosphere/values-tile-gen-g16-radm2.yaml @@ -15,3 +15,9 @@ destination: enabled: true storageClass: "longhorn" existingClaim: "geosphere-tile-gen-shapefiles" +database: + postgresHost: "geosphere-postgis-postgresql" + postgresPort: 5432 + postgresDatabaseName: "postgres" + postgresUser: "postgres" + postgresPasswordSecret: "geosphere-postgis-postgresql" diff --git a/ci_tests/run_basic_postgres_test.sh b/ci_tests/run_basic_postgres_test.sh index c07b54b4ca59b2443ef606540f2598bb4ce3a530..1c1bdd33350aafe4717f356c8beed1fa31ae713e 100755 --- a/ci_tests/run_basic_postgres_test.sh +++ b/ci_tests/run_basic_postgres_test.sh @@ -20,7 +20,7 @@ install_basic_postgres_charts "ci_tests/basic_postgres" # give kubernetes a bit to create the resources debug "Waiting for Kubernetes to deploy and schedule components..." -sleep 60 +sleep 90 debug "Done waiting" debug "Getting GRB pod name"