diff --git a/.github/workflows/lint-and-test-charts.yml b/.github/workflows/lint-and-test-charts.yml index 6a24a1f5..ea6b21ca 100644 --- a/.github/workflows/lint-and-test-charts.yml +++ b/.github/workflows/lint-and-test-charts.yml @@ -34,6 +34,9 @@ jobs: - name: Run chart-testing (lint) run: ct lint --all + - name: Check migration scale-down rendering + run: scripts/test-migration-scale-down.sh + test: runs-on: ubuntu-latest timeout-minutes: 35 diff --git a/charts/lightdash/Chart.yaml b/charts/lightdash/Chart.yaml index 03576f97..7997c44c 100644 --- a/charts/lightdash/Chart.yaml +++ b/charts/lightdash/Chart.yaml @@ -6,7 +6,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 2.16.283 +version: 2.16.284 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/lightdash/README.md b/charts/lightdash/README.md index f53536c7..0cfff64e 100644 --- a/charts/lightdash/README.md +++ b/charts/lightdash/README.md @@ -2,7 +2,7 @@ A Helm chart to deploy lightdash on kubernetes -![Version: 2.16.283](https://img.shields.io/badge/Version-2.16.283-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.77.0](https://img.shields.io/badge/AppVersion-2.77.0-informational?style=flat-square) +![Version: 2.16.284](https://img.shields.io/badge/Version-2.16.284-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.77.0](https://img.shields.io/badge/AppVersion-2.77.0-informational?style=flat-square) ## Prerequisites @@ -131,6 +131,31 @@ Set `minAvailable` to pin a minimum number of available pods instead. When both **Important:** With `replicaCount: 1`, the default permits the only pod to be evicted, so it does not prevent downtime. +## Database migrations during upgrades + +Set one release-wide Deployment strategy when an external upgrade check decides whether an application version can roll safely: + +```yaml +upgrade: + mode: Recreate +migrationJob: + enabled: true +``` + +Map a `true` upgrade-check result to `upgrade.mode: RollingUpdate`. Map `false` or an unknown result to `upgrade.mode: Recreate`. The chart does not call the upgrade-check service or fetch its verdict. + +An explicit mode is authoritative for the backend and all four worker Deployments. `Recreate` removes any per-component `rollingUpdate` block. `RollingUpdate` keeps valid per-component `rollingUpdate` tuning. Leave `upgrade.mode` empty to preserve every existing per-component strategy exactly. + +When `migrationJob.enabled` is `true`, a Helm upgrade automatically runs the scale-down sequence if the explicit mode is `Recreate`. For backward compatibility, an empty mode also runs it when the backend or any enabled worker has `strategy.type: Recreate`. It never scales workloads down during installation. + +Before an upgrade migration that requires `Recreate`, the Job removes the backend HPA, scales the backend and every worker Deployment to zero, and waits for all matching application pods to terminate. The migration starts only after that wait succeeds. The migration Job pod is not part of the wait selector. + +A successful upgrade applies the normal release manifests after the hook. Those manifests restore configured worker replicas and either `replicaCount` or `autoscaling.minReplicas` for the backend, then recreate the backend HPA when autoscaling is enabled. Application downtime lasts from the scale-down until the new pods become ready. + +If the migration, wait, or any Kubernetes command fails, the upgrade fails closed. The Job does not restore the HPA or replicas, so the application remains stopped until an operator fixes the problem and retries or rolls back the release. + +By default, the chart creates temporary namespace-scoped Role and RoleBinding hooks before a pre-upgrade scale-down. Set `migrationJob.scaleDownWorkloads.rbac.create: false` only when the migration service account can get and delete the named backend HPA, list Deployments, patch the scale subresource for the five named Lightdash Deployments, and get, list, and watch pods. The image, timeout, resources, and RBAC settings remain under `migrationJob.scaleDownWorkloads`. When `migrationJob.serviceAccount.create` is `false`, the named custom service account must exist before the migration hook starts. When it is `true`, the chart creates the migration service account as an earlier hook. + ## Values Note The `secret.*` values are used to create [kubernetes secrets](https://kubernetes.io/docs/concepts/configuration/secret/). @@ -245,6 +270,12 @@ If you don't want helm to manage this, you may wish to separately create a secre | migrationJob.inheritGlobalEnv | bool | `false` | When true, the migration Job also receives the top-level extraEnv and existingSecret, so env supplied globally (for example LIGHTDASH_LICENSE_KEY) reaches the migrator. Default false keeps current behaviour. | | migrationJob.podAnnotations | object | `{}` | | | migrationJob.resources | object | `{}` | | +| migrationJob.scaleDownWorkloads.image.pullPolicy | string | `"IfNotPresent"` | | +| migrationJob.scaleDownWorkloads.image.repository | string | `"registry.k8s.io/kubectl"` | | +| migrationJob.scaleDownWorkloads.image.tag | string | `"v1.33.4"` | | +| migrationJob.scaleDownWorkloads.rbac.create | bool | `true` | | +| migrationJob.scaleDownWorkloads.resources | object | `{}` | | +| migrationJob.scaleDownWorkloads.timeoutSeconds | int | `300` | | | migrationJob.serviceAccount.annotations | object | `{}` | | | migrationJob.serviceAccount.create | bool | `true` | | | migrationJob.serviceAccount.name | string | `""` | | @@ -381,6 +412,7 @@ If you don't want helm to manage this, you may wish to separately create a secre | ssl.enabled | bool | `false` | | | ssl.mountPath | string | `"/etc/ssl/certs"` | | | tolerations | list | `[]` | | +| upgrade.mode | string | `""` | | | warehouseNatsWorker.command[0] | string | `"node"` | | | warehouseNatsWorker.command[1] | string | `"dist/natsWorker.js"` | | | warehouseNatsWorker.command[2] | string | `"--stream"` | | diff --git a/charts/lightdash/README.md.gotmpl b/charts/lightdash/README.md.gotmpl index b027a816..7015305c 100644 --- a/charts/lightdash/README.md.gotmpl +++ b/charts/lightdash/README.md.gotmpl @@ -125,6 +125,31 @@ Set `minAvailable` to pin a minimum number of available pods instead. When both **Important:** With `replicaCount: 1`, the default permits the only pod to be evicted, so it does not prevent downtime. +## Database migrations during upgrades + +Set one release-wide Deployment strategy when an external upgrade check decides whether an application version can roll safely: + +```yaml +upgrade: + mode: Recreate +migrationJob: + enabled: true +``` + +Map a `true` upgrade-check result to `upgrade.mode: RollingUpdate`. Map `false` or an unknown result to `upgrade.mode: Recreate`. The chart does not call the upgrade-check service or fetch its verdict. + +An explicit mode is authoritative for the backend and all four worker Deployments. `Recreate` removes any per-component `rollingUpdate` block. `RollingUpdate` keeps valid per-component `rollingUpdate` tuning. Leave `upgrade.mode` empty to preserve every existing per-component strategy exactly. + +When `migrationJob.enabled` is `true`, a Helm upgrade automatically runs the scale-down sequence if the explicit mode is `Recreate`. For backward compatibility, an empty mode also runs it when the backend or any enabled worker has `strategy.type: Recreate`. It never scales workloads down during installation. + +Before an upgrade migration that requires `Recreate`, the Job removes the backend HPA, scales the backend and every worker Deployment to zero, and waits for all matching application pods to terminate. The migration starts only after that wait succeeds. The migration Job pod is not part of the wait selector. + +A successful upgrade applies the normal release manifests after the hook. Those manifests restore configured worker replicas and either `replicaCount` or `autoscaling.minReplicas` for the backend, then recreate the backend HPA when autoscaling is enabled. Application downtime lasts from the scale-down until the new pods become ready. + +If the migration, wait, or any Kubernetes command fails, the upgrade fails closed. The Job does not restore the HPA or replicas, so the application remains stopped until an operator fixes the problem and retries or rolls back the release. + +By default, the chart creates temporary namespace-scoped Role and RoleBinding hooks before a pre-upgrade scale-down. Set `migrationJob.scaleDownWorkloads.rbac.create: false` only when the migration service account can get and delete the named backend HPA, list Deployments, patch the scale subresource for the five named Lightdash Deployments, and get, list, and watch pods. The image, timeout, resources, and RBAC settings remain under `migrationJob.scaleDownWorkloads`. When `migrationJob.serviceAccount.create` is `false`, the named custom service account must exist before the migration hook starts. When it is `true`, the chart creates the migration service account as an earlier hook. + ## Values Note The `secret.*` values are used to create [kubernetes secrets](https://kubernetes.io/docs/concepts/configuration/secret/). diff --git a/charts/lightdash/templates/_helpers.tpl b/charts/lightdash/templates/_helpers.tpl index 338231df..a8618563 100644 --- a/charts/lightdash/templates/_helpers.tpl +++ b/charts/lightdash/templates/_helpers.tpl @@ -154,6 +154,13 @@ If using an external database, the password will be stored in the lightdash secr {{- end -}} {{- end -}} +{{- define "lightdash.validateUpgradeMode" -}} +{{- $mode := default "" .Values.upgrade.mode -}} +{{- if and (ne $mode "") (ne $mode "RollingUpdate") (ne $mode "Recreate") -}} +{{- fail "upgrade.mode must be one of: RollingUpdate, Recreate" -}} +{{- end -}} +{{- end -}} + {{/* Configuration for postgres credentials */}} @@ -220,6 +227,47 @@ Add environment variables to configure database values {{- end -}} {{- end -}} +{{- define "lightdash.deploymentStrategy" -}} +{{- $mode := default "" .root.Values.upgrade.mode -}} +{{- $strategy := .strategy -}} +{{- if eq $mode "Recreate" }} +type: Recreate +{{- else if eq $mode "RollingUpdate" }} +type: RollingUpdate +{{- with $strategy.rollingUpdate }} +rollingUpdate: +{{- toYaml . | nindent 2 }} +{{- end }} +{{- else }} +{{- with $strategy }} +{{- toYaml . }} +{{- end }} +{{- end }} +{{- end -}} + +{{- define "lightdash.requiresPreMigrationScaleDown" -}} +{{- $mode := default "" .Values.upgrade.mode -}} +{{- $backendRecreate := eq (default "" .Values.lightdashBackend.strategy.type) "Recreate" -}} +{{- $schedulerRecreate := and .Values.scheduler.enabled (eq (default "" .Values.scheduler.strategy.type) "Recreate") -}} +{{- $appBuildRecreate := and .Values.appBuildWorker.enabled (eq (default "" .Values.appBuildWorker.strategy.type) "Recreate") -}} +{{- $warehouseRecreate := and .Values.warehouseNatsWorker.enabled (eq (default "" .Values.warehouseNatsWorker.strategy.type) "Recreate") -}} +{{- $preAggregateRecreate := and .Values.preAggregateNatsWorker.enabled (eq (default "" .Values.preAggregateNatsWorker.strategy.type) "Recreate") -}} +{{- $legacyRecreate := or $backendRecreate $schedulerRecreate $appBuildRecreate $warehouseRecreate $preAggregateRecreate -}} +{{- if or (eq $mode "Recreate") (and (eq $mode "") $legacyRecreate) -}}true{{- else -}}false{{- end -}} +{{- end -}} + +{{- define "lightdash.applicationDeploymentNames" -}} +- {{ include "lightdash.fullname" . }}-backend +- {{ include "lightdash.fullname" . }}-worker +- {{ include "lightdash.fullname" . }}-app-build-worker +- {{ include "lightdash.fullname" . }}-warehouse-nats-worker +- {{ include "lightdash.fullname" . }}-pre-aggregate-nats-worker +{{- end -}} + +{{- define "lightdash.applicationWorkloadSelector" -}} +app.kubernetes.io/name={{ include "lightdash.name" . }},app.kubernetes.io/instance={{ .Release.Name }},app.kubernetes.io/component in (backend,worker,app-build-worker,warehouse-nats-worker,pre-aggregate-nats-worker) +{{- end -}} + {{/* Create the name of the backend configuration diff --git a/charts/lightdash/templates/_worker-deployment.tpl b/charts/lightdash/templates/_worker-deployment.tpl index b675118b..5e526184 100644 --- a/charts/lightdash/templates/_worker-deployment.tpl +++ b/charts/lightdash/templates/_worker-deployment.tpl @@ -8,6 +8,7 @@ Usage: {{- include "lightdash.workerDeployment" (dict "root" . "component" "work {{- $workerConfig := .workerConfig -}} {{- $volumes := $workerConfig.extraVolumes }} {{- $volumeMounts := $workerConfig.extraVolumeMounts }} +{{- $strategy := include "lightdash.deploymentStrategy" (dict "root" $root "strategy" $workerConfig.strategy) | trim }} {{- if $workerConfig.enabled }} apiVersion: apps/v1 kind: Deployment @@ -18,9 +19,9 @@ metadata: app.kubernetes.io/component: {{ $component }} spec: replicas: {{ $workerConfig.replicas }} - {{- with $workerConfig.strategy }} + {{- if $strategy }} strategy: - {{- toYaml . | nindent 4 }} + {{- $strategy | nindent 4 }} {{- end }} selector: matchLabels: diff --git a/charts/lightdash/templates/backendDeployment.yaml b/charts/lightdash/templates/backendDeployment.yaml index 19f392d2..242a12d6 100644 --- a/charts/lightdash/templates/backendDeployment.yaml +++ b/charts/lightdash/templates/backendDeployment.yaml @@ -1,5 +1,7 @@ {{- $volumes := .Values.lightdashBackend.extraVolumes }} {{- $volumeMounts := .Values.lightdashBackend.extraVolumeMounts }} +{{- $requiresPreMigrationScaleDown := eq (include "lightdash.requiresPreMigrationScaleDown" .) "true" }} +{{- $strategy := include "lightdash.deploymentStrategy" (dict "root" . "strategy" .Values.lightdashBackend.strategy) | trim }} apiVersion: apps/v1 kind: Deployment metadata: @@ -8,12 +10,14 @@ metadata: {{- include "lightdash.labels" . | nindent 4 }} app.kubernetes.io/component: backend spec: - {{- if not .Values.autoscaling.enabled }} + {{- if and .Values.autoscaling.enabled .Release.IsUpgrade .Values.migrationJob.enabled $requiresPreMigrationScaleDown }} + replicas: {{ .Values.autoscaling.minReplicas }} + {{- else if not .Values.autoscaling.enabled }} replicas: {{ .Values.replicaCount }} {{- end }} - {{- with .Values.lightdashBackend.strategy }} + {{- if $strategy }} strategy: - {{- toYaml . | nindent 4 }} + {{- $strategy | nindent 4 }} {{- end }} selector: matchLabels: diff --git a/charts/lightdash/templates/configmap.yaml b/charts/lightdash/templates/configmap.yaml index e9679feb..e80e9f3b 100644 --- a/charts/lightdash/templates/configmap.yaml +++ b/charts/lightdash/templates/configmap.yaml @@ -1,4 +1,5 @@ {{- include "lightdash.validateS3Config" . }} +{{- include "lightdash.validateUpgradeMode" . }} {{- if .Values.configMap }} apiVersion: v1 kind: ConfigMap diff --git a/charts/lightdash/templates/migrationJob.yaml b/charts/lightdash/templates/migrationJob.yaml index 104d6a57..2a5466d0 100644 --- a/charts/lightdash/templates/migrationJob.yaml +++ b/charts/lightdash/templates/migrationJob.yaml @@ -1,3 +1,4 @@ +{{- $scaleDownWorkloads := and .Release.IsUpgrade (eq (include "lightdash.requiresPreMigrationScaleDown" .) "true") -}} {{- if .Values.migrationJob.enabled }} apiVersion: batch/v1 kind: Job @@ -33,9 +34,56 @@ spec: affinity: {{- toYaml . | nindent 8 }} {{- end }} - {{- if .Values.initContainers }} + {{- if or $scaleDownWorkloads .Values.initContainers }} initContainers: - {{- toYaml .Values.initContainers | nindent 8 }} + {{- if $scaleDownWorkloads }} + - name: remove-backend-hpa + image: "{{ .Values.migrationJob.scaleDownWorkloads.image.repository }}:{{ .Values.migrationJob.scaleDownWorkloads.image.tag }}" + imagePullPolicy: {{ .Values.migrationJob.scaleDownWorkloads.image.pullPolicy }} + command: + - kubectl + args: + - delete + - horizontalpodautoscaler + - {{ include "lightdash.fullname" . }} + - --ignore-not-found=true + {{- with .Values.migrationJob.scaleDownWorkloads.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + - name: scale-down-workloads + image: "{{ .Values.migrationJob.scaleDownWorkloads.image.repository }}:{{ .Values.migrationJob.scaleDownWorkloads.image.tag }}" + imagePullPolicy: {{ .Values.migrationJob.scaleDownWorkloads.image.pullPolicy }} + command: + - kubectl + args: + - scale + - deployment + - --selector={{ include "lightdash.applicationWorkloadSelector" . }} + - --replicas=0 + {{- with .Values.migrationJob.scaleDownWorkloads.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + - name: wait-for-workload-termination + image: "{{ .Values.migrationJob.scaleDownWorkloads.image.repository }}:{{ .Values.migrationJob.scaleDownWorkloads.image.tag }}" + imagePullPolicy: {{ .Values.migrationJob.scaleDownWorkloads.image.pullPolicy }} + command: + - kubectl + args: + - wait + - --for=delete + - pod + - --selector={{ include "lightdash.applicationWorkloadSelector" . }} + - --timeout={{ .Values.migrationJob.scaleDownWorkloads.timeoutSeconds }}s + {{- with .Values.migrationJob.scaleDownWorkloads.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- with .Values.initContainers }} + {{- toYaml . | nindent 8 }} + {{- end }} {{- end }} containers: - name: migrate diff --git a/charts/lightdash/templates/migrationScaleDownRbac.yaml b/charts/lightdash/templates/migrationScaleDownRbac.yaml new file mode 100644 index 00000000..a9f18170 --- /dev/null +++ b/charts/lightdash/templates/migrationScaleDownRbac.yaml @@ -0,0 +1,63 @@ +{{- if and .Release.IsUpgrade .Values.migrationJob.enabled (eq (include "lightdash.requiresPreMigrationScaleDown" .) "true") .Values.migrationJob.scaleDownWorkloads.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "lightdash.fullname" . }}-migration-scale-down + labels: + {{- include "lightdash.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-upgrade + helm.sh/hook-weight: "-3" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded,hook-failed +rules: + - apiGroups: + - apps + resources: + - deployments + verbs: + - list + - apiGroups: + - apps + resources: + - deployments/scale + resourceNames: + {{- include "lightdash.applicationDeploymentNames" . | nindent 6 }} + verbs: + - patch + - apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + resourceNames: + - {{ include "lightdash.fullname" . }} + verbs: + - delete + - get + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "lightdash.fullname" . }}-migration-scale-down + labels: + {{- include "lightdash.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-upgrade + helm.sh/hook-weight: "-1" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded,hook-failed +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "lightdash.fullname" . }}-migration-scale-down +subjects: + - kind: ServiceAccount + name: {{ include "lightdash.migrationServiceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/charts/lightdash/values.yaml b/charts/lightdash/values.yaml index 915a6bbb..68db58ec 100644 --- a/charts/lightdash/values.yaml +++ b/charts/lightdash/values.yaml @@ -65,6 +65,10 @@ schedulerExtraEnv: [] # -- Specify the number of lightdash instances. replicaCount: 1 +## @param upgrade.mode Release-wide Deployment strategy. Accepted values are RollingUpdate and Recreate. Empty preserves each component strategy. +upgrade: + mode: "" + image: repository: lightdash/lightdash pullPolicy: IfNotPresent @@ -352,6 +356,12 @@ ssl: ## @param migrationJob.serviceAccount.create Create a dedicated ServiceAccount for the migration hook Job (pre-install/pre-upgrade) ## @param migrationJob.serviceAccount.name ServiceAccount name (defaults to `-migration` when create is true, or `default` when create is false) ## @param migrationJob.serviceAccount.annotations Annotations on the migration ServiceAccount (e.g. workload identity) +## @param migrationJob.scaleDownWorkloads.timeoutSeconds Maximum time to wait for Lightdash application pods to terminate +## @param migrationJob.scaleDownWorkloads.image.repository kubectl image repository +## @param migrationJob.scaleDownWorkloads.image.tag kubectl image tag +## @param migrationJob.scaleDownWorkloads.image.pullPolicy kubectl image pull policy +## @param migrationJob.scaleDownWorkloads.resources Resource requests and limits for the scale-down containers +## @param migrationJob.scaleDownWorkloads.rbac.create Create temporary pre-upgrade Role and RoleBinding hooks for the migration service account migrationJob: enabled: false # -- When true, the migration Job also receives the top-level extraEnv and existingSecret, so env supplied globally (for example LIGHTDASH_LICENSE_KEY) reaches the migrator. Default false keeps current behaviour. @@ -364,6 +374,15 @@ migrationJob: create: true name: "" annotations: {} + scaleDownWorkloads: + timeoutSeconds: 300 + image: + repository: registry.k8s.io/kubectl + tag: "v1.33.4" + pullPolicy: IfNotPresent + resources: {} + rbac: + create: true ssl: enabled: false mountPath: "/etc/ssl/certs" @@ -462,6 +481,7 @@ lightdashBackend: exec: command: ["sh", "-c", "sleep 10"] ## Deployment update strategy. Empty uses the Kubernetes default RollingUpdate. + ## Ignored when upgrade.mode is set. ## Set type: Recreate for an upgrade declared breaking. strategy: {} extraVolumeMounts: [] @@ -513,6 +533,7 @@ scheduler: ## Set a hook only if your topology needs one. lifecycle: {} ## Deployment update strategy. Empty uses the Kubernetes default RollingUpdate. + ## Ignored when upgrade.mode is set. ## Set type: Recreate for an upgrade declared breaking. strategy: {} tasks: @@ -566,6 +587,7 @@ appBuildWorker: ## Set a hook only if your topology needs one. lifecycle: {} ## Deployment update strategy. Empty uses the Kubernetes default RollingUpdate. + ## Ignored when upgrade.mode is set. ## Set type: Recreate for an upgrade declared breaking. strategy: {} tasks: @@ -615,6 +637,7 @@ warehouseNatsWorker: ## Set a hook only if your topology needs one. lifecycle: {} ## Deployment update strategy. Empty uses the Kubernetes default RollingUpdate. + ## Ignored when upgrade.mode is set. ## Set type: Recreate for an upgrade declared breaking. strategy: {} extraVolumeMounts: [] @@ -661,6 +684,7 @@ preAggregateNatsWorker: ## Set a hook only if your topology needs one. lifecycle: {} ## Deployment update strategy. Empty uses the Kubernetes default RollingUpdate. + ## Ignored when upgrade.mode is set. ## Set type: Recreate for an upgrade declared breaking. strategy: {} extraVolumeMounts: [] diff --git a/scripts/test-migration-scale-down.sh b/scripts/test-migration-scale-down.sh new file mode 100755 index 00000000..997e2840 --- /dev/null +++ b/scripts/test-migration-scale-down.sh @@ -0,0 +1,431 @@ +#!/usr/bin/env bash + +set -euo pipefail + +repo_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +chart_dir="$repo_dir/charts/lightdash" +temp_dir="$(mktemp -d)" +trap 'rm -rf "$temp_dir"' EXIT + +fail() { + printf 'migration scale-down rendering check failed: %s\n' "$1" >&2 + exit 1 +} + +assert_contains() { + local file="$1" + local value="$2" + grep -Fq -- "$value" "$file" || fail "expected '$value' in $file" +} + +assert_not_contains() { + local file="$1" + local value="$2" + if grep -Fq -- "$value" "$file"; then + fail "did not expect '$value' in $file" + fi +} + +extract_source() { + local source="$1" + local input="$2" + local output="$3" + awk -v source="# Source: $source" '$0 == source { capture = 1; next } capture && /^---$/ { exit } capture { print }' "$input" > "$output" +} + +extract_kind() { + local kind="$1" + local input="$2" + local output="$3" + awk -v expected_kind="kind: $kind" ' + function emit() { + if (found) { + printf "%s", document + emitted = 1 + exit + } + } + /^---$/ { + emit() + document = "" + found = 0 + next + } + { + document = document $0 ORS + if ($0 == expected_kind) { + found = 1 + } + } + END { + if (found && !emitted) { + printf "%s", document + } + } + ' "$input" > "$output" +} + +render() { + local output="$1" + local lifecycle="$2" + shift 2 + local args=( + safety "$chart_dir" + --namespace safety-ns + --set postgresql.enabled=false + --set browserless-chrome.enabled=false + --set nats.enabled=false + --set s3.endpoint=http://object-store + --set s3.bucket=test + --set s3.region=local + "$@" + ) + if [[ "$lifecycle" == "upgrade" ]]; then + args+=(--is-upgrade) + fi + helm template "${args[@]}" > "$output" +} + +assert_strategy() { + local render_file="$1" + local source="$2" + local strategy_type="$3" + local manifest="$temp_dir/deployment.yaml" + extract_source "$source" "$render_file" "$manifest" + assert_contains "$manifest" ' strategy:' + assert_contains "$manifest" " type: $strategy_type" +} + +assert_no_strategy() { + local render_file="$1" + local source="$2" + local manifest="$temp_dir/deployment.yaml" + extract_source "$source" "$render_file" "$manifest" + assert_not_contains "$manifest" ' strategy:' +} + +assert_no_rolling_update() { + local render_file="$1" + local source="$2" + local manifest="$temp_dir/deployment.yaml" + extract_source "$source" "$render_file" "$manifest" + assert_not_contains "$manifest" ' rollingUpdate:' +} + +assert_tuning() { + local render_file="$1" + local source="$2" + local value="$3" + local manifest="$temp_dir/deployment.yaml" + extract_source "$source" "$render_file" "$manifest" + assert_contains "$manifest" ' rollingUpdate:' + assert_contains "$manifest" " maxSurge: $value" +} + +expected_rules='rules: + - apiGroups: + - apps + resources: + - deployments + verbs: + - list + - apiGroups: + - apps + resources: + - deployments/scale + resourceNames: + - safety-lightdash-backend + - safety-lightdash-worker + - safety-lightdash-app-build-worker + - safety-lightdash-warehouse-nats-worker + - safety-lightdash-pre-aggregate-nats-worker + verbs: + - patch + - apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + resourceNames: + - safety-lightdash + verbs: + - delete + - get + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch' + +assert_scale_down() { + local render_file="$1" + local job="$temp_dir/job.yaml" + local role="$temp_dir/role.yaml" + local role_binding="$temp_dir/role-binding.yaml" + local service_account="$temp_dir/service-account.yaml" + extract_source lightdash/templates/migrationJob.yaml "$render_file" "$job" + extract_source lightdash/templates/migrationScaleDownRbac.yaml "$render_file" "$role" + extract_kind RoleBinding "$render_file" "$role_binding" + extract_source lightdash/templates/migrationServiceAccount.yaml "$render_file" "$service_account" + assert_contains "$job" 'name: remove-backend-hpa' + assert_contains "$job" 'name: scale-down-workloads' + assert_contains "$job" 'name: wait-for-workload-termination' + assert_contains "$job" '--replicas=0' + assert_contains "$job" 'app.kubernetes.io/component in (backend,worker,app-build-worker,warehouse-nats-worker,pre-aggregate-nats-worker)' + assert_not_contains "$job" 'app.kubernetes.io/component=migration' + assert_contains "$job" 'serviceAccountName: safety-lightdash-migration' + assert_contains "$role" 'kind: Role' + assert_not_contains "$role" 'kind: ClusterRole' + assert_contains "$role" 'helm.sh/hook: pre-upgrade' + assert_contains "$role" 'helm.sh/hook-weight: "-3"' + assert_contains "$role" 'helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded,hook-failed' + assert_contains "$service_account" 'helm.sh/hook: pre-install,pre-upgrade' + assert_contains "$service_account" 'helm.sh/hook-weight: "-2"' + assert_contains "$role_binding" 'helm.sh/hook: pre-upgrade' + assert_contains "$role_binding" 'helm.sh/hook-weight: "-1"' + assert_contains "$role_binding" 'helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded,hook-failed' + assert_contains "$role_binding" 'name: safety-lightdash-migration' + assert_contains "$role_binding" 'namespace: safety-ns' + local actual_rules + actual_rules="$(awk '/^rules:$/ { capture = 1 } capture { print }' "$role")" + if ! diff -u <(printf '%s\n' "$expected_rules") <(printf '%s\n' "$actual_rules"); then + fail "rendered unexpected RBAC rules" + fi +} + +assert_no_scale_down() { + local render_file="$1" + assert_not_contains "$render_file" 'name: safety-lightdash-migration-scale-down' + assert_not_contains "$render_file" 'name: remove-backend-hpa' + assert_not_contains "$render_file" 'name: scale-down-workloads' + assert_not_contains "$render_file" 'name: wait-for-workload-termination' +} + +assert_migration_identity() { + local render_file="$1" + local job="$temp_dir/job.yaml" + local service_account="$temp_dir/service-account.yaml" + extract_source lightdash/templates/migrationJob.yaml "$render_file" "$job" + extract_source lightdash/templates/migrationServiceAccount.yaml "$render_file" "$service_account" + assert_contains "$job" 'serviceAccountName: safety-lightdash-migration' + assert_contains "$service_account" 'name: safety-lightdash-migration' + assert_contains "$service_account" 'helm.sh/hook: pre-install,pre-upgrade' + assert_contains "$service_account" 'helm.sh/hook-weight: "-2"' +} + +deployment_sources=( + lightdash/templates/backendDeployment.yaml + lightdash/templates/workerDeployment.yaml + lightdash/templates/appBuildWorkerDeployment.yaml + lightdash/templates/warehouseNatsWorkerDeployment.yaml + lightdash/templates/preAggregateNatsWorkerDeployment.yaml +) + +for lifecycle in install upgrade; do + for mode in unset RollingUpdate Recreate; do + output="$temp_dir/mode-${mode}-${lifecycle}.yaml" + mode_value="" + if [[ "$mode" != "unset" ]]; then + mode_value="$mode" + fi + render "$output" "$lifecycle" \ + --set-string upgrade.mode="$mode_value" \ + --set migrationJob.enabled=true \ + --set autoscaling.enabled=true \ + --set autoscaling.minReplicas=3 \ + --set scheduler.enabled=true \ + --set appBuildWorker.enabled=true \ + --set warehouseNatsWorker.enabled=true \ + --set preAggregateNatsWorker.enabled=true + + for source in "${deployment_sources[@]}"; do + if [[ "$mode" == "unset" ]]; then + assert_no_strategy "$output" "$source" + else + assert_strategy "$output" "$source" "$mode" + fi + if [[ "$mode" == "Recreate" ]]; then + assert_no_rolling_update "$output" "$source" + fi + done + + backend="$temp_dir/backend.yaml" + extract_source lightdash/templates/backendDeployment.yaml "$output" "$backend" + assert_migration_identity "$output" + if [[ "$lifecycle" == "upgrade" && "$mode" == "Recreate" ]]; then + assert_scale_down "$output" + assert_contains "$backend" ' replicas: 3' + else + assert_no_scale_down "$output" + assert_not_contains "$backend" ' replicas:' + fi + done +done + +legacy_components=( + lightdashBackend + scheduler + appBuildWorker + warehouseNatsWorker + preAggregateNatsWorker +) + +for component in "${legacy_components[@]}"; do + output="$temp_dir/legacy-${component}.yaml" + args=( + --set migrationJob.enabled=true + --set "$component.strategy.type=Recreate" + ) + if [[ "$component" != "lightdashBackend" ]]; then + args+=(--set "$component.enabled=true") + fi + render "$output" upgrade "${args[@]}" + assert_scale_down "$output" +done + +disabled_legacy="$temp_dir/disabled-legacy.yaml" +render "$disabled_legacy" upgrade \ + --set migrationJob.enabled=true \ + --set scheduler.enabled=false \ + --set scheduler.strategy.type=Recreate +assert_no_scale_down "$disabled_legacy" + +legacy_preserved="$temp_dir/legacy-preserved.yaml" +render "$legacy_preserved" upgrade \ + --set migrationJob.enabled=true \ + --set scheduler.enabled=true \ + --set appBuildWorker.enabled=true \ + --set warehouseNatsWorker.enabled=true \ + --set preAggregateNatsWorker.enabled=true \ + --set lightdashBackend.strategy.type=RollingUpdate \ + --set lightdashBackend.strategy.rollingUpdate.maxSurge=11 \ + --set scheduler.strategy.type=Recreate \ + --set appBuildWorker.strategy.type=RollingUpdate \ + --set appBuildWorker.strategy.rollingUpdate.maxSurge=12 \ + --set warehouseNatsWorker.strategy.type=RollingUpdate \ + --set warehouseNatsWorker.strategy.rollingUpdate.maxSurge=13 +assert_strategy "$legacy_preserved" lightdash/templates/backendDeployment.yaml RollingUpdate +assert_tuning "$legacy_preserved" lightdash/templates/backendDeployment.yaml 11 +assert_strategy "$legacy_preserved" lightdash/templates/workerDeployment.yaml Recreate +assert_no_rolling_update "$legacy_preserved" lightdash/templates/workerDeployment.yaml +assert_strategy "$legacy_preserved" lightdash/templates/appBuildWorkerDeployment.yaml RollingUpdate +assert_tuning "$legacy_preserved" lightdash/templates/appBuildWorkerDeployment.yaml 12 +assert_strategy "$legacy_preserved" lightdash/templates/warehouseNatsWorkerDeployment.yaml RollingUpdate +assert_tuning "$legacy_preserved" lightdash/templates/warehouseNatsWorkerDeployment.yaml 13 +assert_no_strategy "$legacy_preserved" lightdash/templates/preAggregateNatsWorkerDeployment.yaml +assert_scale_down "$legacy_preserved" + +rolling_override="$temp_dir/rolling-override.yaml" +render "$rolling_override" upgrade \ + --set upgrade.mode=RollingUpdate \ + --set migrationJob.enabled=true \ + --set scheduler.enabled=true \ + --set appBuildWorker.enabled=true \ + --set warehouseNatsWorker.enabled=true \ + --set preAggregateNatsWorker.enabled=true \ + --set lightdashBackend.strategy.type=Recreate \ + --set lightdashBackend.strategy.rollingUpdate.maxSurge=21 \ + --set scheduler.strategy.type=Recreate \ + --set scheduler.strategy.rollingUpdate.maxSurge=22 \ + --set appBuildWorker.strategy.type=Recreate \ + --set appBuildWorker.strategy.rollingUpdate.maxSurge=23 \ + --set warehouseNatsWorker.strategy.type=Recreate \ + --set warehouseNatsWorker.strategy.rollingUpdate.maxSurge=24 \ + --set preAggregateNatsWorker.strategy.type=Recreate \ + --set preAggregateNatsWorker.strategy.rollingUpdate.maxSurge=25 +surge=21 +for source in "${deployment_sources[@]}"; do + assert_strategy "$rolling_override" "$source" RollingUpdate + assert_tuning "$rolling_override" "$source" "$surge" + surge=$((surge + 1)) +done +assert_no_scale_down "$rolling_override" + +recreate_override="$temp_dir/recreate-override.yaml" +render "$recreate_override" upgrade \ + --set upgrade.mode=Recreate \ + --set migrationJob.enabled=true \ + --set scheduler.enabled=true \ + --set appBuildWorker.enabled=true \ + --set warehouseNatsWorker.enabled=true \ + --set preAggregateNatsWorker.enabled=true \ + --set lightdashBackend.strategy.rollingUpdate.maxSurge=31 \ + --set scheduler.strategy.rollingUpdate.maxSurge=32 \ + --set appBuildWorker.strategy.rollingUpdate.maxSurge=33 \ + --set warehouseNatsWorker.strategy.rollingUpdate.maxSurge=34 \ + --set preAggregateNatsWorker.strategy.rollingUpdate.maxSurge=35 +for source in "${deployment_sources[@]}"; do + assert_strategy "$recreate_override" "$source" Recreate + assert_no_rolling_update "$recreate_override" "$source" +done +assert_scale_down "$recreate_override" + +for mask in {0..15}; do + scheduler_enabled=false + app_build_enabled=false + warehouse_enabled=false + pre_aggregate_enabled=false + ((mask & 1)) && scheduler_enabled=true + ((mask & 2)) && app_build_enabled=true + ((mask & 4)) && warehouse_enabled=true + ((mask & 8)) && pre_aggregate_enabled=true + output="$temp_dir/workers-${mask}.yaml" + render "$output" upgrade \ + --set upgrade.mode=Recreate \ + --set migrationJob.enabled=true \ + --set scheduler.enabled="$scheduler_enabled" \ + --set appBuildWorker.enabled="$app_build_enabled" \ + --set warehouseNatsWorker.enabled="$warehouse_enabled" \ + --set preAggregateNatsWorker.enabled="$pre_aggregate_enabled" + expected_count=$((1 + (mask & 1) + ((mask >> 1) & 1) + ((mask >> 2) & 1) + ((mask >> 3) & 1))) + actual_count="$(grep -c '^kind: Deployment$' "$output" || true)" + [[ "$actual_count" == "$expected_count" ]] || fail "worker mask $mask rendered $actual_count Deployments instead of $expected_count" + assert_scale_down "$output" +done + +migration_off="$temp_dir/migration-off.yaml" +render "$migration_off" upgrade \ + --set upgrade.mode=Recreate \ + --set migrationJob.enabled=false \ + --set autoscaling.enabled=true \ + --set scheduler.enabled=true \ + --set appBuildWorker.enabled=true \ + --set warehouseNatsWorker.enabled=true \ + --set preAggregateNatsWorker.enabled=true +assert_no_scale_down "$migration_off" +assert_not_contains "$migration_off" 'name: safety-lightdash-migrate' +for source in "${deployment_sources[@]}"; do + assert_strategy "$migration_off" "$source" Recreate +done +backend="$temp_dir/backend.yaml" +extract_source lightdash/templates/backendDeployment.yaml "$migration_off" "$backend" +assert_not_contains "$backend" ' replicas:' + +custom_rbac="$temp_dir/custom-rbac.yaml" +render "$custom_rbac" upgrade \ + --set upgrade.mode=Recreate \ + --set migrationJob.enabled=true \ + --set migrationJob.scaleDownWorkloads.rbac.create=false \ + --set migrationJob.serviceAccount.create=false \ + --set migrationJob.serviceAccount.name=existing-migrator +assert_not_contains "$custom_rbac" 'name: safety-lightdash-migration-scale-down' +job="$temp_dir/job.yaml" +extract_source lightdash/templates/migrationJob.yaml "$custom_rbac" "$job" +assert_contains "$job" 'serviceAccountName: existing-migrator' +assert_contains "$job" 'name: scale-down-workloads' + +invalid_output="$temp_dir/invalid.yaml" +invalid_error="$temp_dir/invalid.err" +if render "$invalid_output" upgrade --set upgrade.mode=BlueGreen 2> "$invalid_error"; then + fail "invalid upgrade.mode rendered successfully" +fi +assert_contains "$invalid_error" 'upgrade.mode must be one of: RollingUpdate, Recreate' + +removed_key="migrationJob.scaleDownWorkloads."'enabled' +if grep -R -Fq -- "$removed_key" "$chart_dir"; then + fail "removed scale-down enabled key is still present" +fi + +printf 'migration scale-down rendering checks passed\n'