Stop workloads before pre-upgrade migrations

.github/workflows/lint-and-test-charts.yml

@@ -34,6 +34,9 @@ jobs:
       - name: Run chart-testing (lint)
         run: ct lint --all
 
+      - name: Check migration scale-down rendering
+        run: scripts/test-migration-scale-down.sh
+
   test:
     runs-on: ubuntu-latest
     timeout-minutes: 35

charts/lightdash/Chart.yaml

@@ -6,7 +6,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 2.16.283
+version: 2.16.284
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

charts/lightdash/README.md

@@ -2,7 +2,7 @@
 
 A Helm chart to deploy lightdash on kubernetes
 
-![Version: 2.16.283](https://img.shields.io/badge/Version-2.16.283-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.77.0](https://img.shields.io/badge/AppVersion-2.77.0-informational?style=flat-square)
+![Version: 2.16.284](https://img.shields.io/badge/Version-2.16.284-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.77.0](https://img.shields.io/badge/AppVersion-2.77.0-informational?style=flat-square)
 
 ## Prerequisites
 
@@ -131,6 +131,31 @@ Set `minAvailable` to pin a minimum number of available pods instead. When both
 
 **Important:** With `replicaCount: 1`, the default permits the only pod to be evicted, so it does not prevent downtime.
 
+## Database migrations during upgrades
+
+Set one release-wide Deployment strategy when an external upgrade check decides whether an application version can roll safely:
+
+```yaml
+upgrade:
+  mode: Recreate
+migrationJob:
+  enabled: true
+```
+
+Map a `true` upgrade-check result to `upgrade.mode: RollingUpdate`. Map `false` or an unknown result to `upgrade.mode: Recreate`. The chart does not call the upgrade-check service or fetch its verdict.
+
+An explicit mode is authoritative for the backend and all four worker Deployments. `Recreate` removes any per-component `rollingUpdate` block. `RollingUpdate` keeps valid per-component `rollingUpdate` tuning. Leave `upgrade.mode` empty to preserve every existing per-component strategy exactly.
+
+When `migrationJob.enabled` is `true`, a Helm upgrade automatically runs the scale-down sequence if the explicit mode is `Recreate`. For backward compatibility, an empty mode also runs it when the backend or any enabled worker has `strategy.type: Recreate`. It never scales workloads down during installation.
+
+Before an upgrade migration that requires `Recreate`, the Job removes the backend HPA, scales the backend and every worker Deployment to zero, and waits for all matching application pods to terminate. The migration starts only after that wait succeeds. The migration Job pod is not part of the wait selector.
+
+A successful upgrade applies the normal release manifests after the hook. Those manifests restore configured worker replicas and either `replicaCount` or `autoscaling.minReplicas` for the backend, then recreate the backend HPA when autoscaling is enabled. Application downtime lasts from the scale-down until the new pods become ready.
+
+If the migration, wait, or any Kubernetes command fails, the upgrade fails closed. The Job does not restore the HPA or replicas, so the application remains stopped until an operator fixes the problem and retries or rolls back the release.
+
+By default, the chart creates temporary namespace-scoped Role and RoleBinding hooks before a pre-upgrade scale-down. Set `migrationJob.scaleDownWorkloads.rbac.create: false` only when the migration service account can get and delete the named backend HPA, list Deployments, patch the scale subresource for the five named Lightdash Deployments, and get, list, and watch pods. The image, timeout, resources, and RBAC settings remain under `migrationJob.scaleDownWorkloads`. When `migrationJob.serviceAccount.create` is `false`, the named custom service account must exist before the migration hook starts. When it is `true`, the chart creates the migration service account as an earlier hook.
+
 ## Values
 
 Note The `secret.*` values are used to create [kubernetes secrets](https://kubernetes.io/docs/concepts/configuration/secret/).
@@ -245,6 +270,12 @@ If you don't want helm to manage this, you may wish to separately create a secre
 | migrationJob.inheritGlobalEnv | bool | `false` | When true, the migration Job also receives the top-level extraEnv and existingSecret, so env supplied globally (for example LIGHTDASH_LICENSE_KEY) reaches the migrator. Default false keeps current behaviour. |
 | migrationJob.podAnnotations | object | `{}` |  |
 | migrationJob.resources | object | `{}` |  |
+| migrationJob.scaleDownWorkloads.image.pullPolicy | string | `"IfNotPresent"` |  |
+| migrationJob.scaleDownWorkloads.image.repository | string | `"registry.k8s.io/kubectl"` |  |
+| migrationJob.scaleDownWorkloads.image.tag | string | `"v1.33.4"` |  |
+| migrationJob.scaleDownWorkloads.rbac.create | bool | `true` |  |
+| migrationJob.scaleDownWorkloads.resources | object | `{}` |  |
+| migrationJob.scaleDownWorkloads.timeoutSeconds | int | `300` |  |
 | migrationJob.serviceAccount.annotations | object | `{}` |  |
 | migrationJob.serviceAccount.create | bool | `true` |  |
 | migrationJob.serviceAccount.name | string | `""` |  |
@@ -381,6 +412,7 @@ If you don't want helm to manage this, you may wish to separately create a secre
 | ssl.enabled | bool | `false` |  |
 | ssl.mountPath | string | `"/etc/ssl/certs"` |  |
 | tolerations | list | `[]` |  |
+| upgrade.mode | string | `""` |  |
 | warehouseNatsWorker.command[0] | string | `"node"` |  |
 | warehouseNatsWorker.command[1] | string | `"dist/natsWorker.js"` |  |
 | warehouseNatsWorker.command[2] | string | `"--stream"` |  |

charts/lightdash/README.md.gotmpl

@@ -125,6 +125,31 @@ Set `minAvailable` to pin a minimum number of available pods instead. When both
 
 **Important:** With `replicaCount: 1`, the default permits the only pod to be evicted, so it does not prevent downtime.
 
+## Database migrations during upgrades
+
+Set one release-wide Deployment strategy when an external upgrade check decides whether an application version can roll safely:
+
+```yaml
+upgrade:
+  mode: Recreate
+migrationJob:
+  enabled: true
+```
+
+Map a `true` upgrade-check result to `upgrade.mode: RollingUpdate`. Map `false` or an unknown result to `upgrade.mode: Recreate`. The chart does not call the upgrade-check service or fetch its verdict.
+
+An explicit mode is authoritative for the backend and all four worker Deployments. `Recreate` removes any per-component `rollingUpdate` block. `RollingUpdate` keeps valid per-component `rollingUpdate` tuning. Leave `upgrade.mode` empty to preserve every existing per-component strategy exactly.
+
+When `migrationJob.enabled` is `true`, a Helm upgrade automatically runs the scale-down sequence if the explicit mode is `Recreate`. For backward compatibility, an empty mode also runs it when the backend or any enabled worker has `strategy.type: Recreate`. It never scales workloads down during installation.
+
+Before an upgrade migration that requires `Recreate`, the Job removes the backend HPA, scales the backend and every worker Deployment to zero, and waits for all matching application pods to terminate. The migration starts only after that wait succeeds. The migration Job pod is not part of the wait selector.
+
+A successful upgrade applies the normal release manifests after the hook. Those manifests restore configured worker replicas and either `replicaCount` or `autoscaling.minReplicas` for the backend, then recreate the backend HPA when autoscaling is enabled. Application downtime lasts from the scale-down until the new pods become ready.
+
+If the migration, wait, or any Kubernetes command fails, the upgrade fails closed. The Job does not restore the HPA or replicas, so the application remains stopped until an operator fixes the problem and retries or rolls back the release.
+
+By default, the chart creates temporary namespace-scoped Role and RoleBinding hooks before a pre-upgrade scale-down. Set `migrationJob.scaleDownWorkloads.rbac.create: false` only when the migration service account can get and delete the named backend HPA, list Deployments, patch the scale subresource for the five named Lightdash Deployments, and get, list, and watch pods. The image, timeout, resources, and RBAC settings remain under `migrationJob.scaleDownWorkloads`. When `migrationJob.serviceAccount.create` is `false`, the named custom service account must exist before the migration hook starts. When it is `true`, the chart creates the migration service account as an earlier hook.
+
 ## Values
 
 Note The `secret.*` values are used to create [kubernetes secrets](https://kubernetes.io/docs/concepts/configuration/secret/).

charts/lightdash/templates/_helpers.tpl

@@ -154,6 +154,13 @@ If using an external database, the password will be stored in the lightdash secr
 {{- end -}}
 {{- end -}}
 
+{{- define "lightdash.validateUpgradeMode" -}}
+{{- $mode := default "" .Values.upgrade.mode -}}
+{{- if and (ne $mode "") (ne $mode "RollingUpdate") (ne $mode "Recreate") -}}
+{{- fail "upgrade.mode must be one of: RollingUpdate, Recreate" -}}
+{{- end -}}
+{{- end -}}
+
 {{/*
 Configuration for postgres credentials
 */}}
@@ -220,6 +227,47 @@ Add environment variables to configure database values
 {{- end -}}
 {{- end -}}
 
+{{- define "lightdash.deploymentStrategy" -}}
+{{- $mode := default "" .root.Values.upgrade.mode -}}
+{{- $strategy := .strategy -}}
+{{- if eq $mode "Recreate" }}
+type: Recreate
+{{- else if eq $mode "RollingUpdate" }}
+type: RollingUpdate
+{{- with $strategy.rollingUpdate }}
+rollingUpdate:
+{{- toYaml . | nindent 2 }}
+{{- end }}
+{{- else }}
+{{- with $strategy }}
+{{- toYaml . }}
+{{- end }}
+{{- end }}
+{{- end -}}
+
+{{- define "lightdash.requiresPreMigrationScaleDown" -}}
+{{- $mode := default "" .Values.upgrade.mode -}}
+{{- $backendRecreate := eq (default "" .Values.lightdashBackend.strategy.type) "Recreate" -}}
+{{- $schedulerRecreate := and .Values.scheduler.enabled (eq (default "" .Values.scheduler.strategy.type) "Recreate") -}}
+{{- $appBuildRecreate := and .Values.appBuildWorker.enabled (eq (default "" .Values.appBuildWorker.strategy.type) "Recreate") -}}
+{{- $warehouseRecreate := and .Values.warehouseNatsWorker.enabled (eq (default "" .Values.warehouseNatsWorker.strategy.type) "Recreate") -}}
+{{- $preAggregateRecreate := and .Values.preAggregateNatsWorker.enabled (eq (default "" .Values.preAggregateNatsWorker.strategy.type) "Recreate") -}}
+{{- $legacyRecreate := or $backendRecreate $schedulerRecreate $appBuildRecreate $warehouseRecreate $preAggregateRecreate -}}
+{{- if or (eq $mode "Recreate") (and (eq $mode "") $legacyRecreate) -}}true{{- else -}}false{{- end -}}
+{{- end -}}
+
+{{- define "lightdash.applicationDeploymentNames" -}}
+- {{ include "lightdash.fullname" . }}-backend
+- {{ include "lightdash.fullname" . }}-worker
+- {{ include "lightdash.fullname" . }}-app-build-worker
+- {{ include "lightdash.fullname" . }}-warehouse-nats-worker
+- {{ include "lightdash.fullname" . }}-pre-aggregate-nats-worker
+{{- end -}}
+
+{{- define "lightdash.applicationWorkloadSelector" -}}
+app.kubernetes.io/name={{ include "lightdash.name" . }},app.kubernetes.io/instance={{ .Release.Name }},app.kubernetes.io/component in (backend,worker,app-build-worker,warehouse-nats-worker,pre-aggregate-nats-worker)
+{{- end -}}
+
 
 {{/*
  Create the name of the backend configuration

charts/lightdash/templates/_worker-deployment.tpl

@@ -8,6 +8,7 @@ Usage: {{- include "lightdash.workerDeployment" (dict "root" . "component" "work
 {{- $workerConfig := .workerConfig -}}
 {{- $volumes := $workerConfig.extraVolumes }}
 {{- $volumeMounts := $workerConfig.extraVolumeMounts }}
+{{- $strategy := include "lightdash.deploymentStrategy" (dict "root" $root "strategy" $workerConfig.strategy) | trim }}
 {{- if $workerConfig.enabled }}
 apiVersion: apps/v1
 kind: Deployment
@@ -18,9 +19,9 @@ metadata:
     app.kubernetes.io/component: {{ $component }}
 spec:
   replicas: {{ $workerConfig.replicas }}
-  {{- with $workerConfig.strategy }}
+  {{- if $strategy }}
   strategy:
-    {{- toYaml . | nindent 4 }}
+    {{- $strategy | nindent 4 }}
   {{- end }}
   selector:
     matchLabels:

charts/lightdash/templates/backendDeployment.yaml

@@ -1,5 +1,7 @@
 {{- $volumes := .Values.lightdashBackend.extraVolumes }}
 {{- $volumeMounts := .Values.lightdashBackend.extraVolumeMounts }}
+{{- $requiresPreMigrationScaleDown := eq (include "lightdash.requiresPreMigrationScaleDown" .) "true" }}
+{{- $strategy := include "lightdash.deploymentStrategy" (dict "root" . "strategy" .Values.lightdashBackend.strategy) | trim }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -8,12 +10,14 @@ metadata:
     {{- include "lightdash.labels" . | nindent 4 }}
     app.kubernetes.io/component: backend
 spec:
-  {{- if not .Values.autoscaling.enabled }}
+  {{- if and .Values.autoscaling.enabled .Release.IsUpgrade .Values.migrationJob.enabled $requiresPreMigrationScaleDown }}
+  replicas: {{ .Values.autoscaling.minReplicas }}
+  {{- else if not .Values.autoscaling.enabled }}
   replicas: {{ .Values.replicaCount }}
   {{- end }}
-  {{- with .Values.lightdashBackend.strategy }}
+  {{- if $strategy }}
   strategy:
-    {{- toYaml . | nindent 4 }}
+    {{- $strategy | nindent 4 }}
   {{- end }}
   selector:
     matchLabels:

charts/lightdash/templates/configmap.yaml

@@ -1,4 +1,5 @@
 {{- include "lightdash.validateS3Config" . }}
+{{- include "lightdash.validateUpgradeMode" . }}
 {{- if .Values.configMap }}
 apiVersion: v1
 kind: ConfigMap

charts/lightdash/templates/migrationJob.yaml

@@ -1,3 +1,4 @@
+{{- $scaleDownWorkloads := and .Release.IsUpgrade (eq (include "lightdash.requiresPreMigrationScaleDown" .) "true") -}}
 {{- if .Values.migrationJob.enabled }}
 apiVersion: batch/v1
 kind: Job
@@ -33,9 +34,56 @@ spec:
       affinity:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      {{- if .Values.initContainers }}
+      {{- if or $scaleDownWorkloads .Values.initContainers }}
       initContainers:
-        {{- toYaml .Values.initContainers | nindent 8 }}
+        {{- if $scaleDownWorkloads }}
+        - name: remove-backend-hpa
+          image: "{{ .Values.migrationJob.scaleDownWorkloads.image.repository }}:{{ .Values.migrationJob.scaleDownWorkloads.image.tag }}"
+          imagePullPolicy: {{ .Values.migrationJob.scaleDownWorkloads.image.pullPolicy }}
+          command:
+            - kubectl
+          args:
+            - delete
+            - horizontalpodautoscaler
+            - {{ include "lightdash.fullname" . }}
+            - --ignore-not-found=true
+          {{- with .Values.migrationJob.scaleDownWorkloads.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+        - name: scale-down-workloads
+          image: "{{ .Values.migrationJob.scaleDownWorkloads.image.repository }}:{{ .Values.migrationJob.scaleDownWorkloads.image.tag }}"
+          imagePullPolicy: {{ .Values.migrationJob.scaleDownWorkloads.image.pullPolicy }}
+          command:
+            - kubectl
+          args:
+            - scale
+            - deployment
+            - --selector={{ include "lightdash.applicationWorkloadSelector" . }}
+            - --replicas=0
+          {{- with .Values.migrationJob.scaleDownWorkloads.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+        - name: wait-for-workload-termination
+          image: "{{ .Values.migrationJob.scaleDownWorkloads.image.repository }}:{{ .Values.migrationJob.scaleDownWorkloads.image.tag }}"
+          imagePullPolicy: {{ .Values.migrationJob.scaleDownWorkloads.image.pullPolicy }}
+          command:
+            - kubectl
+          args:
+            - wait
+            - --for=delete
+            - pod
+            - --selector={{ include "lightdash.applicationWorkloadSelector" . }}
+            - --timeout={{ .Values.migrationJob.scaleDownWorkloads.timeoutSeconds }}s
+          {{- with .Values.migrationJob.scaleDownWorkloads.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+        {{- end }}
+        {{- with .Values.initContainers }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
       {{- end }}
       containers:
         - name: migrate

charts/lightdash/templates/migrationScaleDownRbac.yaml

@@ -0,0 +1,63 @@
+{{- if and .Release.IsUpgrade .Values.migrationJob.enabled (eq (include "lightdash.requiresPreMigrationScaleDown" .) "true") .Values.migrationJob.scaleDownWorkloads.rbac.create }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "lightdash.fullname" . }}-migration-scale-down
+  labels:
+    {{- include "lightdash.labels" . | nindent 4 }}
+  annotations:
+    helm.sh/hook: pre-upgrade
+    helm.sh/hook-weight: "-3"
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded,hook-failed
+rules:
+  - apiGroups:
+      - apps
+    resources:
+      - deployments
+    verbs:
+      - list
+  - apiGroups:
+      - apps
+    resources:
+      - deployments/scale
+    resourceNames:
+      {{- include "lightdash.applicationDeploymentNames" . | nindent 6 }}
+    verbs:
+      - patch
+  - apiGroups:
+      - autoscaling
+    resources:
+      - horizontalpodautoscalers
+    resourceNames:
+      - {{ include "lightdash.fullname" . }}
+    verbs:
+      - delete
+      - get
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+      - list
+      - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ include "lightdash.fullname" . }}-migration-scale-down
+  labels:
+    {{- include "lightdash.labels" . | nindent 4 }}
+  annotations:
+    helm.sh/hook: pre-upgrade
+    helm.sh/hook-weight: "-1"
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded,hook-failed
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ include "lightdash.fullname" . }}-migration-scale-down
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "lightdash.migrationServiceAccountName" . }}
+    namespace: {{ .Release.Namespace }}
+{{- end }}