From 1b6ea3ae8f7a690ed27b7221058792c19120ef10 Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Fri, 21 Aug 2026 12:31:43 +0200 Subject: [PATCH 1/2] fix: clamp controller replicas to 1 with warning (HA not yet supported) The controller uses in-memory state for gRPC stream coordination (Dial/Listen pairing via sync.Map), so only one replica can serve traffic correctly. Multiple replicas would cause connection failures when Dial and Listen land on different pods. Additionally, with leader-election-aware readiness (#1012), multiple replicas cause a rolling update deadlock: Kubernetes cannot terminate the leader (it is the only available pod) and new pods cannot become ready (they are not the leader). Changes: - Clamp controller.replicas to 1 in the reconciler with a warning log and a Kubernetes event (ReplicasClamped) - Update the CRD default from 2 to 1 - Update API type comments to document the limitation Tracking issue for HA controller support: #1013 Relates to #1012 --- .../operator/api/v1alpha1/jumpstarter_types.go | 7 +++++-- .../operator.jumpstarter.dev_jumpstarters.yaml | 6 ++++-- .../jumpstarter/jumpstarter_controller.go | 14 ++++++++++++++ 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/controller/deploy/operator/api/v1alpha1/jumpstarter_types.go b/controller/deploy/operator/api/v1alpha1/jumpstarter_types.go index 63ef1f274..1055bd127 100644 --- a/controller/deploy/operator/api/v1alpha1/jumpstarter_types.go +++ b/controller/deploy/operator/api/v1alpha1/jumpstarter_types.go @@ -331,8 +331,11 @@ type ControllerConfig struct { Resources corev1.ResourceRequirements `json:"resources,omitempty"` // Number of controller replicas to run. - // Must be a positive integer. Minimum recommended value is 2 for high availability. - // +kubebuilder:default=2 + // Currently only 1 replica is supported because the controller uses in-memory + // state for gRPC stream coordination (Dial/Listen). Values greater than 1 will + // be clamped to 1 with a warning. See https://github.com/jumpstarter-dev/jumpstarter/issues/1013 + // for the tracking issue on HA controller support. + // +kubebuilder:default=1 // +kubebuilder:validation:Minimum=1 Replicas int32 `json:"replicas,omitempty"` diff --git a/controller/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml b/controller/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml index 47a74f86e..40c0fe5c4 100644 --- a/controller/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml +++ b/controller/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml @@ -1082,10 +1082,12 @@ spec: description: Custom annotations to add to controller pod templates. type: object replicas: - default: 2 + default: 1 description: |- Number of controller replicas to run. - Must be a positive integer. Minimum recommended value is 2 for high availability. + Currently only 1 replica is supported because the controller uses in-memory + state for gRPC stream coordination (Dial/Listen). Values greater than 1 will + be clamped to 1 with a warning. See issue 1013 for HA controller support. format: int32 minimum: 1 type: integer diff --git a/controller/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go b/controller/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go index 21c5c2147..b778bb3ba 100644 --- a/controller/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go +++ b/controller/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go @@ -160,6 +160,20 @@ func (r *JumpstarterReconciler) Reconcile(ctx context.Context, req ctrl.Request) // Static defaults are handled by kubebuilder annotations in the CRD schema r.EndpointReconciler.ApplyDefaults(&jumpstarter.Spec, jumpstarter.Namespace) + // Clamp controller replicas to 1: the controller uses in-memory state for + // gRPC stream coordination (Dial/Listen pairing), so only one replica can + // serve traffic correctly. Multiple replicas would cause connection failures + // when Dial and Listen land on different pods. + if jumpstarter.Spec.Controller.Replicas > 1 { + log.Info("WARNING: controller.replicas > 1 is not yet supported — the controller "+ + "uses in-memory state for gRPC stream coordination. Clamping to 1.", + "requested", jumpstarter.Spec.Controller.Replicas) + r.emitEventf(&jumpstarter, corev1.EventTypeWarning, "ReplicasClamped", + "controller.replicas=%d is not yet supported (in-memory gRPC state requires a single replica), clamping to 1", + jumpstarter.Spec.Controller.Replicas) + jumpstarter.Spec.Controller.Replicas = 1 + } + // Reconcile RBAC resources first if err := r.reconcileRBAC(ctx, &jumpstarter); err != nil { log.Error(err, "Failed to reconcile RBAC") From aa96a258c80da17283ee94a29902c6d5caebf63c Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Mon, 24 Aug 2026 15:54:45 +0200 Subject: [PATCH 2/2] test: update operator e2e to verify replica clamping and use podAnnotations for update event Split the controller spec change test into two: 1. 'should clamp controller replicas > 1 to 1 with a warning event' - Sets replicas to 3, verifies deployment stays at 1 - Verifies ReplicasClamped warning event is emitted 2. 'should emit controller update events when controller spec changes' - Uses podAnnotations instead of replicas to trigger a deployment update - Verifies the annotation appears in the pod template - Verifies ControllerDeploymentUpdated event is emitted The previous test set replicas to originalReplicas+1 and expected the deployment to reflect that, but the new clamping logic always forces replicas back to 1, causing the assertion to fail. --- .../deploy/operator/test/e2e/e2e_test.go | 71 +++++++++++++++++-- 1 file changed, 64 insertions(+), 7 deletions(-) diff --git a/controller/deploy/operator/test/e2e/e2e_test.go b/controller/deploy/operator/test/e2e/e2e_test.go index 948776dc1..0eb6a96dd 100644 --- a/controller/deploy/operator/test/e2e/e2e_test.go +++ b/controller/deploy/operator/test/e2e/e2e_test.go @@ -514,8 +514,8 @@ provisioning: Eventually(verifyConfigMap, 1*time.Minute).Should(Succeed()) }) - It("should emit controller update events when controller spec changes", func() { - By("updating Jumpstarter controller replicas to trigger a deployment update") + It("should clamp controller replicas > 1 to 1 with a warning event", func() { + By("updating Jumpstarter controller replicas to 3") jumpstarter := &operatorv1alpha1.Jumpstarter{} err := k8sClient.Get(ctx, types.NamespacedName{ Name: "jumpstarter", @@ -523,8 +523,7 @@ provisioning: }, jumpstarter) Expect(err).NotTo(HaveOccurred()) - originalReplicas := jumpstarter.Spec.Controller.Replicas - jumpstarter.Spec.Controller.Replicas = originalReplicas + 1 + jumpstarter.Spec.Controller.Replicas = 3 Expect(k8sClient.Update(ctx, jumpstarter)).To(Succeed()) DeferCleanup(func() { restore := &operatorv1alpha1.Jumpstarter{} @@ -535,11 +534,11 @@ provisioning: if getErr != nil { return } - restore.Spec.Controller.Replicas = originalReplicas + restore.Spec.Controller.Replicas = 1 _ = k8sClient.Update(ctx, restore) }) - By("verifying the controller deployment reflects the updated replica count") + By("verifying the controller deployment still has 1 replica (clamped)") Eventually(func(g Gomega) { deployment := &appsv1.Deployment{} getErr := k8sClient.Get(ctx, types.NamespacedName{ @@ -548,7 +547,65 @@ provisioning: }, deployment) g.Expect(getErr).NotTo(HaveOccurred()) g.Expect(deployment.Spec.Replicas).NotTo(BeNil()) - g.Expect(*deployment.Spec.Replicas).To(Equal(originalReplicas + 1)) + g.Expect(*deployment.Spec.Replicas).To(Equal(int32(1))) + }, 2*time.Minute).Should(Succeed()) + + By("verifying ReplicasClamped warning event was emitted") + Eventually(func(g Gomega) { + eventList := &corev1.EventList{} + listErr := k8sClient.List(ctx, eventList, client.InNamespace(dynamicTestNamespace)) + g.Expect(listErr).NotTo(HaveOccurred()) + + found := false + for _, event := range eventList.Items { + if event.InvolvedObject.Kind == "Jumpstarter" && + event.InvolvedObject.Name == "jumpstarter" && + event.Reason == "ReplicasClamped" && + event.Type == "Warning" { + found = true + break + } + } + g.Expect(found).To(BeTrue(), "expected ReplicasClamped warning event for jumpstarter") + }, 2*time.Minute).Should(Succeed()) + }) + + It("should emit controller update events when controller spec changes", func() { + By("adding a pod annotation to trigger a controller deployment update") + jumpstarter := &operatorv1alpha1.Jumpstarter{} + err := k8sClient.Get(ctx, types.NamespacedName{ + Name: "jumpstarter", + Namespace: dynamicTestNamespace, + }, jumpstarter) + Expect(err).NotTo(HaveOccurred()) + + if jumpstarter.Spec.Controller.PodAnnotations == nil { + jumpstarter.Spec.Controller.PodAnnotations = map[string]string{} + } + jumpstarter.Spec.Controller.PodAnnotations["e2e-test/trigger"] = "deployment-update" + Expect(k8sClient.Update(ctx, jumpstarter)).To(Succeed()) + DeferCleanup(func() { + restore := &operatorv1alpha1.Jumpstarter{} + getErr := k8sClient.Get(ctx, types.NamespacedName{ + Name: "jumpstarter", + Namespace: dynamicTestNamespace, + }, restore) + if getErr != nil { + return + } + delete(restore.Spec.Controller.PodAnnotations, "e2e-test/trigger") + _ = k8sClient.Update(ctx, restore) + }) + + By("verifying the controller deployment reflects the new pod annotation") + Eventually(func(g Gomega) { + deployment := &appsv1.Deployment{} + getErr := k8sClient.Get(ctx, types.NamespacedName{ + Name: "jumpstarter-controller", + Namespace: dynamicTestNamespace, + }, deployment) + g.Expect(getErr).NotTo(HaveOccurred()) + g.Expect(deployment.Spec.Template.Annotations).To(HaveKeyWithValue("e2e-test/trigger", "deployment-update")) }, 2*time.Minute).Should(Succeed()) By("verifying ControllerDeploymentUpdated event was emitted on Jumpstarter resource")