fix: report the control plane as unavailable when it cannot be reached

api/controlplane/v1beta2/conditions_consts.go

@@ -23,6 +23,15 @@ import (
 const (
 	// ControlPlaneAvailableReason documents the fact that the control plane is reachable.
 	ControlPlaneAvailableReason = "Available"
+	// ControlPlaneNotAvailableReason surfaces when the workload cluster API did
+	// not answer.
+	ControlPlaneNotAvailableReason = clusterv1.NotAvailableReason
+	// ControlPlaneConnectionDownReason surfaces while reads of the workload cluster
+	// API are failing but have not failed for long enough to report an outage.
+	ControlPlaneConnectionDownReason = clusterv1.ConnectionDownReason
+	// ControlPlaneAvailableUnknownReason surfaces while the workload cluster API has
+	// not been reached even once, so nothing can be said about its availability.
+	ControlPlaneAvailableUnknownReason = clusterv1.AvailableUnknownReason
 	// K0sControlPlaneScalingUpCondition is true if actual replicas < desired replicas.
 	// Note: In case a K0sControlPlane preflight check is preventing scale up, this will surface in the condition message.
 	K0sControlPlaneScalingUpCondition = clusterv1.ScalingUpCondition

internal/controller/bootstrap/worker_bootstrap_controller_env_test.go

@@ -679,7 +679,9 @@ func TestReconcileGenerateBootstrapData(t *testing.T) {
 
 		fmt.Println(updatedK0sWorkerConfig.Status)
 		assert.NotNil(c, updatedK0sWorkerConfig.Status.Initialization.DataSecretCreated)
-		assert.True(c, *updatedK0sWorkerConfig.Status.Initialization.DataSecretCreated)
+		if updatedK0sWorkerConfig.Status.Initialization.DataSecretCreated != nil {
+			assert.True(c, *updatedK0sWorkerConfig.Status.Initialization.DataSecretCreated)
+		}
 		assert.NotNil(c, updatedK0sWorkerConfig.Status.DataSecretName)
 		if updatedK0sWorkerConfig.Status.DataSecretName != nil {
 			assert.Equal(c, *updatedK0sWorkerConfig.Status.DataSecretName, updatedK0sWorkerConfig.Name)

internal/controller/controlplane/inplace.go

@@ -45,14 +45,22 @@ import (
 )
 
 func (c *K0sController) reconcileInplaceK0sVersionUpdate(ctx context.Context, scope *controlplane) (ctrl.Result, error) {
+	controlplaneRequiresUpdate := scope.hasMachinesWithOnlyVersionOutdated && scope.kcp.Spec.UpdateStrategy == cpv1beta2.UpdateInPlace
+
 	if !conditions.IsTrue(scope.kcp, cpv1beta2.ControlPlaneAvailableCondition) {
 		// If the control plane is not available, we cannot proceed with the in-place update, as access to the
 		// workload cluster is required to manage the autopilot plan.
+
+		// Falling through would replace machines an in place update should upgrade
+		// where they are. Only the rollout is held, never a count that does not match.
+		countMatches := scope.activeMachines.Len() == int(scope.kcp.Spec.Replicas)
+		if controlplaneRequiresUpdate && countMatches {
+			return ctrl.Result{RequeueAfter: 10 * time.Second}, nil
+		}
+
 		return ctrl.Result{}, nil
 	}
 
-	controlplaneRequiresUpdate := scope.hasMachinesWithOnlyVersionOutdated && scope.kcp.Spec.UpdateStrategy == cpv1beta2.UpdateInPlace
-
 	logger := log.FromContext(ctx).WithValues("version", scope.kcp.Spec.Version)
 
 	kubeClient, err := c.getWorkloadClusterClientset(ctx, scope.cluster)

internal/controller/controlplane/inplace_test.go

@@ -0,0 +1,150 @@
+//go:build !envtest
+
+/*
+Copyright 2025.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controlplane
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	cpv1beta2 "github.com/k0sproject/k0smotron/v2/api/controlplane/v1beta2"
+	"github.com/stretchr/testify/require"
+	clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2"
+	"sigs.k8s.io/cluster-api/util/collections"
+	"sigs.k8s.io/cluster-api/util/conditions"
+)
+
+// TestReconcileInplaceK0sVersionUpdateWhenUnavailable covers the gate that runs
+// before any workload cluster call, so no client is needed here.
+func TestReconcileInplaceK0sVersionUpdateWhenUnavailable(t *testing.T) {
+	for _, tc := range []struct {
+		name         string
+		strategy     cpv1beta2.UpdateStrategy
+		onlyVersion  bool
+		wantRequeue  bool
+		wantedReason string
+	}{
+		{
+			name:         "an in place update in flight must hold the machines still",
+			strategy:     cpv1beta2.UpdateInPlace,
+			onlyVersion:  true,
+			wantRequeue:  true,
+			wantedReason: "scaling here would recreate the machines the update is upgrading where they are",
+		},
+		{
+			name:         "with nothing to update the scaling logic still runs",
+			strategy:     cpv1beta2.UpdateInPlace,
+			wantRequeue:  false,
+			wantedReason: "bring up needs the scaling logic to create the first machines",
+		},
+		{
+			name:         "a recreating strategy is expected to replace machines",
+			strategy:     cpv1beta2.UpdateRecreate,
+			onlyVersion:  true,
+			wantRequeue:  false,
+			wantedReason: "recreation is what the user asked for",
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			kcp := &cpv1beta2.K0sControlPlane{
+				Spec: cpv1beta2.K0sControlPlaneSpec{UpdateStrategy: tc.strategy},
+			}
+			conditions.Set(kcp, metav1.Condition{
+				Type:   string(cpv1beta2.ControlPlaneAvailableCondition),
+				Status: metav1.ConditionFalse,
+				Reason: cpv1beta2.ControlPlaneNotAvailableReason,
+			})
+
+			scope := &controlplane{
+				kcp:                                kcp,
+				cluster:                            &clusterv1.Cluster{},
+				hasMachinesWithOnlyVersionOutdated: tc.onlyVersion,
+			}
+
+			res, err := (&K0sController{}).reconcileInplaceK0sVersionUpdate(context.Background(), scope)
+
+			require.NoError(t, err)
+			require.Equal(t, tc.wantRequeue, !res.IsZero(), tc.wantedReason)
+		})
+	}
+}
+
+// TestReconcileInplaceK0sVersionUpdateHoldsOnlyTheRollout covers the gate letting
+// through anything that changes the machine count, so nothing can be livelocked.
+func TestReconcileInplaceK0sVersionUpdateHoldsOnlyTheRollout(t *testing.T) {
+	newScope := func(active int, replicas int32) *controlplane {
+		kcp := &cpv1beta2.K0sControlPlane{
+			ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default"},
+			Spec: cpv1beta2.K0sControlPlaneSpec{
+				UpdateStrategy: cpv1beta2.UpdateInPlace,
+				Replicas:       replicas,
+			},
+		}
+		conditions.Set(kcp, metav1.Condition{
+			Type:   string(cpv1beta2.ControlPlaneAvailableCondition),
+			Status: metav1.ConditionFalse,
+			Reason: cpv1beta2.ControlPlaneNotAvailableReason,
+		})
+
+		machines := collections.Machines{}
+		for i := range active {
+			machines.Insert(&clusterv1.Machine{ObjectMeta: metav1.ObjectMeta{
+				Name:      fmt.Sprintf("cp-%d", i),
+				Namespace: "default",
+			}})
+		}
+
+		return &controlplane{
+			kcp:                                kcp,
+			cluster:                            &clusterv1.Cluster{},
+			hasMachinesWithOnlyVersionOutdated: true,
+			activeMachines:                     machines,
+			upToDateMachines:                   collections.Machines{},
+			deletedMachines:                    collections.Machines{},
+		}
+	}
+
+	t.Run("a rollout with the right machine count is held", func(t *testing.T) {
+		res, err := (&K0sController{}).reconcileInplaceK0sVersionUpdate(context.Background(), newScope(3, 3))
+
+		require.NoError(t, err)
+		require.False(t, res.IsZero(),
+			"falling through would replace machines the update should upgrade in place")
+	})
+
+	t.Run("being short of machines is let through", func(t *testing.T) {
+		// Remediation deleted one, or an operator raised replicas. Either way the
+		// scale up is the only thing that can fix it.
+		res, err := (&K0sController{}).reconcileInplaceK0sVersionUpdate(context.Background(), newScope(2, 3))
+
+		require.NoError(t, err)
+		require.True(t, res.IsZero(),
+			"holding here livelocks a control plane that cannot recover on its own")
+	})
+
+	t.Run("having too many machines is let through", func(t *testing.T) {
+		// An operator lowered replicas, or wants a wedged machine gone.
+		res, err := (&K0sController{}).reconcileInplaceK0sVersionUpdate(context.Background(), newScope(3, 1))
+
+		require.NoError(t, err)
+		require.True(t, res.IsZero(), "an operator has to be able to remove a machine")
+	})
+}

internal/controller/controlplane/k0s_controlplane_controller.go

@@ -24,7 +24,9 @@ import (
 	"sync"
 	"time"
 
+	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
 
 	"github.com/google/uuid"
 	autopilot "github.com/k0sproject/k0s/pkg/apis/autopilot/v1beta2"
@@ -51,6 +53,7 @@ import (
 	"sigs.k8s.io/cluster-api/util/collections"
 	"sigs.k8s.io/cluster-api/util/kubeconfig"
 	"sigs.k8s.io/cluster-api/util/patch"
+	"sigs.k8s.io/cluster-api/util/predicates"
 	"sigs.k8s.io/cluster-api/util/secret"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -99,6 +102,9 @@ type controlplane struct {
 	controllerConfigs                  map[string]*bootstrapv2.K0sControllerConfig
 	infraMachines                      map[string]*unstructured.Unstructured
 	hasMachinesWithOnlyVersionOutdated bool
+	// availabilityUndecided records that availability was left as it was, so the
+	// reconcile has to come back instead of waiting for an unrelated event.
+	availabilityUndecided bool
 }
 
 // K0sController is responsible for reconciling K0sControlPlane objects.
@@ -117,6 +123,9 @@ type K0sController struct {
 	// autopilotUpdateCancels holds the cancel function for the running updateMachineVersions
 	// goroutine of each control plane, keyed by its NamespacedName.
 	autopilotUpdateCancels sync.Map
+	// availabilityFailures counts failed availability reads in a row per control
+	// plane. A restart starts the count again, reporting an outage late not early.
+	availabilityFailures sync.Map
 }
 
 // +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=k0scontrolplanes/status,verbs=get;list;watch;create;update;patch;delete
@@ -220,11 +229,9 @@ func (c *K0sController) Reconcile(ctx context.Context, req ctrl.Request) (res ct
 			}
 		}
 
-		if needsRequeue(controlplane.kcp) {
-			if res.IsZero() {
-				res = ctrl.Result{RequeueAfter: 20 * time.Second, Requeue: true}
-			}
-		}
+		// A settled control plane gets no events of its own, so this is what brings
+		// the reconcile back to look at availability again.
+		res = requeueResult(err, res, controlplane)
 	}()
 
 	if !controlplane.kcp.ObjectMeta.DeletionTimestamp.IsZero() {
@@ -670,6 +677,8 @@ token = ` + frpToken + `
 func (c *K0sController) reconcileDelete(ctx context.Context, controlplane *controlplane) (ctrl.Result, error) {
 	logger := log.FromContext(ctx)
 
+	clearAvailabilityFailures(&c.availabilityFailures, controlplane.kcp)
+
 	// Read from the API server: an empty list makes the finalizer be removed, which cannot be undone,
 	// so it must not be decided from a cache that may not have observed the Machines yet.
 	allMachines, err := collections.GetFilteredMachinesForCluster(ctx, c.APIReader, controlplane.cluster)
@@ -862,16 +871,53 @@ func (c *K0sController) calculateMachineState(ctx context.Context, kcp *cpv1beta
 	return ms, nil
 }
 
+// clusterToK0sControlPlane maps a Cluster event onto the control plane it points
+// at, and ignores clusters running any other kind of control plane.
+func clusterToK0sControlPlane(_ context.Context, o client.Object) []ctrl.Request {
+	cluster, ok := o.(*clusterv1.Cluster)
+	if !ok {
+		return nil
+	}
+
+	ref := cluster.Spec.ControlPlaneRef
+	if ref.Kind != "K0sControlPlane" || ref.Name == "" {
+		return nil
+	}
+
+	return []ctrl.Request{{
+		NamespacedName: client.ObjectKey{Namespace: cluster.Namespace, Name: ref.Name},
+	}}
+}
+
 // SetupWithManager sets up the controller with the Manager.
 func (c *K0sController) SetupWithManager(mgr ctrl.Manager, opts controller.Options) error {
 	if c.APIReader == nil {
 		return errors.New("APIReader must not be nil")
 	}
 
+	log := mgr.GetLogger().WithValues("controller", "k0scontrolplane")
+
 	// Check if the cluster.x-k8s.io API is available and if not, don't try to watch for Machine objects
-	return ctrl.NewControllerManagedBy(mgr).
+	ctrlBuilder := ctrl.NewControllerManagedBy(mgr).
 		WithOptions(opts).
 		For(&cpv1beta2.K0sControlPlane{}).
 		Owns(&clusterv1.Machine{}).
-		Complete(c)
+		// Reconcile returns before the requeue is armed while paused, so unpausing the
+		// owning Cluster leaves nothing to bring this control plane back.
+		Watches(
+			&clusterv1.Cluster{},
+			handler.EnqueueRequestsFromMapFunc(clusterToK0sControlPlane),
+			builder.WithPredicates(predicates.ClusterPausedTransitionsOrInfrastructureProvisioned(mgr.GetScheme(), log)),
+		)
+
+	// Nothing schedules a reconcile while the control plane reports available, so
+	// without this a cluster whose connection drops is only noticed by chance.
+
+	// Connect and disconnect events need no option. WatchForProbeFailure keys off the
+	// cache's own probe, which says nothing about one read through a tunnel.
+	if c.ClusterCache != nil {
+		ctrlBuilder = ctrlBuilder.WatchesRawSource(c.ClusterCache.GetClusterSource("k0scontrolplane", clusterToK0sControlPlane))
+	}
+
+	return ctrlBuilder.Complete(c)
 }