diff --git a/docs/reference/api.md b/docs/reference/api.md index e6086574301..5cd8f294611 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -780,6 +780,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `workersToDelete` _string array_ | WorkersToDelete workers to be deleted | | | +| `scaleGate` _string array_ | ScaleGate contains a list of scaling gates blocking this worker group from
scaling up. When Kueue encounters a quota-exceeded error for this worker
group, it appends "kueue.k8s.io/quota-exceeded" to ScaleGate. The Autoscaler
initiates fallback behavior whenever ScaleGate is non-empty. KubeRay defines
this field but does not read or write it, and preserves its value across
reconciles. | | | #### SubmitterConfig diff --git a/helm-chart/kuberay-operator/crds/ray.io_rayclusters.yaml b/helm-chart/kuberay-operator/crds/ray.io_rayclusters.yaml index 7fbd0e19d6e..f736f617fc0 100644 --- a/helm-chart/kuberay-operator/crds/ray.io_rayclusters.yaml +++ b/helm-chart/kuberay-operator/crds/ray.io_rayclusters.yaml @@ -5363,6 +5363,10 @@ spec: type: object scaleStrategy: properties: + scaleGate: + items: + type: string + type: array workersToDelete: items: type: string diff --git a/helm-chart/kuberay-operator/crds/ray.io_raycronjobs.yaml b/helm-chart/kuberay-operator/crds/ray.io_raycronjobs.yaml index bc8375d655b..63430f3cc33 100644 --- a/helm-chart/kuberay-operator/crds/ray.io_raycronjobs.yaml +++ b/helm-chart/kuberay-operator/crds/ray.io_raycronjobs.yaml @@ -5457,6 +5457,10 @@ spec: type: object scaleStrategy: properties: + scaleGate: + items: + type: string + type: array workersToDelete: items: type: string diff --git a/helm-chart/kuberay-operator/crds/ray.io_rayjobs.yaml b/helm-chart/kuberay-operator/crds/ray.io_rayjobs.yaml index 27a7e1894de..6d7ddee853a 100644 --- a/helm-chart/kuberay-operator/crds/ray.io_rayjobs.yaml +++ b/helm-chart/kuberay-operator/crds/ray.io_rayjobs.yaml @@ -5458,6 +5458,10 @@ spec: type: object scaleStrategy: properties: + scaleGate: + items: + type: string + type: array workersToDelete: items: type: string diff --git a/helm-chart/kuberay-operator/crds/ray.io_rayservices.yaml b/helm-chart/kuberay-operator/crds/ray.io_rayservices.yaml index f20ce4d6e03..67c827abcb4 100644 --- a/helm-chart/kuberay-operator/crds/ray.io_rayservices.yaml +++ b/helm-chart/kuberay-operator/crds/ray.io_rayservices.yaml @@ -5351,6 +5351,10 @@ spec: type: object scaleStrategy: properties: + scaleGate: + items: + type: string + type: array workersToDelete: items: type: string diff --git a/ray-operator/apis/ray/v1/raycluster_types.go b/ray-operator/apis/ray/v1/raycluster_types.go index d71ae42a1aa..67b490893a3 100644 --- a/ray-operator/apis/ray/v1/raycluster_types.go +++ b/ray-operator/apis/ray/v1/raycluster_types.go @@ -489,6 +489,14 @@ type WorkerGroupSpec struct { type ScaleStrategy struct { // WorkersToDelete workers to be deleted WorkersToDelete []string `json:"workersToDelete,omitempty"` + // ScaleGate contains a list of scaling gates blocking this worker group from + // scaling up. When Kueue encounters a quota-exceeded error for this worker + // group, it appends "kueue.k8s.io/quota-exceeded" to ScaleGate. The Autoscaler + // initiates fallback behavior whenever ScaleGate is non-empty. KubeRay defines + // this field but does not read or write it, and preserves its value across + // reconciles. + // +optional + ScaleGate []string `json:"scaleGate,omitempty"` } // AutoscalerOptions specifies optional configuration for the Ray autoscaler. diff --git a/ray-operator/apis/ray/v1/zz_generated.deepcopy.go b/ray-operator/apis/ray/v1/zz_generated.deepcopy.go index ec1d842f6ba..54d98083f60 100644 --- a/ray-operator/apis/ray/v1/zz_generated.deepcopy.go +++ b/ray-operator/apis/ray/v1/zz_generated.deepcopy.go @@ -1275,6 +1275,11 @@ func (in *ScaleStrategy) DeepCopyInto(out *ScaleStrategy) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.ScaleGate != nil { + in, out := &in.ScaleGate, &out.ScaleGate + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScaleStrategy. diff --git a/ray-operator/config/crd/bases/ray.io_rayclusters.yaml b/ray-operator/config/crd/bases/ray.io_rayclusters.yaml index 7fbd0e19d6e..f736f617fc0 100644 --- a/ray-operator/config/crd/bases/ray.io_rayclusters.yaml +++ b/ray-operator/config/crd/bases/ray.io_rayclusters.yaml @@ -5363,6 +5363,10 @@ spec: type: object scaleStrategy: properties: + scaleGate: + items: + type: string + type: array workersToDelete: items: type: string diff --git a/ray-operator/config/crd/bases/ray.io_raycronjobs.yaml b/ray-operator/config/crd/bases/ray.io_raycronjobs.yaml index bc8375d655b..63430f3cc33 100644 --- a/ray-operator/config/crd/bases/ray.io_raycronjobs.yaml +++ b/ray-operator/config/crd/bases/ray.io_raycronjobs.yaml @@ -5457,6 +5457,10 @@ spec: type: object scaleStrategy: properties: + scaleGate: + items: + type: string + type: array workersToDelete: items: type: string diff --git a/ray-operator/config/crd/bases/ray.io_rayjobs.yaml b/ray-operator/config/crd/bases/ray.io_rayjobs.yaml index 27a7e1894de..6d7ddee853a 100644 --- a/ray-operator/config/crd/bases/ray.io_rayjobs.yaml +++ b/ray-operator/config/crd/bases/ray.io_rayjobs.yaml @@ -5458,6 +5458,10 @@ spec: type: object scaleStrategy: properties: + scaleGate: + items: + type: string + type: array workersToDelete: items: type: string diff --git a/ray-operator/config/crd/bases/ray.io_rayservices.yaml b/ray-operator/config/crd/bases/ray.io_rayservices.yaml index f20ce4d6e03..67c827abcb4 100644 --- a/ray-operator/config/crd/bases/ray.io_rayservices.yaml +++ b/ray-operator/config/crd/bases/ray.io_rayservices.yaml @@ -5351,6 +5351,10 @@ spec: type: object scaleStrategy: properties: + scaleGate: + items: + type: string + type: array workersToDelete: items: type: string diff --git a/ray-operator/controllers/ray/utils/util.go b/ray-operator/controllers/ray/utils/util.go index b0faac855a6..4acea0df92e 100644 --- a/ray-operator/controllers/ray/utils/util.go +++ b/ray-operator/controllers/ray/utils/util.go @@ -658,6 +658,9 @@ func GenerateHashWithoutReplicasAndWorkersToDelete(rayClusterSpec rayv1.RayClust updatedRayClusterSpec.WorkerGroupSpecs[i].MaxReplicas = nil updatedRayClusterSpec.WorkerGroupSpecs[i].MinReplicas = nil updatedRayClusterSpec.WorkerGroupSpecs[i].ScaleStrategy.WorkersToDelete = nil + // External controllers like Kueue may set ScaleGate to signal quota exhaustion, + // which should not trigger a new RayCluster preparation. + updatedRayClusterSpec.WorkerGroupSpecs[i].ScaleStrategy.ScaleGate = nil updatedRayClusterSpec.WorkerGroupSpecs[i].Template.Spec.Tolerations = nil updatedRayClusterSpec.WorkerGroupSpecs[i].Template.Spec.SchedulingGates = nil } diff --git a/ray-operator/pkg/client/applyconfiguration/ray/v1/scalestrategy.go b/ray-operator/pkg/client/applyconfiguration/ray/v1/scalestrategy.go index 7ce8ee06ad6..ca8c5721109 100644 --- a/ray-operator/pkg/client/applyconfiguration/ray/v1/scalestrategy.go +++ b/ray-operator/pkg/client/applyconfiguration/ray/v1/scalestrategy.go @@ -9,6 +9,13 @@ package v1 type ScaleStrategyApplyConfiguration struct { // WorkersToDelete workers to be deleted WorkersToDelete []string `json:"workersToDelete,omitempty"` + // ScaleGate contains a list of scaling gates blocking this worker group from + // scaling up. When Kueue encounters a quota-exceeded error for this worker + // group, it appends "kueue.k8s.io/quota-exceeded" to ScaleGate. The Autoscaler + // initiates fallback behavior whenever ScaleGate is non-empty. KubeRay defines + // this field but does not read or write it, and preserves its value across + // reconciles. + ScaleGate []string `json:"scaleGate,omitempty"` } // ScaleStrategyApplyConfiguration constructs a declarative configuration of the ScaleStrategy type for use with @@ -26,3 +33,13 @@ func (b *ScaleStrategyApplyConfiguration) WithWorkersToDelete(values ...string) } return b } + +// WithScaleGate adds the given value to the ScaleGate field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the ScaleGate field. +func (b *ScaleStrategyApplyConfiguration) WithScaleGate(values ...string) *ScaleStrategyApplyConfiguration { + for i := range values { + b.ScaleGate = append(b.ScaleGate, values[i]) + } + return b +}