From 7895b02d508fdf431bf21f13999389507662b414 Mon Sep 17 00:00:00 2001 From: Aaron Liang Date: Tue, 4 Aug 2026 05:02:53 +0000 Subject: [PATCH 01/14] Update history server examples --- historyserver/cmd/collector/main.go | 4 +- .../config/historyserver-azureblob.yaml | 2 +- historyserver/config/historyserver.yaml | 2 +- .../config/raycluster-azureblob.yaml | 30 ++++++++++--- historyserver/config/raycluster-gcs.yaml | 38 ++++++++++------- historyserver/config/raycluster.yaml | 42 +++++++++++-------- 6 files changed, 74 insertions(+), 44 deletions(-) diff --git a/historyserver/cmd/collector/main.go b/historyserver/cmd/collector/main.go index b2ec4f2de83..7648a2c9014 100644 --- a/historyserver/cmd/collector/main.go +++ b/historyserver/cmd/collector/main.go @@ -30,7 +30,7 @@ func main() { rayClusterNamespace := "" rayRootDir := "" logBatching := 1000 - eventsPort := 8080 + eventsPort := 8084 pushInterval := time.Minute ownerKind := "" ownerName := "" @@ -53,7 +53,7 @@ func main() { flag.StringVar(&rayClusterNamespace, "ray-cluster-namespace", "default", "") flag.StringVar(&rayRootDir, "ray-root-dir", "", "") flag.IntVar(&logBatching, "log-batching", 1000, "") - flag.IntVar(&eventsPort, "events-port", 8080, "") + flag.IntVar(&eventsPort, "events-port", 8084, "") flag.StringVar(&runtimeClassConfigPath, "runtime-class-config-path", "", "") flag.DurationVar(&pushInterval, "push-interval", time.Minute, "") flag.StringVar(&ownerKind, "owner-kind", "", "") diff --git a/historyserver/config/historyserver-azureblob.yaml b/historyserver/config/historyserver-azureblob.yaml index f00b98bbe34..cdc6d52a871 100644 --- a/historyserver/config/historyserver-azureblob.yaml +++ b/historyserver/config/historyserver-azureblob.yaml @@ -38,7 +38,7 @@ spec: value: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite-service.azurite-dev.svc.cluster.local:10000/devstoreaccount1;" - name: AZURE_STORAGE_CONTAINER value: "ray-historyserver" - image: historyserver:v0.1.0 + image: quay.io/kuberay/historyserver:nightly imagePullPolicy: IfNotPresent command: - historyserver diff --git a/historyserver/config/historyserver.yaml b/historyserver/config/historyserver.yaml index f07d45d34ee..6247439988c 100644 --- a/historyserver/config/historyserver.yaml +++ b/historyserver/config/historyserver.yaml @@ -50,7 +50,7 @@ spec: value: "test" - name: S3FORCE_PATH_STYLE value: "true" - image: historyserver:v0.1.0 + image: quay.io/kuberay/historyserver:nightly imagePullPolicy: IfNotPresent command: - historyserver diff --git a/historyserver/config/raycluster-azureblob.yaml b/historyserver/config/raycluster-azureblob.yaml index 9de3a42e8b7..5cb49e7bff5 100644 --- a/historyserver/config/raycluster-azureblob.yaml +++ b/historyserver/config/raycluster-azureblob.yaml @@ -49,23 +49,32 @@ spec: - name: historyserver mountPath: *rayTmpRoot - name: collector - image: collector:v0.1.0 + image: quay.io/kuberay/collector:nightly imagePullPolicy: IfNotPresent env: + - name: RAY_CLUSTER_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['ray.io/cluster'] - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: FQ_RAY_IP - value: raycluster-historyserver-head-svc.default.svc.cluster.local + value: $(RAY_CLUSTER_NAME)-head-svc.default.svc.cluster.local - name: RAY_TMP_ROOT value: *rayTmpRoot + - name: RAY_ROLE + value: "Head" + - name: STORAGE_BACKEND + value: "azureblob" + - name: RAY_ROOT_DIR + value: "log" # reference: https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite#connect-to-the-emulator-by-using-the-azure-storage-explorer - name: AZURE_STORAGE_CONNECTION_STRING value: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite-service.azurite-dev.svc.cluster.local:10000/devstoreaccount1;" - name: AZURE_STORAGE_CONTAINER value: ray-historyserver - command: [collector, --role=Head, --runtime-class-name=azureblob, --ray-cluster-name=raycluster-historyserver, --ray-root-dir=log, --events-port=8084] volumeMounts: - name: historyserver mountPath: *rayTmpRoot @@ -120,23 +129,32 @@ spec: - name: historyserver mountPath: *rayTmpRoot - name: collector - image: collector:v0.1.0 + image: quay.io/kuberay/collector:nightly imagePullPolicy: IfNotPresent env: + - name: RAY_CLUSTER_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['ray.io/cluster'] - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: FQ_RAY_IP - value: raycluster-historyserver-head-svc.default.svc.cluster.local + value: $(RAY_CLUSTER_NAME)-head-svc.default.svc.cluster.local - name: RAY_TMP_ROOT value: *rayTmpRoot + - name: RAY_ROLE + value: "Worker" + - name: STORAGE_BACKEND + value: "azureblob" + - name: RAY_ROOT_DIR + value: "log" # reference: https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite#connect-to-the-emulator-by-using-the-azure-storage-explorer - name: AZURE_STORAGE_CONNECTION_STRING value: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite-service.azurite-dev.svc.cluster.local:10000/devstoreaccount1;" - name: AZURE_STORAGE_CONTAINER value: ray-historyserver - command: [collector, --role=Worker, --runtime-class-name=azureblob, --ray-cluster-name=raycluster-historyserver, --ray-root-dir=log, --events-port=8084] volumeMounts: - name: historyserver mountPath: *rayTmpRoot diff --git a/historyserver/config/raycluster-gcs.yaml b/historyserver/config/raycluster-gcs.yaml index 4739b79d509..4ef88636ed8 100644 --- a/historyserver/config/raycluster-gcs.yaml +++ b/historyserver/config/raycluster-gcs.yaml @@ -53,12 +53,16 @@ spec: image: ${COLLECTOR_IMAGE} imagePullPolicy: IfNotPresent env: + - name: RAY_CLUSTER_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['ray.io/cluster'] - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: FQ_RAY_IP - value: raycluster-historyserver-head-svc.default.svc.cluster.local + value: $(RAY_CLUSTER_NAME)-head-svc.default.svc.cluster.local - name: RAY_TMP_ROOT value: *rayTmpRoot - name: GCS_BUCKET @@ -84,13 +88,12 @@ spec: # endpoints above. Accepts Go duration format (e.g., "30s", "1m", "5m"). - name: RAY_COLLECTOR_POLL_INTERVAL value: "30s" - command: - - collector - - --role=Head - - --runtime-class-name=gcs - - --ray-cluster-name=${RAY_CLUSTER} - - --ray-root-dir=log - - --events-port=8084 + - name: RAY_ROLE + value: "Head" + - name: STORAGE_BACKEND + value: "gcs" + - name: RAY_ROOT_DIR + value: "log" volumeMounts: - name: historyserver mountPath: *rayTmpRoot @@ -148,23 +151,26 @@ spec: image: ${COLLECTOR_IMAGE} imagePullPolicy: IfNotPresent env: + - name: RAY_CLUSTER_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['ray.io/cluster'] - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: FQ_RAY_IP - value: raycluster-historyserver-head-svc.default.svc.cluster.local + value: $(RAY_CLUSTER_NAME)-head-svc.default.svc.cluster.local - name: RAY_TMP_ROOT value: *rayTmpRoot - name: GCS_BUCKET value: "${GCS_BUCKET}" - command: - - collector - - --role=Worker - - --runtime-class-name=gcs - - --ray-cluster-name=${RAY_CLUSTER} - - --ray-root-dir=log - - --events-port=8084 + - name: RAY_ROLE + value: "Worker" + - name: STORAGE_BACKEND + value: "gcs" + - name: RAY_ROOT_DIR + value: "log" volumeMounts: - name: historyserver mountPath: *rayTmpRoot diff --git a/historyserver/config/raycluster.yaml b/historyserver/config/raycluster.yaml index 1acf4b94462..cebf3dfc73a 100644 --- a/historyserver/config/raycluster.yaml +++ b/historyserver/config/raycluster.yaml @@ -55,15 +55,19 @@ spec: - name: historyserver mountPath: *rayTmpRoot - name: collector - image: collector:v0.1.0 + image: quay.io/kuberay/collector:nightly imagePullPolicy: IfNotPresent env: + - name: RAY_CLUSTER_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['ray.io/cluster'] - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: FQ_RAY_IP - value: raycluster-historyserver-head-svc.default.svc.cluster.local + value: $(RAY_CLUSTER_NAME)-head-svc.default.svc.cluster.local - name: RAY_TMP_ROOT value: *rayTmpRoot # RAY_DASHBOARD_ADDRESS is used by the head collector to fetch endpoints' results @@ -103,13 +107,12 @@ spec: value: "test" - name: S3FORCE_PATH_STYLE value: "true" - command: - - collector - - --role=Head - - --runtime-class-name=s3 - - --ray-cluster-name=raycluster-historyserver - - --ray-root-dir=log - - --events-port=8084 + - name: RAY_ROLE + value: "Head" + - name: STORAGE_BACKEND + value: "s3" + - name: RAY_ROOT_DIR + value: "log" volumeMounts: - name: historyserver mountPath: *rayTmpRoot @@ -166,15 +169,19 @@ spec: - name: historyserver mountPath: *rayTmpRoot - name: collector - image: collector:v0.1.0 + image: quay.io/kuberay/collector:nightly imagePullPolicy: IfNotPresent env: + - name: RAY_CLUSTER_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['ray.io/cluster'] - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: FQ_RAY_IP - value: raycluster-historyserver-head-svc.default.svc.cluster.local + value: $(RAY_CLUSTER_NAME)-head-svc.default.svc.cluster.local - name: RAY_TMP_ROOT value: *rayTmpRoot - name: AWS_ACCESS_KEY_ID @@ -193,13 +200,12 @@ spec: value: "true" - name: S3DISABLE_SSL value: "true" - command: - - collector - - --role=Worker - - --runtime-class-name=s3 - - --ray-cluster-name=raycluster-historyserver - - --ray-root-dir=log - - --events-port=8084 + - name: RAY_ROLE + value: "Worker" + - name: STORAGE_BACKEND + value: "s3" + - name: RAY_ROOT_DIR + value: "log" volumeMounts: - name: historyserver mountPath: *rayTmpRoot From fe35cd51d9d978ffb6bd02934ea0cb7bdff4970c Mon Sep 17 00:00:00 2001 From: Aaron Liang Date: Wed, 5 Aug 2026 00:28:10 +0000 Subject: [PATCH 02/14] update raycluster ray version for history server samples --- historyserver/config/raycluster-azureblob.yaml | 8 ++++---- historyserver/config/raycluster-gcs.yaml | 8 ++++---- historyserver/config/raycluster.yaml | 14 +++++--------- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/historyserver/config/raycluster-azureblob.yaml b/historyserver/config/raycluster-azureblob.yaml index 5cb49e7bff5..6cbf30cc59b 100644 --- a/historyserver/config/raycluster-azureblob.yaml +++ b/historyserver/config/raycluster-azureblob.yaml @@ -28,11 +28,11 @@ spec: value: "true" - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR value: "http://localhost:8084/v1/events" - - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES + - name: RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES value: "TASK_DEFINITION_EVENT,TASK_LIFECYCLE_EVENT,ACTOR_TASK_DEFINITION_EVENT, TASK_PROFILE_EVENT,DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT, ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT,NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT" - image: rayproject/ray:2.52.0 + image: rayproject/ray:2.55.0 imagePullPolicy: IfNotPresent securityContext: allowPrivilegeEscalation: true @@ -108,11 +108,11 @@ spec: value: "true" - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR value: "http://localhost:8084/v1/events" - - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES + - name: RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES value: "TASK_DEFINITION_EVENT,TASK_LIFECYCLE_EVENT,ACTOR_TASK_DEFINITION_EVENT, TASK_PROFILE_EVENT,DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT, ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT,NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT" - image: rayproject/ray:2.52.0 + image: rayproject/ray:2.55.0 imagePullPolicy: IfNotPresent name: ray-worker securityContext: diff --git a/historyserver/config/raycluster-gcs.yaml b/historyserver/config/raycluster-gcs.yaml index 4ef88636ed8..f75a9829385 100644 --- a/historyserver/config/raycluster-gcs.yaml +++ b/historyserver/config/raycluster-gcs.yaml @@ -29,11 +29,11 @@ spec: value: "true" - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR value: "http://localhost:8084/v1/events" - - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES + - name: RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES value: "TASK_DEFINITION_EVENT,TASK_LIFECYCLE_EVENT,ACTOR_TASK_DEFINITION_EVENT, TASK_PROFILE_EVENT,DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT, ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT,NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT" - image: rayproject/ray:2.52.0 + image: rayproject/ray:2.55.0 imagePullPolicy: IfNotPresent securityContext: allowPrivilegeEscalation: true @@ -127,11 +127,11 @@ spec: value: "true" - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR value: "http://localhost:8084/v1/events" - - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES + - name: RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES value: "TASK_DEFINITION_EVENT,TASK_LIFECYCLE_EVENT,ACTOR_TASK_DEFINITION_EVENT, TASK_PROFILE_EVENT,DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT, ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT,NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT" - image: rayproject/ray:2.52.0 + image: rayproject/ray:2.55.0 imagePullPolicy: IfNotPresent name: ray-worker securityContext: diff --git a/historyserver/config/raycluster.yaml b/historyserver/config/raycluster.yaml index cebf3dfc73a..a4f983d6c44 100644 --- a/historyserver/config/raycluster.yaml +++ b/historyserver/config/raycluster.yaml @@ -7,7 +7,7 @@ metadata: namespace: default spec: # Enable Ray token authentication. Requires Ray 2.52.0+ - # rayVersion: "2.52.0" + # rayVersion: "2.55.0" # authOptions: # mode: token headGroupSpec: @@ -32,13 +32,11 @@ spec: value: "true" - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR value: "http://localhost:8084/v1/events" - # in ray 2.52.0, we need to set RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES - # in ray 2.53.0 (noy yet done). we need to set RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES - - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES + - name: RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES value: "TASK_DEFINITION_EVENT,TASK_LIFECYCLE_EVENT,ACTOR_TASK_DEFINITION_EVENT, TASK_PROFILE_EVENT,DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT, ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT,NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT" - image: rayproject/ray:2.52.0 + image: rayproject/ray:2.55.0 imagePullPolicy: IfNotPresent securityContext: allowPrivilegeEscalation: true @@ -146,13 +144,11 @@ spec: value: "true" - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR value: "http://localhost:8084/v1/events" - # in ray 2.52.0, we need to set RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES - # in ray 2.53.0 (not yet done). we need to set RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES - - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES + - name: RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES value: "TASK_DEFINITION_EVENT,TASK_LIFECYCLE_EVENT,ACTOR_TASK_DEFINITION_EVENT, TASK_PROFILE_EVENT,DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT, ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT,NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT" - image: rayproject/ray:2.52.0 + image: rayproject/ray:2.55.0 imagePullPolicy: IfNotPresent name: ray-worker securityContext: From 65999ffe376f91dfc8b9a08f58a695c77c429073 Mon Sep 17 00:00:00 2001 From: Aaron Liang Date: Wed, 5 Aug 2026 00:39:35 +0000 Subject: [PATCH 03/14] fix ci --- ray-operator/controllers/ray/rayservice_controller_unit_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ray-operator/controllers/ray/rayservice_controller_unit_test.go b/ray-operator/controllers/ray/rayservice_controller_unit_test.go index 5065fc76080..e5f31ce92a1 100644 --- a/ray-operator/controllers/ray/rayservice_controller_unit_test.go +++ b/ray-operator/controllers/ray/rayservice_controller_unit_test.go @@ -2634,7 +2634,7 @@ func TestHandleSuspendResumeResetsReadyToInitializing(t *testing.T) { r := &RayServiceReconciler{ Client: fakeClient, Scheme: scheme.Scheme, - Recorder: events.NewFakeRecorder(10), + Recorder: &events.FakeRecorder{}, } _, err := r.handleSuspend(ctx, rs) From 4ee49e6298fe8f14b7b261b6ab2c9d888e9101e4 Mon Sep 17 00:00:00 2001 From: Aaron Liang Date: Wed, 5 Aug 2026 01:02:50 +0000 Subject: [PATCH 04/14] update ray version --- historyserver/DEVELOPMENT.md | 2 +- historyserver/config/raycluster-azureblob.yaml | 4 ++-- historyserver/config/raycluster-gcs.yaml | 4 ++-- historyserver/config/raycluster.yaml | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/historyserver/DEVELOPMENT.md b/historyserver/DEVELOPMENT.md index bf906e0bfeb..23dcf612435 100644 --- a/historyserver/DEVELOPMENT.md +++ b/historyserver/DEVELOPMENT.md @@ -120,7 +120,7 @@ Install Ray locally. Make sure to use at least Ray `v2.55`. ```bash pip uninstall -y ray -pip install -U "ray[default]==2.55.0" +pip install -U "ray[default]==2.56.0" ``` Run the `ray start` command: diff --git a/historyserver/config/raycluster-azureblob.yaml b/historyserver/config/raycluster-azureblob.yaml index 6cbf30cc59b..c12a83391e1 100644 --- a/historyserver/config/raycluster-azureblob.yaml +++ b/historyserver/config/raycluster-azureblob.yaml @@ -32,7 +32,7 @@ spec: value: "TASK_DEFINITION_EVENT,TASK_LIFECYCLE_EVENT,ACTOR_TASK_DEFINITION_EVENT, TASK_PROFILE_EVENT,DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT, ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT,NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT" - image: rayproject/ray:2.55.0 + image: rayproject/ray:2.56.0 imagePullPolicy: IfNotPresent securityContext: allowPrivilegeEscalation: true @@ -112,7 +112,7 @@ spec: value: "TASK_DEFINITION_EVENT,TASK_LIFECYCLE_EVENT,ACTOR_TASK_DEFINITION_EVENT, TASK_PROFILE_EVENT,DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT, ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT,NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT" - image: rayproject/ray:2.55.0 + image: rayproject/ray:2.56.0 imagePullPolicy: IfNotPresent name: ray-worker securityContext: diff --git a/historyserver/config/raycluster-gcs.yaml b/historyserver/config/raycluster-gcs.yaml index f75a9829385..50228ee09e3 100644 --- a/historyserver/config/raycluster-gcs.yaml +++ b/historyserver/config/raycluster-gcs.yaml @@ -33,7 +33,7 @@ spec: value: "TASK_DEFINITION_EVENT,TASK_LIFECYCLE_EVENT,ACTOR_TASK_DEFINITION_EVENT, TASK_PROFILE_EVENT,DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT, ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT,NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT" - image: rayproject/ray:2.55.0 + image: rayproject/ray:2.56.0 imagePullPolicy: IfNotPresent securityContext: allowPrivilegeEscalation: true @@ -131,7 +131,7 @@ spec: value: "TASK_DEFINITION_EVENT,TASK_LIFECYCLE_EVENT,ACTOR_TASK_DEFINITION_EVENT, TASK_PROFILE_EVENT,DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT, ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT,NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT" - image: rayproject/ray:2.55.0 + image: rayproject/ray:2.56.0 imagePullPolicy: IfNotPresent name: ray-worker securityContext: diff --git a/historyserver/config/raycluster.yaml b/historyserver/config/raycluster.yaml index a4f983d6c44..940a5c7e8b7 100644 --- a/historyserver/config/raycluster.yaml +++ b/historyserver/config/raycluster.yaml @@ -7,7 +7,7 @@ metadata: namespace: default spec: # Enable Ray token authentication. Requires Ray 2.52.0+ - # rayVersion: "2.55.0" + # rayVersion: "2.56.0" # authOptions: # mode: token headGroupSpec: @@ -36,7 +36,7 @@ spec: value: "TASK_DEFINITION_EVENT,TASK_LIFECYCLE_EVENT,ACTOR_TASK_DEFINITION_EVENT, TASK_PROFILE_EVENT,DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT, ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT,NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT" - image: rayproject/ray:2.55.0 + image: rayproject/ray:2.56.0 imagePullPolicy: IfNotPresent securityContext: allowPrivilegeEscalation: true @@ -148,7 +148,7 @@ spec: value: "TASK_DEFINITION_EVENT,TASK_LIFECYCLE_EVENT,ACTOR_TASK_DEFINITION_EVENT, TASK_PROFILE_EVENT,DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT, ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT,NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT" - image: rayproject/ray:2.55.0 + image: rayproject/ray:2.56.0 imagePullPolicy: IfNotPresent name: ray-worker securityContext: From e1501f1e53203797c608fcde230e135155584f89 Mon Sep 17 00:00:00 2001 From: Aaron Liang Date: Wed, 5 Aug 2026 18:10:33 +0000 Subject: [PATCH 05/14] Update historyserver and convery raycluster-gcs to rayjob-gcs --- historyserver/cmd/historyserver/main.go | 6 + .../config/historyserver-azureblob.yaml | 2 +- historyserver/config/historyserver-gcs.yaml | 13 +- historyserver/config/historyserver.yaml | 15 +- .../config/raycluster-azureblob.yaml | 28 +- historyserver/config/raycluster-gcs.yaml | 183 ----------- historyserver/config/raycluster.yaml | 28 +- historyserver/config/rayjob-gcs.yaml | 292 ++++++++++++++++++ historyserver/test/support/raycluster.go | 13 +- 9 files changed, 355 insertions(+), 225 deletions(-) delete mode 100644 historyserver/config/raycluster-gcs.yaml create mode 100644 historyserver/config/rayjob-gcs.yaml diff --git a/historyserver/cmd/historyserver/main.go b/historyserver/cmd/historyserver/main.go index bb13e119f9e..888f64d0471 100644 --- a/historyserver/cmd/historyserver/main.go +++ b/historyserver/cmd/historyserver/main.go @@ -54,6 +54,12 @@ func main() { } runtimeClassName = strings.ToLower(runtimeClassName) + if rayRootDir == "" { + if val := os.Getenv("RAY_ROOT_DIR"); val != "" { + rayRootDir = val + } + } + if qps <= 0 { logrus.Fatalf("--kube-api-qps must be > 0, got %v", qps) } diff --git a/historyserver/config/historyserver-azureblob.yaml b/historyserver/config/historyserver-azureblob.yaml index cdc6d52a871..f00b98bbe34 100644 --- a/historyserver/config/historyserver-azureblob.yaml +++ b/historyserver/config/historyserver-azureblob.yaml @@ -38,7 +38,7 @@ spec: value: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite-service.azurite-dev.svc.cluster.local:10000/devstoreaccount1;" - name: AZURE_STORAGE_CONTAINER value: "ray-historyserver" - image: quay.io/kuberay/historyserver:nightly + image: historyserver:v0.1.0 imagePullPolicy: IfNotPresent command: - historyserver diff --git a/historyserver/config/historyserver-gcs.yaml b/historyserver/config/historyserver-gcs.yaml index 104ab5edd66..5beb0c40210 100644 --- a/historyserver/config/historyserver-gcs.yaml +++ b/historyserver/config/historyserver-gcs.yaml @@ -1,4 +1,5 @@ -# Ensure that the env var GCS_BUCKET is set before applying +# Apply with envsubst to substitute environment variables: +# envsubst < historyserver/config/historyserver-gcs.yaml | kubectl apply -f - apiVersion: v1 kind: Service metadata: @@ -37,12 +38,12 @@ spec: env: - name: GCS_BUCKET value: "${GCS_BUCKET}" - image: ${HISTORY_SERVER_IMAGE} + - name: STORAGE_BACKEND + value: "gcs" + - name: RAY_ROOT_DIR + value: "root" + image: historyserver:v0.1.0 imagePullPolicy: IfNotPresent - command: - - historyserver - - --runtime-class-name=gcs - - --ray-root-dir=log ports: - containerPort: 8080 resources: diff --git a/historyserver/config/historyserver.yaml b/historyserver/config/historyserver.yaml index 6247439988c..473ec1f1562 100644 --- a/historyserver/config/historyserver.yaml +++ b/historyserver/config/historyserver.yaml @@ -50,17 +50,12 @@ spec: value: "test" - name: S3FORCE_PATH_STYLE value: "true" - image: quay.io/kuberay/historyserver:nightly + - name: STORAGE_BACKEND + value: "s3" + - name: RAY_ROOT_DIR + value: "log" + image: historyserver:v0.1.0 imagePullPolicy: IfNotPresent - command: - - historyserver - - --runtime-class-name=s3 - - --ray-root-dir=log - # Enable proxying to live RayClusters with token authentication enabled. - # The history server reads the auth Secret and forwards requests with the - # appropriate authorization header. Requires the RBAC in - # service_account_auth_token_mode.yaml. - # - --use-auth-token-mode=true ports: - containerPort: 8080 resources: diff --git a/historyserver/config/raycluster-azureblob.yaml b/historyserver/config/raycluster-azureblob.yaml index c12a83391e1..4bb84f7e805 100644 --- a/historyserver/config/raycluster-azureblob.yaml +++ b/historyserver/config/raycluster-azureblob.yaml @@ -28,10 +28,8 @@ spec: value: "true" - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR value: "http://localhost:8084/v1/events" - - name: RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES - value: "TASK_DEFINITION_EVENT,TASK_LIFECYCLE_EVENT,ACTOR_TASK_DEFINITION_EVENT, - TASK_PROFILE_EVENT,DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT, - ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT,NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT" + - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES + value: "ALL" image: rayproject/ray:2.56.0 imagePullPolicy: IfNotPresent securityContext: @@ -49,13 +47,17 @@ spec: - name: historyserver mountPath: *rayTmpRoot - name: collector - image: quay.io/kuberay/collector:nightly + image: collector:v0.1.0 imagePullPolicy: IfNotPresent env: - name: RAY_CLUSTER_NAME valueFrom: fieldRef: fieldPath: metadata.labels['ray.io/cluster'] + - name: RAY_CLUSTER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace - name: POD_IP valueFrom: fieldRef: @@ -75,6 +77,8 @@ spec: value: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite-service.azurite-dev.svc.cluster.local:10000/devstoreaccount1;" - name: AZURE_STORAGE_CONTAINER value: ray-historyserver + command: + - collector volumeMounts: - name: historyserver mountPath: *rayTmpRoot @@ -108,10 +112,8 @@ spec: value: "true" - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR value: "http://localhost:8084/v1/events" - - name: RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES - value: "TASK_DEFINITION_EVENT,TASK_LIFECYCLE_EVENT,ACTOR_TASK_DEFINITION_EVENT, - TASK_PROFILE_EVENT,DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT, - ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT,NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT" + - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES + value: "ALL" image: rayproject/ray:2.56.0 imagePullPolicy: IfNotPresent name: ray-worker @@ -129,13 +131,17 @@ spec: - name: historyserver mountPath: *rayTmpRoot - name: collector - image: quay.io/kuberay/collector:nightly + image: collector:v0.1.0 imagePullPolicy: IfNotPresent env: - name: RAY_CLUSTER_NAME valueFrom: fieldRef: fieldPath: metadata.labels['ray.io/cluster'] + - name: RAY_CLUSTER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace - name: POD_IP valueFrom: fieldRef: @@ -155,6 +161,8 @@ spec: value: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite-service.azurite-dev.svc.cluster.local:10000/devstoreaccount1;" - name: AZURE_STORAGE_CONTAINER value: ray-historyserver + command: + - collector volumeMounts: - name: historyserver mountPath: *rayTmpRoot diff --git a/historyserver/config/raycluster-gcs.yaml b/historyserver/config/raycluster-gcs.yaml deleted file mode 100644 index 50228ee09e3..00000000000 --- a/historyserver/config/raycluster-gcs.yaml +++ /dev/null @@ -1,183 +0,0 @@ -apiVersion: ray.io/v1 -kind: RayCluster -metadata: - labels: - ray.io/cluster: raycluster-historyserver - name: raycluster-historyserver - namespace: default -spec: - headGroupSpec: - rayStartParams: - dashboard-host: 0.0.0.0 - num-cpus: "0" - serviceType: ClusterIP - template: - metadata: - labels: - test: raycluster-historyserver - spec: - serviceAccountName: historyserver - imagePullSecrets: - affinity: - containers: - - env: - - name: RAY_TMP_ROOT - value: &rayTmpRoot /tmp/ray - - name: RAY_enable_ray_event - value: "true" - - name: RAY_enable_core_worker_ray_event_to_aggregator - value: "true" - - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR - value: "http://localhost:8084/v1/events" - - name: RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES - value: "TASK_DEFINITION_EVENT,TASK_LIFECYCLE_EVENT,ACTOR_TASK_DEFINITION_EVENT, - TASK_PROFILE_EVENT,DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT, - ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT,NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT" - image: rayproject/ray:2.56.0 - imagePullPolicy: IfNotPresent - securityContext: - allowPrivilegeEscalation: true - privileged: true - name: ray-head - resources: - limits: - cpu: "6" - memory: 10G - requests: - cpu: "50m" - memory: 1G - volumeMounts: - - name: historyserver - mountPath: *rayTmpRoot - - name: collector - image: ${COLLECTOR_IMAGE} - imagePullPolicy: IfNotPresent - env: - - name: RAY_CLUSTER_NAME - valueFrom: - fieldRef: - fieldPath: metadata.labels['ray.io/cluster'] - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: FQ_RAY_IP - value: $(RAY_CLUSTER_NAME)-head-svc.default.svc.cluster.local - - name: RAY_TMP_ROOT - value: *rayTmpRoot - - name: GCS_BUCKET - value: "${GCS_BUCKET}" - # RAY_DASHBOARD_ADDRESS is used by the head collector to fetch endpoints' results - # (e.g., /api/v0/cluster_metadata) from the Ray Dashboard running in the same pod. - # Only the head collector uses this; worker collectors do not need it. - # If your Ray Dashboard uses a non-default port (not 8265), update this value accordingly. - - name: RAY_DASHBOARD_ADDRESS - value: "http://localhost:8265" - # RAY_COLLECTOR_ADDITIONAL_ENDPOINTS is a comma-separated list of Ray Dashboard - # API endpoint paths that the head collector will periodically poll and store. - # Use this for endpoints whose data cannot be obtained via Ray events. - # You can add more endpoints, e.g., "/api/v0/placement_groups,/api/serve/applications/" - # Note: Only static endpoints (no dynamic path parameters like {job_id}) are supported. - - name: RAY_COLLECTOR_ADDITIONAL_ENDPOINTS - value: "/api/v0/placement_groups?detail=1&limit=10000" - # Query params must match the Ray Dashboard frontend request exactly - # (see https://github.com/ray-project/ray/blob/cb9c80fee6a700efe61ea97987248ce82e3fa2e2/python/ray/dashboard/client/src/service/placementGroup.ts): - # detail=1 — include bundles and stats fields required by PlacementGroupTable - # limit=10000 — match the frontend's default limit to avoid truncation - # RAY_COLLECTOR_POLL_INTERVAL controls how often the collector polls the additional - # endpoints above. Accepts Go duration format (e.g., "30s", "1m", "5m"). - - name: RAY_COLLECTOR_POLL_INTERVAL - value: "30s" - - name: RAY_ROLE - value: "Head" - - name: STORAGE_BACKEND - value: "gcs" - - name: RAY_ROOT_DIR - value: "log" - volumeMounts: - - name: historyserver - mountPath: *rayTmpRoot - tolerations: - - key: ray - operator: Equal - value: cpu - volumes: - - name: historyserver - emptyDir: {} - workerGroupSpecs: - - groupName: cpu - maxReplicas: 1000 - minReplicas: 1 - numOfHosts: 1 - rayStartParams: {} - replicas: 1 - template: - metadata: - labels: - test: raycluster-historyserver - spec: - serviceAccountName: historyserver - containers: - - env: - - name: RAY_TMP_ROOT - value: *rayTmpRoot - - name: RAY_enable_ray_event - value: "true" - - name: RAY_enable_core_worker_ray_event_to_aggregator - value: "true" - - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR - value: "http://localhost:8084/v1/events" - - name: RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES - value: "TASK_DEFINITION_EVENT,TASK_LIFECYCLE_EVENT,ACTOR_TASK_DEFINITION_EVENT, - TASK_PROFILE_EVENT,DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT, - ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT,NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT" - image: rayproject/ray:2.56.0 - imagePullPolicy: IfNotPresent - name: ray-worker - securityContext: - allowPrivilegeEscalation: true - privileged: true - resources: - limits: - cpu: "2" - memory: 2G - requests: - cpu: "50m" - memory: 1G - volumeMounts: - - name: historyserver - mountPath: *rayTmpRoot - - name: collector - image: ${COLLECTOR_IMAGE} - imagePullPolicy: IfNotPresent - env: - - name: RAY_CLUSTER_NAME - valueFrom: - fieldRef: - fieldPath: metadata.labels['ray.io/cluster'] - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: FQ_RAY_IP - value: $(RAY_CLUSTER_NAME)-head-svc.default.svc.cluster.local - - name: RAY_TMP_ROOT - value: *rayTmpRoot - - name: GCS_BUCKET - value: "${GCS_BUCKET}" - - name: RAY_ROLE - value: "Worker" - - name: STORAGE_BACKEND - value: "gcs" - - name: RAY_ROOT_DIR - value: "log" - volumeMounts: - - name: historyserver - mountPath: *rayTmpRoot - tolerations: - - key: ray - operator: Equal - value: cpu - volumes: - - name: historyserver - emptyDir: {} diff --git a/historyserver/config/raycluster.yaml b/historyserver/config/raycluster.yaml index 940a5c7e8b7..420293cd572 100644 --- a/historyserver/config/raycluster.yaml +++ b/historyserver/config/raycluster.yaml @@ -32,10 +32,8 @@ spec: value: "true" - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR value: "http://localhost:8084/v1/events" - - name: RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES - value: "TASK_DEFINITION_EVENT,TASK_LIFECYCLE_EVENT,ACTOR_TASK_DEFINITION_EVENT, - TASK_PROFILE_EVENT,DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT, - ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT,NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT" + - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES + value: "ALL" image: rayproject/ray:2.56.0 imagePullPolicy: IfNotPresent securityContext: @@ -53,13 +51,17 @@ spec: - name: historyserver mountPath: *rayTmpRoot - name: collector - image: quay.io/kuberay/collector:nightly + image: collector:v0.1.0 imagePullPolicy: IfNotPresent env: - name: RAY_CLUSTER_NAME valueFrom: fieldRef: fieldPath: metadata.labels['ray.io/cluster'] + - name: RAY_CLUSTER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace - name: POD_IP valueFrom: fieldRef: @@ -111,6 +113,8 @@ spec: value: "s3" - name: RAY_ROOT_DIR value: "log" + command: + - collector volumeMounts: - name: historyserver mountPath: *rayTmpRoot @@ -144,10 +148,8 @@ spec: value: "true" - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR value: "http://localhost:8084/v1/events" - - name: RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES - value: "TASK_DEFINITION_EVENT,TASK_LIFECYCLE_EVENT,ACTOR_TASK_DEFINITION_EVENT, - TASK_PROFILE_EVENT,DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT, - ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT,NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT" + - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES + value: "ALL" image: rayproject/ray:2.56.0 imagePullPolicy: IfNotPresent name: ray-worker @@ -165,13 +167,17 @@ spec: - name: historyserver mountPath: *rayTmpRoot - name: collector - image: quay.io/kuberay/collector:nightly + image: collector:v0.1.0 imagePullPolicy: IfNotPresent env: - name: RAY_CLUSTER_NAME valueFrom: fieldRef: fieldPath: metadata.labels['ray.io/cluster'] + - name: RAY_CLUSTER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace - name: POD_IP valueFrom: fieldRef: @@ -202,6 +208,8 @@ spec: value: "s3" - name: RAY_ROOT_DIR value: "log" + command: + - collector volumeMounts: - name: historyserver mountPath: *rayTmpRoot diff --git a/historyserver/config/rayjob-gcs.yaml b/historyserver/config/rayjob-gcs.yaml new file mode 100644 index 00000000000..58b0b19c6f2 --- /dev/null +++ b/historyserver/config/rayjob-gcs.yaml @@ -0,0 +1,292 @@ +# Apply with envsubst to substitute environment variables: +# envsubst < historyserver/config/rayjob-gcs.yaml | kubectl apply -f - +apiVersion: ray.io/v1 +kind: RayJob +metadata: + name: rayjob-historyserver-gcs + namespace: default +spec: + entrypoint: python /home/ray/samples/sample_code.py + shutdownAfterJobFinishes: true + ttlSecondsAfterFinished: 600 + rayClusterSpec: + rayVersion: "2.56.0" + headGroupSpec: + rayStartParams: + dashboard-host: 0.0.0.0 + num-cpus: "0" + serviceType: ClusterIP + template: + metadata: + labels: + test: rayjob-historyserver-gcs + spec: + serviceAccountName: historyserver + imagePullSecrets: + affinity: + containers: + - env: + - name: RAY_TMP_ROOT + value: &rayTmpRoot /tmp/ray + - name: RAY_enable_ray_event + value: "true" + - name: RAY_enable_core_worker_ray_event_to_aggregator + value: "true" + - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR + value: "http://localhost:8084/v1/events" + - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES + value: "ALL" + image: rayproject/ray:2.56.0 + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: true + privileged: true + name: ray-head + resources: + limits: + cpu: "6" + memory: 10G + requests: + cpu: "50m" + memory: 1G + volumeMounts: + - name: historyserver + mountPath: *rayTmpRoot + - name: code-sample + mountPath: /home/ray/samples + - name: collector + image: collector:v0.1.0 + imagePullPolicy: IfNotPresent + env: + - name: RAY_CLUSTER_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['ray.io/cluster'] + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: FQ_RAY_IP + value: $(RAY_CLUSTER_NAME)-head-svc.default.svc.cluster.local + - name: RAY_TMP_ROOT + value: *rayTmpRoot + - name: GCS_BUCKET + value: "${GCS_BUCKET}" + # RAY_DASHBOARD_ADDRESS is used by the head collector to fetch endpoints' results + # (e.g., /api/v0/cluster_metadata) from the Ray Dashboard running in the same pod. + # Only the head collector uses this; worker collectors do not need it. + # If your Ray Dashboard uses a non-default port (not 8265), update this value accordingly. + - name: RAY_DASHBOARD_ADDRESS + value: "http://localhost:8265" + # RAY_COLLECTOR_ADDITIONAL_ENDPOINTS is a comma-separated list of Ray Dashboard + # API endpoint paths that the head collector will periodically poll and store. + # Use this for endpoints whose data cannot be obtained via Ray events. + # You can add more endpoints, e.g., "/api/v0/placement_groups,/api/serve/applications/" + # Note: Only static endpoints (no dynamic path parameters like {job_id}) are supported. + - name: RAY_COLLECTOR_ADDITIONAL_ENDPOINTS + value: "/api/v0/placement_groups?detail=1&limit=10000" + # Query params must match the Ray Dashboard frontend request exactly + # (see https://github.com/ray-project/ray/blob/cb9c80fee6a700efe61ea97987248ce82e3fa2e2/python/ray/dashboard/client/src/service/placementGroup.ts): + # detail=1 — include bundles and stats fields required by PlacementGroupTable + # limit=10000 — match the frontend's default limit to avoid truncation + # RAY_COLLECTOR_POLL_INTERVAL controls how often the collector polls the additional + # endpoints above. Accepts Go duration format (e.g., "30s", "1m", "5m"). + - name: RAY_COLLECTOR_POLL_INTERVAL + value: "30s" + - name: RAY_ROLE + value: "Head" + - name: RAY_CLUSTER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OWNER_KIND + value: "RayJob" + - name: OWNER_NAME + value: "rayjob-historyserver-gcs" + - name: STORAGE_BACKEND + value: "gcs" + - name: RAY_ROOT_DIR + value: "root" + - name: EVENTS_PORT + value: "8084" + volumeMounts: + - name: historyserver + mountPath: *rayTmpRoot + tolerations: + - key: ray + operator: Equal + value: cpu + volumes: + - name: historyserver + emptyDir: {} + - name: code-sample + configMap: + name: ray-job-code-sample + items: + - key: sample_code.py + path: sample_code.py + workerGroupSpecs: + - groupName: cpu + maxReplicas: 1000 + minReplicas: 1 + numOfHosts: 1 + rayStartParams: {} + replicas: 1 + template: + metadata: + labels: + test: rayjob-historyserver-gcs + spec: + serviceAccountName: historyserver + containers: + - env: + - name: RAY_TMP_ROOT + value: *rayTmpRoot + - name: RAY_enable_ray_event + value: "true" + - name: RAY_enable_core_worker_ray_event_to_aggregator + value: "true" + - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR + value: "http://localhost:8084/v1/events" + - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES + value: "ALL" + image: rayproject/ray:2.56.0 + imagePullPolicy: IfNotPresent + name: ray-worker + securityContext: + allowPrivilegeEscalation: true + privileged: true + resources: + limits: + cpu: "2" + memory: 2G + requests: + cpu: "50m" + memory: 1G + volumeMounts: + - name: historyserver + mountPath: *rayTmpRoot + - name: collector + image: collector:v0.1.0 + imagePullPolicy: IfNotPresent + env: + - name: RAY_CLUSTER_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['ray.io/cluster'] + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: FQ_RAY_IP + value: $(RAY_CLUSTER_NAME)-head-svc.default.svc.cluster.local + - name: RAY_TMP_ROOT + value: *rayTmpRoot + - name: GCS_BUCKET + value: "${GCS_BUCKET}" + - name: RAY_ROLE + value: "Worker" + - name: RAY_CLUSTER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OWNER_KIND + value: "RayJob" + - name: OWNER_NAME + value: "rayjob-historyserver-gcs" + - name: STORAGE_BACKEND + value: "gcs" + - name: RAY_ROOT_DIR + value: "root" + - name: EVENTS_PORT + value: "8084" + volumeMounts: + - name: historyserver + mountPath: *rayTmpRoot + tolerations: + - key: ray + operator: Equal + value: cpu + volumes: + - name: historyserver + emptyDir: {} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ray-job-code-sample +data: + sample_code.py: | + import ray + import time + from ray.util.placement_group import placement_group + + ray.init() + + # --- Scenario 1: Root-level NORMAL_TASK (single call) --- + @ray.remote(num_cpus=0.1) + def my_task(x): + # NOTE: Add this to ensure will produce log.out + print(f"Processing {x}", flush=True) + return x * 2 + + # --- Scenario 2: Nested tasks (parent spawns child) --- + # Tests lineage tree depth > 1: child_task is nested under parent_task. + @ray.remote(num_cpus=0.1) + def parent_task(): + result = ray.get(child_task.remote()) + return result + + @ray.remote(num_cpus=0.1) + def child_task(): + print("Child task", flush=True) + return 42 + + # --- Scenario 3: Multiple same-name tasks --- + # Tests GROUP merging in lineage: 3 calls to the same function + # are merged into a single GROUP node. + @ray.remote(num_cpus=0.1) + def repeated_task(x): + return x + + # --- Scenario 4: Actor (creation + method calls) --- + @ray.remote(num_cpus=0.1) + class Counter: + def __init__(self): + self.count = 0 + + def increment(self): + self.count += 1 + return self.count + + def get_count(self): + return self.count + + # Execute all scenarios: + task_result = ray.get(my_task.remote(1)) + print(f"Task result: {task_result}") + + nested_result = ray.get(parent_task.remote()) + print(f"Nested result: {nested_result}") + + refs = [repeated_task.remote(i) for i in range(3)] + results = ray.get(refs) + print(f"Repeated results: {results}") + + counter = Counter.remote() + for i in range(2): + count = ray.get(counter.increment.remote()) + print(f"Counter: {count}") + final_count = ray.get(counter.get_count.remote()) + print(f"Final count: {final_count}") + + print(f"Cluster resources: {ray.cluster_resources()}") + + # Create a detached placement group so it persists after the job exits. + # This ensures the collector captures non-empty data when polling /api/v0/placement_groups. + pg = placement_group([{"CPU": 0.5}], strategy="SPREAD", lifetime="detached", name="test_pg") + ray.get(pg.ready()) + print(f"Placement group created: {pg.bundle_specs}") + + # Wait for events to flush to the collector + time.sleep(5) diff --git a/historyserver/test/support/raycluster.go b/historyserver/test/support/raycluster.go index ce4aede74fb..1d5b5494c0d 100644 --- a/historyserver/test/support/raycluster.go +++ b/historyserver/test/support/raycluster.go @@ -59,18 +59,21 @@ func ApplyRayClusterWithCollectorWithEnvs(test Test, g *WithT, namespace *corev1 return rayCluster } -// injectCollectorRayClusterNamespaceAndEnvVar injects the ray-cluster-namespace argument and required environment variables (POD_IP, FQ_RAY_IP) into all collector containers. +// injectCollectorRayClusterNamespaceAndEnvVar injects the ray-cluster-namespace and required environment variables (RAY_CLUSTER_NAMESPACE, POD_IP, FQ_RAY_IP) into all collector containers. func injectCollectorRayClusterNamespaceAndEnvVar(containers []corev1.Container, rayClusterName string, rayClusterNamespace string) { fqdnRayIP := fmt.Sprintf("%s-head-svc.%s.svc.cluster.local", rayClusterName, rayClusterNamespace) for i := range containers { if containers[i].Name == "collector" { - containers[i].Command = append( - containers[i].Command, - fmt.Sprintf("--ray-cluster-namespace=%s", rayClusterNamespace), - ) + if len(containers[i].Command) > 0 { + containers[i].Command = append( + containers[i].Command, + fmt.Sprintf("--ray-cluster-namespace=%s", rayClusterNamespace), + ) + } if containers[i].Env == nil { containers[i].Env = []corev1.EnvVar{} } + setOrAppendEnv(&containers[i], "RAY_CLUSTER_NAMESPACE", rayClusterNamespace, nil) setOrAppendEnv(&containers[i], "POD_IP", "", &corev1.EnvVarSource{ FieldRef: &corev1.ObjectFieldSelector{ FieldPath: "status.podIP", From 6e5efb12097197bcbf0d800987eb7c676d3dd5ca Mon Sep 17 00:00:00 2001 From: Aaron Liang Date: Wed, 5 Aug 2026 21:59:57 +0000 Subject: [PATCH 06/14] Added 2.56 updated expected grafana health response --- historyserver/test/support/historyserver.go | 1 + 1 file changed, 1 insertion(+) diff --git a/historyserver/test/support/historyserver.go b/historyserver/test/support/historyserver.go index 4aadd189861..d50d7fa050f 100644 --- a/historyserver/test/support/historyserver.go +++ b/historyserver/test/support/historyserver.go @@ -39,6 +39,7 @@ const ( "serveDeployment": "rayServeDeploymentDashboard", "serveLlm": "rayServeLlmDashboard", "data": "rayDataDashboard", + "dataLlm": "rayDataLlmDashboard", "train": "rayTrainDashboard" }, "dashboardDatasource": "Prometheus", From b3086557fd5a0ca2e9fa21ddd93fb1861031d825 Mon Sep 17 00:00:00 2001 From: Aaron Liang Date: Thu, 6 Aug 2026 17:23:30 +0000 Subject: [PATCH 07/14] rebase and update --- historyserver/cmd/historyserver/main.go | 6 ++---- historyserver/config/raycluster-azureblob.yaml | 8 ++------ historyserver/config/raycluster.yaml | 8 ++------ historyserver/config/rayjob-gcs.yaml | 16 ++++++++++------ historyserver/test/support/raycluster.go | 6 ------ 5 files changed, 16 insertions(+), 28 deletions(-) diff --git a/historyserver/cmd/historyserver/main.go b/historyserver/cmd/historyserver/main.go index 888f64d0471..2403ecf397e 100644 --- a/historyserver/cmd/historyserver/main.go +++ b/historyserver/cmd/historyserver/main.go @@ -54,10 +54,8 @@ func main() { } runtimeClassName = strings.ToLower(runtimeClassName) - if rayRootDir == "" { - if val := os.Getenv("RAY_ROOT_DIR"); val != "" { - rayRootDir = val - } + if val := os.Getenv("RAY_ROOT_DIR"); val != "" { + rayRootDir = val } if qps <= 0 { diff --git a/historyserver/config/raycluster-azureblob.yaml b/historyserver/config/raycluster-azureblob.yaml index 4bb84f7e805..ea0f934a51e 100644 --- a/historyserver/config/raycluster-azureblob.yaml +++ b/historyserver/config/raycluster-azureblob.yaml @@ -63,7 +63,7 @@ spec: fieldRef: fieldPath: status.podIP - name: FQ_RAY_IP - value: $(RAY_CLUSTER_NAME)-head-svc.default.svc.cluster.local + value: $(RAY_CLUSTER_NAME)-head-svc.$(RAY_CLUSTER_NAMESPACE).svc.cluster.local - name: RAY_TMP_ROOT value: *rayTmpRoot - name: RAY_ROLE @@ -77,8 +77,6 @@ spec: value: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite-service.azurite-dev.svc.cluster.local:10000/devstoreaccount1;" - name: AZURE_STORAGE_CONTAINER value: ray-historyserver - command: - - collector volumeMounts: - name: historyserver mountPath: *rayTmpRoot @@ -147,7 +145,7 @@ spec: fieldRef: fieldPath: status.podIP - name: FQ_RAY_IP - value: $(RAY_CLUSTER_NAME)-head-svc.default.svc.cluster.local + value: $(RAY_CLUSTER_NAME)-head-svc.$(RAY_CLUSTER_NAMESPACE).svc.cluster.local - name: RAY_TMP_ROOT value: *rayTmpRoot - name: RAY_ROLE @@ -161,8 +159,6 @@ spec: value: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite-service.azurite-dev.svc.cluster.local:10000/devstoreaccount1;" - name: AZURE_STORAGE_CONTAINER value: ray-historyserver - command: - - collector volumeMounts: - name: historyserver mountPath: *rayTmpRoot diff --git a/historyserver/config/raycluster.yaml b/historyserver/config/raycluster.yaml index 420293cd572..a7a8b134e91 100644 --- a/historyserver/config/raycluster.yaml +++ b/historyserver/config/raycluster.yaml @@ -67,7 +67,7 @@ spec: fieldRef: fieldPath: status.podIP - name: FQ_RAY_IP - value: $(RAY_CLUSTER_NAME)-head-svc.default.svc.cluster.local + value: $(RAY_CLUSTER_NAME)-head-svc.$(RAY_CLUSTER_NAMESPACE).svc.cluster.local - name: RAY_TMP_ROOT value: *rayTmpRoot # RAY_DASHBOARD_ADDRESS is used by the head collector to fetch endpoints' results @@ -113,8 +113,6 @@ spec: value: "s3" - name: RAY_ROOT_DIR value: "log" - command: - - collector volumeMounts: - name: historyserver mountPath: *rayTmpRoot @@ -183,7 +181,7 @@ spec: fieldRef: fieldPath: status.podIP - name: FQ_RAY_IP - value: $(RAY_CLUSTER_NAME)-head-svc.default.svc.cluster.local + value: $(RAY_CLUSTER_NAME)-head-svc.$(RAY_CLUSTER_NAMESPACE).svc.cluster.local - name: RAY_TMP_ROOT value: *rayTmpRoot - name: AWS_ACCESS_KEY_ID @@ -208,8 +206,6 @@ spec: value: "s3" - name: RAY_ROOT_DIR value: "log" - command: - - collector volumeMounts: - name: historyserver mountPath: *rayTmpRoot diff --git a/historyserver/config/rayjob-gcs.yaml b/historyserver/config/rayjob-gcs.yaml index 58b0b19c6f2..3ed939605f7 100644 --- a/historyserver/config/rayjob-gcs.yaml +++ b/historyserver/config/rayjob-gcs.yaml @@ -62,12 +62,16 @@ spec: valueFrom: fieldRef: fieldPath: metadata.labels['ray.io/cluster'] + - name: RAY_CLUSTER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: FQ_RAY_IP - value: $(RAY_CLUSTER_NAME)-head-svc.default.svc.cluster.local + value: $(RAY_CLUSTER_NAME)-head-svc.$(RAY_CLUSTER_NAMESPACE).svc.cluster.local - name: RAY_TMP_ROOT value: *rayTmpRoot - name: GCS_BUCKET @@ -174,22 +178,22 @@ spec: valueFrom: fieldRef: fieldPath: metadata.labels['ray.io/cluster'] + - name: RAY_CLUSTER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: FQ_RAY_IP - value: $(RAY_CLUSTER_NAME)-head-svc.default.svc.cluster.local + value: $(RAY_CLUSTER_NAME)-head-svc.$(RAY_CLUSTER_NAMESPACE).svc.cluster.local - name: RAY_TMP_ROOT value: *rayTmpRoot - name: GCS_BUCKET value: "${GCS_BUCKET}" - name: RAY_ROLE value: "Worker" - - name: RAY_CLUSTER_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - name: OWNER_KIND value: "RayJob" - name: OWNER_NAME diff --git a/historyserver/test/support/raycluster.go b/historyserver/test/support/raycluster.go index 1d5b5494c0d..dac77e17aed 100644 --- a/historyserver/test/support/raycluster.go +++ b/historyserver/test/support/raycluster.go @@ -64,12 +64,6 @@ func injectCollectorRayClusterNamespaceAndEnvVar(containers []corev1.Container, fqdnRayIP := fmt.Sprintf("%s-head-svc.%s.svc.cluster.local", rayClusterName, rayClusterNamespace) for i := range containers { if containers[i].Name == "collector" { - if len(containers[i].Command) > 0 { - containers[i].Command = append( - containers[i].Command, - fmt.Sprintf("--ray-cluster-namespace=%s", rayClusterNamespace), - ) - } if containers[i].Env == nil { containers[i].Env = []corev1.EnvVar{} } From bb1e2a70a82a09a8ca926efa93ee210dbd03727d Mon Sep 17 00:00:00 2001 From: Aaron Liang Date: Thu, 6 Aug 2026 18:07:22 +0000 Subject: [PATCH 08/14] reorder env var --- historyserver/config/rayjob-gcs.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/historyserver/config/rayjob-gcs.yaml b/historyserver/config/rayjob-gcs.yaml index 3ed939605f7..26c34cd64d2 100644 --- a/historyserver/config/rayjob-gcs.yaml +++ b/historyserver/config/rayjob-gcs.yaml @@ -99,10 +99,6 @@ spec: value: "30s" - name: RAY_ROLE value: "Head" - - name: RAY_CLUSTER_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - name: OWNER_KIND value: "RayJob" - name: OWNER_NAME From c9f2fd590047955aceee2277b01ab43c87954463 Mon Sep 17 00:00:00 2001 From: Aaron Liang Date: Thu, 6 Aug 2026 22:54:59 +0000 Subject: [PATCH 09/14] add back raycluster-gcs example to not break doc --- historyserver/config/raycluster-gcs.yaml | 197 +++++++++++++++++++++++ historyserver/config/rayjob-gcs.yaml | 28 +++- 2 files changed, 217 insertions(+), 8 deletions(-) create mode 100644 historyserver/config/raycluster-gcs.yaml diff --git a/historyserver/config/raycluster-gcs.yaml b/historyserver/config/raycluster-gcs.yaml new file mode 100644 index 00000000000..da64e3dacd6 --- /dev/null +++ b/historyserver/config/raycluster-gcs.yaml @@ -0,0 +1,197 @@ +# Apply with envsubst to substitute environment variables: +# envsubst < historyserver/config/raycluster-gcs.yaml | kubectl apply -f - +apiVersion: ray.io/v1 +kind: RayCluster +metadata: + labels: + ray.io/cluster: raycluster-historyserver + name: raycluster-historyserver + namespace: default +spec: + headGroupSpec: + rayStartParams: + dashboard-host: 0.0.0.0 + num-cpus: "0" + serviceType: ClusterIP + template: + metadata: + labels: + test: raycluster-historyserver + spec: + serviceAccountName: historyserver + imagePullSecrets: + affinity: + containers: + - env: + - name: RAY_TMP_ROOT + value: &rayTmpRoot /tmp/ray + - name: RAY_enable_ray_event + value: "true" + - name: RAY_enable_core_worker_ray_event_to_aggregator + value: "true" + - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR + value: "http://localhost:8084/v1/events" + - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES + value: "ALL" + - name: RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES + value: "ALL" + image: rayproject/ray:2.56.0 + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: true + privileged: true + name: ray-head + resources: + limits: + cpu: "6" + memory: 10G + requests: + cpu: "50m" + memory: 1G + volumeMounts: + - name: historyserver + mountPath: *rayTmpRoot + - name: collector + image: collector:v0.1.0 + imagePullPolicy: IfNotPresent + env: + - name: RAY_CLUSTER_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['ray.io/cluster'] + - name: RAY_CLUSTER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: FQ_RAY_IP + value: $(RAY_CLUSTER_NAME)-head-svc.$(RAY_CLUSTER_NAMESPACE).svc.cluster.local + - name: RAY_TMP_ROOT + value: *rayTmpRoot + - name: GCS_BUCKET + value: "${GCS_BUCKET}" + # RAY_DASHBOARD_ADDRESS is used by the head collector to fetch endpoints' results + # (e.g., /api/v0/cluster_metadata) from the Ray Dashboard running in the same pod. + # Only the head collector uses this; worker collectors do not need it. + # If your Ray Dashboard uses a non-default port (not 8265), update this value accordingly. + - name: RAY_DASHBOARD_ADDRESS + value: "http://localhost:8265" + # RAY_COLLECTOR_ADDITIONAL_ENDPOINTS is a comma-separated list of Ray Dashboard + # API endpoint paths that the head collector will periodically poll and store. + # Use this for endpoints whose data cannot be obtained via Ray events. + # You can add more endpoints, e.g., "/api/v0/placement_groups,/api/serve/applications/" + # Note: Only static endpoints (no dynamic path parameters like {job_id}) are supported. + - name: RAY_COLLECTOR_ADDITIONAL_ENDPOINTS + value: "/api/v0/placement_groups?detail=1&limit=10000" + # Query params must match the Ray Dashboard frontend request exactly + # (see https://github.com/ray-project/ray/blob/cb9c80fee6a700efe61ea97987248ce82e3fa2e2/python/ray/dashboard/client/src/service/placementGroup.ts): + # detail=1 — include bundles and stats fields required by PlacementGroupTable + # limit=10000 — match the frontend's default limit to avoid truncation + # RAY_COLLECTOR_POLL_INTERVAL controls how often the collector polls the additional + # endpoints above. Accepts Go duration format (e.g., "30s", "1m", "5m"). + - name: RAY_COLLECTOR_POLL_INTERVAL + value: "30s" + - name: RAY_ROLE + value: "Head" + - name: STORAGE_BACKEND + value: "gcs" + - name: RAY_ROOT_DIR + value: "log" + - name: EVENTS_PORT + value: "8084" + volumeMounts: + - name: historyserver + mountPath: *rayTmpRoot + tolerations: + - key: ray + operator: Equal + value: cpu + volumes: + - name: historyserver + emptyDir: {} + workerGroupSpecs: + - groupName: cpu + maxReplicas: 1000 + minReplicas: 1 + numOfHosts: 1 + rayStartParams: {} + replicas: 1 + template: + metadata: + labels: + test: raycluster-historyserver + spec: + serviceAccountName: historyserver + containers: + - env: + - name: RAY_TMP_ROOT + value: *rayTmpRoot + - name: RAY_enable_ray_event + value: "true" + - name: RAY_enable_core_worker_ray_event_to_aggregator + value: "true" + - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR + value: "http://localhost:8084/v1/events" + - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES + value: "ALL" + - name: RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES + value: "ALL" + image: rayproject/ray:2.56.0 + imagePullPolicy: IfNotPresent + name: ray-worker + securityContext: + allowPrivilegeEscalation: true + privileged: true + resources: + limits: + cpu: "2" + memory: 2G + requests: + cpu: "50m" + memory: 1G + volumeMounts: + - name: historyserver + mountPath: *rayTmpRoot + - name: collector + image: collector:v0.1.0 + imagePullPolicy: IfNotPresent + env: + - name: RAY_CLUSTER_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['ray.io/cluster'] + - name: RAY_CLUSTER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: FQ_RAY_IP + value: $(RAY_CLUSTER_NAME)-head-svc.$(RAY_CLUSTER_NAMESPACE).svc.cluster.local + - name: RAY_TMP_ROOT + value: *rayTmpRoot + - name: GCS_BUCKET + value: "${GCS_BUCKET}" + - name: RAY_ROLE + value: "Worker" + - name: STORAGE_BACKEND + value: "gcs" + - name: RAY_ROOT_DIR + value: "log" + - name: EVENTS_PORT + value: "8084" + volumeMounts: + - name: historyserver + mountPath: *rayTmpRoot + tolerations: + - key: ray + operator: Equal + value: cpu + volumes: + - name: historyserver + emptyDir: {} diff --git a/historyserver/config/rayjob-gcs.yaml b/historyserver/config/rayjob-gcs.yaml index 26c34cd64d2..dbfd5e7fd04 100644 --- a/historyserver/config/rayjob-gcs.yaml +++ b/historyserver/config/rayjob-gcs.yaml @@ -36,6 +36,8 @@ spec: value: "http://localhost:8084/v1/events" - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES value: "ALL" + - name: RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES + value: "ALL" image: rayproject/ray:2.56.0 imagePullPolicy: IfNotPresent securityContext: @@ -55,8 +57,8 @@ spec: - name: code-sample mountPath: /home/ray/samples - name: collector - image: collector:v0.1.0 - imagePullPolicy: IfNotPresent + image: quay.io/kuberay/collector:nightly + imagePullPolicy: Always env: - name: RAY_CLUSTER_NAME valueFrom: @@ -100,9 +102,13 @@ spec: - name: RAY_ROLE value: "Head" - name: OWNER_KIND - value: "RayJob" + valueFrom: + fieldRef: + fieldPath: metadata.labels['ray.io/originated-from-crd'] - name: OWNER_NAME - value: "rayjob-historyserver-gcs" + valueFrom: + fieldRef: + fieldPath: metadata.labels['ray.io/originated-from-cr-name'] - name: STORAGE_BACKEND value: "gcs" - name: RAY_ROOT_DIR @@ -150,6 +156,8 @@ spec: value: "http://localhost:8084/v1/events" - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES value: "ALL" + - name: RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES + value: "ALL" image: rayproject/ray:2.56.0 imagePullPolicy: IfNotPresent name: ray-worker @@ -167,8 +175,8 @@ spec: - name: historyserver mountPath: *rayTmpRoot - name: collector - image: collector:v0.1.0 - imagePullPolicy: IfNotPresent + image: quay.io/kuberay/collector:nightly + imagePullPolicy: Always env: - name: RAY_CLUSTER_NAME valueFrom: @@ -191,9 +199,13 @@ spec: - name: RAY_ROLE value: "Worker" - name: OWNER_KIND - value: "RayJob" + valueFrom: + fieldRef: + fieldPath: metadata.labels['ray.io/originated-from-crd'] - name: OWNER_NAME - value: "rayjob-historyserver-gcs" + valueFrom: + fieldRef: + fieldPath: metadata.labels['ray.io/originated-from-cr-name'] - name: STORAGE_BACKEND value: "gcs" - name: RAY_ROOT_DIR From 2c63b0014e9fb517851836cdf6a27231f8f6e5dd Mon Sep 17 00:00:00 2001 From: Aaron Liang Date: Fri, 7 Aug 2026 00:17:24 +0000 Subject: [PATCH 10/14] Update storage root env var value to make it clear what it is for --- historyserver/config/historyserver-gcs.yaml | 2 +- historyserver/config/raycluster-gcs.yaml | 4 ++-- historyserver/config/rayjob-gcs.yaml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/historyserver/config/historyserver-gcs.yaml b/historyserver/config/historyserver-gcs.yaml index 5beb0c40210..c6fd0522929 100644 --- a/historyserver/config/historyserver-gcs.yaml +++ b/historyserver/config/historyserver-gcs.yaml @@ -41,7 +41,7 @@ spec: - name: STORAGE_BACKEND value: "gcs" - name: RAY_ROOT_DIR - value: "root" + value: "storage-root" image: historyserver:v0.1.0 imagePullPolicy: IfNotPresent ports: diff --git a/historyserver/config/raycluster-gcs.yaml b/historyserver/config/raycluster-gcs.yaml index da64e3dacd6..d4dba92bc05 100644 --- a/historyserver/config/raycluster-gcs.yaml +++ b/historyserver/config/raycluster-gcs.yaml @@ -99,7 +99,7 @@ spec: - name: STORAGE_BACKEND value: "gcs" - name: RAY_ROOT_DIR - value: "log" + value: "storage-root" - name: EVENTS_PORT value: "8084" volumeMounts: @@ -182,7 +182,7 @@ spec: - name: STORAGE_BACKEND value: "gcs" - name: RAY_ROOT_DIR - value: "log" + value: "storage-root" - name: EVENTS_PORT value: "8084" volumeMounts: diff --git a/historyserver/config/rayjob-gcs.yaml b/historyserver/config/rayjob-gcs.yaml index dbfd5e7fd04..3103318de01 100644 --- a/historyserver/config/rayjob-gcs.yaml +++ b/historyserver/config/rayjob-gcs.yaml @@ -112,7 +112,7 @@ spec: - name: STORAGE_BACKEND value: "gcs" - name: RAY_ROOT_DIR - value: "root" + value: "storage-root" - name: EVENTS_PORT value: "8084" volumeMounts: @@ -209,7 +209,7 @@ spec: - name: STORAGE_BACKEND value: "gcs" - name: RAY_ROOT_DIR - value: "root" + value: "storage-root" - name: EVENTS_PORT value: "8084" volumeMounts: From aa941eeaec71311dbfa00c8f374dcce2523049e9 Mon Sep 17 00:00:00 2001 From: Aaron Liang Date: Fri, 7 Aug 2026 07:17:25 +0000 Subject: [PATCH 11/14] Update gcs yaml --- historyserver/config/historyserver-gcs.yaml | 2 +- historyserver/config/historyserver.yaml | 7 +++++++ historyserver/config/raycluster-gcs.yaml | 4 ++-- historyserver/config/rayjob-gcs.yaml | 16 ++++------------ .../ray/rayservice_controller_unit_test.go | 2 +- 5 files changed, 15 insertions(+), 16 deletions(-) diff --git a/historyserver/config/historyserver-gcs.yaml b/historyserver/config/historyserver-gcs.yaml index c6fd0522929..279a69fd33f 100644 --- a/historyserver/config/historyserver-gcs.yaml +++ b/historyserver/config/historyserver-gcs.yaml @@ -42,7 +42,7 @@ spec: value: "gcs" - name: RAY_ROOT_DIR value: "storage-root" - image: historyserver:v0.1.0 + image: ${HISTORY_SERVER_IMAGE} imagePullPolicy: IfNotPresent ports: - containerPort: 8080 diff --git a/historyserver/config/historyserver.yaml b/historyserver/config/historyserver.yaml index 473ec1f1562..09366353bff 100644 --- a/historyserver/config/historyserver.yaml +++ b/historyserver/config/historyserver.yaml @@ -56,6 +56,13 @@ spec: value: "log" image: historyserver:v0.1.0 imagePullPolicy: IfNotPresent + # Enable proxying to live RayClusters with token authentication enabled. + # The history server reads the auth Secret and forwards requests with the + # appropriate authorization header. Requires the RBAC in + # service_account_auth_token_mode.yaml. + # command: + # - historyserver + # - --use-auth-token-mode=true ports: - containerPort: 8080 resources: diff --git a/historyserver/config/raycluster-gcs.yaml b/historyserver/config/raycluster-gcs.yaml index d4dba92bc05..13c6c6aa2d5 100644 --- a/historyserver/config/raycluster-gcs.yaml +++ b/historyserver/config/raycluster-gcs.yaml @@ -52,7 +52,7 @@ spec: - name: historyserver mountPath: *rayTmpRoot - name: collector - image: collector:v0.1.0 + image: ${COLLECTOR_IMAGE} imagePullPolicy: IfNotPresent env: - name: RAY_CLUSTER_NAME @@ -156,7 +156,7 @@ spec: - name: historyserver mountPath: *rayTmpRoot - name: collector - image: collector:v0.1.0 + image: ${COLLECTOR_IMAGE} imagePullPolicy: IfNotPresent env: - name: RAY_CLUSTER_NAME diff --git a/historyserver/config/rayjob-gcs.yaml b/historyserver/config/rayjob-gcs.yaml index 3103318de01..5cfac971592 100644 --- a/historyserver/config/rayjob-gcs.yaml +++ b/historyserver/config/rayjob-gcs.yaml @@ -102,13 +102,9 @@ spec: - name: RAY_ROLE value: "Head" - name: OWNER_KIND - valueFrom: - fieldRef: - fieldPath: metadata.labels['ray.io/originated-from-crd'] + value: "rayjob" - name: OWNER_NAME - valueFrom: - fieldRef: - fieldPath: metadata.labels['ray.io/originated-from-cr-name'] + value: "rayjob-historyserver-gcs" - name: STORAGE_BACKEND value: "gcs" - name: RAY_ROOT_DIR @@ -199,13 +195,9 @@ spec: - name: RAY_ROLE value: "Worker" - name: OWNER_KIND - valueFrom: - fieldRef: - fieldPath: metadata.labels['ray.io/originated-from-crd'] + value: "rayjob" - name: OWNER_NAME - valueFrom: - fieldRef: - fieldPath: metadata.labels['ray.io/originated-from-cr-name'] + value: "rayjob-historyserver-gcs" - name: STORAGE_BACKEND value: "gcs" - name: RAY_ROOT_DIR diff --git a/ray-operator/controllers/ray/rayservice_controller_unit_test.go b/ray-operator/controllers/ray/rayservice_controller_unit_test.go index e5f31ce92a1..5065fc76080 100644 --- a/ray-operator/controllers/ray/rayservice_controller_unit_test.go +++ b/ray-operator/controllers/ray/rayservice_controller_unit_test.go @@ -2634,7 +2634,7 @@ func TestHandleSuspendResumeResetsReadyToInitializing(t *testing.T) { r := &RayServiceReconciler{ Client: fakeClient, Scheme: scheme.Scheme, - Recorder: &events.FakeRecorder{}, + Recorder: events.NewFakeRecorder(10), } _, err := r.handleSuspend(ctx, rs) From f983057e50af12fa7a159151982b38ec8b8a9123 Mon Sep 17 00:00:00 2001 From: Aaron Liang Date: Fri, 7 Aug 2026 17:28:15 +0000 Subject: [PATCH 12/14] remove unused configuration --- .../config/raycluster-azureblob.yaml | 21 ++++------------ historyserver/config/raycluster-gcs.yaml | 16 ------------- historyserver/config/raycluster.yaml | 21 ++++------------ historyserver/config/rayjob-gcs.yaml | 24 ++++--------------- 4 files changed, 12 insertions(+), 70 deletions(-) diff --git a/historyserver/config/raycluster-azureblob.yaml b/historyserver/config/raycluster-azureblob.yaml index ea0f934a51e..88494dc2411 100644 --- a/historyserver/config/raycluster-azureblob.yaml +++ b/historyserver/config/raycluster-azureblob.yaml @@ -16,8 +16,6 @@ spec: labels: test: raycluster-historyserver spec: - imagePullSecrets: - affinity: containers: - env: - name: RAY_TMP_ROOT @@ -30,11 +28,10 @@ spec: value: "http://localhost:8084/v1/events" - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES value: "ALL" + - name: RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES + value: "ALL" image: rayproject/ray:2.56.0 imagePullPolicy: IfNotPresent - securityContext: - allowPrivilegeEscalation: true - privileged: true name: ray-head resources: limits: @@ -80,10 +77,6 @@ spec: volumeMounts: - name: historyserver mountPath: *rayTmpRoot - tolerations: - - key: ray - operator: Equal - value: cpu volumes: - name: historyserver emptyDir: {} @@ -99,7 +92,6 @@ spec: labels: test: raycluster-historyserver spec: - imagePullSecrets: containers: - env: - name: RAY_TMP_ROOT @@ -112,12 +104,11 @@ spec: value: "http://localhost:8084/v1/events" - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES value: "ALL" + - name: RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES + value: "ALL" image: rayproject/ray:2.56.0 imagePullPolicy: IfNotPresent name: ray-worker - securityContext: - allowPrivilegeEscalation: true - privileged: true resources: limits: cpu: "2" @@ -162,10 +153,6 @@ spec: volumeMounts: - name: historyserver mountPath: *rayTmpRoot - tolerations: - - key: ray - operator: Equal - value: cpu volumes: - name: historyserver emptyDir: {} diff --git a/historyserver/config/raycluster-gcs.yaml b/historyserver/config/raycluster-gcs.yaml index 13c6c6aa2d5..f6cddc509ee 100644 --- a/historyserver/config/raycluster-gcs.yaml +++ b/historyserver/config/raycluster-gcs.yaml @@ -19,8 +19,6 @@ spec: test: raycluster-historyserver spec: serviceAccountName: historyserver - imagePullSecrets: - affinity: containers: - env: - name: RAY_TMP_ROOT @@ -37,9 +35,6 @@ spec: value: "ALL" image: rayproject/ray:2.56.0 imagePullPolicy: IfNotPresent - securityContext: - allowPrivilegeEscalation: true - privileged: true name: ray-head resources: limits: @@ -105,10 +100,6 @@ spec: volumeMounts: - name: historyserver mountPath: *rayTmpRoot - tolerations: - - key: ray - operator: Equal - value: cpu volumes: - name: historyserver emptyDir: {} @@ -142,9 +133,6 @@ spec: image: rayproject/ray:2.56.0 imagePullPolicy: IfNotPresent name: ray-worker - securityContext: - allowPrivilegeEscalation: true - privileged: true resources: limits: cpu: "2" @@ -188,10 +176,6 @@ spec: volumeMounts: - name: historyserver mountPath: *rayTmpRoot - tolerations: - - key: ray - operator: Equal - value: cpu volumes: - name: historyserver emptyDir: {} diff --git a/historyserver/config/raycluster.yaml b/historyserver/config/raycluster.yaml index a7a8b134e91..ef33840fa64 100644 --- a/historyserver/config/raycluster.yaml +++ b/historyserver/config/raycluster.yaml @@ -20,8 +20,6 @@ spec: labels: test: raycluster-historyserver spec: - imagePullSecrets: - affinity: containers: - env: - name: RAY_TMP_ROOT @@ -34,11 +32,10 @@ spec: value: "http://localhost:8084/v1/events" - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES value: "ALL" + - name: RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES + value: "ALL" image: rayproject/ray:2.56.0 imagePullPolicy: IfNotPresent - securityContext: - allowPrivilegeEscalation: true - privileged: true name: ray-head resources: limits: @@ -116,10 +113,6 @@ spec: volumeMounts: - name: historyserver mountPath: *rayTmpRoot - tolerations: - - key: ray - operator: Equal - value: cpu volumes: - name: historyserver emptyDir: {} @@ -135,7 +128,6 @@ spec: labels: test: raycluster-historyserver spec: - imagePullSecrets: containers: - env: - name: RAY_TMP_ROOT @@ -148,12 +140,11 @@ spec: value: "http://localhost:8084/v1/events" - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES value: "ALL" + - name: RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES + value: "ALL" image: rayproject/ray:2.56.0 imagePullPolicy: IfNotPresent name: ray-worker - securityContext: - allowPrivilegeEscalation: true - privileged: true resources: limits: cpu: "2" @@ -209,10 +200,6 @@ spec: volumeMounts: - name: historyserver mountPath: *rayTmpRoot - tolerations: - - key: ray - operator: Equal - value: cpu volumes: - name: historyserver emptyDir: {} diff --git a/historyserver/config/rayjob-gcs.yaml b/historyserver/config/rayjob-gcs.yaml index 5cfac971592..6452bd3cc41 100644 --- a/historyserver/config/rayjob-gcs.yaml +++ b/historyserver/config/rayjob-gcs.yaml @@ -22,8 +22,6 @@ spec: test: rayjob-historyserver-gcs spec: serviceAccountName: historyserver - imagePullSecrets: - affinity: containers: - env: - name: RAY_TMP_ROOT @@ -40,9 +38,6 @@ spec: value: "ALL" image: rayproject/ray:2.56.0 imagePullPolicy: IfNotPresent - securityContext: - allowPrivilegeEscalation: true - privileged: true name: ray-head resources: limits: @@ -57,8 +52,8 @@ spec: - name: code-sample mountPath: /home/ray/samples - name: collector - image: quay.io/kuberay/collector:nightly - imagePullPolicy: Always + image: ${COLLECTOR_IMAGE} + imagePullPolicy: IfNotPresent env: - name: RAY_CLUSTER_NAME valueFrom: @@ -114,10 +109,6 @@ spec: volumeMounts: - name: historyserver mountPath: *rayTmpRoot - tolerations: - - key: ray - operator: Equal - value: cpu volumes: - name: historyserver emptyDir: {} @@ -157,9 +148,6 @@ spec: image: rayproject/ray:2.56.0 imagePullPolicy: IfNotPresent name: ray-worker - securityContext: - allowPrivilegeEscalation: true - privileged: true resources: limits: cpu: "2" @@ -171,8 +159,8 @@ spec: - name: historyserver mountPath: *rayTmpRoot - name: collector - image: quay.io/kuberay/collector:nightly - imagePullPolicy: Always + image: ${COLLECTOR_IMAGE} + imagePullPolicy: IfNotPresent env: - name: RAY_CLUSTER_NAME valueFrom: @@ -207,10 +195,6 @@ spec: volumeMounts: - name: historyserver mountPath: *rayTmpRoot - tolerations: - - key: ray - operator: Equal - value: cpu volumes: - name: historyserver emptyDir: {} From bda7aeb58c8a04c57ccc346d557bbeb221385d90 Mon Sep 17 00:00:00 2001 From: Aaron Liang Date: Fri, 7 Aug 2026 17:34:13 +0000 Subject: [PATCH 13/14] Updated env var for event exporter --- historyserver/config/raycluster-azureblob.yaml | 8 ++------ historyserver/config/raycluster-gcs.yaml | 4 ---- historyserver/config/raycluster.yaml | 10 +++------- historyserver/config/rayjob-gcs.yaml | 4 ---- 4 files changed, 5 insertions(+), 21 deletions(-) diff --git a/historyserver/config/raycluster-azureblob.yaml b/historyserver/config/raycluster-azureblob.yaml index 88494dc2411..80150df58db 100644 --- a/historyserver/config/raycluster-azureblob.yaml +++ b/historyserver/config/raycluster-azureblob.yaml @@ -28,9 +28,7 @@ spec: value: "http://localhost:8084/v1/events" - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES value: "ALL" - - name: RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES - value: "ALL" - image: rayproject/ray:2.56.0 + image: rayproject/ray:2.55.0 imagePullPolicy: IfNotPresent name: ray-head resources: @@ -104,9 +102,7 @@ spec: value: "http://localhost:8084/v1/events" - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES value: "ALL" - - name: RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES - value: "ALL" - image: rayproject/ray:2.56.0 + image: rayproject/ray:2.55.0 imagePullPolicy: IfNotPresent name: ray-worker resources: diff --git a/historyserver/config/raycluster-gcs.yaml b/historyserver/config/raycluster-gcs.yaml index f6cddc509ee..202f7b9d882 100644 --- a/historyserver/config/raycluster-gcs.yaml +++ b/historyserver/config/raycluster-gcs.yaml @@ -31,8 +31,6 @@ spec: value: "http://localhost:8084/v1/events" - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES value: "ALL" - - name: RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES - value: "ALL" image: rayproject/ray:2.56.0 imagePullPolicy: IfNotPresent name: ray-head @@ -128,8 +126,6 @@ spec: value: "http://localhost:8084/v1/events" - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES value: "ALL" - - name: RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES - value: "ALL" image: rayproject/ray:2.56.0 imagePullPolicy: IfNotPresent name: ray-worker diff --git a/historyserver/config/raycluster.yaml b/historyserver/config/raycluster.yaml index ef33840fa64..fc7fea4d32c 100644 --- a/historyserver/config/raycluster.yaml +++ b/historyserver/config/raycluster.yaml @@ -7,7 +7,7 @@ metadata: namespace: default spec: # Enable Ray token authentication. Requires Ray 2.52.0+ - # rayVersion: "2.56.0" + # rayVersion: "2.55.0" # authOptions: # mode: token headGroupSpec: @@ -32,9 +32,7 @@ spec: value: "http://localhost:8084/v1/events" - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES value: "ALL" - - name: RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES - value: "ALL" - image: rayproject/ray:2.56.0 + image: rayproject/ray:2.55.0 imagePullPolicy: IfNotPresent name: ray-head resources: @@ -140,9 +138,7 @@ spec: value: "http://localhost:8084/v1/events" - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES value: "ALL" - - name: RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES - value: "ALL" - image: rayproject/ray:2.56.0 + image: rayproject/ray:2.55.0 imagePullPolicy: IfNotPresent name: ray-worker resources: diff --git a/historyserver/config/rayjob-gcs.yaml b/historyserver/config/rayjob-gcs.yaml index 6452bd3cc41..d2bf190c0b3 100644 --- a/historyserver/config/rayjob-gcs.yaml +++ b/historyserver/config/rayjob-gcs.yaml @@ -34,8 +34,6 @@ spec: value: "http://localhost:8084/v1/events" - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES value: "ALL" - - name: RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES - value: "ALL" image: rayproject/ray:2.56.0 imagePullPolicy: IfNotPresent name: ray-head @@ -143,8 +141,6 @@ spec: value: "http://localhost:8084/v1/events" - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES value: "ALL" - - name: RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES - value: "ALL" image: rayproject/ray:2.56.0 imagePullPolicy: IfNotPresent name: ray-worker From 9ea2bde36afeb7dbdeb3b03abf5a81fb004253a5 Mon Sep 17 00:00:00 2001 From: Aaron Liang Date: Fri, 7 Aug 2026 23:28:00 +0000 Subject: [PATCH 14/14] remove storage root and use bucket as root dir --- historyserver/config/historyserver-gcs.yaml | 2 -- historyserver/config/raycluster-gcs.yaml | 4 ---- historyserver/config/rayjob-gcs.yaml | 4 ---- 3 files changed, 10 deletions(-) diff --git a/historyserver/config/historyserver-gcs.yaml b/historyserver/config/historyserver-gcs.yaml index 279a69fd33f..a678356a2aa 100644 --- a/historyserver/config/historyserver-gcs.yaml +++ b/historyserver/config/historyserver-gcs.yaml @@ -40,8 +40,6 @@ spec: value: "${GCS_BUCKET}" - name: STORAGE_BACKEND value: "gcs" - - name: RAY_ROOT_DIR - value: "storage-root" image: ${HISTORY_SERVER_IMAGE} imagePullPolicy: IfNotPresent ports: diff --git a/historyserver/config/raycluster-gcs.yaml b/historyserver/config/raycluster-gcs.yaml index 202f7b9d882..6cd8dc50bbf 100644 --- a/historyserver/config/raycluster-gcs.yaml +++ b/historyserver/config/raycluster-gcs.yaml @@ -91,8 +91,6 @@ spec: value: "Head" - name: STORAGE_BACKEND value: "gcs" - - name: RAY_ROOT_DIR - value: "storage-root" - name: EVENTS_PORT value: "8084" volumeMounts: @@ -165,8 +163,6 @@ spec: value: "Worker" - name: STORAGE_BACKEND value: "gcs" - - name: RAY_ROOT_DIR - value: "storage-root" - name: EVENTS_PORT value: "8084" volumeMounts: diff --git a/historyserver/config/rayjob-gcs.yaml b/historyserver/config/rayjob-gcs.yaml index d2bf190c0b3..3f598ce2e1a 100644 --- a/historyserver/config/rayjob-gcs.yaml +++ b/historyserver/config/rayjob-gcs.yaml @@ -100,8 +100,6 @@ spec: value: "rayjob-historyserver-gcs" - name: STORAGE_BACKEND value: "gcs" - - name: RAY_ROOT_DIR - value: "storage-root" - name: EVENTS_PORT value: "8084" volumeMounts: @@ -184,8 +182,6 @@ spec: value: "rayjob-historyserver-gcs" - name: STORAGE_BACKEND value: "gcs" - - name: RAY_ROOT_DIR - value: "storage-root" - name: EVENTS_PORT value: "8084" volumeMounts: