diff --git a/historyserver/DEVELOPMENT.md b/historyserver/DEVELOPMENT.md index bf906e0bfeb..23dcf612435 100644 --- a/historyserver/DEVELOPMENT.md +++ b/historyserver/DEVELOPMENT.md @@ -120,7 +120,7 @@ Install Ray locally. Make sure to use at least Ray `v2.55`. ```bash pip uninstall -y ray -pip install -U "ray[default]==2.55.0" +pip install -U "ray[default]==2.56.0" ``` Run the `ray start` command: diff --git a/historyserver/cmd/collector/main.go b/historyserver/cmd/collector/main.go index b2ec4f2de83..7648a2c9014 100644 --- a/historyserver/cmd/collector/main.go +++ b/historyserver/cmd/collector/main.go @@ -30,7 +30,7 @@ func main() { rayClusterNamespace := "" rayRootDir := "" logBatching := 1000 - eventsPort := 8080 + eventsPort := 8084 pushInterval := time.Minute ownerKind := "" ownerName := "" @@ -53,7 +53,7 @@ func main() { flag.StringVar(&rayClusterNamespace, "ray-cluster-namespace", "default", "") flag.StringVar(&rayRootDir, "ray-root-dir", "", "") flag.IntVar(&logBatching, "log-batching", 1000, "") - flag.IntVar(&eventsPort, "events-port", 8080, "") + flag.IntVar(&eventsPort, "events-port", 8084, "") flag.StringVar(&runtimeClassConfigPath, "runtime-class-config-path", "", "") flag.DurationVar(&pushInterval, "push-interval", time.Minute, "") flag.StringVar(&ownerKind, "owner-kind", "", "") diff --git a/historyserver/cmd/historyserver/main.go b/historyserver/cmd/historyserver/main.go index bb13e119f9e..2403ecf397e 100644 --- a/historyserver/cmd/historyserver/main.go +++ b/historyserver/cmd/historyserver/main.go @@ -54,6 +54,10 @@ func main() { } runtimeClassName = strings.ToLower(runtimeClassName) + if val := os.Getenv("RAY_ROOT_DIR"); val != "" { + rayRootDir = val + } + if qps <= 0 { logrus.Fatalf("--kube-api-qps must be > 0, got %v", qps) } diff --git a/historyserver/config/historyserver-gcs.yaml b/historyserver/config/historyserver-gcs.yaml index 104ab5edd66..a678356a2aa 100644 --- a/historyserver/config/historyserver-gcs.yaml +++ b/historyserver/config/historyserver-gcs.yaml @@ -1,4 +1,5 @@ -# Ensure that the env var GCS_BUCKET is set before applying +# Apply with envsubst to substitute environment variables: +# envsubst < historyserver/config/historyserver-gcs.yaml | kubectl apply -f - apiVersion: v1 kind: Service metadata: @@ -37,12 +38,10 @@ spec: env: - name: GCS_BUCKET value: "${GCS_BUCKET}" + - name: STORAGE_BACKEND + value: "gcs" image: ${HISTORY_SERVER_IMAGE} imagePullPolicy: IfNotPresent - command: - - historyserver - - --runtime-class-name=gcs - - --ray-root-dir=log ports: - containerPort: 8080 resources: diff --git a/historyserver/config/historyserver.yaml b/historyserver/config/historyserver.yaml index f07d45d34ee..09366353bff 100644 --- a/historyserver/config/historyserver.yaml +++ b/historyserver/config/historyserver.yaml @@ -50,16 +50,18 @@ spec: value: "test" - name: S3FORCE_PATH_STYLE value: "true" + - name: STORAGE_BACKEND + value: "s3" + - name: RAY_ROOT_DIR + value: "log" image: historyserver:v0.1.0 imagePullPolicy: IfNotPresent - command: - - historyserver - - --runtime-class-name=s3 - - --ray-root-dir=log # Enable proxying to live RayClusters with token authentication enabled. # The history server reads the auth Secret and forwards requests with the # appropriate authorization header. Requires the RBAC in # service_account_auth_token_mode.yaml. + # command: + # - historyserver # - --use-auth-token-mode=true ports: - containerPort: 8080 diff --git a/historyserver/config/raycluster-azureblob.yaml b/historyserver/config/raycluster-azureblob.yaml index 9de3a42e8b7..80150df58db 100644 --- a/historyserver/config/raycluster-azureblob.yaml +++ b/historyserver/config/raycluster-azureblob.yaml @@ -16,8 +16,6 @@ spec: labels: test: raycluster-historyserver spec: - imagePullSecrets: - affinity: containers: - env: - name: RAY_TMP_ROOT @@ -29,14 +27,9 @@ spec: - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR value: "http://localhost:8084/v1/events" - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES - value: "TASK_DEFINITION_EVENT,TASK_LIFECYCLE_EVENT,ACTOR_TASK_DEFINITION_EVENT, - TASK_PROFILE_EVENT,DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT, - ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT,NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT" - image: rayproject/ray:2.52.0 + value: "ALL" + image: rayproject/ray:2.55.0 imagePullPolicy: IfNotPresent - securityContext: - allowPrivilegeEscalation: true - privileged: true name: ray-head resources: limits: @@ -52,27 +45,36 @@ spec: image: collector:v0.1.0 imagePullPolicy: IfNotPresent env: + - name: RAY_CLUSTER_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['ray.io/cluster'] + - name: RAY_CLUSTER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: FQ_RAY_IP - value: raycluster-historyserver-head-svc.default.svc.cluster.local + value: $(RAY_CLUSTER_NAME)-head-svc.$(RAY_CLUSTER_NAMESPACE).svc.cluster.local - name: RAY_TMP_ROOT value: *rayTmpRoot + - name: RAY_ROLE + value: "Head" + - name: STORAGE_BACKEND + value: "azureblob" + - name: RAY_ROOT_DIR + value: "log" # reference: https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite#connect-to-the-emulator-by-using-the-azure-storage-explorer - name: AZURE_STORAGE_CONNECTION_STRING value: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite-service.azurite-dev.svc.cluster.local:10000/devstoreaccount1;" - name: AZURE_STORAGE_CONTAINER value: ray-historyserver - command: [collector, --role=Head, --runtime-class-name=azureblob, --ray-cluster-name=raycluster-historyserver, --ray-root-dir=log, --events-port=8084] volumeMounts: - name: historyserver mountPath: *rayTmpRoot - tolerations: - - key: ray - operator: Equal - value: cpu volumes: - name: historyserver emptyDir: {} @@ -88,7 +90,6 @@ spec: labels: test: raycluster-historyserver spec: - imagePullSecrets: containers: - env: - name: RAY_TMP_ROOT @@ -100,15 +101,10 @@ spec: - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR value: "http://localhost:8084/v1/events" - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES - value: "TASK_DEFINITION_EVENT,TASK_LIFECYCLE_EVENT,ACTOR_TASK_DEFINITION_EVENT, - TASK_PROFILE_EVENT,DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT, - ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT,NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT" - image: rayproject/ray:2.52.0 + value: "ALL" + image: rayproject/ray:2.55.0 imagePullPolicy: IfNotPresent name: ray-worker - securityContext: - allowPrivilegeEscalation: true - privileged: true resources: limits: cpu: "2" @@ -123,27 +119,36 @@ spec: image: collector:v0.1.0 imagePullPolicy: IfNotPresent env: + - name: RAY_CLUSTER_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['ray.io/cluster'] + - name: RAY_CLUSTER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: FQ_RAY_IP - value: raycluster-historyserver-head-svc.default.svc.cluster.local + value: $(RAY_CLUSTER_NAME)-head-svc.$(RAY_CLUSTER_NAMESPACE).svc.cluster.local - name: RAY_TMP_ROOT value: *rayTmpRoot + - name: RAY_ROLE + value: "Worker" + - name: STORAGE_BACKEND + value: "azureblob" + - name: RAY_ROOT_DIR + value: "log" # reference: https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite#connect-to-the-emulator-by-using-the-azure-storage-explorer - name: AZURE_STORAGE_CONNECTION_STRING value: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite-service.azurite-dev.svc.cluster.local:10000/devstoreaccount1;" - name: AZURE_STORAGE_CONTAINER value: ray-historyserver - command: [collector, --role=Worker, --runtime-class-name=azureblob, --ray-cluster-name=raycluster-historyserver, --ray-root-dir=log, --events-port=8084] volumeMounts: - name: historyserver mountPath: *rayTmpRoot - tolerations: - - key: ray - operator: Equal - value: cpu volumes: - name: historyserver emptyDir: {} diff --git a/historyserver/config/raycluster-gcs.yaml b/historyserver/config/raycluster-gcs.yaml index 4739b79d509..6cd8dc50bbf 100644 --- a/historyserver/config/raycluster-gcs.yaml +++ b/historyserver/config/raycluster-gcs.yaml @@ -1,3 +1,5 @@ +# Apply with envsubst to substitute environment variables: +# envsubst < historyserver/config/raycluster-gcs.yaml | kubectl apply -f - apiVersion: ray.io/v1 kind: RayCluster metadata: @@ -17,8 +19,6 @@ spec: test: raycluster-historyserver spec: serviceAccountName: historyserver - imagePullSecrets: - affinity: containers: - env: - name: RAY_TMP_ROOT @@ -30,14 +30,9 @@ spec: - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR value: "http://localhost:8084/v1/events" - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES - value: "TASK_DEFINITION_EVENT,TASK_LIFECYCLE_EVENT,ACTOR_TASK_DEFINITION_EVENT, - TASK_PROFILE_EVENT,DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT, - ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT,NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT" - image: rayproject/ray:2.52.0 + value: "ALL" + image: rayproject/ray:2.56.0 imagePullPolicy: IfNotPresent - securityContext: - allowPrivilegeEscalation: true - privileged: true name: ray-head resources: limits: @@ -53,12 +48,20 @@ spec: image: ${COLLECTOR_IMAGE} imagePullPolicy: IfNotPresent env: + - name: RAY_CLUSTER_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['ray.io/cluster'] + - name: RAY_CLUSTER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: FQ_RAY_IP - value: raycluster-historyserver-head-svc.default.svc.cluster.local + value: $(RAY_CLUSTER_NAME)-head-svc.$(RAY_CLUSTER_NAMESPACE).svc.cluster.local - name: RAY_TMP_ROOT value: *rayTmpRoot - name: GCS_BUCKET @@ -84,20 +87,15 @@ spec: # endpoints above. Accepts Go duration format (e.g., "30s", "1m", "5m"). - name: RAY_COLLECTOR_POLL_INTERVAL value: "30s" - command: - - collector - - --role=Head - - --runtime-class-name=gcs - - --ray-cluster-name=${RAY_CLUSTER} - - --ray-root-dir=log - - --events-port=8084 + - name: RAY_ROLE + value: "Head" + - name: STORAGE_BACKEND + value: "gcs" + - name: EVENTS_PORT + value: "8084" volumeMounts: - name: historyserver mountPath: *rayTmpRoot - tolerations: - - key: ray - operator: Equal - value: cpu volumes: - name: historyserver emptyDir: {} @@ -125,15 +123,10 @@ spec: - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR value: "http://localhost:8084/v1/events" - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES - value: "TASK_DEFINITION_EVENT,TASK_LIFECYCLE_EVENT,ACTOR_TASK_DEFINITION_EVENT, - TASK_PROFILE_EVENT,DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT, - ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT,NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT" - image: rayproject/ray:2.52.0 + value: "ALL" + image: rayproject/ray:2.56.0 imagePullPolicy: IfNotPresent name: ray-worker - securityContext: - allowPrivilegeEscalation: true - privileged: true resources: limits: cpu: "2" @@ -148,30 +141,33 @@ spec: image: ${COLLECTOR_IMAGE} imagePullPolicy: IfNotPresent env: + - name: RAY_CLUSTER_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['ray.io/cluster'] + - name: RAY_CLUSTER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: FQ_RAY_IP - value: raycluster-historyserver-head-svc.default.svc.cluster.local + value: $(RAY_CLUSTER_NAME)-head-svc.$(RAY_CLUSTER_NAMESPACE).svc.cluster.local - name: RAY_TMP_ROOT value: *rayTmpRoot - name: GCS_BUCKET value: "${GCS_BUCKET}" - command: - - collector - - --role=Worker - - --runtime-class-name=gcs - - --ray-cluster-name=${RAY_CLUSTER} - - --ray-root-dir=log - - --events-port=8084 + - name: RAY_ROLE + value: "Worker" + - name: STORAGE_BACKEND + value: "gcs" + - name: EVENTS_PORT + value: "8084" volumeMounts: - name: historyserver mountPath: *rayTmpRoot - tolerations: - - key: ray - operator: Equal - value: cpu volumes: - name: historyserver emptyDir: {} diff --git a/historyserver/config/raycluster.yaml b/historyserver/config/raycluster.yaml index 1acf4b94462..fc7fea4d32c 100644 --- a/historyserver/config/raycluster.yaml +++ b/historyserver/config/raycluster.yaml @@ -7,7 +7,7 @@ metadata: namespace: default spec: # Enable Ray token authentication. Requires Ray 2.52.0+ - # rayVersion: "2.52.0" + # rayVersion: "2.55.0" # authOptions: # mode: token headGroupSpec: @@ -20,8 +20,6 @@ spec: labels: test: raycluster-historyserver spec: - imagePullSecrets: - affinity: containers: - env: - name: RAY_TMP_ROOT @@ -32,17 +30,10 @@ spec: value: "true" - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR value: "http://localhost:8084/v1/events" - # in ray 2.52.0, we need to set RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES - # in ray 2.53.0 (noy yet done). we need to set RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES - value: "TASK_DEFINITION_EVENT,TASK_LIFECYCLE_EVENT,ACTOR_TASK_DEFINITION_EVENT, - TASK_PROFILE_EVENT,DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT, - ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT,NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT" - image: rayproject/ray:2.52.0 + value: "ALL" + image: rayproject/ray:2.55.0 imagePullPolicy: IfNotPresent - securityContext: - allowPrivilegeEscalation: true - privileged: true name: ray-head resources: limits: @@ -58,12 +49,20 @@ spec: image: collector:v0.1.0 imagePullPolicy: IfNotPresent env: + - name: RAY_CLUSTER_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['ray.io/cluster'] + - name: RAY_CLUSTER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: FQ_RAY_IP - value: raycluster-historyserver-head-svc.default.svc.cluster.local + value: $(RAY_CLUSTER_NAME)-head-svc.$(RAY_CLUSTER_NAMESPACE).svc.cluster.local - name: RAY_TMP_ROOT value: *rayTmpRoot # RAY_DASHBOARD_ADDRESS is used by the head collector to fetch endpoints' results @@ -103,20 +102,15 @@ spec: value: "test" - name: S3FORCE_PATH_STYLE value: "true" - command: - - collector - - --role=Head - - --runtime-class-name=s3 - - --ray-cluster-name=raycluster-historyserver - - --ray-root-dir=log - - --events-port=8084 + - name: RAY_ROLE + value: "Head" + - name: STORAGE_BACKEND + value: "s3" + - name: RAY_ROOT_DIR + value: "log" volumeMounts: - name: historyserver mountPath: *rayTmpRoot - tolerations: - - key: ray - operator: Equal - value: cpu volumes: - name: historyserver emptyDir: {} @@ -132,7 +126,6 @@ spec: labels: test: raycluster-historyserver spec: - imagePullSecrets: containers: - env: - name: RAY_TMP_ROOT @@ -143,18 +136,11 @@ spec: value: "true" - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR value: "http://localhost:8084/v1/events" - # in ray 2.52.0, we need to set RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES - # in ray 2.53.0 (not yet done). we need to set RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES - value: "TASK_DEFINITION_EVENT,TASK_LIFECYCLE_EVENT,ACTOR_TASK_DEFINITION_EVENT, - TASK_PROFILE_EVENT,DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT, - ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT,NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT" - image: rayproject/ray:2.52.0 + value: "ALL" + image: rayproject/ray:2.55.0 imagePullPolicy: IfNotPresent name: ray-worker - securityContext: - allowPrivilegeEscalation: true - privileged: true resources: limits: cpu: "2" @@ -169,12 +155,20 @@ spec: image: collector:v0.1.0 imagePullPolicy: IfNotPresent env: + - name: RAY_CLUSTER_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['ray.io/cluster'] + - name: RAY_CLUSTER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: FQ_RAY_IP - value: raycluster-historyserver-head-svc.default.svc.cluster.local + value: $(RAY_CLUSTER_NAME)-head-svc.$(RAY_CLUSTER_NAMESPACE).svc.cluster.local - name: RAY_TMP_ROOT value: *rayTmpRoot - name: AWS_ACCESS_KEY_ID @@ -193,20 +187,15 @@ spec: value: "true" - name: S3DISABLE_SSL value: "true" - command: - - collector - - --role=Worker - - --runtime-class-name=s3 - - --ray-cluster-name=raycluster-historyserver - - --ray-root-dir=log - - --events-port=8084 + - name: RAY_ROLE + value: "Worker" + - name: STORAGE_BACKEND + value: "s3" + - name: RAY_ROOT_DIR + value: "log" volumeMounts: - name: historyserver mountPath: *rayTmpRoot - tolerations: - - key: ray - operator: Equal - value: cpu volumes: - name: historyserver emptyDir: {} diff --git a/historyserver/config/rayjob-gcs.yaml b/historyserver/config/rayjob-gcs.yaml new file mode 100644 index 00000000000..3f598ce2e1a --- /dev/null +++ b/historyserver/config/rayjob-gcs.yaml @@ -0,0 +1,272 @@ +# Apply with envsubst to substitute environment variables: +# envsubst < historyserver/config/rayjob-gcs.yaml | kubectl apply -f - +apiVersion: ray.io/v1 +kind: RayJob +metadata: + name: rayjob-historyserver-gcs + namespace: default +spec: + entrypoint: python /home/ray/samples/sample_code.py + shutdownAfterJobFinishes: true + ttlSecondsAfterFinished: 600 + rayClusterSpec: + rayVersion: "2.56.0" + headGroupSpec: + rayStartParams: + dashboard-host: 0.0.0.0 + num-cpus: "0" + serviceType: ClusterIP + template: + metadata: + labels: + test: rayjob-historyserver-gcs + spec: + serviceAccountName: historyserver + containers: + - env: + - name: RAY_TMP_ROOT + value: &rayTmpRoot /tmp/ray + - name: RAY_enable_ray_event + value: "true" + - name: RAY_enable_core_worker_ray_event_to_aggregator + value: "true" + - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR + value: "http://localhost:8084/v1/events" + - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES + value: "ALL" + image: rayproject/ray:2.56.0 + imagePullPolicy: IfNotPresent + name: ray-head + resources: + limits: + cpu: "6" + memory: 10G + requests: + cpu: "50m" + memory: 1G + volumeMounts: + - name: historyserver + mountPath: *rayTmpRoot + - name: code-sample + mountPath: /home/ray/samples + - name: collector + image: ${COLLECTOR_IMAGE} + imagePullPolicy: IfNotPresent + env: + - name: RAY_CLUSTER_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['ray.io/cluster'] + - name: RAY_CLUSTER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: FQ_RAY_IP + value: $(RAY_CLUSTER_NAME)-head-svc.$(RAY_CLUSTER_NAMESPACE).svc.cluster.local + - name: RAY_TMP_ROOT + value: *rayTmpRoot + - name: GCS_BUCKET + value: "${GCS_BUCKET}" + # RAY_DASHBOARD_ADDRESS is used by the head collector to fetch endpoints' results + # (e.g., /api/v0/cluster_metadata) from the Ray Dashboard running in the same pod. + # Only the head collector uses this; worker collectors do not need it. + # If your Ray Dashboard uses a non-default port (not 8265), update this value accordingly. + - name: RAY_DASHBOARD_ADDRESS + value: "http://localhost:8265" + # RAY_COLLECTOR_ADDITIONAL_ENDPOINTS is a comma-separated list of Ray Dashboard + # API endpoint paths that the head collector will periodically poll and store. + # Use this for endpoints whose data cannot be obtained via Ray events. + # You can add more endpoints, e.g., "/api/v0/placement_groups,/api/serve/applications/" + # Note: Only static endpoints (no dynamic path parameters like {job_id}) are supported. + - name: RAY_COLLECTOR_ADDITIONAL_ENDPOINTS + value: "/api/v0/placement_groups?detail=1&limit=10000" + # Query params must match the Ray Dashboard frontend request exactly + # (see https://github.com/ray-project/ray/blob/cb9c80fee6a700efe61ea97987248ce82e3fa2e2/python/ray/dashboard/client/src/service/placementGroup.ts): + # detail=1 — include bundles and stats fields required by PlacementGroupTable + # limit=10000 — match the frontend's default limit to avoid truncation + # RAY_COLLECTOR_POLL_INTERVAL controls how often the collector polls the additional + # endpoints above. Accepts Go duration format (e.g., "30s", "1m", "5m"). + - name: RAY_COLLECTOR_POLL_INTERVAL + value: "30s" + - name: RAY_ROLE + value: "Head" + - name: OWNER_KIND + value: "rayjob" + - name: OWNER_NAME + value: "rayjob-historyserver-gcs" + - name: STORAGE_BACKEND + value: "gcs" + - name: EVENTS_PORT + value: "8084" + volumeMounts: + - name: historyserver + mountPath: *rayTmpRoot + volumes: + - name: historyserver + emptyDir: {} + - name: code-sample + configMap: + name: ray-job-code-sample + items: + - key: sample_code.py + path: sample_code.py + workerGroupSpecs: + - groupName: cpu + maxReplicas: 1000 + minReplicas: 1 + numOfHosts: 1 + rayStartParams: {} + replicas: 1 + template: + metadata: + labels: + test: rayjob-historyserver-gcs + spec: + serviceAccountName: historyserver + containers: + - env: + - name: RAY_TMP_ROOT + value: *rayTmpRoot + - name: RAY_enable_ray_event + value: "true" + - name: RAY_enable_core_worker_ray_event_to_aggregator + value: "true" + - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR + value: "http://localhost:8084/v1/events" + - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES + value: "ALL" + image: rayproject/ray:2.56.0 + imagePullPolicy: IfNotPresent + name: ray-worker + resources: + limits: + cpu: "2" + memory: 2G + requests: + cpu: "50m" + memory: 1G + volumeMounts: + - name: historyserver + mountPath: *rayTmpRoot + - name: collector + image: ${COLLECTOR_IMAGE} + imagePullPolicy: IfNotPresent + env: + - name: RAY_CLUSTER_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['ray.io/cluster'] + - name: RAY_CLUSTER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: FQ_RAY_IP + value: $(RAY_CLUSTER_NAME)-head-svc.$(RAY_CLUSTER_NAMESPACE).svc.cluster.local + - name: RAY_TMP_ROOT + value: *rayTmpRoot + - name: GCS_BUCKET + value: "${GCS_BUCKET}" + - name: RAY_ROLE + value: "Worker" + - name: OWNER_KIND + value: "rayjob" + - name: OWNER_NAME + value: "rayjob-historyserver-gcs" + - name: STORAGE_BACKEND + value: "gcs" + - name: EVENTS_PORT + value: "8084" + volumeMounts: + - name: historyserver + mountPath: *rayTmpRoot + volumes: + - name: historyserver + emptyDir: {} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ray-job-code-sample +data: + sample_code.py: | + import ray + import time + from ray.util.placement_group import placement_group + + ray.init() + + # --- Scenario 1: Root-level NORMAL_TASK (single call) --- + @ray.remote(num_cpus=0.1) + def my_task(x): + # NOTE: Add this to ensure will produce log.out + print(f"Processing {x}", flush=True) + return x * 2 + + # --- Scenario 2: Nested tasks (parent spawns child) --- + # Tests lineage tree depth > 1: child_task is nested under parent_task. + @ray.remote(num_cpus=0.1) + def parent_task(): + result = ray.get(child_task.remote()) + return result + + @ray.remote(num_cpus=0.1) + def child_task(): + print("Child task", flush=True) + return 42 + + # --- Scenario 3: Multiple same-name tasks --- + # Tests GROUP merging in lineage: 3 calls to the same function + # are merged into a single GROUP node. + @ray.remote(num_cpus=0.1) + def repeated_task(x): + return x + + # --- Scenario 4: Actor (creation + method calls) --- + @ray.remote(num_cpus=0.1) + class Counter: + def __init__(self): + self.count = 0 + + def increment(self): + self.count += 1 + return self.count + + def get_count(self): + return self.count + + # Execute all scenarios: + task_result = ray.get(my_task.remote(1)) + print(f"Task result: {task_result}") + + nested_result = ray.get(parent_task.remote()) + print(f"Nested result: {nested_result}") + + refs = [repeated_task.remote(i) for i in range(3)] + results = ray.get(refs) + print(f"Repeated results: {results}") + + counter = Counter.remote() + for i in range(2): + count = ray.get(counter.increment.remote()) + print(f"Counter: {count}") + final_count = ray.get(counter.get_count.remote()) + print(f"Final count: {final_count}") + + print(f"Cluster resources: {ray.cluster_resources()}") + + # Create a detached placement group so it persists after the job exits. + # This ensures the collector captures non-empty data when polling /api/v0/placement_groups. + pg = placement_group([{"CPU": 0.5}], strategy="SPREAD", lifetime="detached", name="test_pg") + ray.get(pg.ready()) + print(f"Placement group created: {pg.bundle_specs}") + + # Wait for events to flush to the collector + time.sleep(5) diff --git a/historyserver/test/support/historyserver.go b/historyserver/test/support/historyserver.go index 4aadd189861..d50d7fa050f 100644 --- a/historyserver/test/support/historyserver.go +++ b/historyserver/test/support/historyserver.go @@ -39,6 +39,7 @@ const ( "serveDeployment": "rayServeDeploymentDashboard", "serveLlm": "rayServeLlmDashboard", "data": "rayDataDashboard", + "dataLlm": "rayDataLlmDashboard", "train": "rayTrainDashboard" }, "dashboardDatasource": "Prometheus", diff --git a/historyserver/test/support/raycluster.go b/historyserver/test/support/raycluster.go index ce4aede74fb..dac77e17aed 100644 --- a/historyserver/test/support/raycluster.go +++ b/historyserver/test/support/raycluster.go @@ -59,18 +59,15 @@ func ApplyRayClusterWithCollectorWithEnvs(test Test, g *WithT, namespace *corev1 return rayCluster } -// injectCollectorRayClusterNamespaceAndEnvVar injects the ray-cluster-namespace argument and required environment variables (POD_IP, FQ_RAY_IP) into all collector containers. +// injectCollectorRayClusterNamespaceAndEnvVar injects the ray-cluster-namespace and required environment variables (RAY_CLUSTER_NAMESPACE, POD_IP, FQ_RAY_IP) into all collector containers. func injectCollectorRayClusterNamespaceAndEnvVar(containers []corev1.Container, rayClusterName string, rayClusterNamespace string) { fqdnRayIP := fmt.Sprintf("%s-head-svc.%s.svc.cluster.local", rayClusterName, rayClusterNamespace) for i := range containers { if containers[i].Name == "collector" { - containers[i].Command = append( - containers[i].Command, - fmt.Sprintf("--ray-cluster-namespace=%s", rayClusterNamespace), - ) if containers[i].Env == nil { containers[i].Env = []corev1.EnvVar{} } + setOrAppendEnv(&containers[i], "RAY_CLUSTER_NAMESPACE", rayClusterNamespace, nil) setOrAppendEnv(&containers[i], "POD_IP", "", &corev1.EnvVarSource{ FieldRef: &corev1.ObjectFieldSelector{ FieldPath: "status.podIP",