[History Server] Poll Serve, placement group, and Ray Data endpoints by default

historyserver/cmd/collector/main.go

@@ -160,6 +160,8 @@ func main() {
 		}
 	}
 
+	// RAY_COLLECTOR_ADDITIONAL_ENDPOINTS is optional: the head collector always
+	// polls its built-in endpoints, and anything listed here is polled on top.
 	var additionalEndpoints []string
 	if epStr := os.Getenv("RAY_COLLECTOR_ADDITIONAL_ENDPOINTS"); epStr != "" {
 		for _, ep := range strings.Split(epStr, ",") {
@@ -170,16 +172,15 @@ func main() {
 		}
 	}
 
+	// Fall back instead of exiting: crash-looping this sidecar would take the head pod
+	// out of its Service endpoints.
 	endpointPollInterval := 30 * time.Second
-	if intervalStr := os.Getenv("RAY_COLLECTOR_POLL_INTERVAL"); intervalStr != "" {
-		parsed, parseErr := time.ParseDuration(intervalStr)
-		if parseErr != nil {
-			logrus.Fatalf("Failed to parse RAY_COLLECTOR_POLL_INTERVAL: %v", parseErr)
-		}
-		if parsed <= 0 {
-			logrus.Fatalf("RAY_COLLECTOR_POLL_INTERVAL must be positive, got: %s", intervalStr)
+	if v := os.Getenv("RAY_COLLECTOR_POLL_INTERVAL"); v != "" {
+		if parsed, err := time.ParseDuration(v); err == nil && parsed > 0 {
+			endpointPollInterval = parsed
+		} else {
+			logrus.Warnf("Invalid RAY_COLLECTOR_POLL_INTERVAL=%s, using default %s", v, endpointPollInterval)
 		}
-		endpointPollInterval = parsed
 	}
 
 	jsonData := make(map[string]interface{})
@@ -229,6 +230,12 @@ func main() {
 
 	sessionName := path.Base(activeSessionDir)
 
+	// The head collector shares a pod with the dashboard; override for non-default ports.
+	dashboardAddress := "http://localhost:8265"
+	if v := os.Getenv("RAY_DASHBOARD_ADDRESS"); v != "" {
+		dashboardAddress = v
+	}
+
 	globalConfig := types.RayCollectorConfig{
 		RootDir:             rayRootDir,
 		SessionDir:          activeSessionDir,
@@ -238,7 +245,7 @@ func main() {
 		RayClusterNamespace: rayClusterNamespace,
 		PushInterval:        pushInterval,
 		LogBatching:         logBatching,
-		DashboardAddress:    os.Getenv("RAY_DASHBOARD_ADDRESS"),
+		DashboardAddress:    dashboardAddress,
 		OwnerKind:           ownerKind,
 		OwnerName:           ownerName,
 

historyserver/config/ray-data.yaml

@@ -0,0 +1,112 @@
+apiVersion: ray.io/v1
+kind: RayJob
+metadata:
+  name: rayjob-ray-data
+spec:
+  # Self-contained: brings up its own cluster; also exercises the collector's shutdown path.
+  shutdownAfterJobFinishes: true
+  # Long enough for one polling cycle after the job succeeds.
+  ttlSecondsAfterFinished: 30
+  entrypoint: |
+    python -c "
+    import ray
+    ray.init()
+
+    # materialize() is required: an unexecuted Dataset produces no stats.
+    ds = ray.data.range(100).map_batches(lambda batch: batch).materialize()
+    print(f'Dataset rows: {ds.count()}')
+    "
+  rayClusterSpec:
+    # Head-only: worker collectors would need the generated head Service FQDN in FQ_RAY_IP.
+    headGroupSpec:
+      rayStartParams:
+        dashboard-host: 0.0.0.0
+      serviceType: ClusterIP
+      template:
+        spec:
+          containers:
+          - env:
+            - name: RAY_TMP_ROOT
+              value: &rayTmpRoot /tmp/ray
+            - name: RAY_enable_ray_event
+              value: "true"
+            - name: RAY_enable_core_worker_ray_event_to_aggregator
+              value: "true"
+            - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR
+              value: "http://localhost:8084/v1/events"
+            - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES
+              value: "ALL"
+            image: rayproject/ray:2.56.0
+            imagePullPolicy: IfNotPresent
+            name: ray-head
+            securityContext:
+              allowPrivilegeEscalation: true
+              privileged: true
+            resources:
+              limits:
+                cpu: "5"
+                memory: 10G
+              requests:
+                cpu: "50m"
+                memory: 1G
+            volumeMounts:
+            - name: historyserver
+              mountPath: *rayTmpRoot
+          - name: collector
+            image: collector:v0.1.0
+            imagePullPolicy: IfNotPresent
+            env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            # KubeRay generates the cluster name; read it back from the pod label.
+            - name: RAY_CLUSTER_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.labels['ray.io/cluster']
+            - name: RAY_CLUSTER_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            # ray.io/originated-from-* labels are on the RayCluster, not the pod; must match metadata.name.
+            - name: OWNER_KIND
+              value: "RayJob"
+            - name: OWNER_NAME
+              value: "rayjob-ray-data"
+            # Only used to look up this pod's Ray NodeID; the dashboard is in this pod.
+            - name: FQ_RAY_IP
+              value: "localhost"
+            - name: RAY_TMP_ROOT
+              value: *rayTmpRoot
+            # Shorter than the 30s default so a cycle fits inside ttlSecondsAfterFinished.
+            - name: RAY_COLLECTOR_POLL_INTERVAL
+              value: "5s"
+            - name: S3DISABLE_SSL
+              value: "true"
+            - name: AWS_ACCESS_KEY_ID
+              value: minioadmin
+            - name: AWS_SECRET_ACCESS_KEY
+              value: minioadmin
+            - name: AWS_SESSION_TOKEN
+              value: ""
+            - name: S3_BUCKET
+              value: "ray-historyserver"
+            - name: S3_ENDPOINT
+              value: "minio-service.minio-dev:9000"
+            - name: S3_REGION
+              value: "test"
+            - name: S3FORCE_PATH_STYLE
+              value: "true"
+            command:
+            - collector
+            - --role=Head
+            - --runtime-class-name=s3
+            - --ray-root-dir=log
+            - --events-port=8084
+            volumeMounts:
+            - name: historyserver
+              mountPath: *rayTmpRoot
+          volumes:
+          - name: historyserver
+            emptyDir: {}

historyserver/config/raycluster-azureblob.yaml

@@ -61,6 +61,20 @@ spec:
             value: $(RAY_CLUSTER_NAME)-head-svc.$(RAY_CLUSTER_NAMESPACE).svc.cluster.local
           - name: RAY_TMP_ROOT
             value: *rayTmpRoot
+          # Optional; defaults to http://localhost:8265 (head only).
+          # - name: RAY_DASHBOARD_ADDRESS
+          #   value: "http://localhost:9265"
+          # Optional; defaults to 30s.
+          # - name: RAY_COLLECTOR_POLL_INTERVAL
+          #   value: "1m"
+          # Optional extras on top of the built-in endpoints (Serve, placement groups, Ray Data).
+          # Paths must match the History Server replay request URI, query string included.
+          # For example, preserve Ray Train V2 run details for a future History Server view.
+          # Requires the Ray Train workload/driver (not the collector) to set
+          # RAY_TRAIN_V2_ENABLED=1 and RAY_TRAIN_ENABLE_STATE_TRACKING=1;
+          # verify this DeveloperAPI URI when changing Ray versions.
+          # - name: RAY_COLLECTOR_ADDITIONAL_ENDPOINTS
+          #   value: "/api/train/v2/runs/v1"
           - name: RAY_ROLE
             value: "Head"
           - name: STORAGE_BACKEND

historyserver/config/raycluster-gcs.yaml

@@ -66,27 +66,20 @@ spec:
             value: *rayTmpRoot
           - name: GCS_BUCKET
             value: "${GCS_BUCKET}"
-          # RAY_DASHBOARD_ADDRESS is used by the head collector to fetch endpoints' results
-          # (e.g., /api/v0/cluster_metadata) from the Ray Dashboard running in the same pod.
-          # Only the head collector uses this; worker collectors do not need it.
-          # If your Ray Dashboard uses a non-default port (not 8265), update this value accordingly.
-          - name: RAY_DASHBOARD_ADDRESS
-            value: "http://localhost:8265"
-          # RAY_COLLECTOR_ADDITIONAL_ENDPOINTS is a comma-separated list of Ray Dashboard
-          # API endpoint paths that the head collector will periodically poll and store.
-          # Use this for endpoints whose data cannot be obtained via Ray events.
-          # You can add more endpoints, e.g., "/api/v0/placement_groups,/api/serve/applications/"
-          # Note: Only static endpoints (no dynamic path parameters like {job_id}) are supported.
-          - name: RAY_COLLECTOR_ADDITIONAL_ENDPOINTS
-            value: "/api/v0/placement_groups?detail=1&limit=10000"
-            # Query params must match the Ray Dashboard frontend request exactly
-            # (see https://github.com/ray-project/ray/blob/cb9c80fee6a700efe61ea97987248ce82e3fa2e2/python/ray/dashboard/client/src/service/placementGroup.ts):
-            #   detail=1    — include bundles and stats fields required by PlacementGroupTable
-            #   limit=10000 — match the frontend's default limit to avoid truncation
-          # RAY_COLLECTOR_POLL_INTERVAL controls how often the collector polls the additional
-          # endpoints above. Accepts Go duration format (e.g., "30s", "1m", "5m").
-          - name: RAY_COLLECTOR_POLL_INTERVAL
-            value: "30s"
+          # Optional; defaults to http://localhost:8265 (head only).
+          # - name: RAY_DASHBOARD_ADDRESS
+          #   value: "http://localhost:9265"
+          # Optional; defaults to 30s.
+          # - name: RAY_COLLECTOR_POLL_INTERVAL
+          #   value: "1m"
+          # Optional extras on top of the built-in endpoints (Serve, placement groups, Ray Data).
+          # Paths must match the History Server replay request URI, query string included.
+          # For example, preserve Ray Train V2 run details for a future History Server view.
+          # Requires the Ray Train workload/driver (not the collector) to set
+          # RAY_TRAIN_V2_ENABLED=1 and RAY_TRAIN_ENABLE_STATE_TRACKING=1;
+          # verify this DeveloperAPI URI when changing Ray versions.
+          # - name: RAY_COLLECTOR_ADDITIONAL_ENDPOINTS
+          #   value: "/api/train/v2/runs/v1"
           - name: RAY_ROLE
             value: "Head"
           - name: STORAGE_BACKEND

historyserver/config/raycluster.yaml

@@ -65,27 +65,20 @@ spec:
             value: $(RAY_CLUSTER_NAME)-head-svc.$(RAY_CLUSTER_NAMESPACE).svc.cluster.local
           - name: RAY_TMP_ROOT
             value: *rayTmpRoot
-          # RAY_DASHBOARD_ADDRESS is used by the head collector to fetch endpoints' results
-          # (e.g., /api/v0/cluster_metadata) from the Ray Dashboard running in the same pod.
-          # Only the head collector uses this; worker collectors do not need it.
-          # If your Ray Dashboard uses a non-default port (not 8265), update this value accordingly.
-          - name: RAY_DASHBOARD_ADDRESS
-            value: "http://localhost:8265"
-          # RAY_COLLECTOR_ADDITIONAL_ENDPOINTS is a comma-separated list of Ray Dashboard
-          # API endpoint paths that the head collector will periodically poll and store.
-          # Use this for endpoints whose data cannot be obtained via Ray events.
-          # You can add more endpoints, e.g., "/api/v0/placement_groups,/api/serve/applications/"
-          # Note: Only static endpoints (no dynamic path parameters like {job_id}) are supported.
-          - name: RAY_COLLECTOR_ADDITIONAL_ENDPOINTS
-            # Query params must match the Ray Dashboard frontend request exactly
-            # (see https://github.com/ray-project/ray/blob/cb9c80fee6a700efe61ea97987248ce82e3fa2e2/python/ray/dashboard/client/src/service/placementGroup.ts):
-            #   detail=1    — include bundles and stats fields required by PlacementGroupTable
-            #   limit=10000 — match the frontend's default limit to avoid truncation
-            value: "/api/v0/placement_groups?detail=1&limit=10000"
-          # RAY_COLLECTOR_POLL_INTERVAL controls how often the collector polls the additional
-          # endpoints above. Accepts Go duration format (e.g., "30s", "1m", "5m").
-          - name: RAY_COLLECTOR_POLL_INTERVAL
-            value: "30s"
+          # Optional; defaults to http://localhost:8265 (head only).
+          # - name: RAY_DASHBOARD_ADDRESS
+          #   value: "http://localhost:9265"
+          # Optional; defaults to 30s.
+          # - name: RAY_COLLECTOR_POLL_INTERVAL
+          #   value: "1m"
+          # Optional extras on top of the built-in endpoints (Serve, placement groups, Ray Data).
+          # Paths must match the History Server replay request URI, query string included.
+          # For example, preserve Ray Train V2 run details for a future History Server view.
+          # Requires the Ray Train workload/driver (not the collector) to set
+          # RAY_TRAIN_V2_ENABLED=1 and RAY_TRAIN_ENABLE_STATE_TRACKING=1;
+          # verify this DeveloperAPI URI when changing Ray versions.
+          # - name: RAY_COLLECTOR_ADDITIONAL_ENDPOINTS
+          #   value: "/api/train/v2/runs/v1"
           - name: S3DISABLE_SSL
             value: "true"
           - name: AWS_ACCESS_KEY_ID

historyserver/config/rayservice.yaml

@@ -0,0 +1,126 @@
+apiVersion: ray.io/v1
+kind: RayService
+metadata:
+  name: rayservice-historyserver
+spec:
+  serveConfigV2: |
+    proxy_location: EveryNode
+    applications:
+      - name: history-e2e
+        route_prefix: /
+        import_path: microbenchmarks.no_ops:app_builder
+        args:
+          num_forwards: 0
+        runtime_env:
+          working_dir: https://github.com/ray-project/serve_workloads/archive/a9f184f4d9ddb7f9a578502ae106470f87a702ef.zip
+        deployments:
+          - name: NoOp
+            num_replicas: 1
+            ray_actor_options:
+              num_cpus: 0.5
+  rayClusterConfig:
+    # Head-only: worker collectors would need the generated head Service FQDN in FQ_RAY_IP.
+    headGroupSpec:
+      rayStartParams:
+        dashboard-host: 0.0.0.0
+      serviceType: ClusterIP
+      template:
+        spec:
+          containers:
+          - env:
+            - name: RAY_TMP_ROOT
+              value: &rayTmpRoot /tmp/ray
+            - name: RAY_enable_ray_event
+              value: "true"
+            - name: RAY_enable_core_worker_ray_event_to_aggregator
+              value: "true"
+            - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR
+              value: "http://localhost:8084/v1/events"
+              # in ray 2.52.0, we need to set RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES
+              # in ray 2.53.0 (noy yet done). we need to set RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER_HTTP_ENDPOINT_EXPOSABLE_EVENT_TYPES
+            - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES
+              value: "TASK_DEFINITION_EVENT,TASK_LIFECYCLE_EVENT,ACTOR_TASK_DEFINITION_EVENT,
+                      TASK_PROFILE_EVENT,DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT,
+                      ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT,NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT"
+            image: rayproject/ray:2.52.0
+            imagePullPolicy: IfNotPresent
+            name: ray-head
+            securityContext:
+              allowPrivilegeEscalation: true
+              privileged: true
+            ports:
+            - containerPort: 6379
+              name: gcs-server
+            - containerPort: 8265
+              name: dashboard
+            - containerPort: 10001
+              name: client
+            - containerPort: 8000
+              name: serve
+            resources:
+              limits:
+                cpu: "5"
+                memory: 10G
+              requests:
+                cpu: "50m"
+                memory: 1G
+            volumeMounts:
+            - name: historyserver
+              mountPath: *rayTmpRoot
+          - name: collector
+            image: collector:v0.1.0
+            imagePullPolicy: IfNotPresent
+            env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            # KubeRay generates the cluster name; read it back from the pod label.
+            - name: RAY_CLUSTER_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.labels['ray.io/cluster']
+            - name: RAY_CLUSTER_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            # ray.io/originated-from-* labels are on the RayCluster, not the pod; must match metadata.name.
+            - name: OWNER_KIND
+              value: "RayService"
+            - name: OWNER_NAME
+              value: "rayservice-historyserver"
+            # Only used to look up this pod's Ray NodeID; the dashboard is in this pod.
+            - name: FQ_RAY_IP
+              value: "localhost"
+            - name: RAY_TMP_ROOT
+              value: *rayTmpRoot
+            - name: S3DISABLE_SSL
+              value: "true"
+            - name: AWS_ACCESS_KEY_ID
+              value: minioadmin
+            - name: AWS_SECRET_ACCESS_KEY
+              value: minioadmin
+            - name: AWS_SESSION_TOKEN
+              value: ""
+            - name: S3_BUCKET
+              value: "ray-historyserver"
+            - name: S3_ENDPOINT
+              value: "minio-service.minio-dev:9000"
+            - name: S3_REGION
+              value: "test"
+            - name: S3FORCE_PATH_STYLE
+              value: "true"
+            - name: RAY_COLLECTOR_POLL_INTERVAL
+              value: "30s"
+            command:
+            - collector
+            - --role=Head
+            - --runtime-class-name=s3
+            - --ray-root-dir=log
+            - --events-port=8084
+            volumeMounts:
+            - name: historyserver
+              mountPath: *rayTmpRoot
+          volumes:
+          - name: historyserver
+            emptyDir: {}