[History server] Update history server examples and defaults

historyserver/DEVELOPMENT.md

@@ -120,7 +120,7 @@ Install Ray locally. Make sure to use at least Ray `v2.55`.
 
 ```bash
 pip uninstall -y ray
-pip install -U "ray[default]==2.55.0"
+pip install -U "ray[default]==2.56.0"
 ```
 
 Run the `ray start` command:

historyserver/cmd/collector/main.go

@@ -30,7 +30,7 @@ func main() {
 	rayClusterNamespace := ""
 	rayRootDir := ""
 	logBatching := 1000
-	eventsPort := 8080
+	eventsPort := 8084
 	pushInterval := time.Minute
 	ownerKind := ""
 	ownerName := ""
@@ -53,7 +53,7 @@ func main() {
 	flag.StringVar(&rayClusterNamespace, "ray-cluster-namespace", "default", "")
 	flag.StringVar(&rayRootDir, "ray-root-dir", "", "")
 	flag.IntVar(&logBatching, "log-batching", 1000, "")
-	flag.IntVar(&eventsPort, "events-port", 8080, "")
+	flag.IntVar(&eventsPort, "events-port", 8084, "")
 	flag.StringVar(&runtimeClassConfigPath, "runtime-class-config-path", "", "")
 	flag.DurationVar(&pushInterval, "push-interval", time.Minute, "")
 	flag.StringVar(&ownerKind, "owner-kind", "", "")

historyserver/cmd/historyserver/main.go

@@ -54,6 +54,10 @@ func main() {
 	}
 	runtimeClassName = strings.ToLower(runtimeClassName)
 
+	if val := os.Getenv("RAY_ROOT_DIR"); val != "" {
+		rayRootDir = val
+	}
+
 	if qps <= 0 {
 		logrus.Fatalf("--kube-api-qps must be > 0, got %v", qps)
 	}

historyserver/config/historyserver-gcs.yaml

@@ -1,4 +1,5 @@
-# Ensure that the env var GCS_BUCKET is set before applying
+# Apply with envsubst to substitute environment variables:
+#   envsubst < historyserver/config/historyserver-gcs.yaml | kubectl apply -f -
 apiVersion: v1
 kind: Service
 metadata:
@@ -37,12 +38,10 @@ spec:
         env:
           - name: GCS_BUCKET
             value: "${GCS_BUCKET}"
+          - name: STORAGE_BACKEND
+            value: "gcs"
         image: ${HISTORY_SERVER_IMAGE}
         imagePullPolicy: IfNotPresent
-        command:
-        - historyserver
-        - --runtime-class-name=gcs
-        - --ray-root-dir=log
         ports:
         - containerPort: 8080
         resources:

historyserver/config/historyserver.yaml

@@ -50,16 +50,18 @@ spec:
             value: "test"
           - name: S3FORCE_PATH_STYLE
             value: "true"
+          - name: STORAGE_BACKEND
+            value: "s3"
+          - name: RAY_ROOT_DIR
+            value: "log"
         image: historyserver:v0.1.0
         imagePullPolicy: IfNotPresent
-        command:
-        - historyserver
-        - --runtime-class-name=s3
-        - --ray-root-dir=log
         # Enable proxying to live RayClusters with token authentication enabled.
         # The history server reads the auth Secret and forwards requests with the
         # appropriate authorization header. Requires the RBAC in
         # service_account_auth_token_mode.yaml.
+        # command:
+        # - historyserver
         # - --use-auth-token-mode=true
         ports:
         - containerPort: 8080

historyserver/config/raycluster-azureblob.yaml

@@ -16,8 +16,6 @@ spec:
         labels:
           test: raycluster-historyserver
       spec:
-        imagePullSecrets:
-        affinity:
         containers:
         - env:
           - name: RAY_TMP_ROOT
@@ -29,14 +27,9 @@ spec:
           - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR
             value: "http://localhost:8084/v1/events"
           - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES
-            value: "TASK_DEFINITION_EVENT,TASK_LIFECYCLE_EVENT,ACTOR_TASK_DEFINITION_EVENT,
-                    TASK_PROFILE_EVENT,DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT,
-                    ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT,NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT"
-          image: rayproject/ray:2.52.0
+            value: "ALL"
+          image: rayproject/ray:2.55.0
           imagePullPolicy: IfNotPresent
-          securityContext:
-            allowPrivilegeEscalation: true
-            privileged: true
           name: ray-head
           resources:
             limits:
@@ -52,27 +45,36 @@ spec:
           image: collector:v0.1.0
           imagePullPolicy: IfNotPresent
           env:
+          - name: RAY_CLUSTER_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.labels['ray.io/cluster']
+          - name: RAY_CLUSTER_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
           - name: POD_IP
             valueFrom:
               fieldRef:
                 fieldPath: status.podIP
           - name: FQ_RAY_IP
-            value: raycluster-historyserver-head-svc.default.svc.cluster.local
+            value: $(RAY_CLUSTER_NAME)-head-svc.$(RAY_CLUSTER_NAMESPACE).svc.cluster.local
           - name: RAY_TMP_ROOT
             value: *rayTmpRoot
+          - name: RAY_ROLE
+            value: "Head"
+          - name: STORAGE_BACKEND
+            value: "azureblob"
+          - name: RAY_ROOT_DIR
+            value: "log"
           # reference: https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite#connect-to-the-emulator-by-using-the-azure-storage-explorer
           - name: AZURE_STORAGE_CONNECTION_STRING
             value: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite-service.azurite-dev.svc.cluster.local:10000/devstoreaccount1;"
           - name: AZURE_STORAGE_CONTAINER
             value: ray-historyserver
-          command: [collector, --role=Head, --runtime-class-name=azureblob, --ray-cluster-name=raycluster-historyserver, --ray-root-dir=log, --events-port=8084]
           volumeMounts:
           - name: historyserver
             mountPath: *rayTmpRoot
-        tolerations:
-        - key: ray
-          operator: Equal
-          value: cpu
         volumes:
         - name: historyserver
           emptyDir: {}
@@ -88,7 +90,6 @@ spec:
         labels:
           test: raycluster-historyserver
       spec:
-        imagePullSecrets:
         containers:
         - env:
           - name: RAY_TMP_ROOT
@@ -100,15 +101,10 @@ spec:
           - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR
             value: "http://localhost:8084/v1/events"
           - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES
-            value: "TASK_DEFINITION_EVENT,TASK_LIFECYCLE_EVENT,ACTOR_TASK_DEFINITION_EVENT,
-                    TASK_PROFILE_EVENT,DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT,
-                    ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT,NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT"
-          image: rayproject/ray:2.52.0
+            value: "ALL"
+          image: rayproject/ray:2.55.0
           imagePullPolicy: IfNotPresent
           name: ray-worker
-          securityContext:
-            allowPrivilegeEscalation: true
-            privileged: true
           resources:
             limits:
               cpu: "2"
@@ -123,27 +119,36 @@ spec:
           image: collector:v0.1.0
           imagePullPolicy: IfNotPresent
           env:
+          - name: RAY_CLUSTER_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.labels['ray.io/cluster']
+          - name: RAY_CLUSTER_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
           - name: POD_IP
             valueFrom:
               fieldRef:
                 fieldPath: status.podIP
           - name: FQ_RAY_IP
-            value: raycluster-historyserver-head-svc.default.svc.cluster.local
+            value: $(RAY_CLUSTER_NAME)-head-svc.$(RAY_CLUSTER_NAMESPACE).svc.cluster.local
           - name: RAY_TMP_ROOT
             value: *rayTmpRoot
+          - name: RAY_ROLE
+            value: "Worker"
+          - name: STORAGE_BACKEND
+            value: "azureblob"
+          - name: RAY_ROOT_DIR
+            value: "log"
           # reference: https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite#connect-to-the-emulator-by-using-the-azure-storage-explorer
           - name: AZURE_STORAGE_CONNECTION_STRING
             value: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite-service.azurite-dev.svc.cluster.local:10000/devstoreaccount1;"
           - name: AZURE_STORAGE_CONTAINER
             value: ray-historyserver
-          command: [collector, --role=Worker, --runtime-class-name=azureblob, --ray-cluster-name=raycluster-historyserver, --ray-root-dir=log, --events-port=8084]
           volumeMounts:
           - name: historyserver
             mountPath: *rayTmpRoot
-        tolerations:
-        - key: ray
-          operator: Equal
-          value: cpu
         volumes:
         - name: historyserver
           emptyDir: {}

historyserver/config/raycluster-gcs.yaml

@@ -1,3 +1,5 @@
+# Apply with envsubst to substitute environment variables:
+#   envsubst < historyserver/config/raycluster-gcs.yaml | kubectl apply -f -
 apiVersion: ray.io/v1
 kind: RayCluster
 metadata:
@@ -17,8 +19,6 @@ spec:
           test: raycluster-historyserver
       spec:
         serviceAccountName: historyserver
-        imagePullSecrets:
-        affinity:
         containers:
         - env:
           - name: RAY_TMP_ROOT
@@ -30,14 +30,9 @@ spec:
           - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR
             value: "http://localhost:8084/v1/events"
           - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES
-            value: "TASK_DEFINITION_EVENT,TASK_LIFECYCLE_EVENT,ACTOR_TASK_DEFINITION_EVENT,
-                    TASK_PROFILE_EVENT,DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT,
-                    ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT,NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT"
-          image: rayproject/ray:2.52.0
+            value: "ALL"
+          image: rayproject/ray:2.56.0
           imagePullPolicy: IfNotPresent
-          securityContext:
-            allowPrivilegeEscalation: true
-            privileged: true
           name: ray-head
           resources:
             limits:
@@ -53,12 +48,20 @@ spec:
           image: ${COLLECTOR_IMAGE}
           imagePullPolicy: IfNotPresent
           env:
+          - name: RAY_CLUSTER_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.labels['ray.io/cluster']
+          - name: RAY_CLUSTER_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
           - name: POD_IP
             valueFrom:
               fieldRef:
                 fieldPath: status.podIP
           - name: FQ_RAY_IP
-            value: raycluster-historyserver-head-svc.default.svc.cluster.local
+            value: $(RAY_CLUSTER_NAME)-head-svc.$(RAY_CLUSTER_NAMESPACE).svc.cluster.local
           - name: RAY_TMP_ROOT
             value: *rayTmpRoot
           - name: GCS_BUCKET
@@ -84,20 +87,15 @@ spec:
           # endpoints above. Accepts Go duration format (e.g., "30s", "1m", "5m").
           - name: RAY_COLLECTOR_POLL_INTERVAL
             value: "30s"
-          command:
-          - collector
-          - --role=Head
-          - --runtime-class-name=gcs
-          - --ray-cluster-name=${RAY_CLUSTER}
-          - --ray-root-dir=log
-          - --events-port=8084
+          - name: RAY_ROLE
+            value: "Head"
+          - name: STORAGE_BACKEND
+            value: "gcs"
+          - name: EVENTS_PORT
+            value: "8084"
           volumeMounts:
           - name: historyserver
             mountPath: *rayTmpRoot
-        tolerations:
-        - key: ray
-          operator: Equal
-          value: cpu
         volumes:
         - name: historyserver
           emptyDir: {}
@@ -125,15 +123,10 @@ spec:
           - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR
             value: "http://localhost:8084/v1/events"
           - name: RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES
-            value: "TASK_DEFINITION_EVENT,TASK_LIFECYCLE_EVENT,ACTOR_TASK_DEFINITION_EVENT,
-                    TASK_PROFILE_EVENT,DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT,
-                    ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT,NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT"
-          image: rayproject/ray:2.52.0
+            value: "ALL"
+          image: rayproject/ray:2.56.0
           imagePullPolicy: IfNotPresent
           name: ray-worker
-          securityContext:
-            allowPrivilegeEscalation: true
-            privileged: true
           resources:
             limits:
               cpu: "2"
@@ -148,30 +141,33 @@ spec:
           image: ${COLLECTOR_IMAGE}
           imagePullPolicy: IfNotPresent
           env:
+          - name: RAY_CLUSTER_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.labels['ray.io/cluster']
+          - name: RAY_CLUSTER_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
           - name: POD_IP
             valueFrom:
               fieldRef:
                 fieldPath: status.podIP
           - name: FQ_RAY_IP
-            value: raycluster-historyserver-head-svc.default.svc.cluster.local
+            value: $(RAY_CLUSTER_NAME)-head-svc.$(RAY_CLUSTER_NAMESPACE).svc.cluster.local
           - name: RAY_TMP_ROOT
             value: *rayTmpRoot
           - name: GCS_BUCKET
             value: "${GCS_BUCKET}"
-          command:
-          - collector
-          - --role=Worker
-          - --runtime-class-name=gcs
-          - --ray-cluster-name=${RAY_CLUSTER}
-          - --ray-root-dir=log
-          - --events-port=8084
+          - name: RAY_ROLE
+            value: "Worker"
+          - name: STORAGE_BACKEND
+            value: "gcs"
+          - name: EVENTS_PORT
+            value: "8084"
           volumeMounts:
           - name: historyserver
             mountPath: *rayTmpRoot
-        tolerations:
-        - key: ray
-          operator: Equal
-          value: cpu
         volumes:
         - name: historyserver
           emptyDir: {}