From 281ba9aec9d02a533c05d77d5715de199fe0ebed Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Wed, 5 Aug 2026 20:30:53 -0500 Subject: [PATCH] [History Server] Fix cold-load CPU starvation and per-event logging Loading a dead session is CPU-bound, but the sample manifest sets only `limits.cpu: "500m"`, so Kubernetes pins requests there too and the load saturates the quota for its entire duration. Measured on kind across 13 runs, the container sat at 0.42-0.50 cores every time. Raising the limit to 4 (requests stay at 500m, so scheduling cost is unchanged): | tasks in session | 500m | 4 cores | |---|---|---| | 50,000 | 97.9s | 30.5s | | 100,000 | 907.3s | 62.7s | This also removes what looked like superlinear degradation past 50k tasks: at 500m the per-task cost went from 1.96ms to 9.07ms between 50k and 100k, while at 4 cores it is flat (0.61ms and 0.63ms). The load uses ~1.2 cores on average and peaks at ~2.2, because Go's GC runs concurrently and needs cores of its own. The second change drops the per-event log line in storeEvent to Debug. It runs once per event, so a 100k-task session writes ~436,000 INFO lines per cold load, and the binary never calls logrus.SetLevel, so there is no way to turn it off. Worth ~13% of load time at 50k tasks (97.9s -> 85.3s). --- historyserver/config/historyserver.yaml | 7 ++++++- historyserver/pkg/eventserver/eventserver.go | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/historyserver/config/historyserver.yaml b/historyserver/config/historyserver.yaml index f07d45d34ee..46000d59e33 100644 --- a/historyserver/config/historyserver.yaml +++ b/historyserver/config/historyserver.yaml @@ -64,5 +64,10 @@ spec: ports: - containerPort: 8080 resources: - limits: + requests: cpu: "500m" + limits: + # Loading a dead session is CPU-bound: it uses ~1.2 cores with bursts + # to ~2.2. A 500m cap is saturated for the whole load, which made a + # 100k-task session take 907s instead of 63s. + cpu: "4" diff --git a/historyserver/pkg/eventserver/eventserver.go b/historyserver/pkg/eventserver/eventserver.go index 1b1f81ddfa6..920bc25d385 100644 --- a/historyserver/pkg/eventserver/eventserver.go +++ b/historyserver/pkg/eventserver/eventserver.go @@ -133,7 +133,7 @@ func (h *EventHandler) storeEvent(clusterSessionKey string, eventMap map[string] } eventType := types.EventType(eventTypeStr) - logrus.Infof("current eventType: %v", eventType) + logrus.Debugf("current eventType: %v", eventType) switch eventType { case types.TASK_DEFINITION_EVENT: return h.handleTaskDefinitionEvent(eventMap, clusterSessionKey, false)