diff --git a/historyserver/pkg/collector/logcollector/runtime/logcollector/collector.go b/historyserver/pkg/collector/logcollector/runtime/logcollector/collector.go index 8d3e96274af..9d2bc3a53ea 100644 --- a/historyserver/pkg/collector/logcollector/runtime/logcollector/collector.go +++ b/historyserver/pkg/collector/logcollector/runtime/logcollector/collector.go @@ -21,6 +21,79 @@ import ( "github.com/ray-project/kuberay/historyserver/pkg/utils" ) +// sessionLatestLinkName is the symlink Ray keeps pointing at the active session. It +// is never a session name, which is why resolveSessionIdentity refuses to use it as +// one. +const sessionLatestLinkName = "session_latest" + +// defaultSessionPollInterval is how often session_latest is re-read for a change. +const defaultSessionPollInterval = 5 * time.Second + +// transitionGate decides whether a session transition may run. +// +// The session poller outlives the start of shutdown: ShutdownChan cannot close until +// after the final endpoint poll, which is after the final legacy walk. Without this +// gate a tick landing in that window could rediscover a node ID, retire a collector, +// or relocate the live tree out from under processSessionLatestLogs while it walks it. +// Freezing the supervisor does not prevent any of those, because they are the +// handler's side effects rather than the collector's. +// +// Its mutex is a leaf: held only to inspect and update the two counters, never across +// a transition, a supervisor call or a filesystem operation. +type transitionGate struct { + cond *sync.Cond + mu sync.Mutex + running int + closed bool +} + +// enter reports whether a transition may start, and counts it in when it may. +func (g *transitionGate) enter() bool { + g.mu.Lock() + defer g.mu.Unlock() + if g.closed { + return false + } + g.running++ + return true +} + +// leave counts a transition out and wakes a close that is waiting for it. +func (g *transitionGate) leave() { + g.mu.Lock() + defer g.mu.Unlock() + g.running-- + if g.running == 0 && g.cond != nil { + g.cond.Broadcast() + } +} + +// close refuses every future transition and waits for the ones already admitted. +// +// The wait has no timeout because there is nothing to cancel: every step an admitted +// transition can still be inside terminates on its own. The node-ID query is an HTTP +// call with a one-second client timeout and the relocation is a rename. The handover +// is the longest step — stopRun reconciles the outgoing collector once before draining +// it, and that reconciliation is a synchronous walk of the logs tree, so it is bounded +// by the size of that tree rather than by the drain budget; the drain budget and the +// upload worker's stop grace bound everything after it. +// +// Returning early would not shorten any of those. It would only let shutdown run the +// rotated retirement and the legacy walk concurrently with a transition still free to +// change the node identity they write under, move the tree the walk is reading, or +// take the supervisor's lifecycle lock the retirement needs. +func (g *transitionGate) close() { + g.mu.Lock() + defer g.mu.Unlock() + g.closed = true + if g.cond == nil { + g.cond = sync.NewCond(&g.mu) + } + for g.running > 0 { + g.cond.Wait() + } +} + type RayLogHandler struct { Writer storage.StorageWriter LogFiles chan string @@ -38,13 +111,34 @@ type RayLogHandler struct { SessionDir string prevLogsDir string persistCompleteLogsDir string - PushInterval time.Duration - LogBatching int - IsHead bool - DashboardAddress string - AdditionalEndpoints []string - EndpointPollInterval time.Duration - mu sync.RWMutex + // rotated owns the rotated-log subsystem. Run installs it before any other + // goroutine starts; every method on it is nil-safe, so a handler that never ran — + // which is how the legacy tests construct one — behaves exactly as it did before. + // It is read through rotatedCollection() and written only by + // startRotatedCollection, both under mu. + rotated *rotatedSupervisor + // discoverNodeID resolves this pod's current Ray node ID. It is nil in production, + // where currentNodeID queries the dashboard; tests set it to make a session change + // carry a node change without a network. + discoverNodeID func() (string, bool) + // beforeRelocation runs just before an outgoing session's logs are moved. It is nil + // in production and exists so a test can hold a transition in its last step, where + // a shutdown that did not wait would walk a tree that is being moved. + beforeRelocation func() + // transitions gates session transitions against shutdown. Its zero value is an + // open gate, so a handler that was never run behaves exactly as it did before. + transitions transitionGate + // sessionPollInterval overrides how often session_latest is re-read. Zero means + // defaultSessionPollInterval, which is what production uses; tests set it so that + // waiting for several polling cycles does not mean waiting several tens of seconds. + sessionPollInterval time.Duration + PushInterval time.Duration + LogBatching int + IsHead bool + DashboardAddress string + AdditionalEndpoints []string + EndpointPollInterval time.Duration + mu sync.RWMutex } func (r *RayLogHandler) GetRayNodeName() string { @@ -76,6 +170,12 @@ func (r *RayLogHandler) Run(stop <-chan struct{}) error { } defer watcher.Close() + // Rotated-log protection comes up first, and long before the final shutdown walk. + // A segment Ray rotates away in the first seconds of a session is already gone by + // the time that walk runs, so starting the legacy flow first would leave exactly + // the window this subsystem exists to close. + r.startRotatedCollection() + // WatchPrevLogsLoops performs an initial scan of the prev-logs directory on startup // to process leftover log files in prev-logs/{sessionID}/{nodeID}/logs/ directories. // After scanning, it watches for new directories and files. This ensures incomplete @@ -91,7 +191,7 @@ func (r *RayLogHandler) Run(stop <-chan struct{}) error { <-stop logrus.Info("Received stop signal, processing all logs...") - r.processSessionLatestLogs() + r.shutdownLogCollection() // Perform one final poll of additional endpoints before shutting down. // This must happen before close(r.ShutdownChan) because pollSingleEndpoint // uses ShutdownChan to cancel in-flight HTTP requests. @@ -103,21 +203,473 @@ func (r *RayLogHandler) Run(stop <-chan struct{}) error { return nil } -// processSessionLatestLogs processes logs in the configured session_latest/logs directory -// on shutdown, using the real session ID and node ID -func (r *RayLogHandler) processSessionLatestLogs() { - logrus.Info("Processing session_latest logs on shutdown...") +// shutdownLogCollection brings log collection down in the only order that is safe, and +// each step must complete before the next begins. +// +// 1. Close session-transition admission and wait for one that was already admitted. +// The poller outlives this point — ShutdownChan cannot close until after the final +// endpoint poll — so anything less would leave a transition free to relocate the +// tree the walk below reads, change the node identity it writes under, or hold the +// supervisor's lifecycle lock that step 2 needs. +// 2. Retire the rotated subsystem: one last reconciliation of the live tree, then a +// bounded drain. Bounded because storage.StorageWriter.WriteFile cannot be +// canceled and the walk below is the pre-existing data path, which must still get +// its share of the pod's termination grace period. +// 3. Walk the live tree, from an immutable snapshot of what "the live tree" meant +// when this began. +// +// The walk is untouched by the rotated subsystem: the two write disjoint object keys, +// so neither suppresses the other; see startRotatedCollection. +func (r *RayLogHandler) shutdownLogCollection() { + r.transitions.close() + r.rotatedCollection().shutdown() + r.processSessionLatestLogs() +} + +// clusterIdentity is the owner-aware cluster address every object this handler writes +// lives under. It is assembled from the same fields the legacy upload paths use, so a +// captured segment and its uncaptured siblings land beside each other and RayJob and +// RayService clusters keep nesting their logs under the owner name. +func (r *RayLogHandler) clusterIdentity() clusterIdentity { + return clusterIdentity{ + RootDir: r.RootDir, + OwnerKind: r.OwnerKind, + OwnerName: r.OwnerName, + Namespace: r.RayClusterNamespace, + ClusterName: r.RayClusterName, + } +} + +// rotatedCollection returns the rotated-log subsystem, or nil when this handler never +// started one. Every method on the result is nil-safe. +// +// The pointer is read under mu and the supervisor is called with mu released. That +// order is the whole reason there can be no lock inversion between the handler's node +// lock and the supervisor's: the supervisor never calls back into the handler, and the +// handler never calls into the supervisor while holding its own lock. +func (r *RayLogHandler) rotatedCollection() *rotatedSupervisor { + r.mu.RLock() + defer r.mu.RUnlock() + return r.rotated +} + +// startRotatedCollection installs the rotated-log subsystem and points it at the +// session this handler started with. +// +// What it adds is strictly additive to the legacy flow, and that is a property of the +// object keys rather than of any coordination between the two. A captured segment is +// written as ".rotated." — a capture ID is what makes two +// generations of "raylet.out.1" two distinct objects — while the legacy walk writes +// "". The two key spaces cannot intersect, so neither half can overwrite or +// stand in for the other, and nothing the rotated subsystem does may suppress a legacy +// upload: doing so would delete a key the legacy walk has always produced, in exchange +// for an object with a different name. +func (r *RayLogHandler) startRotatedCollection() { + sup := newRotatedSupervisor(r.clusterIdentity(), r.Writer, utils.GetRayRotatedStagingPath()) + + r.mu.Lock() + if r.rotated == nil { + r.rotated = sup + } + r.mu.Unlock() + + r.ensureRotatedCollection(r.SessionDir) +} + +// ensureRotatedCollection points rotated-log protection at the session directory that +// is active now, under the node ID the handler currently holds. +// +// It is idempotent, so the session poller can call it on every tick: that is what +// picks up a session directory that appeared late, a session that was replaced, and a +// node ID that was rediscovered. +func (r *RayLogHandler) ensureRotatedCollection(sessionDir string) { + r.ensureRotatedCollectionForNode(sessionDir, r.GetRayNodeName()) +} + +// ensureRotatedCollectionForNode points rotated-log protection at one session on one +// explicitly named node. +// +// The node is a parameter rather than something read back out of the handler because a +// session change has to build the new collector under the *new* session's node ID, and +// that ID is discovered during the changeover. Writing it into the handler first, only +// so that ensure could read it, would make the handler's node ID mean "the node of the +// collector I am about to build" for the duration of the changeover — and every other +// reader of GetRayNodeName, including the relocation of the old session's logs, would +// see the wrong answer. +// +// Getting this wrong is not recoverable by restarting the collector. The node ID is +// baked into the staged entry, the staging path and the object key at capture time, and +// cross-session adoption deliberately preserves a record's original identity, so a +// segment captured under the previous session's node stays addressed to it forever. +func (r *RayLogHandler) ensureRotatedCollectionForNode(sessionDir, nodeID string) { + sup := r.rotatedCollection() + if sup == nil { + return + } + session, logsDir, ok := resolveSessionIdentity(sessionDir) + if !ok { + return + } + sup.ensure(session, strings.TrimSpace(nodeID), logsDir) +} + +// sessionTransition is what the session poller knows about where the runtime is +// between two Ray sessions. +// +// The four facts are kept apart because they become true at different times and fail +// independently. Inferring them from one "last directory" variable plus the handler's +// mutable node ID is what let a session be protected under its predecessor's node, and +// let a failed relocation retry retire a collector that was already correct. +type sessionTransition struct { + // dir is the session directory session_latest resolves to. + dir string + // node is the node ID this runtime is running the session at dir under. Empty means + // not established yet, and empty is never filled in from the handler's current + // node: that value belongs to whichever session was active last, and carrying it + // forward is what addressed a session's objects to its predecessor's node. + // + // "Verified" here means only that the dashboard answered after this session was + // observed — see currentNodeID for exactly how weak that binding is. + node string + // handedOff mirrors the supervisor: it records that an exact run for dir+node is + // attached. It is refreshed from the supervisor on every observation and is never + // trusted across one, because a run can fail at any point after it is attached. + handedOff bool + // pending is every outgoing session observed by this poller whose logs have not + // reached prev-logs yet, oldest first, each with the node it actually ran under. + // One shared node ID would file them all under whichever node was verified last. + pending []pendingRelocation + // sweepUnobserved records that the broad legacy sweep is still owed. It catches + // session directories this poller never observed — ones already stale when the + // collector started, or created and replaced between two ticks — which have no + // node of their own to be filed under. + sweepUnobserved bool +} + +// pendingRelocation is one outgoing session and the node it ran under. +type pendingRelocation struct { + dir string + node string +} + +// maxPendingRelocations bounds the queue. Each entry is one observed session change +// whose relocation has not succeeded, so reaching this means the filesystem has been +// refusing renames across many sessions; the oldest are handed to the broad sweep, +// which can still move them, rather than accumulating without limit. +const maxPendingRelocations = 16 + +// advanceSession moves the transition state on by one observation of session_latest. +// +// It is one function for the poller's first look and for every tick, because the two +// were the same problem and diverged: the startup path used to relocate and hand off +// under whatever node the handler happened to hold. +// +// The three steps are independent and individually idempotent, which is what makes a +// retry of one of them not a repeat of the others: +// +// 1. verify the node ID for the session that is active now; +// 2. hand rotated protection over, once, when that identity is known; +// 3. relocate the previous session's logs, retried until it succeeds. +// +// The ordering constraint between 2 and 3 is that nothing may own a tree that is about +// to move: step 2 retires the old collector as part of building the new one, and when +// step 2 cannot run — the node is unknown — step 3 retires it explicitly instead, but +// only if it belongs to a session that is no longer current. +func (r *RayLogHandler) advanceSession(st *sessionTransition, newDir string) { + if newDir == "" { + return + } + if !r.transitions.enter() { + // Shutdown has begun. A transition from here could move the live tree out from + // under the final legacy walk, or change the node identity that walk writes + // under, so none of its three steps may start. + return + } + defer r.transitions.leave() + + if newDir != st.dir { + if st.dir != "" { + logrus.Infof("PollActiveSessionChanges: session changed from %s to %s. Relocating old logs.", st.dir, newDir) + // Each outgoing session is queued with the node it actually ran under, so + // that a session whose relocation fails is still filed under its own node + // when it is retried, however many sessions come and go in between. + st.enqueueRelocation(pendingRelocation{dir: st.dir, node: st.node}) + st.sweepUnobserved = true + } + st.dir, st.node, st.handedOff = newDir, "", false + } + + // 1. Node identity. A rediscovery that disagrees is a genuine node change and needs + // a collector of its own; a rediscovery that fails leaves everything as it is, + // because failing to reach the dashboard says nothing about the collector that + // is already running. + if id, ok := r.currentNodeID(); ok && id != st.node { + st.node, st.handedOff = id, false + r.SetRayNodeName(id) + } + + // 2. Hand over rotated protection until the supervisor actually holds a run for + // this exact identity. handedOff is re-derived from the supervisor rather than + // remembered, because ensure returning is not the same as a collector existing, + // and a collector that attached can still fail on any startup step afterwards. + // Rebuilding is cheap to attempt and the supervisor's own backoff decides when + // it may actually happen. + st.handedOff = r.rotatedHandedOff(st.dir, st.node) + if st.node != "" && !st.handedOff { + r.ensureRotatedCollectionForNode(st.dir, st.node) + st.handedOff = r.rotatedHandedOff(st.dir, st.node) + } + + // 3. Relocate what earlier sessions left behind. + r.relocateOutgoingSessions(st) +} - // Resolve the session_latest symlink to get the real session directory - sessionLatestDir := utils.GetRaySessionLatestPath() - sessionRealDir, err := filepath.EvalSymlinks(sessionLatestDir) +// enqueueRelocation records one outgoing session, oldest first and bounded. +func (st *sessionTransition) enqueueRelocation(p pendingRelocation) { + for _, existing := range st.pending { + if existing.dir == p.dir { + return + } + } + st.pending = append(st.pending, p) + for len(st.pending) > maxPendingRelocations { + dropped := st.pending[0] + st.pending = st.pending[1:] + // The broad sweep can still move it; only its exact node is lost. + st.sweepUnobserved = true + logrus.Warnf("PollActiveSessionChanges: %d session relocations are outstanding, so %s is left to the broad sweep and may be filed under a later node.", + maxPendingRelocations, dropped.dir) + } +} + +// rotatedHandedOff reports whether the supervisor holds a run for this exact identity +// that is either starting or ready. A run whose goroutine has exited is not a handover: +// it is one that has to be made again. +func (r *RayLogHandler) rotatedHandedOff(sessionDir, nodeID string) bool { + sup := r.rotatedCollection() + if sup == nil || nodeID == "" { + return false + } + session, logsDir, ok := resolveSessionIdentity(sessionDir) + if !ok { + return false + } + switch sup.statusFor(session, strings.TrimSpace(nodeID), logsDir) { + case runStarting, runReady: + return true + case runAbsent, runFinished: + return false + default: + return false + } +} + +// relocateOutgoingSessions moves each observed outgoing session's logs into prev-logs, +// under the node that session actually ran on, and only then lets the broad legacy +// sweep collect whatever this poller never observed. +// +// The order is what keeps the labels honest. MoveLeftoverSessionLogs takes a single +// node ID and files *every* inactive session_* directory beneath it, so running it +// while a known session is still waiting would put that session's logs under a node +// that never wrote them. Each known session is therefore moved by itself first, and +// the sweep runs only once none are left. +// +// None of this is conditional on rotated collection having started. These logs are the +// legacy path's, this subsystem has no claim on them, and a dashboard that is briefly +// unreachable must not strand a whole session outside prev-logs. +func (r *RayLogHandler) relocateOutgoingSessions(st *sessionTransition) { + if len(st.pending) == 0 && !st.sweepUnobserved { + return + } + if r.beforeRelocation != nil { + r.beforeRelocation() + } + + // Nothing may own a tree that is about to move: a collector still gets its final + // reconciliation, and it gets it before the move rather than after, when there + // would be nothing left to scan. A collector for the *current* session is exempt — + // its tree is not the one moving, and retiring it because an unrelated relocation + // is being retried would cost a drain, a watcher and a staging reconstruction for + // nothing. + r.retireRotatedCollectionUnless(st.dir) + + // The label for anything whose own node is not known. The node verified for the + // current session is preferred; failing that — no dashboard has answered since this + // process started — it is the handler's node ID, which is what the legacy sweep has + // always used and is the only identity the runtime has. It is never an older + // session's verified node: that would file logs under a node they demonstrably did + // not run on. + fallbackNode := st.node + if fallbackNode == "" { + fallbackNode = strings.TrimSpace(r.GetRayNodeName()) + } + + var stuck []pendingRelocation + for _, p := range st.pending { + if p.node == "" { + // Its own node was never established, so it can only be filed under the + // fallback, which is exactly what the broad sweep below does. + st.sweepUnobserved = true + continue + } + if err := utils.MoveSessionLogsToPrevLogs(p.dir, p.node); err != nil { + logrus.Warnf("PollActiveSessionChanges: failed to relocate the logs of %s under node %s: %v. Retrying on next tick.", p.dir, p.node, err) + stuck = append(stuck, p) + continue + } + logrus.Infof("PollActiveSessionChanges: relocated the logs of %s under node %s", p.dir, p.node) + } + st.pending = stuck + + if len(stuck) > 0 { + // The sweep would file those sessions under the wrong node. It waits. + return + } + if !st.sweepUnobserved { + return + } + if fallbackNode == "" { + logrus.Warnf("PollActiveSessionChanges: no node ID is known, so any unobserved session directories stay in %s for now.", utils.GetTmpRayRoot()) + return + } + // Whatever is left was never observed by this poller — a directory that was already + // stale when the collector started, or one created and replaced between two ticks — + // so there is no node it can be said to belong to and the fallback is the only label + // available. This is the same choice the legacy sweep has always made. + if err := utils.MoveLeftoverSessionLogs(st.dir, fallbackNode); err != nil { + logrus.Warnf("PollActiveSessionChanges: failed to relocate leftover session logs: %v. Retrying on next tick.", err) + return + } + st.sweepUnobserved = false +} + +// retireRotatedCollectionUnless stops rotated collection unless it is already running +// for the session at sessionDir. A session directory that cannot be resolved names +// nothing, so nothing can be exempt from retirement. +func (r *RayLogHandler) retireRotatedCollectionUnless(sessionDir string) { + sup := r.rotatedCollection() + if sup == nil { + return + } + session, _, ok := resolveSessionIdentity(sessionDir) + if !ok { + session = "" + } + sup.retireUnless(session) +} + +// currentNodeID rediscovers this pod's Ray node ID, normalized to hex. The query is a +// live HTTP call with its own one-second timeout, and it fails routinely in the seconds +// after a session restart. +// +// utils.FetchCurrentNodeID asks "which ALIVE node does the dashboard report for this +// pod's IP?". It takes no session, so the answer is the freshest identity available +// rather than proof of which session it belongs to: a session change seen on the +// filesystem can precede the dashboard dropping the outgoing node, and the outgoing ID +// is then accepted for the new session. That is deliberate — refusing an unchanged ID +// would stop rotated collection entirely on a deployment where node IDs legitimately +// persist, which is worse than a mislabeled changeover window. +// +// Because it is the only identity the runtime has, callers must treat a failure as +// "not now" and wait rather than substitute a guess. +func (r *RayLogHandler) currentNodeID() (string, bool) { + if r.discoverNodeID != nil { + return r.discoverNodeID() + } + rawID, err := utils.FetchCurrentNodeID() + if err != nil || rawID == "" { + logrus.Debugf("Cannot discover the current Ray node ID: %v", err) + return "", false + } + hexID, err := utils.ConvertBase64ToHex(rawID) + if err != nil || hexID == "" { + logrus.Debugf("Cannot normalize the Ray node ID %q: %v", rawID, err) + return "", false + } + return hexID, true +} + +// resolveSessionIdentity turns a session directory into the real session name and the +// logs directory beneath it. +// +// The symlink is resolved first, and that is load-bearing rather than defensive. Ray's +// active session is reached through /tmp/ray/session_latest, and the session name is +// half of every staging path and every object key this subsystem writes. Taking the +// base of an unresolved path would name the session "session_latest": captures would +// be staged under a subtree no later collector recognizes as a session, and uploaded +// under a prefix the History Server never lists, while the legacy walk — which does +// resolve the symlink — kept writing the real session ID beside it. +// +// Failure to resolve is reported as "not now" rather than as a fault. A session +// directory that is briefly absent is exactly what the poller sees between two Ray +// sessions, and the next tick tries again. +func resolveSessionIdentity(sessionDir string) (session, logsDir string, ok bool) { + if strings.TrimSpace(sessionDir) == "" { + return "", "", false + } + resolved, err := filepath.EvalSymlinks(sessionDir) + if err != nil { + logrus.Debugf("Rotated log collection: cannot resolve session directory %s yet: %v", sessionDir, err) + return "", "", false + } + session = filepath.Base(resolved) + if session == sessionLatestLinkName || session == "." || session == string(filepath.Separator) { + logrus.Warnf("Rotated log collection: %s did not resolve to a real Ray session directory (got %q), so it is not started for it", sessionDir, session) + return "", "", false + } + return session, filepath.Join(resolved, utils.RAY_SESSIONDIR_LOGDIR_NAME), true +} + +// shutdownSnapshot is what "the live session" meant at the moment shutdown began. +// +// Every field is taken once and then never re-read. session_latest is a symlink Ray +// repoints whenever it restarts a session, and the node ID is a mutable field the +// session poller writes; re-reading either one per file would let a session change +// during the walk split one shutdown across two identities — some objects under the +// old session, some under the new, and relative paths computed against a logs +// directory that is no longer the one being walked. +type shutdownSnapshot struct { + // sessionDir is the resolved real session directory, never the symlink. + sessionDir string + sessionID string + nodeID string + // logsDir is beneath sessionDir, so the walk stays inside the real tree even if + // session_latest is repointed while it runs. + logsDir string +} + +// takeShutdownSnapshot resolves session_latest and the node ID exactly once. +func (r *RayLogHandler) takeShutdownSnapshot() (shutdownSnapshot, bool) { + sessionRealDir, err := filepath.EvalSymlinks(utils.GetRaySessionLatestPath()) if err != nil { logrus.Errorf("Failed to resolve session_latest symlink: %v", err) + return shutdownSnapshot{}, false + } + return shutdownSnapshot{ + sessionDir: sessionRealDir, + sessionID: filepath.Base(sessionRealDir), + // Use the already discovered node ID instead of retrying network requests + // during shutdown. + nodeID: strings.TrimSpace(r.GetRayNodeName()), + logsDir: filepath.Join(sessionRealDir, utils.RAY_SESSIONDIR_LOGDIR_NAME), + }, true +} + +// processSessionLatestLogs processes logs in the active session's logs directory on +// shutdown, using the real session ID and node ID. +func (r *RayLogHandler) processSessionLatestLogs() { + snap, ok := r.takeShutdownSnapshot() + if !ok { return } + r.processSessionLogs(snap) +} + +// processSessionLogs walks one immutable snapshot of the live tree. +func (r *RayLogHandler) processSessionLogs(snap shutdownSnapshot) { + logrus.Infof("Processing logs of session %s on shutdown...", snap.sessionID) - // Extract the real session ID from the resolved path - sessionID := filepath.Base(sessionRealDir) + sessionID := snap.sessionID if r.IsHead { metafile := clustermetadata.EncodePath( utils.ClusterInfo{ @@ -138,11 +690,8 @@ func (r *RayLogHandler) processSessionLatestLogs() { } } - // Use already discovered node ID (RayNodeName) instead of retrying network requests during shutdown - nodeID := strings.TrimSpace(r.GetRayNodeName()) - - // Process logs in session_latest/logs - logsDir := filepath.Join(sessionLatestDir, utils.RAY_SESSIONDIR_LOGDIR_NAME) + // Process the logs of the snapshotted session, beneath its real directory. + logsDir := snap.logsDir dirExist := false for i := 0; i < 10; i++ { if _, err := os.Stat(logsDir); os.IsNotExist(err) { @@ -159,7 +708,7 @@ func (r *RayLogHandler) processSessionLatestLogs() { } // Walk through the logs directory and process all files - err = filepath.WalkDir(logsDir, func(path string, info fs.DirEntry, err error) error { + err := filepath.WalkDir(logsDir, func(path string, info fs.DirEntry, err error) error { if err != nil { logrus.Errorf("Error walking logs path %s: %v", path, err) return nil @@ -170,9 +719,16 @@ func (r *RayLogHandler) processSessionLatestLogs() { return nil } - // Process log file with the real session ID and node ID - if err := r.processSessionLatestLogFile(path, sessionID, nodeID); err != nil { - logrus.Errorf("Failed to process session_latest log file %s: %v", path, err) + // Every regular file in the live tree is uploaded here, exactly as it was + // before rotated collection existed. A file the rotated subsystem has already + // captured is not a duplicate of anything this walk writes: that capture went + // to ".rotated.", and the key below is "". Skipping it + // would not save a write, it would delete a key. + // + // Process log file against the snapshot, so every object of this shutdown + // belongs to one session, one node and one logs directory. + if err := r.processSessionLogFile(snap, path); err != nil { + logrus.Errorf("Failed to process session log file %s: %v", path, err) } return nil @@ -181,16 +737,15 @@ func (r *RayLogHandler) processSessionLatestLogs() { logrus.Errorf("Error walking logs directory %s: %v", logsDir, err) } - logrus.Info("Finished processing session_latest logs") + logrus.Infof("Finished processing logs of session %s", snap.sessionID) } -// processSessionLatestLogFile processes a single log file from session_latest -func (r *RayLogHandler) processSessionLatestLogFile(absoluteLogPathName, sessionID, nodeID string) error { - // Calculate relative path within logs directory - // The logsDir is the configured session_latest/logs directory. - sessionLatestDir := utils.GetRaySessionLatestPath() - logsDir := filepath.Join(sessionLatestDir, utils.RAY_SESSIONDIR_LOGDIR_NAME) - relativePath, err := filepath.Rel(logsDir, absoluteLogPathName) +// processSessionLogFile processes a single log file from the snapshotted session. +func (r *RayLogHandler) processSessionLogFile(snap shutdownSnapshot, absoluteLogPathName string) error { + sessionID, nodeID := snap.sessionID, snap.nodeID + // The relative path is computed against the same real logs directory the walk + // used, never against session_latest, which may point elsewhere by now. + relativePath, err := filepath.Rel(snap.logsDir, absoluteLogPathName) if err != nil { return fmt.Errorf("failed to get relative path for %s: %w", absoluteLogPathName, err) } @@ -794,31 +1349,56 @@ func (r *RayLogHandler) WatchSessionLatestLoops() { // Polls if the active session changes, when it does, it moves the old session logs to a prev-logs/ folder. func (r *RayLogHandler) PollActiveSessionChanges() { tmpRayRoot := utils.GetTmpRayRoot() - symlinkPath := filepath.Join(tmpRayRoot, "session_latest") + symlinkPath := filepath.Join(tmpRayRoot, sessionLatestLinkName) + + // Run has already started rotated protection for the session the handler was + // configured with, under the node ID main.go discovered and validated for it before + // the handler existed, so the transition starts life already handed off for it. + // + // The startup directory is resolved first, because it is compared against a + // resolved symlink target on every observation. The configured value and that + // target are routinely two spellings of one directory — a symlinked /tmp, or macOS + // putting /var behind /private/var — and treating that as a session change would + // discard a node ID that was verified for exactly this session and leave the + // runtime pretending it has never seen one. + startupDir := strings.TrimSpace(r.SessionDir) + if resolved, err := filepath.EvalSymlinks(startupDir); err == nil && resolved != "" { + startupDir = resolved + } + st := sessionTransition{ + dir: startupDir, + node: strings.TrimSpace(r.GetRayNodeName()), + handedOff: true, + } - var lastResolvedDir string + // The first observation is not deferred by a tick. A session that changed between + // the handler being configured and this goroutine starting has logs to relocate + // now, which is what the startup block here has always done. + // + // The fallback is startupDir, not the raw SessionDir it was derived from, so that it + // is the same string st.dir holds. advanceSession compares the two directly, and the + // raw value is routinely a second spelling of the resolved one — utils.GetSessionDir + // falls back to os.Readlink, which resolves only the session_latest link and leaves + // any symlinked component above it in place. Passing that spelling here would look + // like a session change and relocate the logs of the session that is still running. + // Nothing is lost by declining to act: SessionDir names the session this poller + // started with, so it can never be evidence of a new one, and the first tick that + // resolves session_latest observes any real change. currentActiveDir, err := filepath.EvalSymlinks(symlinkPath) - if err == nil && currentActiveDir != "" { - if r.SessionDir != "" && currentActiveDir != r.SessionDir { - logrus.Infof("PollActiveSessionChanges: detected startup session change from %s to %s. Relocating startup session logs.", r.SessionDir, currentActiveDir) - if err := utils.MoveLeftoverSessionLogs(currentActiveDir, r.GetRayNodeName()); err != nil { - logrus.Warnf("PollActiveSessionChanges: failed to relocate startup session logs: %v. Retrying on next poll tick.", err) - lastResolvedDir = r.SessionDir - } else { - lastResolvedDir = currentActiveDir - } - } else { - lastResolvedDir = currentActiveDir - } - } else { - logrus.Warnf("PollActiveSessionChanges: failed to resolve initial session_latest target: %v. Falling back to startup SessionDir %s", err, r.SessionDir) - lastResolvedDir = r.SessionDir + if err != nil || currentActiveDir == "" { + logrus.Warnf("PollActiveSessionChanges: failed to resolve initial session_latest target: %v. Falling back to startup session directory %s", err, startupDir) + currentActiveDir = startupDir } + r.advanceSession(&st, currentActiveDir) - ticker := time.NewTicker(5 * time.Second) + interval := r.sessionPollInterval + if interval <= 0 { + interval = defaultSessionPollInterval + } + ticker := time.NewTicker(interval) defer ticker.Stop() - logrus.Infof("Started polling active session changes at: %s (initial target: %s)", symlinkPath, lastResolvedDir) + logrus.Infof("Started polling active session changes at: %s (initial target: %s)", symlinkPath, st.dir) for { select { case <-r.ShutdownChan: @@ -829,21 +1409,7 @@ func (r *RayLogHandler) PollActiveSessionChanges() { if err != nil || newResolvedDir == "" { continue } - - if lastResolvedDir != "" && newResolvedDir != lastResolvedDir { - logrus.Infof("PollActiveSessionChanges: session changed from %s to %s. Relocating old logs.", lastResolvedDir, newResolvedDir) - if err := utils.MoveLeftoverSessionLogs(newResolvedDir, r.GetRayNodeName()); err != nil { - logrus.Warnf("PollActiveSessionChanges: failed to relocate leftover session logs from %s to %s: %v. Retrying on next tick.", lastResolvedDir, newResolvedDir, err) - continue - } - } - lastResolvedDir = newResolvedDir - - if freshNodeID, err := utils.FetchCurrentNodeID(); err == nil && freshNodeID != "" { - if hexID, err := utils.ConvertBase64ToHex(freshNodeID); err == nil && hexID != "" { - r.SetRayNodeName(hexID) - } - } + r.advanceSession(&st, newResolvedDir) } } } diff --git a/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_collector.go b/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_collector.go new file mode 100644 index 00000000000..541699ac3ef --- /dev/null +++ b/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_collector.go @@ -0,0 +1,992 @@ +package logcollector + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "slices" + "strings" + "syscall" + "time" + + "github.com/fsnotify/fsnotify" + "github.com/sirupsen/logrus" +) + +// defaultReconcileInterval is the backstop sweep period. fsnotify can drop events +// under load, and a watch added to a directory cannot see what was already in it, +// so discovery never relies on events alone. +const defaultReconcileInterval = 30 * time.Second + +// errIntakePaused reports that the staging volume is full. Capture stops adding new +// data but never deletes what it already holds. Uploads, promotions and releases keep +// running, and it is a release — or a probe link that succeeds — that lifts the pause. +var errIntakePaused = errors.New("staging volume full: rotated log intake paused") + +// fsWatcher is the slice of fsnotify the collector uses, so tests can drive events +// deterministically instead of waiting on the kernel. +type fsWatcher interface { + Add(name string) error + Close() error + Events() <-chan fsnotify.Event + Errors() <-chan error +} + +// fsnotifyWatcher adapts *fsnotify.Watcher, whose Events and Errors are fields. +type fsnotifyWatcher struct{ w *fsnotify.Watcher } + +func newFsnotifyWatcher() (fsWatcher, error) { + w, err := fsnotify.NewWatcher() + if err != nil { + return nil, fmt.Errorf("create fsnotify watcher: %w", err) + } + return &fsnotifyWatcher{w: w}, nil +} + +func (f *fsnotifyWatcher) Add(name string) error { return f.w.Add(name) } +func (f *fsnotifyWatcher) Close() error { return f.w.Close() } +func (f *fsnotifyWatcher) Events() <-chan fsnotify.Event { return f.w.Events } +func (f *fsnotifyWatcher) Errors() <-chan error { return f.w.Errors } + +// rotatedCollectorConfig configures one collector. Everything the loop touches that +// is not a plain filesystem operation is injectable, so tests can be deterministic +// while still exercising real files, real hard links and real link counts. +type rotatedCollectorConfig struct { + LogsDir string // the active session's logs directory + StagingRoot string + SessionName string + NodeName string + + // Cluster is where captures land in object storage. Writer is what puts them + // there, and production always supplies one. A nil Writer leaves the collector + // capturing and accounting normally with the upload pipeline switched off, which + // is how tests exercise capture on its own. + Cluster clusterIdentity + Writer objectWriter + + // HighWaterBytes pauses new intake once the staging volume holds that many bytes, + // and LowWaterBytes resumes it once the total falls back to that. Zero + // HighWaterBytes disables the watermarks; a filesystem that reports ENOSPC still + // pauses intake either way. Defaults belong to the tranche that configures this + // from the operator, not here. + HighWaterBytes int64 + LowWaterBytes int64 + + ReconcileInterval time.Duration + NewWatcher func() (fsWatcher, error) + NewTicker func(time.Duration) (<-chan time.Time, func()) + CaptureIDs *captureIDGenerator + + // UploadBackoff is the delay before each successive upload retry, with the last + // entry repeating. Now and NewTimer are the clock the retry schedule runs on, so + // tests can advance time instead of waiting for it. + UploadBackoff []time.Duration + Now func() time.Time + NewTimer func(time.Duration) (<-chan time.Time, func()) + + // WorkerStopGrace bounds how long Stop waits for an upload the storage interface + // gives it no way to cancel. + WorkerStopGrace time.Duration + + // Link creates the staging hard link. Production always uses captureLink; tests + // replace it to make the race between validating a candidate and pinning it + // deterministic. + Link func(src, dst string) error + + // BeforeReconstruct runs after watches are installed and before staging + // reconstruction. Tests hold the collector in that window to prove the tree is + // already covered while reconstruction is in progress. + BeforeReconstruct func() + + // OnReady runs on the owner goroutine once every startup step has succeeded and + // the loop is about to begin. It is how the runtime tells a collector that is + // genuinely protecting its tree from one that merely got as far as being + // constructed. It must not block. + OnReady func() + + // OnIssue receives problems that must not stop discovery: a segment lost to a + // rotation race, an unsupported filesystem, a corrupt staging record. + OnIssue func(error) +} + +// rotatedCollector discovers Ray's rotated log segments and pins them with hard +// links before rotation can delete them. +// +// One goroutine owns all state. fsnotify events, the reconcile ticker, startup +// reconstruction and callers' requests all become work performed by that goroutine, +// so "look up the inode, create the link, register the capture" is indivisible +// without a single mutex. +// +// Object storage is reached only from the upload worker, never from this goroutine. +// An upload is slow and, through storage.StorageWriter, uncancelable, so letting one +// sit between a rotation and its capture would lose exactly the segments this +// collector exists to keep. The owner decides everything about an upload — when it +// starts, whether its result still applies, what happens next — but never waits for +// one. +type rotatedCollector struct { + cfg rotatedCollectorConfig + ix *captureIndex + watcher fsWatcher + + // up, bytes and gate are owned by the same goroutine as ix. Uploading is the one + // slow thing the collector does, so it is pushed onto a worker that returns an + // immutable result; every decision that result leads to is made here. + up *uploadScheduler + bytes *stagedBytes + gate *intakeGate + + // intakePauses and intakeResumes count gate transitions. A wrongly reopened gate + // can be closed again before anyone looks at it, so the counts — not the current + // flag — are what show intake was reopened at all. + intakePauses int + intakeResumes int + + stopCh chan struct{} + doneCh chan struct{} + + snapshotReq chan chan []stagedEntry + reconcileReq chan chan struct{} + statsReq chan chan collectorStats +} + +// collectorStats is a consistent view of the collector's own bookkeeping. Like +// snapshot it is produced by the owner goroutine, because staged bytes, the upload +// queue and the intake gate are owner-owned and must never be read from outside it. +type collectorStats struct { + // StagedBytes is every logical byte the collector has pinned. RetainedBytes is + // the subset the collector alone is keeping allocated, which is what the intake + // watermark is measured against. + StagedBytes int64 + RetainedBytes int64 + Captures int + Pending int + Uploaded int + QueuedUploads int + InFlightUploads int + AwaitingPromotion int + IntakePauses int + IntakeResumes int + IntakePaused bool +} + +func newRotatedCollector(cfg rotatedCollectorConfig) (*rotatedCollector, error) { + if cfg.LogsDir == "" || cfg.StagingRoot == "" { + return nil, fmt.Errorf("rotated collector: LogsDir and StagingRoot are required") + } + if err := validatePathSegment("session name", cfg.SessionName); err != nil { + return nil, err + } + if err := validatePathSegment("node name", cfg.NodeName); err != nil { + return nil, err + } + if cfg.ReconcileInterval <= 0 { + cfg.ReconcileInterval = defaultReconcileInterval + } + if cfg.NewWatcher == nil { + cfg.NewWatcher = newFsnotifyWatcher + } + if cfg.NewTicker == nil { + cfg.NewTicker = func(d time.Duration) (<-chan time.Time, func()) { + t := time.NewTicker(d) + return t.C, t.Stop + } + } + if cfg.CaptureIDs == nil { + cfg.CaptureIDs = newCaptureIDGenerator() + } + if cfg.Link == nil { + cfg.Link = captureLink + } + if cfg.OnIssue == nil { + cfg.OnIssue = func(err error) { logrus.Warnf("Rotated log collector: %v", err) } + } + if cfg.Now == nil { + cfg.Now = time.Now + } + if cfg.NewTimer == nil { + cfg.NewTimer = func(d time.Duration) (<-chan time.Time, func()) { + t := time.NewTimer(d) + return t.C, func() { t.Stop() } + } + } + if len(cfg.UploadBackoff) == 0 { + cfg.UploadBackoff = defaultUploadBackoff + } + for i, d := range cfg.UploadBackoff { + // A zero or negative delay would make a failing upload due the instant it + // failed, turning the retry schedule into a spin against the object store. + if d <= 0 { + return nil, fmt.Errorf("rotated collector: UploadBackoff[%d] is %s, but every retry delay must be positive", i, d) + } + } + if cfg.WorkerStopGrace <= 0 { + cfg.WorkerStopGrace = defaultWorkerStopGrace + } + if err := validateWatermarks(cfg.HighWaterBytes, cfg.LowWaterBytes); err != nil { + return nil, err + } + + return &rotatedCollector{ + cfg: cfg, + ix: newCaptureIndex(), + up: newUploadScheduler(cfg.Writer, cfg.UploadBackoff), + bytes: newStagedBytes(), + gate: &intakeGate{high: cfg.HighWaterBytes, low: cfg.LowWaterBytes}, + stopCh: make(chan struct{}), + doneCh: make(chan struct{}), + snapshotReq: make(chan chan []stagedEntry), + reconcileReq: make(chan chan struct{}), + statsReq: make(chan chan collectorStats), + }, nil +} + +// Run owns the collector's state until Stop is called. Callers run it in its own +// goroutine. +func (rc *rotatedCollector) Run() error { + defer close(rc.doneCh) + + w, err := rc.cfg.NewWatcher() + if err != nil { + // Same boundary, same reasoning as the watch installation below: the kernel + // refusing another inotify instance or descriptor is about what else the + // machine is doing, and it clears. Anything else about constructing a watcher + // is a property of the deployment and is left durable. + if isWatchResourceExhausted(err) { + return retryableRotated(err) + } + return err + } + rc.watcher = w + defer func() { + if err := w.Close(); err != nil { + rc.report(fmt.Errorf("close watcher: %w", err)) + } + }() + + // Startup order is load-bearing. + // + // Watches go on first, and install nothing else: until a directory is watched, + // a segment can be created and deleted inside it without leaving any trace for + // a later scan to find. Reconstruction can take a while on a large staging + // volume, and that whole window would otherwise be blind. + // + // Reconstruction then runs before any capture, so a restart adopts what the + // previous run pinned instead of minting a second ID for it. Only then does the + // full scan capture, and only then are the events that queued in the watcher + // channel since step one processed. + if err := rc.installWatchesRecursive(rc.cfg.LogsDir); err != nil { + err = fmt.Errorf("rotated log collector cannot watch the whole logs tree: %w", err) + // Watch installation is the one startup step whose failure can be about the + // moment rather than the deployment, and it is the only place that can say so: + // the logs tree disappearing here is a session that ended mid-startup, and an + // exhausted inotify or descriptor limit clears when whatever consumed it + // releases it. Both are worth another attempt on a later tick. A permission the + // collector does not have is not, and is left durable by omission. + // + // Nothing later in Run is marked: from here on, an error means the staging + // volume and the index disagree, and retrying that is a restart loop. + if isVanished(err) || isWatchResourceExhausted(err) { + return retryableRotated(err) + } + return err + } + + if rc.cfg.BeforeReconstruct != nil { + rc.cfg.BeforeReconstruct() + } + if err := rc.reconstructStaging(); err != nil { + return err + } + + // Settle the gate on what the previous run left staged before the live scan can + // add to it. A restart that adopts a volume already over its limit must not then + // capture its way further past it. + rc.enforceHighWater() + + rc.scanTree() + + // The worker starts only once reconstruction has established what is already + // staged, so nothing is uploaded before the collector knows whether a previous + // run already sent it. + rc.up.start(rc.cfg.StagingRoot) + defer rc.up.stop(rc.cfg.WorkerStopGrace, rc.report) + + // Adopt the previous run's work: pending captures are queued, uploaded ones are + // never re-sent but are always candidates for release. + rc.sweepUploads() + rc.sweepReleases() + + tick, stopTicker := rc.cfg.NewTicker(rc.cfg.ReconcileInterval) + defer stopTicker() + + // Startup is complete and nothing above it can fail any more: the watcher exists, + // the whole tree is watched, staging has been reconstructed, the live scan has run, + // the uploader is up and the previous run's work has been rescheduled. Only from + // here is the collector actually protecting anything, which is why this — and not + // the moment it was constructed — is what tells the supervisor it recovered. + if rc.cfg.OnReady != nil { + rc.cfg.OnReady() + } + + for { + // Housekeeping runs before any request is served, so a caller that gets a + // snapshot is looking at state the scheduler has already acted on. It never + // blocks and never performs a storage call. It fails only for a condition + // that retrying cannot fix. + if err := rc.pump(); err != nil { + return err + } + + select { + case <-rc.stopCh: + return nil + + case event, ok := <-w.Events(): + if !ok { + return errors.New("rotated log watcher event channel closed") + } + rc.handleEvent(event) + + case err, ok := <-w.Errors(): + if !ok { + return errors.New("rotated log watcher error channel closed") + } + // An overflow means events were dropped, so the only safe response is + // to look at the tree again immediately rather than wait for the tick. + if errors.Is(err, fsnotify.ErrEventOverflow) { + rc.report(fmt.Errorf("watcher overflowed, reconciling immediately: %w", err)) + rc.maintain() + continue + } + rc.report(fmt.Errorf("watcher error: %w", err)) + + case <-tick: + rc.maintain() + + // An upload finished. Applying its result is pure bookkeeping: the slow part + // already happened on the worker. It stops the collector only when the + // worker found the staging volume contradicting the index. + case res := <-rc.uploadResults(): + if err := rc.applyUploadResult(res); err != nil { + return err + } + + case <-rc.retryTimer(): + rc.processDue() + + case reply := <-rc.snapshotReq: + reply <- rc.ix.entries() + + case reply := <-rc.statsReq: + reply <- rc.currentStats() + + case reply := <-rc.reconcileReq: + rc.maintain() + // The caller is answered even when the pump fails, so a reconcileNow in + // flight cannot be left waiting on a collector that is shutting down. + err := rc.pump() + reply <- struct{}{} + if err != nil { + return err + } + } + } +} + +// Stop asks the loop to exit and waits for it. It is safe to call more than once. +func (rc *rotatedCollector) Stop() { + select { + case <-rc.stopCh: + default: + close(rc.stopCh) + } + <-rc.doneCh +} + +// snapshot returns the tracked captures. The answer is produced by the owner +// goroutine, so callers never read the index themselves. +func (rc *rotatedCollector) snapshot() []stagedEntry { + reply := make(chan []stagedEntry, 1) + select { + case rc.snapshotReq <- reply: + return <-reply + case <-rc.doneCh: + return nil + } +} + +// stats returns the collector's bookkeeping, computed by the owner goroutine. +func (rc *rotatedCollector) stats() collectorStats { + reply := make(chan collectorStats, 1) + select { + case rc.statsReq <- reply: + return <-reply + case <-rc.doneCh: + return collectorStats{} + } +} + +// currentStats runs on the owner goroutine. +func (rc *rotatedCollector) currentStats() collectorStats { + s := collectorStats{ + StagedBytes: rc.bytes.total, + RetainedBytes: rc.bytes.retained, + Captures: rc.ix.len(), + QueuedUploads: len(rc.up.queue), + InFlightUploads: rc.up.inFlight, + IntakePauses: rc.intakePauses, + IntakeResumes: rc.intakeResumes, + IntakePaused: rc.intakePaused(), + } + for _, c := range rc.ix.byInode { + if c.Entry.State == stateUploaded { + s.Uploaded++ + } else { + s.Pending++ + } + } + for _, st := range rc.up.states { + if st.phase == phaseAwaitingPromotion { + s.AwaitingPromotion++ + } + } + return s +} + +// reconcileNow runs one full sweep on the owner goroutine and waits for it. +func (rc *rotatedCollector) reconcileNow() { + reply := make(chan struct{}, 1) + select { + case rc.reconcileReq <- reply: + <-reply + case <-rc.doneCh: + } +} + +func (rc *rotatedCollector) report(err error) { + if err != nil { + rc.cfg.OnIssue(err) + } +} + +// handleEvent turns one filesystem event into discovery work. It performs no +// storage calls and no waiting, so a rotation cannot delete a segment while the +// loop is busy elsewhere. +func (rc *rotatedCollector) handleEvent(event fsnotify.Event) { + if !rc.underLogsDir(event.Name) { + return + } + + // Only creations matter. Rotation's rename surfaces as a Create on the + // destination, and removals leave nothing to inspect. + // + // Writes are deliberately ignored: every append to every active log would enter + // this loop, and that traffic can fill the kernel queue and delay the one event + // that matters — the Create of a rotation backup. Active file names are learned + // from the startup scan, from their own Create events, from the scan that + // follows watching a new directory, and from the periodic sweep. + if !event.Op.Has(fsnotify.Create) { + return + } + + fi, err := os.Lstat(event.Name) + if err != nil { + if !isVanished(err) { + rc.report(fmt.Errorf("stat %s: %w", event.Name, err)) + } + return + } + + switch { + case fi.IsDir(): + // A watch cannot see what a directory already contained, so scan it the + // moment it is watched. + rc.watchAndScan(event.Name) + case fi.Mode().IsRegular(): + rc.inspectFile(event.Name, fi) + } +} + +// underLogsDir reports whether path is inside the active logs tree. Events for +// anything else are not this collector's business. +func (rc *rotatedCollector) underLogsDir(path string) bool { + rel, err := filepath.Rel(rc.cfg.LogsDir, path) + if err != nil { + return false + } + return rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) +} + +// scanTree walks the active logs tree, watching every directory and inspecting +// every file. It is the startup scan, the periodic backstop and the overflow +// recovery, and it is idempotent: captures are keyed by inode, so a file seen twice +// is captured once. +func (rc *rotatedCollector) scanTree() { + rc.watchAndScan(rc.cfg.LogsDir) +} + +// installWatchesRecursive watches dir and every directory beneath it and does +// nothing else. It runs before staging reconstruction so that the tree is covered +// while that work happens; capturing at this point would mint IDs for segments the +// previous run may already hold. +// +// The parent is watched before its children are enumerated, so a directory created +// during the walk is reported through its parent's watch even if the enumeration +// missed it. +// +// Incomplete coverage is fatal. Starting with part of the tree unwatched looks +// healthy but silently loses any segment that is created and deleted in the gap +// between sweeps, which is exactly the failure this collector exists to prevent. +// The one tolerated failure is a directory that disappeared after its parent was +// watched: that is an ordinary race, and if it comes back the parent's watch +// reports it. +func (rc *rotatedCollector) installWatchesRecursive(dir string) error { + isRoot := dir == rc.cfg.LogsDir + + if rc.watcher != nil { + if err := rc.watcher.Add(dir); err != nil { + if isVanished(err) && !isRoot { + return nil + } + return fmt.Errorf("watch %s: %w", dir, err) + } + } + + entries, err := os.ReadDir(dir) + if err != nil { + if isVanished(err) && !isRoot { + return nil + } + return fmt.Errorf("read directory %s: %w", dir, err) + } + for _, entry := range entries { + if entry.Type()&fs.ModeSymlink != 0 || !entry.IsDir() { + continue + } + if err := rc.installWatchesRecursive(filepath.Join(dir, entry.Name())); err != nil { + return err + } + } + return nil +} + +// watchAndScan watches dir and everything beneath it, scanning each directory +// immediately after its watch is added so nothing that already existed is missed. +func (rc *rotatedCollector) watchAndScan(dir string) { + if rc.watcher != nil { + if err := rc.watcher.Add(dir); err != nil { + if isVanished(err) { + return + } + rc.report(fmt.Errorf("watch %s: %w", dir, err)) + } + } + + entries, err := os.ReadDir(dir) + if err != nil { + if !isVanished(err) { + rc.report(fmt.Errorf("read directory %s: %w", dir, err)) + } + return + } + + for _, entry := range entries { + path := filepath.Join(dir, entry.Name()) + + // Never follow symlinks: a symlinked directory could point anywhere, and a + // symlinked file is not a log Ray rotated. + if entry.Type()&fs.ModeSymlink != 0 { + continue + } + if entry.IsDir() { + rc.watchAndScan(path) + continue + } + + fi, err := entry.Info() + if err != nil { + if !isVanished(err) { + rc.report(fmt.Errorf("stat %s: %w", path, err)) + } + continue + } + rc.inspectFile(path, fi) + } +} + +// inspectFile records an active log file or captures a rotation backup. +func (rc *rotatedCollector) inspectFile(path string, fi fs.FileInfo) { + if !fi.Mode().IsRegular() { + return + } + dir, name := filepath.Split(path) + dir = filepath.Clean(dir) + + if _, _, isBackup := parseBackupName(name); !isBackup { + // Remembering the active file is what lets a backup still be recognized + // during the moment a rotation cascade leaves the active name unlinked. + rc.ix.observeBase(dir, name) + return + } + + switch status := evaluateCandidate(dir, name, fi.Mode(), baseKnownWith(rc.ix)); status { + case candidateEligible: + rc.report(rc.capture(path, name)) + case candidateNotBackupName, candidateNotRegular, candidateUnknownBase: + logrus.Debugf("Rotated log collector: skipping %s: %v", path, status) + } +} + +// capture pins one rotation backup. +// +// The hard link — not the source path — decides which inode was captured. A +// rotation filename is reused constantly, so between validating the candidate and +// linking it, "raylet.out.1" can already name a different file. Reading the inode +// from the source beforehand would let the staging link pin one file while the index +// records another, which would break deduplication, restart recovery and release. +// So the inode is read back from the link we created, and that value alone is +// registered. +// +// For the same reason there is no dedup shortcut before linking: an early return +// because the source's current inode looks familiar could skip a generation that had +// just replaced it, and rotation would then delete that generation unseen. +// Intake is the only thing backpressure stops. Nothing already captured is evicted, +// and uploads, promotions and releases keep running, which is what eventually brings +// the total back down. The cost is honest and unavoidable: a backup Ray rotates away +// while intake is paused is lost, because there is nowhere to pin it. This is a +// fail-safe against exhausting the volume Ray itself is writing to, not a promise of +// lossless capture under unbounded pressure. +// While paused, one capture per maintenance sweep is still allowed through as a +// capacity probe. Without it a disk-full pause would latch permanently: capture would +// return before ever attempting a link, so if another process filled the volume and +// this collector holds nothing it can release, nothing would ever discover that space +// came back. Using a real eligible backup as the probe means a successful attempt +// preserves data instead of merely testing the filesystem. +func (rc *rotatedCollector) capture(path, name string) error { + if rc.intakePaused() && !rc.gate.takeProbe() { + logrus.Debugf("Rotated log collector: intake is paused, not capturing %s", path) + return nil + } + + fi, err := os.Lstat(path) + if err != nil { + if isVanished(err) { + // Rotation deleted it first. Expected under fast rotation, not an error. + logrus.Debugf("Rotated log collector: %s vanished before capture", path) + return nil + } + return fmt.Errorf("stat %s: %w", path, err) + } + if !fi.Mode().IsRegular() { + return nil + } + // Diagnostics only. This value must never reach captureIndex. + sourceInode, _, inodeErr := inodeFromFileInfo(fi) + if inodeErr != nil { + return fmt.Errorf("read inode of %s: %w", path, inodeErr) + } + + relDir, err := relDirFor(rc.cfg.LogsDir, path) + if err != nil { + return err + } + id, err := rc.cfg.CaptureIDs.next() + if err != nil { + return err + } + entry, err := newStagedEntry(statePending, rc.cfg.SessionName, rc.cfg.NodeName, relDir, name, id) + if err != nil { + return err + } + staged := entry.path(rc.cfg.StagingRoot) + + if err := rc.cfg.Link(path, staged); err != nil { + switch { + case isVanished(err): + logrus.Debugf("Rotated log collector: %s vanished before it could be linked", path) + return nil + case isUnsupportedLinkError(err): + // A different filesystem or a Ray container running as another user. + // Skip the segment and keep discovering; v1 has no copy fallback. + return fmt.Errorf("cannot capture %s on this deployment: %w", path, err) + case errors.Is(err, syscall.ENOSPC): + // The volume is full regardless of what the watermarks think the + // collector is holding: something else may have filled it. Stop taking + // new data until some operation proves there is room again — and discard + // any earlier success from this same sweep, because the filesystem has + // just contradicted it. + // + // Only the transition is reported. Every later sweep spends its probe + // here while the volume stays full, and repeating the same line once per + // sweep would bury the state change that mattered. + if !rc.gate.observedFull() { + logrus.Debugf("Rotated log collector: staging volume is still full, %s was not captured", path) + return nil + } + return fmt.Errorf("%w: %w", errIntakePaused, err) + default: + return err + } + } + + // A link that succeeded is proof the volume has room, whoever freed it. That is + // what lifts a disk-full pause, and it is the only thing that can: elapsed time + // says nothing about a volume another process filled. A later ENOSPC in this same + // sweep discards this observation again. + rc.gate.observedSpace() + + key, size, nlink, err := rc.pinnedInode(staged) + if err != nil { + return errors.Join(err, discardStagingLink(staged)) + } + if key != sourceInode { + logrus.Debugf("Rotated log collector: %s changed from %s to %s before it was pinned; the pinned file wins", + path, sourceInode, key) + } + + if existing, alreadyCaptured := rc.ix.lookup(key); alreadyCaptured { + // Another name for a file we already hold, most often the same segment seen + // again after rotation renamed it. Drop the surplus link and keep the + // original capture: one inode is one capture, with one object key. The + // surplus link was never accounted for, so removing it changes no total. + logrus.Debugf("Rotated log collector: %s is already captured as %s", path, existing.Entry.CaptureID) + return discardStagingLink(staged) + } + + if err := registerStaged(rc.cfg.StagingRoot, rc.ix, key, entry); err != nil { + return err + } + // Accounting and scheduling follow registration, never precede it: a capture the + // index rejected has had its link removed and must leave no trace behind. + // + // os.Link only succeeds against a source that exists, so a fresh capture has at + // least two links and contributes nothing to retained bytes. It starts counting + // later, once Ray rolls the segment off its backup ring and the reconcile sweep + // re-reads the link count. + rc.bytes.observe(key, size, nlink) + // The limit is enforced here, not on the next trip through the event loop. A + // scan registers every backup it finds without returning to that loop, so + // deferring this would let the rest of the scan through after the volume was + // already over its limit. This capture stands; the next one in the same scan + // sees a paused gate and is skipped. + rc.enforceHighWater() + rc.enqueueUpload(key) + return nil +} + +// pinnedInode reads the identity, size and link count of the file the staging link +// actually pinned. The link must still be a regular file: if the source turned into a +// symlink or another non-regular object first, what was linked is not a log segment. +// +// All three come from the same stat, so accounting describes exactly the file that was +// pinned rather than whatever the source path holds a moment later. The link count is +// what tells retained-byte accounting whether Ray still owns this segment. +func (rc *rotatedCollector) pinnedInode(staged string) (inodeKey, int64, uint64, error) { + fi, err := os.Lstat(staged) + if err != nil { + return inodeKey{}, 0, 0, fmt.Errorf("stat staged capture %s: %w", staged, err) + } + if !fi.Mode().IsRegular() { + return inodeKey{}, 0, 0, fmt.Errorf("staged capture %s is not a regular file (%s)", staged, fi.Mode()) + } + key, nlink, err := inodeFromFileInfo(fi) + if err != nil { + return inodeKey{}, 0, 0, fmt.Errorf("read inode of staged capture %s: %w", staged, err) + } + return key, fi.Size(), nlink, nil +} + +// discardStagingLink removes a link the collector created but will not track. An +// untracked link would pin blocks that nothing ever releases. +func discardStagingLink(staged string) error { + if err := os.Remove(staged); err != nil && !isVanished(err) { + return fmt.Errorf("remove surplus staging link %s: %w", staged, err) + } + return nil +} + +// registerStaged records a freshly linked capture, and removes the link it just +// created if that fails. An untracked hard link would pin blocks nothing will ever +// release, so the staging volume must never keep one. +func registerStaged(stagingRoot string, ix *captureIndex, key inodeKey, e stagedEntry) error { + _, added, err := ix.add(key, e) + if err == nil && !added { + err = fmt.Errorf("capture %s: inode %s was already registered", e.CaptureID, key) + } + if err != nil { + if rmErr := os.Remove(e.path(stagingRoot)); rmErr != nil && !isVanished(rmErr) { + return fmt.Errorf("%w (and its staging link could not be removed: %w)", err, rmErr) + } + return err + } + return nil +} + +// stagedRecord is one staging file found during reconstruction, together with the +// inode it was holding at that moment and how many links that inode had. The link +// count is what lets a restart rebuild retained-byte accounting from the filesystem +// alone, with nothing persisted. +type stagedRecord struct { + entry stagedEntry + path string + key inodeKey + size int64 + nlink uint64 +} + +// reconstructStaging rebuilds the index from the staging volume so a restarted +// collector adopts what the previous run pinned instead of capturing it again. +// Capture identity comes from the filenames; no new IDs are minted here. +// +// It returns an error when it cannot establish one staged link per inode. Starting +// with a link nothing tracks would be worse than not starting: that link pins the +// inode forever, so releaseCapture would always see an extra link count and could +// never free the segment. +func (rc *rotatedCollector) reconstructStaging() error { + root := rc.cfg.StagingRoot + if _, err := os.Lstat(root); err != nil { + if isVanished(err) { + return nil + } + return fmt.Errorf("stat staging root %s: %w", root, err) + } + + records, err := rc.collectStagedRecords(root) + if err != nil { + return err + } + + // A total order, so nothing depends on the order the filesystem was walked. + // Pending beats uploaded, keeping the guarantee that captured data is uploaded + // at least once; then the lower capture ID; then the path, so two records that + // are otherwise identical still have one deterministic winner. + slices.SortFunc(records, func(a, b stagedRecord) int { + if a.entry.State != b.entry.State { + if a.entry.State == statePending { + return -1 + } + return 1 + } + if c := strings.Compare(a.entry.CaptureID, b.entry.CaptureID); c != 0 { + return c + } + return strings.Compare(a.path, b.path) + }) + + var failures []error + winners := make(map[inodeKey]stagedRecord, len(records)) + for _, r := range records { + winner, taken := winners[r.key] + if !taken { + if _, err := rc.ix.restore(r.key, r.entry); err != nil { + failures = append(failures, fmt.Errorf("restore staged capture %s: %w", r.path, err)) + continue + } + // One inode, counted once, whether the previous run left it pending or + // uploaded: both states pin the same blocks. Whether those blocks are + // retained *because of* the collector is re-derived from the link count + // this walk read, so a restart recovers retention with nothing persisted. + rc.bytes.observe(r.key, r.size, r.nlink) + winners[r.key] = r + continue + } + + rc.report(fmt.Errorf("staging volume holds a surplus record for %s: keeping %s, removing %s", + r.key, winner.path, r.path)) + if err := removeSurplusLink(r); err != nil { + failures = append(failures, err) + } + } + + if len(failures) > 0 { + return fmt.Errorf("staging volume is inconsistent: %w", errors.Join(failures...)) + } + return nil +} + +// collectStagedRecords reads every usable staging record under root. Unreadable +// subtrees and unusable files are reported and skipped; only failures that would +// leave the index inconsistent are returned. +func (rc *rotatedCollector) collectStagedRecords(root string) ([]stagedRecord, error) { + var records []stagedRecord + + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + if isVanished(err) { + return nil // drained or cleaned up concurrently; nothing to adopt + } + // A subtree we cannot read may hold real staged links. Adopting the + // rest and starting anyway would leave those pinning their inodes with + // no owner, so this has to stop startup. + return fmt.Errorf("read staging volume at %s: %w", path, err) + } + if d.IsDir() || d.Type()&fs.ModeSymlink != 0 { + return nil + } + + // A name that does not parse cannot be a link this collector made: every + // capture is written as ////.rotated.. + // So it pins nothing the index needs to own, and ignoring it is safe. + entry, err := parseStagedPath(root, path) + if err != nil { + rc.report(err) + return nil + } + fi, err := d.Info() + if err != nil { + if isVanished(err) { + return nil + } + return fmt.Errorf("stat staged capture %s: %w", path, err) + } + // Only a regular file can be a captured segment: captures are made with + // os.Link from a regular log file, so a FIFO, socket or device here was + // never ours and pins nothing we must track. Indexing one would also let a + // later uploader block forever trying to read it. + if !fi.Mode().IsRegular() { + rc.report(fmt.Errorf("staged capture %s is not a regular file (%s), ignoring it", path, fi.Mode())) + return nil + } + key, nlink, err := inodeFromFileInfo(fi) + if err != nil { + return fmt.Errorf("read inode of staged capture %s: %w", path, err) + } + records = append(records, stagedRecord{key: key, entry: entry, path: path, size: fi.Size(), nlink: nlink}) + return nil + }) + if err != nil { + return nil, fmt.Errorf("reconstruct staging volume %s: %w", root, err) + } + return records, nil +} + +// removeSurplusLink drops a staging link that lost a conflict, but only after +// confirming it still holds the inode it was recorded with. If the path has since +// become something else, removing it could destroy an unrelated capture, so it is +// reported and left alone instead. +func removeSurplusLink(r stagedRecord) error { + fi, err := os.Lstat(r.path) + if err != nil { + if isVanished(err) { + return nil // already gone; nothing pins the inode through this name + } + return fmt.Errorf("stat surplus staging link %s: %w", r.path, err) + } + if !fi.Mode().IsRegular() { + return fmt.Errorf("surplus staging link %s is no longer a regular file (%s)", r.path, fi.Mode()) + } + current, _, err := inodeFromFileInfo(fi) + if err != nil { + return fmt.Errorf("read inode of surplus staging link %s: %w", r.path, err) + } + if current != r.key { + return fmt.Errorf("surplus staging link %s now holds %s, not %s, so it was left in place", + r.path, current, r.key) + } + if err := os.Remove(r.path); err != nil && !isVanished(err) { + return fmt.Errorf("remove surplus staging link %s: %w", r.path, err) + } + return nil +} diff --git a/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_collector_test.go b/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_collector_test.go new file mode 100644 index 00000000000..0135079b3f0 --- /dev/null +++ b/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_collector_test.go @@ -0,0 +1,1579 @@ +package logcollector + +import ( + "os" + "path/filepath" + "runtime" + "slices" + "strconv" + "strings" + "sync" + "syscall" + "testing" + "time" + + "github.com/fsnotify/fsnotify" +) + +// fakeWatcher lets tests deliver events exactly when they choose. Everything else in +// these tests is a real directory, a real file and a real hard link. +type fakeWatcher struct { + mu sync.Mutex + added []string + closed bool + failAdd map[string]error // path -> error returned by Add + + events chan fsnotify.Event + errs chan error +} + +func newFakeWatcher() *fakeWatcher { return newFakeWatcherBuffered(0) } + +// newFakeWatcherBuffered mirrors fsnotify's buffered queue. Most tests use an +// unbuffered channel so that delivering an event and then round-tripping a snapshot +// proves the event was handled; buffering is only for tests that must queue events +// while the loop is busy. +func newFakeWatcherBuffered(n int) *fakeWatcher { + return &fakeWatcher{ + events: make(chan fsnotify.Event, n), + errs: make(chan error, n), + } +} + +func (w *fakeWatcher) Add(name string) error { + w.mu.Lock() + defer w.mu.Unlock() + if err, ok := w.failAdd[name]; ok { + return err + } + w.added = append(w.added, name) + return nil +} + +func (w *fakeWatcher) Close() error { + w.mu.Lock() + defer w.mu.Unlock() + w.closed = true + return nil +} + +func (w *fakeWatcher) Events() <-chan fsnotify.Event { return w.events } +func (w *fakeWatcher) Errors() <-chan error { return w.errs } + +func (w *fakeWatcher) watched() []string { + w.mu.Lock() + defer w.mu.Unlock() + return append([]string(nil), w.added...) +} + +func (w *fakeWatcher) isClosed() bool { + w.mu.Lock() + defer w.mu.Unlock() + return w.closed +} + +// issueLog records what the collector reported instead of failing on. +type issueLog struct { + mu sync.Mutex + issues []error +} + +func (l *issueLog) add(err error) { + l.mu.Lock() + defer l.mu.Unlock() + l.issues = append(l.issues, err) +} + +func (l *issueLog) all() []error { + l.mu.Lock() + defer l.mu.Unlock() + return append([]error(nil), l.issues...) +} + +func (l *issueLog) matching(substr string) []error { + var out []error + for _, err := range l.all() { + if strings.Contains(err.Error(), substr) { + out = append(out, err) + } + } + return out +} + +// harness is one collector running against real temp directories. +type harness struct { + rc *rotatedCollector + watcher *fakeWatcher + tick chan time.Time + issues *issueLog + logsDir string + stagingRoot string + runErr chan error +} + +// start builds and runs a collector. It returns once startup reconstruction and the +// startup scan have finished, because the first snapshot round-trip is only served +// after the owner loop reaches its select. +func start(t *testing.T, dir string) *harness { + t.Helper() + return startWith(t, dir, func(*rotatedCollectorConfig) {}) +} + +func startWith(t *testing.T, dir string, tweak func(*rotatedCollectorConfig)) *harness { + t.Helper() + logsDir := filepath.Join(dir, "session", "logs") + if err := os.MkdirAll(logsDir, 0o750); err != nil { + t.Fatalf("create logs dir: %v", err) + } + + h := &harness{ + watcher: newFakeWatcher(), + tick: make(chan time.Time), + issues: &issueLog{}, + logsDir: logsDir, + stagingRoot: filepath.Join(dir, "rotated-staging"), + runErr: make(chan error, 1), + } + + cfg := rotatedCollectorConfig{ + LogsDir: logsDir, + StagingRoot: h.stagingRoot, + SessionName: "session-1", + NodeName: "node-1", + NewWatcher: func() (fsWatcher, error) { return h.watcher, nil }, + NewTicker: func(time.Duration) (<-chan time.Time, func()) { return h.tick, func() {} }, + OnIssue: h.issues.add, + } + tweak(&cfg) + + rc, err := newRotatedCollector(cfg) + if err != nil { + t.Fatalf("newRotatedCollector() error: %v", err) + } + h.rc = rc + + go func() { h.runErr <- rc.Run() }() + t.Cleanup(func() { rc.Stop() }) + + rc.snapshot() // wait for startup to finish + return h +} + +// sendEvent delivers an event and returns once the loop has finished handling it. +func (h *harness) sendEvent(t *testing.T, name string) { + t.Helper() + select { + case h.watcher.events <- fsnotify.Event{Name: name, Op: fsnotify.Create}: + case <-time.After(5 * time.Second): + t.Fatal("timed out delivering event") + } + h.rc.snapshot() +} + +// fireTick runs one periodic reconciliation and waits for it to complete. +func (h *harness) fireTick(t *testing.T) { + t.Helper() + select { + case h.tick <- time.Now(): + case <-time.After(5 * time.Second): + t.Fatal("timed out firing tick") + } + h.rc.snapshot() +} + +func (h *harness) writeLog(t *testing.T, rel, content string) string { + t.Helper() + path := filepath.Join(h.logsDir, filepath.FromSlash(rel)) + writeFile(t, path, content) + return path +} + +func (h *harness) stagedPaths(t *testing.T) []string { + t.Helper() + var out []string + err := filepath.WalkDir(h.stagingRoot, func(p string, d os.DirEntry, err error) error { + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + if !d.IsDir() { + rel, relErr := filepath.Rel(h.stagingRoot, p) + if relErr != nil { + return relErr + } + out = append(out, filepath.ToSlash(rel)) + } + return nil + }) + if err != nil && !os.IsNotExist(err) { + t.Fatalf("walk staging root: %v", err) + } + return out +} + +func TestCollectorStartupWatchesTreeAndCapturesExistingBackups(t *testing.T) { + dir := t.TempDir() + logsDir := filepath.Join(dir, "session", "logs") + writeFile(t, filepath.Join(logsDir, "raylet.out"), "active") + writeFile(t, filepath.Join(logsDir, "raylet.out.1"), "rotated") + writeFile(t, filepath.Join(logsDir, "events", "event.log"), "active nested") + writeFile(t, filepath.Join(logsDir, "events", "event.log.2"), "rotated nested") + + h := start(t, dir) + + // 1. Every directory in the tree is watched. + watched := h.watcher.watched() + for _, want := range []string{logsDir, filepath.Join(logsDir, "events")} { + found := false + for _, got := range watched { + if got == want { + found = true + } + } + if !found { + t.Errorf("directory %s was not watched, watched = %v", want, watched) + } + } + + // 3. Backups that already existed are captured by the startup scan. + entries := h.rc.snapshot() + if len(entries) != 2 { + t.Fatalf("captured %d segments, want 2: %+v", len(entries), entries) + } + byName := map[string]stagedEntry{} + for _, e := range entries { + byName[e.OriginalName] = e + } + if e, ok := byName["raylet.out.1"]; !ok || e.RelDir != "" { + t.Errorf("raylet.out.1 captured as %+v", e) + } + if e, ok := byName["event.log.2"]; !ok || e.RelDir != "events" { + t.Errorf("event.log.2 captured as %+v (want RelDir \"events\")", e) + } + for _, e := range entries { + if e.State != statePending { + t.Errorf("capture %s state = %q, want %q", e.CaptureID, e.State, statePending) + } + if _, err := os.Lstat(e.path(h.stagingRoot)); err != nil { + t.Errorf("staged link missing for %s: %v", e.OriginalName, err) + } + } +} + +func TestCollectorRemembersActiveBases(t *testing.T) { + dir := t.TempDir() + logsDir := filepath.Join(dir, "session", "logs") + // The active file exists at startup and is recorded as a base... + writeFile(t, filepath.Join(logsDir, "raylet.out"), "active") + h := start(t, dir) + + // ...then a rotation cascade briefly leaves the active name unlinked while the + // backup appears. Without the remembered base this would look like an unrelated + // file ending in ".1" and be skipped. + if err := os.Remove(filepath.Join(logsDir, "raylet.out")); err != nil { + t.Fatalf("remove active file: %v", err) + } + backup := h.writeLog(t, "raylet.out.1", "rotated") + h.sendEvent(t, backup) + + // A file with no base, remembered or present, stays untouched. + unrelated := h.writeLog(t, "user-data.1", "not a ray log") + h.sendEvent(t, unrelated) + + entries := h.rc.snapshot() + if len(entries) != 1 { + t.Fatalf("captured %d segments, want only the rotation backup: %+v", len(entries), entries) + } + if entries[0].OriginalName != "raylet.out.1" { + t.Errorf("captured %q, want raylet.out.1", entries[0].OriginalName) + } +} + +func TestCollectorCapturesBeforeSourceIsDeleted(t *testing.T) { + dir := t.TempDir() + h := start(t, dir) + h.writeLog(t, "raylet.out", "active") + + const content = "the segment rotation is about to delete" + backup := h.writeLog(t, "raylet.out.1", content) + h.sendEvent(t, backup) + + // Rotation deletes it immediately afterwards; the captured bytes must survive. + if err := os.Remove(backup); err != nil { + t.Fatalf("remove source: %v", err) + } + + entries := h.rc.snapshot() + if len(entries) != 1 { + t.Fatalf("captured %d segments, want 1", len(entries)) + } + got, err := os.ReadFile(entries[0].path(h.stagingRoot)) + if err != nil { + t.Fatalf("read staged capture: %v", err) + } + if string(got) != content { + t.Errorf("staged content = %q, want %q", got, content) + } +} + +func TestCollectorWatchesAndScansNewDirectory(t *testing.T) { + dir := t.TempDir() + h := start(t, dir) + + // Ray creates a subdirectory that already contains an active file and a backup. + // The watch cannot report what was there before it existed, so the scan that + // follows the watch is what finds them. + h.writeLog(t, "serve/replica.log", "active") + h.writeLog(t, "serve/replica.log.1", "rotated") + h.sendEvent(t, filepath.Join(h.logsDir, "serve")) + + watchedServe := false + for _, got := range h.watcher.watched() { + if got == filepath.Join(h.logsDir, "serve") { + watchedServe = true + } + } + if !watchedServe { + t.Errorf("new directory was not watched, watched = %v", h.watcher.watched()) + } + + entries := h.rc.snapshot() + if len(entries) != 1 || entries[0].OriginalName != "replica.log.1" || entries[0].RelDir != "serve" { + t.Fatalf("captured %+v, want replica.log.1 under serve", entries) + } +} + +func TestCollectorCapturesOneSegmentAcrossRotationIndexes(t *testing.T) { + dir := t.TempDir() + h := start(t, dir) + h.writeLog(t, "raylet.out", "active") + + first := h.writeLog(t, "raylet.out.1", "generation one") + h.sendEvent(t, first) + afterFirst := h.rc.snapshot() + if len(afterFirst) != 1 { + t.Fatalf("captured %d segments, want 1", len(afterFirst)) + } + + // The next rotation renames the same physical file to .2. It is the same pinned + // inode, so it must not become a second capture. + second := filepath.Join(h.logsDir, "raylet.out.2") + if err := os.Rename(first, second); err != nil { + t.Fatalf("rotate to .2: %v", err) + } + h.sendEvent(t, second) + + entries := h.rc.snapshot() + if len(entries) != 1 { + t.Fatalf("captured %d segments after rotation, want 1: %+v", len(entries), entries) + } + if entries[0].CaptureID != afterFirst[0].CaptureID { + t.Errorf("capture ID changed across rotation: %q -> %q", afterFirst[0].CaptureID, entries[0].CaptureID) + } + if staged := h.stagedPaths(t); len(staged) != 1 { + t.Errorf("staging holds %v, want one link", staged) + } +} + +func TestCollectorCapturesEachGenerationAtTheSamePath(t *testing.T) { + dir := t.TempDir() + h := start(t, dir) + h.writeLog(t, "raylet.out", "active") + + backup := h.writeLog(t, "raylet.out.1", "generation one") + h.sendEvent(t, backup) + + // Rotation deletes that segment and a later one takes the same name. Our link + // keeps the first inode alive, so the second file is necessarily a new inode. + if err := os.Remove(backup); err != nil { + t.Fatalf("remove first generation: %v", err) + } + backup = h.writeLog(t, "raylet.out.1", "generation two") + h.sendEvent(t, backup) + + entries := h.rc.snapshot() + if len(entries) != 2 { + t.Fatalf("captured %d segments, want 2: %+v", len(entries), entries) + } + if entries[0].CaptureID == entries[1].CaptureID { + t.Error("both generations share a capture ID") + } + identity := clusterIdentity{RootDir: "root", Namespace: "default", ClusterName: "c"} + if entries[0].objectKey(identity) == entries[1].objectKey(identity) { + t.Error("both generations map to one object key") + } + contents := map[string]bool{} + for _, e := range entries { + data, err := os.ReadFile(e.path(h.stagingRoot)) + if err != nil { + t.Fatalf("read staged capture: %v", err) + } + contents[string(data)] = true + } + for _, want := range []string{"generation one", "generation two"} { + if !contents[want] { + t.Errorf("staged captures %v do not include %q", contents, want) + } + } +} + +func TestCollectorReconstructsStagingWithoutNewCaptureIDs(t *testing.T) { + dir := t.TempDir() + logsDir := filepath.Join(dir, "session", "logs") + stagingRoot := filepath.Join(dir, "rotated-staging") + writeFile(t, filepath.Join(logsDir, "raylet.out"), "active") + + // A previous run staged one pending and one uploaded capture. + prior := newCaptureIndex() + ids := newCaptureIDGenerator() + stage := func(name, content string, promote bool) stagedEntry { + t.Helper() + src := filepath.Join(logsDir, name) + writeFile(t, src, content) + id, err := ids.next() + if err != nil { + t.Fatalf("next() error: %v", err) + } + entry, err := newStagedEntry(statePending, "session-1", "node-1", "", name, id) + if err != nil { + t.Fatalf("newStagedEntry() error: %v", err) + } + if err := captureLink(src, entry.path(stagingRoot)); err != nil { + t.Fatalf("captureLink() error: %v", err) + } + key, _, err := statInode(entry.path(stagingRoot)) + if err != nil { + t.Fatalf("statInode() error: %v", err) + } + if _, _, err := prior.add(key, entry); err != nil { + t.Fatalf("add() error: %v", err) + } + if promote { + entry, err = promoteCapture(stagingRoot, prior, key) + if err != nil { + t.Fatalf("promoteCapture() error: %v", err) + } + } + return entry + } + pending := stage("raylet.out.1", "pending segment", false) + uploaded := stage("raylet.out.2", "uploaded segment", true) + before := stagingFiles(t, stagingRoot) + + h := start(t, dir) + + entries := h.rc.snapshot() + if len(entries) != 2 { + t.Fatalf("reconstructed %d captures, want 2: %+v", len(entries), entries) + } + got := map[string]stagedEntry{} + for _, e := range entries { + got[e.CaptureID] = e + } + for _, want := range []stagedEntry{pending, uploaded} { + e, ok := got[want.CaptureID] + if !ok { + t.Fatalf("capture %s was not reconstructed (a new ID was minted)", want.CaptureID) + } + if e != want { + t.Errorf("reconstructed %+v, want %+v", e, want) + } + } + // The startup scan sees the same source files, but their inodes are already + // tracked, so nothing is staged twice. + if after := h.stagedPaths(t); len(after) != len(before) { + t.Errorf("staging changed during reconstruction: %v -> %v", before, after) + } +} + +// stagingFiles lists staging paths before a collector exists. +func stagingFiles(t *testing.T, stagingRoot string) []string { + t.Helper() + var out []string + err := filepath.WalkDir(stagingRoot, func(p string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() { + rel, relErr := filepath.Rel(stagingRoot, p) + if relErr != nil { + return relErr + } + out = append(out, filepath.ToSlash(rel)) + } + return nil + }) + if err != nil { + t.Fatalf("walk staging root: %v", err) + } + return out +} + +func TestCollectorReportsConflictingStagedRecords(t *testing.T) { + dir := t.TempDir() + logsDir := filepath.Join(dir, "session", "logs") + stagingRoot := filepath.Join(dir, "rotated-staging") + src := filepath.Join(logsDir, "raylet.out.1") + writeFile(t, src, "one segment") + + // A corrupt staging tree: one inode recorded under two capture IDs. + for _, id := range []string{"0001780000000000000.aaaaaaaaaaaaaaaa", "0001780000000000001.bbbbbbbbbbbbbbbb"} { + entry, err := newStagedEntry(statePending, "session-1", "node-1", "", "raylet.out.1", id) + if err != nil { + t.Fatalf("newStagedEntry() error: %v", err) + } + if err := captureLink(src, entry.path(stagingRoot)); err != nil { + t.Fatalf("captureLink() error: %v", err) + } + } + // ...plus a file that is not a staging record at all. + writeFile(t, filepath.Join(stagingRoot, "session-1", "node-1", "pending", "garbage.txt"), "junk") + + h := start(t, dir) + + if entries := h.rc.snapshot(); len(entries) != 1 { + t.Fatalf("index holds %d captures, want exactly one of the conflicting records: %+v", len(entries), entries) + } + if got := h.issues.matching("surplus record"); len(got) == 0 { + t.Errorf("conflicting staging record was not reported, issues = %v", h.issues.all()) + } + // Exactly one link may remain for the inode, otherwise the surplus one would + // pin it forever and release could never free the segment. + if _, nlink, err := statInode(src); err != nil || nlink != 2 { + t.Errorf("link count = %d (err %v), want 2 (the source and one staged link)", nlink, err) + } + if got := h.issues.matching("garbage.txt"); len(got) == 0 { + t.Errorf("malformed staging record was not reported, issues = %v", h.issues.all()) + } +} + +func TestCollectorPeriodicReconciliationCatchesMissedEvents(t *testing.T) { + dir := t.TempDir() + h := start(t, dir) + h.writeLog(t, "raylet.out", "active") + + // No event is delivered for this file at all: the sweep is the only thing that + // can find it. + h.writeLog(t, "raylet.out.1", "missed by fsnotify") + if entries := h.rc.snapshot(); len(entries) != 0 { + t.Fatalf("captured %d segments before reconciliation, want 0", len(entries)) + } + + h.fireTick(t) + + if entries := h.rc.snapshot(); len(entries) != 1 { + t.Fatalf("captured %d segments after reconciliation, want 1", len(entries)) + } +} + +func TestCollectorReconcilesImmediatelyAfterOverflow(t *testing.T) { + dir := t.TempDir() + h := start(t, dir) + h.writeLog(t, "raylet.out", "active") + h.writeLog(t, "raylet.out.1", "dropped event") + + select { + case h.watcher.errs <- fsnotify.ErrEventOverflow: + case <-time.After(5 * time.Second): + t.Fatal("timed out delivering overflow") + } + h.rc.snapshot() // round-trip: the loop has finished reconciling + + if entries := h.rc.snapshot(); len(entries) != 1 { + t.Fatalf("captured %d segments after overflow, want 1", len(entries)) + } + if got := h.issues.matching("overflow"); len(got) == 0 { + t.Errorf("overflow was not reported, issues = %v", h.issues.all()) + } +} + +func TestCollectorKeepsRunningAfterWatcherError(t *testing.T) { + dir := t.TempDir() + h := start(t, dir) + + select { + case h.watcher.errs <- os.ErrPermission: + case <-time.After(5 * time.Second): + t.Fatal("timed out delivering watcher error") + } + + // The loop must still serve requests and still capture. + h.writeLog(t, "raylet.out", "active") + backup := h.writeLog(t, "raylet.out.1", "rotated") + h.sendEvent(t, backup) + if entries := h.rc.snapshot(); len(entries) != 1 { + t.Fatalf("collector stopped working after a watcher error: %+v", entries) + } +} + +func TestCollectorSurvivesCaptureFailures(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("running as root: directory permissions do not prevent staging writes") + } + dir := t.TempDir() + // A staging root that cannot be written to stands in for any capture failure + // the deployment can produce, such as EXDEV or EPERM from os.Link. + blocked := filepath.Join(dir, "blocked") + if err := os.MkdirAll(blocked, 0o500); err != nil { + t.Fatalf("create blocked staging root: %v", err) + } + t.Cleanup(func() { _ = os.Chmod(blocked, 0o750) }) + + h := startWith(t, dir, func(cfg *rotatedCollectorConfig) { + cfg.StagingRoot = filepath.Join(blocked, "rotated-staging") + }) + + h.writeLog(t, "raylet.out", "active") + backup := h.writeLog(t, "raylet.out.1", "rotated") + h.sendEvent(t, backup) + + if entries := h.rc.snapshot(); len(entries) != 0 { + t.Errorf("failed capture was registered anyway: %+v", entries) + } + if len(h.issues.all()) == 0 { + t.Error("capture failure was not reported") + } + + // The loop is still alive and still discovering. + select { + case err := <-h.runErr: + t.Fatalf("collector exited after a capture failure: %v", err) + default: + } + h.fireTick(t) +} + +func TestCollectorIgnoresVanishedAndOutOfTreePaths(t *testing.T) { + dir := t.TempDir() + h := start(t, dir) + + // A file that rotation removed between the event and our stat. + h.sendEvent(t, filepath.Join(h.logsDir, "raylet.out.1")) + // A path that is not under the active logs tree at all. + h.sendEvent(t, filepath.Join(dir, "elsewhere", "raylet.out.1")) + + if entries := h.rc.snapshot(); len(entries) != 0 { + t.Errorf("captured %+v, want nothing", entries) + } + if issues := h.issues.all(); len(issues) != 0 { + t.Errorf("expected races and foreign paths to be silent, got %v", issues) + } +} + +func TestCollectorDoesNotFollowSymlinkedDirectories(t *testing.T) { + dir := t.TempDir() + logsDir := filepath.Join(dir, "session", "logs") + writeFile(t, filepath.Join(logsDir, "raylet.out"), "active") + writeFile(t, filepath.Join(logsDir, "raylet.out.1"), "rotated") + // A symlink pointing back at the tree would make a naive walk recurse forever. + if err := os.Symlink(logsDir, filepath.Join(logsDir, "loop")); err != nil { + t.Fatalf("create symlink: %v", err) + } + // A symlinked file is not a log Ray rotated either. + if err := os.Symlink(filepath.Join(logsDir, "raylet.out.1"), filepath.Join(logsDir, "alias.out.1")); err != nil { + t.Fatalf("create symlink: %v", err) + } + + h := start(t, dir) // would hang or overflow the stack if symlinks were followed + + for _, watched := range h.watcher.watched() { + if strings.Contains(watched, "loop") { + t.Errorf("symlinked directory was watched: %s", watched) + } + } + entries := h.rc.snapshot() + if len(entries) != 1 || entries[0].OriginalName != "raylet.out.1" { + t.Errorf("captured %+v, want only the real backup", entries) + } +} + +func TestRegisterStagedRollsBackOnRegistrationFailure(t *testing.T) { + dir := t.TempDir() + stagingRoot := filepath.Join(dir, "rotated-staging") + src := filepath.Join(dir, "logs", "raylet.out.1") + writeFile(t, src, "rotated") + + ix := newCaptureIndex() + first, err := newStagedEntry(statePending, "session-1", "node-1", "", "raylet.out.1", "0001780000000000000.aaaaaaaaaaaaaaaa") + if err != nil { + t.Fatalf("newStagedEntry() error: %v", err) + } + if err := captureLink(src, first.path(stagingRoot)); err != nil { + t.Fatalf("captureLink() error: %v", err) + } + key, _, err := statInode(first.path(stagingRoot)) + if err != nil { + t.Fatalf("statInode() error: %v", err) + } + if err := registerStaged(stagingRoot, ix, key, first); err != nil { + t.Fatalf("registerStaged() error: %v", err) + } + + // A second capture ID for an inode that is already registered: the link must be + // undone, or it would pin blocks that nothing ever releases. + second, err := newStagedEntry(statePending, "session-1", "node-1", "", "raylet.out.1", "0001780000000000001.bbbbbbbbbbbbbbbb") + if err != nil { + t.Fatalf("newStagedEntry() error: %v", err) + } + if err := captureLink(src, second.path(stagingRoot)); err != nil { + t.Fatalf("captureLink() error: %v", err) + } + + if err := registerStaged(stagingRoot, ix, key, second); err == nil { + t.Fatal("registerStaged() accepted a duplicate inode") + } + if _, err := os.Lstat(second.path(stagingRoot)); !isVanished(err) { + t.Errorf("the rejected capture's staging link was left behind: %v", err) + } + if _, err := os.Lstat(first.path(stagingRoot)); err != nil { + t.Errorf("rollback removed the wrong link: %v", err) + } + if ix.len() != 1 { + t.Errorf("index holds %d captures, want 1", ix.len()) + } +} + +func TestCollectorCaptureIsNotBlockedByOtherWork(t *testing.T) { + // The loop performs no storage calls, so discovery latency depends only on the + // filesystem. Capturing a burst of segments must be prompt. + dir := t.TempDir() + h := start(t, dir) + h.writeLog(t, "raylet.out", "active") + + const segments = 50 + for i := range segments { + h.writeLog(t, filepath.Join("burst", "worker.out."+strconv.Itoa(i+1)), "segment") + } + h.writeLog(t, "burst/worker.out", "active") + + deadline := time.Now() + h.fireTick(t) + elapsed := time.Since(deadline) + + entries := h.rc.snapshot() + if len(entries) != segments { + t.Fatalf("captured %d of %d segments", len(entries), segments) + } + if elapsed > 5*time.Second { + t.Errorf("capturing %d segments took %v, which suggests blocking work on the loop", segments, elapsed) + } +} + +func TestCollectorStateIsOnlyTouchedByTheOwnerLoop(t *testing.T) { + // Concurrent readers go through the loop's request channel rather than the + // index. Under -race this proves there is no unsynchronised access. + dir := t.TempDir() + h := start(t, dir) + h.writeLog(t, "raylet.out", "active") + + var wg sync.WaitGroup + for range 8 { + wg.Go(func() { + for range 20 { + h.rc.snapshot() + } + }) + } + for i := range 20 { + h.writeLog(t, "raylet.out."+strconv.Itoa(i+1), "segment") + } + h.rc.reconcileNow() + wg.Wait() + + if entries := h.rc.snapshot(); len(entries) != 20 { + t.Errorf("captured %d segments, want 20", len(entries)) + } +} + +func TestCollectorStopClosesWatcherAndExits(t *testing.T) { + before := runtime.NumGoroutine() + dir := t.TempDir() + h := start(t, dir) + + h.rc.Stop() + h.rc.Stop() // idempotent + + select { + case err := <-h.runErr: + if err != nil { + t.Errorf("Run() returned %v, want nil on a deliberate stop", err) + } + case <-time.After(5 * time.Second): + t.Fatal("Run() did not return after Stop()") + } + if !h.watcher.isClosed() { + t.Error("Stop() did not close the watcher") + } + if entries := h.rc.snapshot(); entries != nil { + t.Errorf("snapshot() after Stop() = %+v, want nil", entries) + } + + // Goroutines settle asynchronously; give the runtime a moment before comparing. + for range 20 { + if runtime.NumGoroutine() <= before { + return + } + time.Sleep(50 * time.Millisecond) + } + t.Errorf("goroutines leaked: %d before, %d after", before, runtime.NumGoroutine()) +} + +func TestCollectorWithRealWatcher(t *testing.T) { + // One end-to-end run against the real fsnotify adapter, so the interface seam + // used by every other test is known to match the kernel's behavior. + dir := t.TempDir() + logsDir := filepath.Join(dir, "session", "logs") + if err := os.MkdirAll(logsDir, 0o750); err != nil { + t.Fatalf("create logs dir: %v", err) + } + writeFile(t, filepath.Join(logsDir, "raylet.out"), "active") + + rc, err := newRotatedCollector(rotatedCollectorConfig{ + LogsDir: logsDir, + StagingRoot: filepath.Join(dir, "rotated-staging"), + SessionName: "session-1", + NodeName: "node-1", + NewTicker: func(time.Duration) (<-chan time.Time, func()) { return make(chan time.Time), func() {} }, + OnIssue: func(error) {}, + }) + if err != nil { + t.Fatalf("newRotatedCollector() error: %v", err) + } + runErr := make(chan error, 1) + go func() { runErr <- rc.Run() }() + t.Cleanup(func() { rc.Stop() }) + rc.snapshot() + + // Rotation: the active file is renamed to .1, which inotify reports as a create. + if err := os.Rename(filepath.Join(logsDir, "raylet.out"), filepath.Join(logsDir, "raylet.out.1")); err != nil { + t.Fatalf("rotate: %v", err) + } + + deadline := time.Now().Add(10 * time.Second) + for { + entries := rc.snapshot() + if len(entries) == 1 && entries[0].OriginalName == "raylet.out.1" { + return + } + if time.Now().After(deadline) { + t.Fatalf("real watcher did not lead to a capture, entries = %+v", entries) + } + time.Sleep(20 * time.Millisecond) + } +} + +// replaceBeforeLink returns a Link function that swaps the source path for a brand +// new file just before the hard link is created. That is exactly what a rotation +// cascade does to a reused name like "raylet.out.1", and doing it inside the seam +// makes the race deterministic instead of timing-dependent. +func replaceBeforeLink(t *testing.T, target, content string) func(string, string) error { + t.Helper() + var once sync.Once + return func(src, dst string) error { + once.Do(func() { + if src != target { + return + } + if err := os.Remove(src); err != nil { + t.Errorf("replace source: %v", err) + return + } + writeFile(t, src, content) + }) + return captureLink(src, dst) + } +} + +func TestCollectorPinsTheInodeItActuallyLinked(t *testing.T) { + dir := t.TempDir() + logsDir := filepath.Join(dir, "session", "logs") + writeFile(t, filepath.Join(logsDir, "raylet.out"), "active") + backup := filepath.Join(logsDir, "raylet.out.1") + writeFile(t, backup, "generation A") + + h := startWith(t, dir, func(cfg *rotatedCollectorConfig) { + cfg.Link = replaceBeforeLink(t, backup, "generation B") + }) + + entries := h.rc.snapshot() + if len(entries) != 1 { + t.Fatalf("captured %d segments, want 1: %+v", len(entries), entries) + } + + // The link pinned whatever the path named at link time, so the index must hold + // that file — not the one that was validated a moment earlier. + staged := entries[0].path(h.stagingRoot) + got, err := os.ReadFile(staged) + if err != nil { + t.Fatalf("read staged capture: %v", err) + } + if string(got) != "generation B" { + t.Errorf("staged content = %q, want the file that was actually linked", got) + } + + stagedKey, _, err := statInode(staged) + if err != nil { + t.Fatalf("statInode(staged) error: %v", err) + } + liveKey, _, err := statInode(backup) + if err != nil { + t.Fatalf("statInode(source) error: %v", err) + } + if stagedKey != liveKey { + t.Errorf("staged link pinned %s but the source is %s", stagedKey, liveKey) + } + // Capturing the same inode again must be recognized as a duplicate, which only + // works if the index holds the pinned inode rather than the pre-link one. + h.rc.reconcileNow() + if after := h.rc.snapshot(); len(after) != 1 { + t.Errorf("index recorded the wrong inode: reconciliation added %d more captures", len(after)-1) + } +} + +func TestCollectorCapturesGenerationThatReplacedACapturedOne(t *testing.T) { + dir := t.TempDir() + logsDir := filepath.Join(dir, "session", "logs") + writeFile(t, filepath.Join(logsDir, "raylet.out"), "active") + backup := filepath.Join(logsDir, "raylet.out.1") + writeFile(t, backup, "generation A") + + // Generation A is captured normally by the startup scan. + h := start(t, dir) + first := h.rc.snapshot() + if len(first) != 1 { + t.Fatalf("captured %d segments at startup, want 1", len(first)) + } + + // Now the path is replaced by a new generation just before the link. A dedup + // shortcut based on the pre-link inode would decide "already captured" and skip + // generation B entirely, losing it at the next rotation. + h.rc.cfg.Link = replaceBeforeLink(t, backup, "generation B") + h.rc.reconcileNow() + + entries := h.rc.snapshot() + if len(entries) != 2 { + t.Fatalf("captured %d segments, want both generations: %+v", len(entries), entries) + } + if entries[0].CaptureID == entries[1].CaptureID { + t.Error("both generations share a capture ID") + } + contents := map[string]bool{} + for _, e := range entries { + data, err := os.ReadFile(e.path(h.stagingRoot)) + if err != nil { + t.Fatalf("read staged capture: %v", err) + } + contents[string(data)] = true + } + for _, want := range []string{"generation A", "generation B"} { + if !contents[want] { + t.Errorf("staged captures %v are missing %q", contents, want) + } + } +} + +func TestCollectorDiscardsSurplusLinkForAnAlreadyCapturedInode(t *testing.T) { + dir := t.TempDir() + h := start(t, dir) + h.writeLog(t, "raylet.out", "active") + + backup := h.writeLog(t, "raylet.out.1", "one segment") + h.sendEvent(t, backup) + first := h.rc.snapshot() + if len(first) != 1 { + t.Fatalf("captured %d segments, want 1", len(first)) + } + + // A second name for the same physical file: the collector links it, discovers + // the pinned inode is already tracked, and must drop the surplus link. + second := filepath.Join(h.logsDir, "raylet.out.2") + if err := os.Link(backup, second); err != nil { + t.Fatalf("create second name: %v", err) + } + h.sendEvent(t, second) + + entries := h.rc.snapshot() + if len(entries) != 1 { + t.Fatalf("index holds %d captures, want 1: %+v", len(entries), entries) + } + if entries[0] != first[0] { + t.Errorf("the original capture changed: %+v -> %+v", first[0], entries[0]) + } + if staged := h.stagedPaths(t); len(staged) != 1 { + t.Errorf("staging holds %v, want exactly one link", staged) + } + if len(h.issues.all()) != 0 { + t.Errorf("a duplicate is normal and must not be reported as a problem: %v", h.issues.all()) + } +} + +func TestCollectorRejectsNonRegularStagedObject(t *testing.T) { + dir := t.TempDir() + logsDir := filepath.Join(dir, "session", "logs") + writeFile(t, filepath.Join(logsDir, "raylet.out"), "active") + writeFile(t, filepath.Join(logsDir, "raylet.out.1"), "rotated") + + // Stand in for a source that became a symlink before it was linked: what ends up + // at the staging path is not a regular file, so it must not be registered. + h := startWith(t, dir, func(cfg *rotatedCollectorConfig) { + cfg.Link = func(src, dst string) error { + if err := os.MkdirAll(filepath.Dir(dst), 0o750); err != nil { + return err + } + return os.Symlink(src, dst) + } + }) + + if entries := h.rc.snapshot(); len(entries) != 0 { + t.Errorf("registered a non-regular staged object: %+v", entries) + } + if staged := h.stagedPaths(t); len(staged) != 0 { + t.Errorf("staging still holds %v, want the rejected object removed", staged) + } + if got := h.issues.matching("not a regular file"); len(got) == 0 { + t.Errorf("rejection was not reported, issues = %v", h.issues.all()) + } +} + +func TestCollectorRemovesLinkWhenPostLinkStatFails(t *testing.T) { + dir := t.TempDir() + logsDir := filepath.Join(dir, "session", "logs") + writeFile(t, filepath.Join(logsDir, "raylet.out"), "active") + writeFile(t, filepath.Join(logsDir, "raylet.out.1"), "rotated") + + // The link disappears before it can be read back. + h := startWith(t, dir, func(cfg *rotatedCollectorConfig) { + cfg.Link = func(string, string) error { return nil } + }) + + if entries := h.rc.snapshot(); len(entries) != 0 { + t.Errorf("registered a capture whose link could not be read back: %+v", entries) + } + if got := h.issues.matching("stat staged capture"); len(got) == 0 { + t.Errorf("post-link stat failure was not reported, issues = %v", h.issues.all()) + } +} + +func TestCollectorIgnoresWriteEvents(t *testing.T) { + dir := t.TempDir() + h := start(t, dir) + active := h.writeLog(t, "raylet.out", "active") + + // A busy Ray node appends constantly. None of that may reach the capture path, + // or the queue fills and the Create that matters is delayed or dropped. + for range 200 { + select { + case h.watcher.events <- fsnotify.Event{Name: active, Op: fsnotify.Write}: + case <-time.After(5 * time.Second): + t.Fatal("timed out delivering write burst: the loop is not draining events") + } + } + h.rc.snapshot() // the loop is still responsive + + if entries := h.rc.snapshot(); len(entries) != 0 { + t.Fatalf("writes to an active file produced captures: %+v", entries) + } + + // The Create for a rotation backup is still handled. + backup := h.writeLog(t, "raylet.out.1", "rotated") + h.sendEvent(t, backup) + if entries := h.rc.snapshot(); len(entries) != 1 { + t.Fatalf("captured %d segments after the write burst, want 1", len(entries)) + } +} + +func TestCollectorStagingConflictResolutionIsDeterministic(t *testing.T) { + // A corrupt staging tree holds a pending and an uploaded record for one inode. + // Whichever the walk happens to reach first must not decide the outcome: pending + // always wins, so the data is still guaranteed to be uploaded at least once. + dir := t.TempDir() + logsDir := filepath.Join(dir, "session", "logs") + stagingRoot := filepath.Join(dir, "rotated-staging") + src := filepath.Join(logsDir, "raylet.out.1") + writeFile(t, src, "one segment") + + pending, err := newStagedEntry(statePending, "session-1", "node-1", "", "raylet.out.1", "0001780000000000009.ffffffffffffffff") + if err != nil { + t.Fatalf("newStagedEntry() error: %v", err) + } + uploaded, err := newStagedEntry(stateUploaded, "session-1", "node-1", "", "raylet.out.1", "0001780000000000000.aaaaaaaaaaaaaaaa") + if err != nil { + t.Fatalf("newStagedEntry() error: %v", err) + } + // The uploaded record sorts first by capture ID and by directory name, so a + // walk-order or ID-order rule would pick it. + for _, e := range []stagedEntry{uploaded, pending} { + if err := captureLink(src, e.path(stagingRoot)); err != nil { + t.Fatalf("captureLink() error: %v", err) + } + } + + h := start(t, dir) + + entries := h.rc.snapshot() + if len(entries) != 1 { + t.Fatalf("index holds %d captures, want 1: %+v", len(entries), entries) + } + if entries[0] != pending { + t.Errorf("kept %+v, want the pending record %+v", entries[0], pending) + } + if got := h.issues.matching("surplus record"); len(got) == 0 { + t.Errorf("the conflict was not reported, issues = %v", h.issues.all()) + } + if _, err := os.Lstat(uploaded.path(stagingRoot)); !isVanished(err) { + t.Errorf("the losing uploaded link was left behind: %v", err) + } +} + +func TestCollectorWatchesTreeBeforeReconstruction(t *testing.T) { + dir := t.TempDir() + logsDir := filepath.Join(dir, "session", "logs") + writeFile(t, filepath.Join(logsDir, "raylet.out"), "active") + writeFile(t, filepath.Join(logsDir, "events", "event.log"), "active nested") + + watcher := newFakeWatcherBuffered(16) + reconstructing := make(chan struct{}) + release := make(chan struct{}) + issues := &issueLog{} + + rc, err := newRotatedCollector(rotatedCollectorConfig{ + LogsDir: logsDir, + StagingRoot: filepath.Join(dir, "rotated-staging"), + SessionName: "session-1", + NodeName: "node-1", + NewWatcher: func() (fsWatcher, error) { return watcher, nil }, + NewTicker: func(time.Duration) (<-chan time.Time, func()) { return make(chan time.Time), func() {} }, + OnIssue: issues.add, + BeforeReconstruct: func() { + close(reconstructing) + <-release + }, + }) + if err != nil { + t.Fatalf("newRotatedCollector() error: %v", err) + } + go func() { _ = rc.Run() }() + t.Cleanup(func() { rc.Stop() }) + + select { + case <-reconstructing: + case <-time.After(5 * time.Second): + t.Fatal("collector never reached staging reconstruction") + } + + // The whole tree must already be watched: this is the window in which a + // short-lived segment could otherwise be created and deleted unseen. + watched := watcher.watched() + for _, want := range []string{logsDir, filepath.Join(logsDir, "events")} { + if !slices.Contains(watched, want) { + t.Errorf("%s was not watched before reconstruction, watched = %v", want, watched) + } + } + + // Ray rotates while reconstruction is still running. The event queues in the + // watcher channel, exactly as the kernel would queue it. + backup := filepath.Join(logsDir, "raylet.out.1") + writeFile(t, backup, "rotated during reconstruction") + nested := filepath.Join(logsDir, "serve") + writeFile(t, filepath.Join(nested, "replica.log"), "active") + writeFile(t, filepath.Join(nested, "replica.log.1"), "rotated in a new directory") + watcher.events <- fsnotify.Event{Name: backup, Op: fsnotify.Create} + watcher.events <- fsnotify.Event{Name: nested, Op: fsnotify.Create} + + close(release) + + // Both segments are captured, and the queued events plus the startup scan must + // not produce duplicates. + deadline := time.Now().Add(5 * time.Second) + for { + entries := rc.snapshot() + if len(entries) == 2 { + names := []string{entries[0].OriginalName, entries[1].OriginalName} + slices.Sort(names) + if !slices.Equal(names, []string{"raylet.out.1", "replica.log.1"}) { + t.Fatalf("captured %v, want both segments once each", names) + } + return + } + if time.Now().After(deadline) { + t.Fatalf("captured %+v, want exactly two segments", entries) + } + time.Sleep(10 * time.Millisecond) + } +} + +func TestCollectorRemovesSurplusStagingLinks(t *testing.T) { + dir := t.TempDir() + logsDir := filepath.Join(dir, "session", "logs") + stagingRoot := filepath.Join(dir, "rotated-staging") + src := filepath.Join(logsDir, "raylet.out.1") + writeFile(t, src, "one segment") + + // Same state and same capture ID, different paths: only the path tie-breaker + // can decide this deterministically. + const id = "0001780000000000000.aaaaaaaaaaaaaaaa" + first, err := newStagedEntry(statePending, "session-1", "node-1", "a", "raylet.out.1", id) + if err != nil { + t.Fatalf("newStagedEntry() error: %v", err) + } + second, err := newStagedEntry(statePending, "session-1", "node-1", "b", "raylet.out.1", id) + if err != nil { + t.Fatalf("newStagedEntry() error: %v", err) + } + for _, e := range []stagedEntry{second, first} { // created in the "wrong" order + if err := captureLink(src, e.path(stagingRoot)); err != nil { + t.Fatalf("captureLink() error: %v", err) + } + } + if _, nlink, err := statInode(src); err != nil || nlink != 3 { + t.Fatalf("link count = %d (err %v), want 3 before reconstruction", nlink, err) + } + + h := start(t, dir) + + entries := h.rc.snapshot() + if len(entries) != 1 { + t.Fatalf("index holds %d captures, want 1: %+v", len(entries), entries) + } + if entries[0] != first { + t.Errorf("kept %+v, want the lexicographically first path %+v", entries[0], first) + } + if _, err := os.Lstat(first.path(stagingRoot)); err != nil { + t.Errorf("the winning link was removed: %v", err) + } + if _, err := os.Lstat(second.path(stagingRoot)); !isVanished(err) { + t.Errorf("the surplus link was left behind: %v", err) + } + // The surplus link no longer pins the inode, so release can eventually work. + if _, nlink, err := statInode(src); err != nil || nlink != 2 { + t.Errorf("link count = %d (err %v), want 2 after cleanup", nlink, err) + } + if got := h.issues.matching("surplus record"); len(got) == 0 { + t.Errorf("the conflict was not reported, issues = %v", h.issues.all()) + } +} + +func TestCollectorFailsWhenSurplusLinkCannotBeRemoved(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("running as root: directory permissions do not prevent unlink") + } + dir := t.TempDir() + logsDir := filepath.Join(dir, "session", "logs") + stagingRoot := filepath.Join(dir, "rotated-staging") + src := filepath.Join(logsDir, "raylet.out.1") + writeFile(t, src, "one segment") + + winner, err := newStagedEntry(statePending, "session-1", "node-1", "", "raylet.out.1", "0001780000000000000.aaaaaaaaaaaaaaaa") + if err != nil { + t.Fatalf("newStagedEntry() error: %v", err) + } + loser, err := newStagedEntry(stateUploaded, "session-1", "node-1", "", "raylet.out.1", "0001780000000000001.bbbbbbbbbbbbbbbb") + if err != nil { + t.Fatalf("newStagedEntry() error: %v", err) + } + for _, e := range []stagedEntry{winner, loser} { + if err := captureLink(src, e.path(stagingRoot)); err != nil { + t.Fatalf("captureLink() error: %v", err) + } + } + loserDir := filepath.Dir(loser.path(stagingRoot)) + if err := os.Chmod(loserDir, 0o500); err != nil { + t.Fatalf("chmod: %v", err) + } + t.Cleanup(func() { _ = os.Chmod(loserDir, 0o750) }) + + rc, err := newRotatedCollector(rotatedCollectorConfig{ + LogsDir: logsDir, + StagingRoot: stagingRoot, + SessionName: "session-1", + NodeName: "node-1", + NewWatcher: func() (fsWatcher, error) { return newFakeWatcher(), nil }, + NewTicker: func(time.Duration) (<-chan time.Time, func()) { return make(chan time.Time), func() {} }, + OnIssue: func(error) {}, + }) + if err != nil { + t.Fatalf("newRotatedCollector() error: %v", err) + } + + // Starting with a link nothing tracks would pin the inode forever, so the + // collector must refuse to start rather than run with a broken invariant. + runErr := runExpectingFailure(t, rc) + if !strings.Contains(runErr.Error(), "staging volume is inconsistent") { + t.Errorf("Run() error = %v, want it to name the inconsistency", runErr) + } +} + +func TestRemoveSurplusLinkLeavesReplacedPathAlone(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "pending", "raylet.out.1.rotated.0001780000000000000.aaaaaaaaaaaaaaaa") + writeFile(t, path, "a different file now lives here") + + // The record was taken when the path held another inode; the file there now is + // not ours to delete. + err := removeSurplusLink(stagedRecord{key: inodeKey{Dev: 1, Ino: 999999}, path: path}) + if err == nil { + t.Fatal("removeSurplusLink() removed a path that no longer holds the recorded inode") + } + if !strings.Contains(err.Error(), "left in place") { + t.Errorf("error = %v, want it to say the link was left in place", err) + } + if _, statErr := os.Lstat(path); statErr != nil { + t.Errorf("the replaced file was removed: %v", statErr) + } +} + +func TestCollectorIgnoresSpecialStagingFiles(t *testing.T) { + dir := t.TempDir() + logsDir := filepath.Join(dir, "session", "logs") + stagingRoot := filepath.Join(dir, "rotated-staging") + writeFile(t, filepath.Join(logsDir, "raylet.out"), "active") + + fifo := filepath.Join(stagingRoot, "session-1", "node-1", "pending", + "raylet.out.1.rotated.0001780000000000000.aaaaaaaaaaaaaaaa") + if err := os.MkdirAll(filepath.Dir(fifo), 0o750); err != nil { + t.Fatalf("create staging directory: %v", err) + } + if err := syscall.Mkfifo(fifo, 0o600); err != nil { + t.Skipf("cannot create a FIFO on this platform: %v", err) + } + + h := start(t, dir) + + // A FIFO with a plausible staging name must never be indexed: an uploader + // opening it could block forever. + if entries := h.rc.snapshot(); len(entries) != 0 { + t.Errorf("a special file was restored into the index: %+v", entries) + } + if got := h.issues.matching("not a regular file"); len(got) == 0 { + t.Errorf("the special file was not reported, issues = %v", h.issues.all()) + } +} + +// runExpectingFailure runs the collector and returns the error startup failed with. +// If startup wrongly succeeds the loop would run forever, so this reports that +// directly instead of letting the test hang until the package timeout. +func runExpectingFailure(t *testing.T, rc *rotatedCollector) error { + t.Helper() + done := make(chan error, 1) + go func() { done <- rc.Run() }() + select { + case err := <-done: + if err == nil { + t.Fatal("Run() returned nil: startup was expected to fail") + } + return err + case <-time.After(10 * time.Second): + rc.Stop() + t.Fatal("Run() entered its event loop: startup was expected to fail") + return nil + } +} + +// startForFailure builds a collector without running it, so a test can observe how +// startup fails. +func startForFailure(t *testing.T, dir string, tweak func(*rotatedCollectorConfig)) (*rotatedCollector, *fakeWatcher) { + t.Helper() + logsDir := filepath.Join(dir, "session", "logs") + if err := os.MkdirAll(logsDir, 0o750); err != nil { + t.Fatalf("create logs dir: %v", err) + } + watcher := newFakeWatcher() + issues := &issueLog{} + cfg := rotatedCollectorConfig{ + LogsDir: logsDir, + StagingRoot: filepath.Join(dir, "rotated-staging"), + SessionName: "session-1", + NodeName: "node-1", + NewWatcher: func() (fsWatcher, error) { return watcher, nil }, + NewTicker: func(time.Duration) (<-chan time.Time, func()) { return make(chan time.Time), func() {} }, + OnIssue: issues.add, + } + tweak(&cfg) + rc, err := newRotatedCollector(cfg) + if err != nil { + t.Fatalf("newRotatedCollector() error: %v", err) + } + return rc, watcher +} + +func TestCollectorFailsWhenWatchCoverageIsIncomplete(t *testing.T) { + // Starting with part of the tree unwatched looks healthy but loses any segment + // created and deleted in the gap, so it must be fatal. + tests := []struct { + name string + prepare func(t *testing.T, dir, logsDir string, w *fakeWatcher) + wantIn string + }{ + { + name: "root watch fails", + prepare: func(_ *testing.T, _, logsDir string, w *fakeWatcher) { + w.failAdd = map[string]error{logsDir: os.ErrPermission} + }, + wantIn: "logs", + }, + { + name: "nested watch fails", + prepare: func(t *testing.T, _, logsDir string, w *fakeWatcher) { + writeFile(t, filepath.Join(logsDir, "events", "event.log"), "active") + w.failAdd = map[string]error{filepath.Join(logsDir, "events"): os.ErrPermission} + }, + wantIn: "events", + }, + { + name: "directory enumeration fails", + prepare: func(t *testing.T, _, logsDir string, _ *fakeWatcher) { + if os.Geteuid() == 0 { + t.Skip("running as root: directory permissions do not prevent reads") + } + nested := filepath.Join(logsDir, "serve") + if err := os.MkdirAll(nested, 0o750); err != nil { + t.Fatalf("create nested dir: %v", err) + } + if err := os.Chmod(nested, 0o000); err != nil { + t.Fatalf("chmod: %v", err) + } + t.Cleanup(func() { _ = os.Chmod(nested, 0o750) }) + }, + wantIn: "serve", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + logsDir := filepath.Join(dir, "session", "logs") + if err := os.MkdirAll(logsDir, 0o750); err != nil { + t.Fatalf("create logs dir: %v", err) + } + // A backup that would be captured if startup wrongly continued. + writeFile(t, filepath.Join(logsDir, "raylet.out"), "active") + writeFile(t, filepath.Join(logsDir, "raylet.out.1"), "rotated") + + reconstructed := false + rc, watcher := startForFailure(t, dir, func(cfg *rotatedCollectorConfig) { + cfg.BeforeReconstruct = func() { reconstructed = true } + }) + tt.prepare(t, dir, logsDir, watcher) + + err := runExpectingFailure(t, rc) + if !strings.Contains(err.Error(), "cannot watch the whole logs tree") || !strings.Contains(err.Error(), tt.wantIn) { + t.Errorf("Run() error = %v, want it to name the failure and the directory %q", err, tt.wantIn) + } + if reconstructed { + t.Error("reconstruction ran even though watch installation failed") + } + if _, statErr := os.Lstat(filepath.Join(dir, "rotated-staging")); !isVanished(statErr) { + t.Error("captures were staged even though watch installation failed") + } + if !watcher.isClosed() { + t.Error("the watcher was not closed when startup failed") + } + }) + } +} + +func TestCollectorFailsWhenStagingCannotBeRead(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("running as root: directory permissions do not prevent reads") + } + dir := t.TempDir() + logsDir := filepath.Join(dir, "session", "logs") + stagingRoot := filepath.Join(dir, "rotated-staging") + writeFile(t, filepath.Join(logsDir, "raylet.out"), "active") + + // An unreadable subtree may hold real staged links; adopting only what is + // readable would leave those pinning inodes with no owner. + unreadable := filepath.Join(stagingRoot, "session-1", "node-1", "pending") + if err := os.MkdirAll(unreadable, 0o750); err != nil { + t.Fatalf("create staging dir: %v", err) + } + if err := os.Chmod(unreadable, 0o000); err != nil { + t.Fatalf("chmod: %v", err) + } + t.Cleanup(func() { _ = os.Chmod(unreadable, 0o750) }) + + rc, watcher := startForFailure(t, dir, func(cfg *rotatedCollectorConfig) { + cfg.StagingRoot = stagingRoot + }) + + err := runExpectingFailure(t, rc) + if !strings.Contains(err.Error(), "read staging volume") { + t.Errorf("Run() error = %v, want it to name the unreadable staging volume", err) + } + if !watcher.isClosed() { + t.Error("the watcher was not closed when startup failed") + } +} + +func TestCollectorFailsWhenSurplusPathWasReplaced(t *testing.T) { + dir := t.TempDir() + logsDir := filepath.Join(dir, "session", "logs") + stagingRoot := filepath.Join(dir, "rotated-staging") + src := filepath.Join(logsDir, "raylet.out.1") + writeFile(t, src, "one segment") + + winner, err := newStagedEntry(statePending, "session-1", "node-1", "", "raylet.out.1", "0001780000000000000.aaaaaaaaaaaaaaaa") + if err != nil { + t.Fatalf("newStagedEntry() error: %v", err) + } + loser, err := newStagedEntry(statePending, "session-1", "node-1", "", "raylet.out.1", "0001780000000000001.bbbbbbbbbbbbbbbb") + if err != nil { + t.Fatalf("newStagedEntry() error: %v", err) + } + for _, e := range []stagedEntry{winner, loser} { + if err := captureLink(src, e.path(stagingRoot)); err != nil { + t.Fatalf("captureLink() error: %v", err) + } + } + + // The conflict is reported immediately before the surplus link is removed, so + // swapping the file from inside OnIssue lands exactly in that window: the + // record says one inode, the path now holds another. + replaced := loser.path(stagingRoot) + var once sync.Once + watcher := newFakeWatcher() + rc, err := newRotatedCollector(rotatedCollectorConfig{ + LogsDir: logsDir, + StagingRoot: stagingRoot, + SessionName: "session-1", + NodeName: "node-1", + NewWatcher: func() (fsWatcher, error) { return watcher, nil }, + NewTicker: func(time.Duration) (<-chan time.Time, func()) { return make(chan time.Time), func() {} }, + OnIssue: func(issue error) { + if !strings.Contains(issue.Error(), "surplus record") { + return + } + once.Do(func() { + if err := os.Remove(replaced); err != nil { + t.Errorf("remove surplus link: %v", err) + return + } + writeFile(t, replaced, "an unrelated file") + }) + }, + }) + if err != nil { + t.Fatalf("newRotatedCollector() error: %v", err) + } + + // The collector must not delete a file it did not link, and must not carry on + // with a staging tree it cannot account for. + runErr := runExpectingFailure(t, rc) + if !strings.Contains(runErr.Error(), "left in place") || !strings.Contains(runErr.Error(), "staging volume is inconsistent") { + t.Errorf("Run() error = %v, want an inconsistency naming the untouched path", runErr) + } + if got, readErr := os.ReadFile(replaced); readErr != nil || string(got) != "an unrelated file" { + t.Errorf("the replacement file was modified: %q (err %v)", got, readErr) + } + if _, statErr := os.Lstat(winner.path(stagingRoot)); statErr != nil { + t.Errorf("the winning link was removed: %v", statErr) + } + if entries := rc.snapshot(); entries != nil { + t.Errorf("the collector began operating after a failed reconstruction: %+v", entries) + } + if !watcher.isClosed() { + t.Error("the watcher was not closed when startup failed") + } +} diff --git a/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_fs.go b/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_fs.go new file mode 100644 index 00000000000..c315d3f4823 --- /dev/null +++ b/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_fs.go @@ -0,0 +1,192 @@ +package logcollector + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "syscall" +) + +// stagingDirPerm keeps the staging tree collector-private. Unlike prev-logs, Ray +// never reads these directories; only the collector writes and drains them. +const stagingDirPerm fs.FileMode = 0o750 + +// statInode returns the device/inode pair and current link count of path without +// following symlinks. A hard link is a real directory entry for the same inode, so +// this reports on the staged link exactly as it does on Ray's own name. +func statInode(path string) (inodeKey, uint64, error) { + fi, err := os.Lstat(path) + if err != nil { + return inodeKey{}, 0, fmt.Errorf("stat %s: %w", path, err) + } + key, nlink, err := inodeFromFileInfo(fi) + if err != nil { + return inodeKey{}, 0, fmt.Errorf("stat %s: %w", path, err) + } + return key, nlink, nil +} + +// captureLink pins src by hard-linking it to dst, creating dst's parent directory. +// +// Pinning is what makes capture safe: from this point the bytes survive Ray +// rotating the name away or deleting it, and no copy is made, so a large segment +// costs no additional blocks while Ray still holds its own link. +func captureLink(src, dst string) error { + dir := filepath.Dir(dst) + if err := os.MkdirAll(dir, stagingDirPerm); err != nil { + return fmt.Errorf("create staging directory %s: %w", dir, err) + } + if err := os.Link(src, dst); err != nil { + if isAlreadyStaged(err) { + return fmt.Errorf("hard link capture: staging path %s already exists, so a capture ID was reused: %w", dst, err) + } + return fmt.Errorf("hard link capture: %w", err) + } + return nil +} + +// isUnsupportedLinkError reports whether err means hard-link capture cannot work in +// this deployment: the staging tree is on a different filesystem than the logs +// (EXDEV), or the collector may not link the Ray container's files (EPERM/EACCES, +// typically a custom image whose Ray user differs from the collector's UID). +// +// v1 warns and skips those segments. The alternative — copying — would need a +// content hash to stay deduplicated, because a dev+inode marker can match a +// completely unrelated file after the kernel reuses an inode number. +func isUnsupportedLinkError(err error) bool { + return errors.Is(err, syscall.EXDEV) || + errors.Is(err, syscall.EPERM) || + errors.Is(err, syscall.EACCES) +} + +// isVanished reports whether err means the file is already gone. Losing that race +// with Ray's rotation delete is expected and is not a capture failure. +func isVanished(err error) bool { + return errors.Is(err, fs.ErrNotExist) +} + +// isWatchResourceExhausted reports whether err means the kernel had no room for +// another watch: inotify reports its per-user watch limit as ENOSPC, and the +// descriptor limits as EMFILE and ENFILE. All three are about what else the machine is +// doing rather than about this deployment, so a later attempt can succeed. +// +// It is only ever consulted for a watch, never for a capture: ENOSPC from os.Link +// means the staging volume is full, which is a different condition entirely and is +// handled by the intake gate. +func isWatchResourceExhausted(err error) bool { + return errors.Is(err, syscall.ENOSPC) || + errors.Is(err, syscall.EMFILE) || + errors.Is(err, syscall.ENFILE) +} + +// isAlreadyStaged reports whether a capture link collided with an existing file. +// +// This is not deduplication: every discovery mints a fresh capture ID and so a +// fresh destination, which is why the index — not EEXIST — is what stops an inode +// being captured twice. A collision here means the same capture ID was used twice, +// which is a bug worth surfacing rather than a condition to swallow. +func isAlreadyStaged(err error) bool { + return errors.Is(err, fs.ErrExist) +} + +// promoteCapture moves a capture from pending to uploaded, on disk and then in the +// index. It is the only way either changes: there is no memory-only promotion, and +// no disk-only one. +// +// Every fallible step happens before the rename. Once the rename succeeds all that +// remains is an assignment to a struct owned by the single event-loop goroutine, +// which cannot fail — so disk and index cannot end up disagreeing. Rolling the +// rename back instead would only add a second operation that can fail on its own. +// +// The rename is atomic, so a crash leaves the capture in exactly one state and a +// restarting collector can tell an unsent capture from a finished one by path alone. +// Capture identity is carried across unchanged: same original name, same capture ID, +// therefore the same object key on any retry. +func promoteCapture(stagingRoot string, ix *captureIndex, key inodeKey) (stagedEntry, error) { + c, ok := ix.lookup(key) + if !ok { + return stagedEntry{}, fmt.Errorf("promote capture: no capture pinned for %s", key) + } + if !validTransition(c.Entry.State, stateUploaded) { + return stagedEntry{}, fmt.Errorf("promote capture %s: cannot move from %q to %q", + c.Entry.CaptureID, c.Entry.State, stateUploaded) + } + + promoted := c.Entry.withState(stateUploaded) + src := c.Entry.path(stagingRoot) + dst := promoted.path(stagingRoot) + + dir := filepath.Dir(dst) + if err := os.MkdirAll(dir, stagingDirPerm); err != nil { + return stagedEntry{}, fmt.Errorf("create staging directory %s: %w", dir, err) + } + if err := os.Rename(src, dst); err != nil { + // Disk and index are both still pending, so the upload can be retried. + return stagedEntry{}, fmt.Errorf("promote capture %s to uploaded: %w", c.Entry.CaptureID, err) + } + + c.Entry = promoted + return promoted, nil +} + +// releaseCapture unlinks a fully uploaded capture and only then forgets it. +// +// It reads the link count itself rather than accepting one, because a caller- +// supplied count is a claim about the past: between the caller's stat and this call +// Ray may have created or dropped a link. Everything the safety decision rests on is +// therefore established here, immediately before the unlink. +// +// Removing the index entry after — never before — the unlink matters too: the kernel +// may hand that inode number to an unrelated file the moment the last link +// disappears, so a stale entry could later match the wrong file. +func releaseCapture(stagingRoot string, ix *captureIndex, key inodeKey) error { + c, ok := ix.lookup(key) + if !ok { + return fmt.Errorf("release capture: no capture pinned for %s", key) + } + if c.Entry.State != stateUploaded { + return fmt.Errorf("release capture %s: not releasable in state %q", c.Entry.CaptureID, c.Entry.State) + } + + p := c.Entry.path(stagingRoot) + staged, nlink, err := statInode(p) + if err != nil { + // The index says the capture is staged here. If it is not, disk and index + // disagree: dropping the entry now could strand a link staged elsewhere, so + // this has to surface rather than look like a completed release. + return fmt.Errorf("release capture %s: index and staging volume disagree about %s: %w", c.Entry.CaptureID, p, err) + } + if staged != c.Inode { + return fmt.Errorf("release capture %s: %s now holds %s, not the pinned %s", c.Entry.CaptureID, p, staged, c.Inode) + } + if !c.releasable(nlink) { + return fmt.Errorf("release capture %s: %d link(s) remain, so Ray still holds the segment", c.Entry.CaptureID, nlink) + } + + if err := os.Remove(p); err != nil { + return fmt.Errorf("release capture %s: %w", c.Entry.CaptureID, err) + } + ix.remove(c.Inode) + return nil +} + +// regularFileExists reports whether path is a regular file, without following +// symlinks. +func regularFileExists(path string) bool { + fi, err := os.Lstat(path) + return err == nil && fi.Mode().IsRegular() +} + +// baseKnownWith answers "does this backup belong to a log file that rotates here?" +// by checking the live directory first and the index's memory second, so a backup +// that appears while its active name is briefly unlinked is still recognized. +func baseKnownWith(ix *captureIndex) baseKnownFunc { + return func(dir, base string) bool { + if regularFileExists(filepath.Join(dir, base)) { + return true + } + return ix != nil && ix.baseObserved(dir, base) + } +} diff --git a/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_fs_test.go b/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_fs_test.go new file mode 100644 index 00000000000..818ce0ce591 --- /dev/null +++ b/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_fs_test.go @@ -0,0 +1,765 @@ +package logcollector + +import ( + "errors" + "io/fs" + "os" + "path/filepath" + "slices" + "sort" + "strings" + "syscall" + "testing" +) + +// writeFile creates a file with content, failing the test on error. +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("create directory for %s: %v", path, err) + } + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + +func TestStatInode(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "raylet.out") + writeFile(t, path, "log line") + + key, nlink, err := statInode(path) + if err != nil { + t.Fatalf("statInode() error: %v", err) + } + if key.Ino == 0 { + t.Error("statInode() returned inode 0") + } + if nlink != 1 { + t.Errorf("statInode() nlink = %d, want 1", nlink) + } + + // A second file in the same directory is a different inode on the same device. + other := filepath.Join(dir, "gcs_server.out") + writeFile(t, other, "log line") + otherKey, _, err := statInode(other) + if err != nil { + t.Fatalf("statInode() error: %v", err) + } + if otherKey == key { + t.Error("statInode() returned the same key for two distinct files") + } + if otherKey.Dev != key.Dev { + t.Errorf("statInode() device differs within one directory: %d vs %d", otherKey.Dev, key.Dev) + } + + if _, _, err := statInode(filepath.Join(dir, "missing.out")); !isVanished(err) { + t.Errorf("statInode() on a missing file = %v, want a not-exist error", err) + } +} + +func TestCaptureLinkPinsInode(t *testing.T) { + dir := t.TempDir() + logsDir := filepath.Join(dir, "logs") + stagingRoot := filepath.Join(dir, "rotated-staging") + + src := filepath.Join(logsDir, "raylet.out.1") + writeFile(t, src, "rotated content") + + entry := stagedEntry{ + State: statePending, SessionName: "session-1", NodeName: "node-1", + OriginalName: "raylet.out.1", CaptureID: "0001780000000000000.a1b2c3d4e5f60718", + } + dst := entry.path(stagingRoot) + if err := captureLink(src, dst); err != nil { + t.Fatalf("captureLink() error: %v", err) + } + + srcKey, srcLinks, err := statInode(src) + if err != nil { + t.Fatalf("statInode(src) error: %v", err) + } + dstKey, dstLinks, err := statInode(dst) + if err != nil { + t.Fatalf("statInode(dst) error: %v", err) + } + if srcKey != dstKey { + t.Errorf("staged link is a different inode: %s vs %s", srcKey, dstKey) + } + if srcLinks != 2 || dstLinks != 2 { + t.Errorf("link count = (%d, %d), want (2, 2)", srcLinks, dstLinks) + } +} + +func TestCaptureSurvivesRotationAndDeletion(t *testing.T) { + dir := t.TempDir() + logsDir := filepath.Join(dir, "logs") + stagingRoot := filepath.Join(dir, "rotated-staging") + + src := filepath.Join(logsDir, "raylet.out.1") + const content = "the segment Ray is about to rotate away" + writeFile(t, src, content) + + entry := stagedEntry{ + State: statePending, SessionName: "session-1", NodeName: "node-1", + OriginalName: "raylet.out.1", CaptureID: "0001780000000000000.a1b2c3d4e5f60718", + } + staged := entry.path(stagingRoot) + if err := captureLink(src, staged); err != nil { + t.Fatalf("captureLink() error: %v", err) + } + pinned, _, err := statInode(staged) + if err != nil { + t.Fatalf("statInode(staged) error: %v", err) + } + + // Ray's next rotation renames the segment to ".2". + rotated := filepath.Join(logsDir, "raylet.out.2") + if err := os.Rename(src, rotated); err != nil { + t.Fatalf("rename src: %v", err) + } + afterRename, links, err := statInode(staged) + if err != nil { + t.Fatalf("statInode(staged) after rename error: %v", err) + } + if afterRename != pinned { + t.Errorf("staged link changed inode after rename: %s -> %s", pinned, afterRename) + } + if links != 2 { + t.Errorf("link count after rename = %d, want 2", links) + } + + // Rotation eventually deletes the segment; the staged bytes must survive. + if err := os.Remove(rotated); err != nil { + t.Fatalf("remove rotated file: %v", err) + } + afterDelete, links, err := statInode(staged) + if err != nil { + t.Fatalf("statInode(staged) after delete error: %v", err) + } + if afterDelete != pinned { + t.Errorf("staged link changed inode after delete: %s -> %s", pinned, afterDelete) + } + if links != 1 { + t.Errorf("link count after delete = %d, want 1", links) + } + got, err := os.ReadFile(staged) + if err != nil { + t.Fatalf("read staged file: %v", err) + } + if string(got) != content { + t.Errorf("staged content = %q, want %q", got, content) + } +} + +func TestCaptureOfSuccessiveSegmentsAtSamePath(t *testing.T) { + dir := t.TempDir() + logsDir := filepath.Join(dir, "logs") + stagingRoot := filepath.Join(dir, "rotated-staging") + path := filepath.Join(logsDir, "raylet.out.1") + + g := newCaptureIDGenerator() + identity := clusterIdentity{RootDir: "root", Namespace: "default", ClusterName: "my-cluster"} + ix := newCaptureIndex() + + captureSegment := func(content string) (inodeKey, stagedEntry) { + t.Helper() + writeFile(t, path, content) + key, _, err := statInode(path) + if err != nil { + t.Fatalf("statInode() error: %v", err) + } + id, err := g.next() + if err != nil { + t.Fatalf("next() error: %v", err) + } + entry := stagedEntry{ + State: statePending, SessionName: "session-1", NodeName: "node-1", + OriginalName: "raylet.out.1", CaptureID: id, + } + if err := captureLink(path, entry.path(stagingRoot)); err != nil { + t.Fatalf("captureLink() error: %v", err) + } + if _, _, err := ix.add(key, entry); err != nil { + t.Fatalf("add() error: %v", err) + } + return key, entry + } + + firstKey, first := captureSegment("first segment") + // Ray deletes the segment and a later rotation puts a new one at the same name. + // Our link keeps the old inode alive, so the new file must be a different inode. + if err := os.Remove(path); err != nil { + t.Fatalf("remove first segment: %v", err) + } + secondKey, second := captureSegment("second segment") + + if firstKey == secondKey { + t.Fatalf("both segments reported inode %s; the staged link should have kept the first alive", firstKey) + } + if first.CaptureID == second.CaptureID { + t.Error("segments sharing a rotation filename were given the same capture ID") + } + if first.objectKey(identity) == second.objectKey(identity) { + t.Errorf("segments sharing a rotation filename map to one object key: %q", first.objectKey(identity)) + } + if ix.len() != 2 { + t.Errorf("index holds %d captures, want 2", ix.len()) + } + + for _, e := range []stagedEntry{first, second} { + content, err := os.ReadFile(e.path(stagingRoot)) + if err != nil { + t.Fatalf("read staged capture %s: %v", e.CaptureID, err) + } + if len(content) == 0 { + t.Errorf("staged capture %s is empty", e.CaptureID) + } + } +} + +func TestBaseKnownWith(t *testing.T) { + dir := t.TempDir() + logsDir := filepath.Join(dir, "logs") + writeFile(t, filepath.Join(logsDir, "raylet.out"), "active") + + ix := newCaptureIndex() + baseKnown := baseKnownWith(ix) + + // The active file is present: the backup beside it is eligible. + if status := evaluateCandidate(logsDir, "raylet.out.1", 0, baseKnown); status != candidateEligible { + t.Errorf("backup with a live base = %v, want %v", status, candidateEligible) + } + + // A file that merely ends in ".1" with no active base is left alone. + if status := evaluateCandidate(logsDir, "user-data.1", 0, baseKnown); status != candidateUnknownBase { + t.Errorf("unrelated numeric suffix = %v, want %v", status, candidateUnknownBase) + } + + // A rotation cascade briefly unlinks the active name. Once observed, a backup + // that appears while the base is missing is still recognized. + ix.observeBase(logsDir, "gcs_server.out") + if status := evaluateCandidate(logsDir, "gcs_server.out.1", 0, baseKnown); status != candidateEligible { + t.Errorf("backup with a previously observed base = %v, want %v", status, candidateEligible) + } + if status := evaluateCandidate(filepath.Join(logsDir, "events"), "gcs_server.out.1", 0, baseKnown); status != candidateUnknownBase { + t.Errorf("observed base leaked into another directory: %v", status) + } + + // A symlink named like an active log file is not an active log file. + linkTarget := filepath.Join(dir, "elsewhere.out") + writeFile(t, linkTarget, "not a log") + if err := os.Symlink(linkTarget, filepath.Join(logsDir, "dashboard.log")); err != nil { + t.Fatalf("create symlink: %v", err) + } + if status := evaluateCandidate(logsDir, "dashboard.log.1", 0, baseKnown); status != candidateUnknownBase { + t.Errorf("symlinked base = %v, want %v", status, candidateUnknownBase) + } +} + +func TestRegularFileExists(t *testing.T) { + dir := t.TempDir() + file := filepath.Join(dir, "raylet.out") + writeFile(t, file, "active") + + if !regularFileExists(file) { + t.Error("regularFileExists() = false for a regular file") + } + if regularFileExists(dir) { + t.Error("regularFileExists() = true for a directory") + } + if regularFileExists(filepath.Join(dir, "missing")) { + t.Error("regularFileExists() = true for a missing file") + } + + link := filepath.Join(dir, "link.out") + if err := os.Symlink(file, link); err != nil { + t.Fatalf("create symlink: %v", err) + } + if regularFileExists(link) { + t.Error("regularFileExists() followed a symlink") + } +} + +func TestIsUnsupportedLinkError(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + { + name: "cross-device link", + err: &os.LinkError{Op: "link", Old: "/tmp/ray/logs/raylet.out.1", New: "/staging/x", Err: syscall.EXDEV}, + want: true, + }, + { + name: "not permitted", + err: &os.LinkError{Op: "link", Old: "/tmp/ray/logs/raylet.out.1", New: "/staging/x", Err: syscall.EPERM}, + want: true, + }, + { + name: "access denied", + err: &os.LinkError{Op: "link", Old: "/tmp/ray/logs/raylet.out.1", New: "/staging/x", Err: syscall.EACCES}, + want: true, + }, + { + name: "wrapped by captureLink", + err: errors.Join(errors.New("hard link capture"), &os.LinkError{Op: "link", Err: syscall.EXDEV}), + want: true, + }, + { + name: "vanished before capture", + err: &os.LinkError{Op: "link", Old: "/tmp/ray/logs/raylet.out.1", New: "/staging/x", Err: syscall.ENOENT}, + want: false, + }, + {name: "unrelated error", err: errors.New("boom"), want: false}, + {name: "no error", err: nil, want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isUnsupportedLinkError(tt.err); got != tt.want { + t.Errorf("isUnsupportedLinkError(%v) = %v, want %v", tt.err, got, tt.want) + } + }) + } +} + +func TestIsVanished(t *testing.T) { + if !isVanished(&os.LinkError{Op: "link", Err: syscall.ENOENT}) { + t.Error("isVanished() = false for ENOENT") + } + if !isVanished(fs.ErrNotExist) { + t.Error("isVanished() = false for fs.ErrNotExist") + } + if isVanished(&os.LinkError{Op: "link", Err: syscall.EXDEV}) { + t.Error("isVanished() = true for EXDEV") + } + if isVanished(nil) { + t.Error("isVanished() = true for nil") + } +} + +func TestCaptureLinkReportsMissingSource(t *testing.T) { + dir := t.TempDir() + err := captureLink(filepath.Join(dir, "logs", "missing.out.1"), filepath.Join(dir, "staging", "x")) + if !isVanished(err) { + t.Errorf("captureLink() on a missing source = %v, want a not-exist error", err) + } + if err != nil && !strings.Contains(err.Error(), "missing.out.1") { + t.Errorf("captureLink() error %q does not name the source path", err) + } +} + +func TestRestartReconstructsStagedCaptures(t *testing.T) { + dir := t.TempDir() + stagingRoot := filepath.Join(dir, "rotated-staging") + logsDir := filepath.Join(dir, "logs") + + g := newCaptureIDGenerator() + original := newCaptureIndex() + stage := func(relDir, name string, promote bool) stagedEntry { + t.Helper() + src := filepath.Join(logsDir, filepath.FromSlash(relDir), name) + writeFile(t, src, "content of "+name) + id, err := g.next() + if err != nil { + t.Fatalf("next() error: %v", err) + } + entry, err := newStagedEntry(statePending, "session-1", "node-1", relDir, name, id) + if err != nil { + t.Fatalf("newStagedEntry() error: %v", err) + } + if err := captureLink(src, entry.path(stagingRoot)); err != nil { + t.Fatalf("captureLink() error: %v", err) + } + key, _, err := statInode(entry.path(stagingRoot)) + if err != nil { + t.Fatalf("statInode() error: %v", err) + } + if _, _, err := original.add(key, entry); err != nil { + t.Fatalf("add() error: %v", err) + } + if promote { + entry, err = promoteCapture(stagingRoot, original, key) + if err != nil { + t.Fatalf("promoteCapture() error: %v", err) + } + } + return entry + } + + before := map[string]stagedEntry{} + for _, e := range []stagedEntry{ + stage("", "raylet.out.1", false), + stage("events", "event.log.2", true), + } { + before[e.CaptureID] = e + } + + // Restart: a fresh index rebuilt from the staging volume alone. + after := newCaptureIndex() + err := filepath.WalkDir(stagingRoot, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + entry, err := parseStagedPath(stagingRoot, p) + if err != nil { + return err + } + key, _, err := statInode(p) + if err != nil { + return err + } + _, err = after.restore(key, entry) + return err + }) + if err != nil { + t.Fatalf("reconstruct staging volume: %v", err) + } + + if after.len() != len(before) { + t.Fatalf("reconstructed %d captures, want %d", after.len(), len(before)) + } + for _, got := range after.entries() { + want, ok := before[got.CaptureID] + if !ok { + t.Errorf("reconstruction minted a new capture ID %q", got.CaptureID) + continue + } + if got != want { + t.Errorf("reconstructed %+v, want %+v", got, want) + } + } +} + +func TestCaptureLinkRejectsReusedCaptureID(t *testing.T) { + dir := t.TempDir() + stagingRoot := filepath.Join(dir, "rotated-staging") + writeFile(t, filepath.Join(dir, "logs", "raylet.out.1"), "first") + writeFile(t, filepath.Join(dir, "logs", "raylet.out.2"), "second") + + entry, err := newStagedEntry(statePending, "session-1", "node-1", "", "raylet.out.1", "0001780000000000000.a1b2c3d4e5f60718") + if err != nil { + t.Fatalf("newStagedEntry() error: %v", err) + } + if err := captureLink(filepath.Join(dir, "logs", "raylet.out.1"), entry.path(stagingRoot)); err != nil { + t.Fatalf("captureLink() error: %v", err) + } + + // Reusing a capture ID would silently overwrite another segment's object, so the + // collision must surface rather than be treated as deduplication. + err = captureLink(filepath.Join(dir, "logs", "raylet.out.2"), entry.path(stagingRoot)) + if !isAlreadyStaged(err) { + t.Fatalf("captureLink() on a reused capture ID = %v, want an already-exists error", err) + } + if !strings.Contains(err.Error(), entry.CaptureID) { + t.Errorf("captureLink() error %q does not name the reused capture ID", err) + } + + // The first capture's bytes must be untouched. + got, err := os.ReadFile(entry.path(stagingRoot)) + if err != nil { + t.Fatalf("read staged capture: %v", err) + } + if string(got) != "first" { + t.Errorf("staged content = %q, want %q", got, "first") + } +} + +// stagedCapture pins one file and returns the index, its key and the source path. +func stagedCapture(t *testing.T, dir, relDir, name string) (*captureIndex, inodeKey, string, string) { + t.Helper() + stagingRoot := filepath.Join(dir, "rotated-staging") + src := filepath.Join(dir, "logs", filepath.FromSlash(relDir), name) + writeFile(t, src, "content of "+name) + + entry, err := newStagedEntry(statePending, "session-1", "node-1", relDir, name, "0001780000000000000.a1b2c3d4e5f60718") + if err != nil { + t.Fatalf("newStagedEntry() error: %v", err) + } + if err := captureLink(src, entry.path(stagingRoot)); err != nil { + t.Fatalf("captureLink() error: %v", err) + } + key, _, err := statInode(entry.path(stagingRoot)) + if err != nil { + t.Fatalf("statInode() error: %v", err) + } + ix := newCaptureIndex() + if _, _, err := ix.add(key, entry); err != nil { + t.Fatalf("add() error: %v", err) + } + return ix, key, stagingRoot, src +} + +func TestPromoteCaptureMovesDiskAndIndexTogether(t *testing.T) { + dir := t.TempDir() + ix, key, stagingRoot, _ := stagedCapture(t, dir, "events", "event.log.1") + pendingPath := mustEntry(t, ix, key).path(stagingRoot) + + promoted, err := promoteCapture(stagingRoot, ix, key) + if err != nil { + t.Fatalf("promoteCapture() error: %v", err) + } + if promoted.State != stateUploaded { + t.Errorf("promoted state = %q, want %q", promoted.State, stateUploaded) + } + + // Disk moved. + if _, err := os.Lstat(pendingPath); !isVanished(err) { + t.Errorf("pending path still exists after promotion: %v", err) + } + if _, err := os.Lstat(promoted.path(stagingRoot)); err != nil { + t.Errorf("uploaded path missing after promotion: %v", err) + } + // Index moved with it, and still points at a path that exists. + tracked := mustEntry(t, ix, key) + if tracked != promoted { + t.Errorf("index entry = %+v, want %+v", tracked, promoted) + } + if _, err := os.Lstat(tracked.path(stagingRoot)); err != nil { + t.Errorf("index points at a path that does not exist: %v", err) + } +} + +func TestPromoteCaptureFailureLeavesDiskAndIndexPending(t *testing.T) { + dir := t.TempDir() + ix, key, stagingRoot, _ := stagedCapture(t, dir, "events", "event.log.1") + pending := mustEntry(t, ix, key) + + // Block the uploaded tree so the rename cannot happen. + writeFile(t, filepath.Join(stagingRoot, "session-1", "node-1", string(stateUploaded)), "blocker") + + if promoted, err := promoteCapture(stagingRoot, ix, key); err == nil { + t.Fatalf("promoteCapture() = %+v, want error", promoted) + } + + if tracked := mustEntry(t, ix, key); tracked != pending { + t.Errorf("index moved to %+v although the rename failed, want %+v", tracked, pending) + } + if _, err := os.Lstat(pending.path(stagingRoot)); err != nil { + t.Errorf("pending link lost after a failed promotion: %v", err) + } +} + +func TestReleaseCaptureRequiresUploadedAndLastLink(t *testing.T) { + dir := t.TempDir() + ix, key, stagingRoot, src := stagedCapture(t, dir, "", "raylet.out.1") + pendingPath := mustEntry(t, ix, key).path(stagingRoot) + + // Pending data is never released, however few links remain. + if err := releaseCapture(stagingRoot, ix, key); err == nil { + t.Error("releaseCapture() released a pending capture") + } + if _, ok := ix.lookup(key); !ok { + t.Fatal("releaseCapture() forgot a capture it refused to release") + } + if _, err := os.Lstat(pendingPath); err != nil { + t.Errorf("pending link was unlinked: %v", err) + } + + uploaded, err := promoteCapture(stagingRoot, ix, key) + if err != nil { + t.Fatalf("promoteCapture() error: %v", err) + } + + // Ray still holds its own link: releaseCapture must read that for itself. + if err := releaseCapture(stagingRoot, ix, key); err == nil { + t.Error("releaseCapture() released a segment Ray still links to") + } + if _, ok := ix.lookup(key); !ok { + t.Fatal("releaseCapture() forgot a capture it refused to release") + } + if _, err := os.Lstat(uploaded.path(stagingRoot)); err != nil { + t.Errorf("staged link was unlinked: %v", err) + } + + // Ray drops its name; ours is now the only link. + if err := os.Remove(src); err != nil { + t.Fatalf("remove source: %v", err) + } + if err := releaseCapture(stagingRoot, ix, key); err != nil { + t.Fatalf("releaseCapture() error: %v", err) + } + if _, ok := ix.lookup(key); ok { + t.Error("releaseCapture() kept the dev/inode entry after unlinking the last link") + } + if _, err := os.Lstat(uploaded.path(stagingRoot)); !isVanished(err) { + t.Errorf("uploaded link still on disk after release: %v", err) + } + // Nothing may be left behind under pending/ either. + if _, err := os.Lstat(pendingPath); !isVanished(err) { + t.Errorf("pending link leaked: %v", err) + } +} + +func TestReleaseCaptureRefusesWhenStagingDisagrees(t *testing.T) { + dir := t.TempDir() + ix, key, stagingRoot, src := stagedCapture(t, dir, "", "raylet.out.1") + + uploaded, err := promoteCapture(stagingRoot, ix, key) + if err != nil { + t.Fatalf("promoteCapture() error: %v", err) + } + if err := os.Remove(src); err != nil { + t.Fatalf("remove source: %v", err) + } + + // Simulate a staging tree that no longer matches the index: the capture is + // tracked as uploaded but its link is back under pending. Releasing must not + // read the missing uploaded path as "already released" and drop the entry, + // which would leak the pending link. + pendingPath := uploaded.withState(statePending).path(stagingRoot) + if err := os.MkdirAll(filepath.Dir(pendingPath), 0o750); err != nil { + t.Fatalf("recreate pending directory: %v", err) + } + if err := os.Rename(uploaded.path(stagingRoot), pendingPath); err != nil { + t.Fatalf("move staged link back to pending: %v", err) + } + + err = releaseCapture(stagingRoot, ix, key) + if err == nil { + t.Fatal("releaseCapture() reported success although the uploaded path was missing") + } + if !strings.Contains(err.Error(), uploaded.CaptureID) { + t.Errorf("releaseCapture() error %q does not name the capture", err) + } + if _, ok := ix.lookup(key); !ok { + t.Error("releaseCapture() discarded the index entry although the staged link still exists") + } + if _, err := os.Lstat(pendingPath); err != nil { + t.Errorf("staged link lost: %v", err) + } +} + +func TestReleaseCaptureRefusesWhenInodeChanged(t *testing.T) { + dir := t.TempDir() + ix, key, stagingRoot, src := stagedCapture(t, dir, "", "raylet.out.1") + uploaded, err := promoteCapture(stagingRoot, ix, key) + if err != nil { + t.Fatalf("promoteCapture() error: %v", err) + } + if err := os.Remove(src); err != nil { + t.Fatalf("remove source: %v", err) + } + + // Replace the staged link with an unrelated file at the same path. + if err := os.Remove(uploaded.path(stagingRoot)); err != nil { + t.Fatalf("remove staged link: %v", err) + } + writeFile(t, uploaded.path(stagingRoot), "someone else's file") + + if err := releaseCapture(stagingRoot, ix, key); err == nil { + t.Error("releaseCapture() unlinked a path that no longer holds the pinned inode") + } + if _, err := os.Lstat(uploaded.path(stagingRoot)); err != nil { + t.Errorf("releaseCapture() removed the unrelated file: %v", err) + } +} + +func TestReleaseCaptureKeepsEntryWhenUnlinkFails(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("running as root: directory permissions do not prevent unlink") + } + dir := t.TempDir() + ix, key, stagingRoot, src := stagedCapture(t, dir, "", "raylet.out.1") + uploaded, err := promoteCapture(stagingRoot, ix, key) + if err != nil { + t.Fatalf("promoteCapture() error: %v", err) + } + if err := os.Remove(src); err != nil { + t.Fatalf("remove source: %v", err) + } + + parent := filepath.Dir(uploaded.path(stagingRoot)) + if err := os.Chmod(parent, 0o500); err != nil { + t.Fatalf("chmod staging directory: %v", err) + } + t.Cleanup(func() { + if err := os.Chmod(parent, 0o750); err != nil { + t.Logf("restore staging directory permissions: %v", err) + } + }) + + if err := releaseCapture(stagingRoot, ix, key); err == nil { + t.Fatal("releaseCapture() reported success although the unlink was denied") + } + // The inode is still pinned, so it must still be tracked: forgetting it here + // would leave a link nobody ever releases. + if _, ok := ix.lookup(key); !ok { + t.Error("releaseCapture() forgot the capture although the unlink failed") + } + if _, err := os.Lstat(uploaded.path(stagingRoot)); err != nil { + t.Errorf("staged link lost: %v", err) + } +} + +func mustEntry(t *testing.T, ix *captureIndex, key inodeKey) stagedEntry { + t.Helper() + c, ok := ix.lookup(key) + if !ok { + t.Fatalf("no capture tracked for %s", key) + } + return c.Entry +} + +func TestPromoteCaptureRejectsBeforeTouchingDisk(t *testing.T) { + // Everything that can fail must fail before the rename, so a rejected promotion + // leaves the staging volume exactly as it was. + snapshot := func(t *testing.T, root string) []string { + t.Helper() + var paths []string + err := filepath.WalkDir(root, func(p string, _ fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(root, p) + if err != nil { + return err + } + paths = append(paths, rel) + return nil + }) + if err != nil { + t.Fatalf("walk staging volume: %v", err) + } + sort.Strings(paths) + return paths + } + + t.Run("already uploaded", func(t *testing.T) { + dir := t.TempDir() + ix, key, stagingRoot, _ := stagedCapture(t, dir, "", "raylet.out.1") + uploaded, err := promoteCapture(stagingRoot, ix, key) + if err != nil { + t.Fatalf("promoteCapture() error: %v", err) + } + before := snapshot(t, stagingRoot) + + if got, err := promoteCapture(stagingRoot, ix, key); err == nil { + t.Fatalf("promoteCapture() = %+v, want error for an already uploaded capture", got) + } + if diff := snapshot(t, stagingRoot); !slices.Equal(diff, before) { + t.Errorf("staging volume changed on a rejected promotion:\n got %v\nwant %v", diff, before) + } + if tracked := mustEntry(t, ix, key); tracked != uploaded { + t.Errorf("index entry changed on a rejected promotion: %+v", tracked) + } + }) + + t.Run("inode not pinned", func(t *testing.T) { + dir := t.TempDir() + ix, _, stagingRoot, _ := stagedCapture(t, dir, "", "raylet.out.1") + before := snapshot(t, stagingRoot) + + if got, err := promoteCapture(stagingRoot, ix, inodeKey{Dev: 9, Ino: 9}); err == nil { + t.Fatalf("promoteCapture() = %+v, want error for an unpinned inode", got) + } + if diff := snapshot(t, stagingRoot); !slices.Equal(diff, before) { + t.Errorf("staging volume changed on a rejected promotion:\n got %v\nwant %v", diff, before) + } + }) +} diff --git a/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_names.go b/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_names.go new file mode 100644 index 00000000000..ac0aa814706 --- /dev/null +++ b/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_names.go @@ -0,0 +1,361 @@ +package logcollector + +import ( + "crypto/rand" + "encoding/hex" + "fmt" + "io" + "io/fs" + "path" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + + "github.com/ray-project/kuberay/historyserver/pkg/storage/clusterlogs" +) + +const ( + // captureIDSeparator joins a rotated file's original name to its capture ID in + // both staging file names and storage object keys. + captureIDSeparator = ".rotated." + + // captureIDRandomBytes is how much crypto/rand entropy each capture ID carries. + captureIDRandomBytes = 8 + + // captureIDNanosDigits zero-pads the timestamp so capture IDs sort in capture + // order for the lifetime of int64 nanoseconds. + captureIDNanosDigits = 19 +) + +// rotationBackupRe matches a Ray rotation backup: the complete name of the active +// file followed by a positive index. +// +// Ray has a single rotation naming convention. Python's RotatingFileHandler appends +// ".1", ".2", ... and Ray patches spdlog to agree +// (thirdparty/patches/spdlog-rotation-file-format.patch rewrites "mylog.3.txt" to +// "mylog.txt.3"), so C++ components rotate the same way: worker-abc.out.1, +// raylet.out.1, dashboard.log.1. +var rotationBackupRe = regexp.MustCompile(`^(.+)\.([1-9][0-9]*)$`) + +// captureIDRe matches the format produced by captureIDGenerator.next. +var captureIDRe = regexp.MustCompile(`^[0-9]{19}\.[0-9a-f]{16}$`) + +// parseBackupName splits a rotation backup's basename into the name of the active +// file it rotated from, plus its backup index. +// +// Ray imposes no upper bound on RAY_ROTATION_BACKUP_COUNT, so no cap is applied +// here; the only rejected numeric form is one too large for an int, which no +// rotation cascade can produce. +func parseBackupName(name string) (base string, index int, ok bool) { + m := rotationBackupRe.FindStringSubmatch(name) + if m == nil { + return "", 0, false + } + index, err := strconv.Atoi(m[2]) + if err != nil { + return "", 0, false + } + return m[1], index, true +} + +// candidateStatus explains why a directory entry is or is not capturable. +type candidateStatus int + +const ( + candidateEligible candidateStatus = iota + candidateNotBackupName + candidateNotRegular + candidateUnknownBase +) + +func (s candidateStatus) String() string { + switch s { + case candidateEligible: + return "eligible" + case candidateNotBackupName: + return "not a rotation backup name" + case candidateNotRegular: + return "not a regular file" + case candidateUnknownBase: + return "no known active base file" + default: + return fmt.Sprintf("unknown status %d", int(s)) + } +} + +// baseKnownFunc reports whether base is, or recently was, the active file that +// backups in dir rotate from. +type baseKnownFunc func(dir, base string) bool + +// evaluateCandidate decides whether dir/name is a Ray rotation backup the collector +// may capture. mode must come from an Lstat, so that symlinks are rejected instead +// of followed. +// +// Requiring a known active base is what keeps unrelated files that merely end in +// "." out of the capture path: rotation always leaves the active file in +// place. The "recently observed" half of baseKnownFunc matters because a rotation +// cascade briefly unlinks the active name before the writer recreates it. +func evaluateCandidate(dir, name string, mode fs.FileMode, baseKnown baseKnownFunc) candidateStatus { + base, _, ok := parseBackupName(name) + if !ok { + return candidateNotBackupName + } + if !mode.IsRegular() { + return candidateNotRegular + } + if baseKnown == nil || !baseKnown(dir, base) { + return candidateUnknownBase + } + return candidateEligible +} + +// captureIDGenerator mints capture IDs. A rotation filename such as "raylet.out.1" +// is reused by every later segment, so the ID — not the filename and not the inode +// — is a captured segment's permanent identity. +// +// IDs must stay unique across collector restarts within a session: a restart that +// reset a counter would mint an ID already used by an earlier object and overwrite +// it. Wall-clock nanoseconds plus 64 bits of crypto/rand survive both a restart and +// a clock that steps backwards. +type captureIDGenerator struct { + now func() time.Time + rand io.Reader +} + +func newCaptureIDGenerator() *captureIDGenerator { + return &captureIDGenerator{now: time.Now, rand: rand.Reader} +} + +// next returns a fresh capture ID, or an error if the entropy source fails. A +// failure must abort the capture rather than fall back to a predictable ID. +func (g *captureIDGenerator) next() (string, error) { + b := make([]byte, captureIDRandomBytes) + if _, err := io.ReadFull(g.rand, b); err != nil { + return "", fmt.Errorf("generate capture ID: read %d random bytes: %w", captureIDRandomBytes, err) + } + return fmt.Sprintf("%0*d.%s", captureIDNanosDigits, g.now().UnixNano(), hex.EncodeToString(b)), nil +} + +// captureFileName is the leaf name a captured backup takes in staging and in +// storage: the original rotation name, so operators can still read it, plus the +// capture ID that makes it unique across segments. +func captureFileName(originalName, captureID string) string { + return originalName + captureIDSeparator + captureID +} + +// parseCaptureFileName is the inverse of captureFileName. It splits on the last +// separator so that an original name containing ".rotated." round-trips. +func parseCaptureFileName(fileName string) (originalName, captureID string, ok bool) { + i := strings.LastIndex(fileName, captureIDSeparator) + if i <= 0 { + return "", "", false + } + originalName = fileName[:i] + captureID = fileName[i+len(captureIDSeparator):] + if !captureIDRe.MatchString(captureID) { + return "", "", false + } + return originalName, captureID, true +} + +// clusterIdentity carries every value clusterlogs.LogsDir needs. RayJob and +// RayService clusters nest their logs under the owner name while RayCluster ones do +// not, so dropping the owner fields would silently write to the wrong prefix. +type clusterIdentity struct { + RootDir string + OwnerKind string + OwnerName string + Namespace string + ClusterName string +} + +// logsPrefix is the node's log directory in storage: every object this collector +// writes for that node lives under it, and nothing it writes may escape it. +func (c clusterIdentity) logsPrefix(sessionName, nodeName string) string { + return clusterlogs.LogsDir(c.RootDir, c.OwnerKind, c.OwnerName, c.Namespace, c.ClusterName, sessionName, nodeName) +} + +// objectKey returns the storage key for a captured backup. Keys stay flat — one +// object per captured segment directly beside the node's other logs — because the +// History Server lists a node's logs non-recursively unless the caller passes a +// "**" glob, so anything nested under an extra directory would be invisible to the +// ordinary listing. +// +// relDir is the one exception, and it is not the capture's doing: Ray already nests +// some logs (events/, serve/), and the legacy shutdown upload mirrors that same +// structure, so a captured segment lands exactly where its uncaptured siblings do. +func (c clusterIdentity) objectKey(sessionName, nodeName, relDir, originalName, captureID string) string { + return path.Join(c.logsPrefix(sessionName, nodeName), relDir, captureFileName(originalName, captureID)) +} + +// stagingState is the durable, on-disk record of how far a capture has progressed. +// It is a directory level rather than in-memory bookkeeping so that a collector +// restart can tell an unsent capture from an already-uploaded one. +type stagingState string + +const ( + statePending stagingState = "pending" + stateUploaded stagingState = "uploaded" +) + +func validStagingState(s stagingState) bool { + return s == statePending || s == stateUploaded +} + +// stagedEntry identifies one captured backup on the staging volume: +// +// /////.rotated. +// +// Build entries with newStagedEntry: path and objectKey join these fields into +// filesystem and object paths, so an unvalidated field would be a traversal. +type stagedEntry struct { + State stagingState + SessionName string + NodeName string + RelDir string // slash-separated, empty when the backup sits directly in logs/ + OriginalName string // the rotation name at capture time, e.g. "worker-abc.out.1" + CaptureID string +} + +// newStagedEntry validates every component that ends up in a path. It is the only +// place a stagedEntry should be constructed outside of parsing. +func newStagedEntry(state stagingState, sessionName, nodeName, relDir, originalName, captureID string) (stagedEntry, error) { + if !validStagingState(state) { + return stagedEntry{}, fmt.Errorf("staged entry: unknown state %q", state) + } + for _, f := range []struct{ label, value string }{ + {"session name", sessionName}, + {"node name", nodeName}, + {"original name", originalName}, + } { + if err := validatePathSegment(f.label, f.value); err != nil { + return stagedEntry{}, err + } + } + if err := validateRelDir(relDir); err != nil { + return stagedEntry{}, err + } + // A capture without a well-formed ID has no stable identity, so it must never + // reach the staging volume: a failed ID generation has to abort the capture. + if !captureIDRe.MatchString(captureID) { + return stagedEntry{}, fmt.Errorf("staged entry: malformed capture ID %q", captureID) + } + return stagedEntry{ + State: state, + SessionName: sessionName, + NodeName: nodeName, + RelDir: relDir, + OriginalName: originalName, + CaptureID: captureID, + }, nil +} + +// validatePathSegment rejects anything that would climb out of, or disappear from, +// the directory it is joined into. +func validatePathSegment(label, value string) error { + if value == "" { + return fmt.Errorf("staged entry: empty %s", label) + } + if value != path.Base(value) || value == "." || value == ".." { + return fmt.Errorf("staged entry: %s %q must be a single path segment", label, value) + } + return nil +} + +// validateRelDir accepts only a clean, relative, slash-separated directory. Ray +// nests some logs (events/, serve/), and that structure is mirrored into staging +// and storage, so it has to be reproduced exactly and without traversal. +func validateRelDir(relDir string) error { + if relDir == "" { + return nil + } + if path.IsAbs(relDir) || filepath.IsAbs(relDir) { + return fmt.Errorf("staged entry: relative directory %q must not be absolute", relDir) + } + if path.Clean(relDir) != relDir { + return fmt.Errorf("staged entry: relative directory %q must be clean", relDir) + } + for seg := range strings.SplitSeq(relDir, "/") { + if seg == "" || seg == "." || seg == ".." { + return fmt.Errorf("staged entry: relative directory %q must not contain %q", relDir, seg) + } + } + return nil +} + +// path returns the entry's absolute location under stagingRoot. +func (e stagedEntry) path(stagingRoot string) string { + return filepath.Join( + stagingRoot, + e.SessionName, + e.NodeName, + string(e.State), + filepath.FromSlash(e.RelDir), + captureFileName(e.OriginalName, e.CaptureID), + ) +} + +// withState returns a copy of the entry in a different staging state, preserving +// capture identity. +func (e stagedEntry) withState(s stagingState) stagedEntry { + e.State = s + return e +} + +// objectKey returns where this entry belongs in storage. +func (e stagedEntry) objectKey(c clusterIdentity) string { + return c.objectKey(e.SessionName, e.NodeName, e.RelDir, e.OriginalName, e.CaptureID) +} + +// parseStagedPath reconstructs an entry from a path found on the staging volume, +// which is how a restarting collector recovers both capture identity and upload +// state without re-deriving either. +func parseStagedPath(stagingRoot, absPath string) (stagedEntry, error) { + rel, err := filepath.Rel(stagingRoot, absPath) + if err != nil { + return stagedEntry{}, fmt.Errorf("staging path %s is not under %s: %w", absPath, stagingRoot, err) + } + parts := strings.Split(filepath.ToSlash(rel), "/") + if len(parts) < 4 || parts[0] == ".." { + return stagedEntry{}, fmt.Errorf("staging path %s is not ///", absPath) + } + + originalName, captureID, ok := parseCaptureFileName(parts[len(parts)-1]) + if !ok { + return stagedEntry{}, fmt.Errorf("staging path %s does not end in %s", absPath, captureIDSeparator) + } + + entry, err := newStagedEntry( + stagingState(parts[2]), + parts[0], + parts[1], + path.Join(parts[3:len(parts)-1]...), + originalName, + captureID, + ) + if err != nil { + return stagedEntry{}, fmt.Errorf("staging path %s: %w", absPath, err) + } + return entry, nil +} + +// relDirFor returns the slash-separated directory of a log file relative to the +// session's logs directory, empty when the file sits directly in it. Ray nests some +// logs (events/, serve/), and that structure is preserved in both staging and +// storage. +func relDirFor(logsDir, filePath string) (string, error) { + rel, err := filepath.Rel(logsDir, filepath.Dir(filePath)) + if err != nil { + return "", fmt.Errorf("relative directory of %s under %s: %w", filePath, logsDir, err) + } + rel = filepath.ToSlash(rel) + if rel == "." { + return "", nil + } + if err := validateRelDir(rel); err != nil { + return "", fmt.Errorf("log file %s is not safely under logs directory %s: %w", filePath, logsDir, err) + } + return rel, nil +} diff --git a/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_names_test.go b/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_names_test.go new file mode 100644 index 00000000000..bbf6e3414d6 --- /dev/null +++ b/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_names_test.go @@ -0,0 +1,486 @@ +package logcollector + +import ( + "bytes" + "errors" + "io" + "io/fs" + "path/filepath" + "strings" + "testing" + "time" +) + +// fixedClock returns a deterministic clock for capture ID tests. +func fixedClock(nanos int64) func() time.Time { + return func() time.Time { return time.Unix(0, nanos) } +} + +// testGenerator builds a capture ID generator with a fixed clock and a repeatable +// entropy source. +func testGenerator(nanos int64, entropy string) *captureIDGenerator { + return &captureIDGenerator{now: fixedClock(nanos), rand: strings.NewReader(entropy)} +} + +func TestParseBackupName(t *testing.T) { + tests := []struct { + name string + wantBase string + wantIndex int + wantOK bool + }{ + // Ray's single rotation convention: the complete active name plus an index. + {name: "worker-abc-01000000-123.out.1", wantBase: "worker-abc-01000000-123.out", wantIndex: 1, wantOK: true}, + {name: "worker-abc-01000000-123.err.2", wantBase: "worker-abc-01000000-123.err", wantIndex: 2, wantOK: true}, + {name: "raylet.out.1", wantBase: "raylet.out", wantIndex: 1, wantOK: true}, + {name: "gcs_server.err.5", wantBase: "gcs_server.err", wantIndex: 5, wantOK: true}, + {name: "dashboard.log.1", wantBase: "dashboard.log", wantIndex: 1, wantOK: true}, + {name: "event_EXPORT_TASK.log.12", wantBase: "event_EXPORT_TASK.log", wantIndex: 12, wantOK: true}, + // Ray patches spdlog so that the index never lands before the extension; + // "raylet.1.out" is not a rotation backup and must not be captured as one. + {name: "raylet.1.out", wantOK: false}, + {name: "worker-abc-01000000-123.1.err", wantOK: false}, + // Active files are never backups. + {name: "raylet.out", wantOK: false}, + {name: "dashboard.log", wantOK: false}, + // Rejected numeric forms. + {name: "raylet.out.0", wantOK: false}, + {name: "raylet.out.01", wantOK: false}, + {name: "raylet.out.-1", wantOK: false}, + {name: "raylet.out.1a", wantOK: false}, + {name: "raylet.out.", wantOK: false}, + {name: ".1", wantOK: false}, + {name: "1", wantOK: false}, + {name: "", wantOK: false}, + // Ray does not cap the backup count, so a large index stays valid. + {name: "raylet.out.4096", wantBase: "raylet.out", wantIndex: 4096, wantOK: true}, + // An index too large for an int cannot come from a rotation cascade. + {name: "raylet.out.99999999999999999999999", wantOK: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + base, index, ok := parseBackupName(tt.name) + if ok != tt.wantOK { + t.Fatalf("parseBackupName(%q) ok = %v, want %v", tt.name, ok, tt.wantOK) + } + if !tt.wantOK { + return + } + if base != tt.wantBase || index != tt.wantIndex { + t.Errorf("parseBackupName(%q) = (%q, %d), want (%q, %d)", tt.name, base, index, tt.wantBase, tt.wantIndex) + } + }) + } +} + +func TestEvaluateCandidate(t *testing.T) { + knownBase := func(_, base string) bool { return base == "raylet.out" } + + tests := []struct { + name string + fileName string + mode fs.FileMode + baseKnown baseKnownFunc + wantStatus candidateStatus + }{ + {name: "regular backup with known base", fileName: "raylet.out.1", mode: 0, baseKnown: knownBase, wantStatus: candidateEligible}, + {name: "not a backup name", fileName: "raylet.out", mode: 0, baseKnown: knownBase, wantStatus: candidateNotBackupName}, + {name: "directory", fileName: "raylet.out.1", mode: fs.ModeDir, baseKnown: knownBase, wantStatus: candidateNotRegular}, + {name: "symlink", fileName: "raylet.out.1", mode: fs.ModeSymlink, baseKnown: knownBase, wantStatus: candidateNotRegular}, + {name: "socket", fileName: "raylet.out.1", mode: fs.ModeSocket, baseKnown: knownBase, wantStatus: candidateNotRegular}, + {name: "unknown base", fileName: "user-data.1", mode: 0, baseKnown: knownBase, wantStatus: candidateUnknownBase}, + {name: "no base oracle", fileName: "raylet.out.1", mode: 0, baseKnown: nil, wantStatus: candidateUnknownBase}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + status := evaluateCandidate("/tmp/ray/session/logs", tt.fileName, tt.mode, tt.baseKnown) + if status != tt.wantStatus { + t.Errorf("evaluateCandidate(%q) = %v, want %v", tt.fileName, status, tt.wantStatus) + } + }) + } +} + +func TestCaptureIDFormatAndUniqueness(t *testing.T) { + g := newCaptureIDGenerator() + + const iterations = 2000 + seen := make(map[string]struct{}, iterations) + for range iterations { + id, err := g.next() + if err != nil { + t.Fatalf("next() error: %v", err) + } + if !captureIDRe.MatchString(id) { + t.Fatalf("capture ID %q does not match %v", id, captureIDRe) + } + if _, dup := seen[id]; dup { + t.Fatalf("capture ID %q generated twice", id) + } + seen[id] = struct{}{} + } +} + +func TestCaptureIDSurvivesRestart(t *testing.T) { + // Two collector instances that restart within the same nanosecond must not mint + // the same ID: a reused ID would overwrite the earlier run's object. + const sameNanos = int64(1780000000000000000) + first := &captureIDGenerator{now: fixedClock(sameNanos), rand: newCaptureIDGenerator().rand} + second := &captureIDGenerator{now: fixedClock(sameNanos), rand: newCaptureIDGenerator().rand} + + idA, err := first.next() + if err != nil { + t.Fatalf("first.next() error: %v", err) + } + idB, err := second.next() + if err != nil { + t.Fatalf("second.next() error: %v", err) + } + if idA == idB { + t.Errorf("restarted collectors reused capture ID %q", idA) + } + + // A clock that steps backwards must not collide either: the random half differs. + rewound := testGenerator(sameNanos-1_000_000, "entropy!") + idC, err := rewound.next() + if err != nil { + t.Fatalf("rewound.next() error: %v", err) + } + if idC == idA || idC == idB { + t.Errorf("capture ID %q collided after the clock stepped backwards", idC) + } +} + +func TestCaptureIDGenerationErrorPropagates(t *testing.T) { + wantErr := errors.New("entropy source unavailable") + g := &captureIDGenerator{now: time.Now, rand: failingReader{err: wantErr}} + + id, err := g.next() + if err == nil { + t.Fatalf("next() = %q, want error", id) + } + if !errors.Is(err, wantErr) { + t.Errorf("next() error = %v, want it to wrap %v", err, wantErr) + } + if id != "" { + t.Errorf("next() returned ID %q alongside an error", id) + } + + // A truncated read is a failure too: a short ID would weaken uniqueness. + short := &captureIDGenerator{now: time.Now, rand: bytes.NewReader([]byte{1, 2, 3})} + if _, err := short.next(); !errors.Is(err, io.ErrUnexpectedEOF) { + t.Errorf("short read error = %v, want io.ErrUnexpectedEOF", err) + } +} + +type failingReader struct{ err error } + +func (r failingReader) Read([]byte) (int, error) { return 0, r.err } + +func TestCaptureFileNameRoundTrip(t *testing.T) { + const id = "0001780000000000000.a1b2c3d4e5f60718" + + tests := []string{ + "raylet.out.1", + "worker-abc-01000000-123.out.2", + "dashboard.log.1", + // An original name that itself contains the separator must round-trip. + "weird.rotated.name.log.3", + } + + for _, original := range tests { + t.Run(original, func(t *testing.T) { + fileName := captureFileName(original, id) + if !strings.HasPrefix(fileName, original) { + t.Errorf("captureFileName(%q) = %q, want it to keep the original name readable", original, fileName) + } + gotName, gotID, ok := parseCaptureFileName(fileName) + if !ok { + t.Fatalf("parseCaptureFileName(%q) not recognized", fileName) + } + if gotName != original || gotID != id { + t.Errorf("parseCaptureFileName(%q) = (%q, %q), want (%q, %q)", fileName, gotName, gotID, original, id) + } + }) + } + + rejected := []string{ + "raylet.out.1", // no capture ID + "raylet.out.1.rotated.", // empty ID + "raylet.out.1.rotated.not-an-id", // malformed ID + ".rotated.0001780000000000000.a1b2c3d4e5f60718", // no original name + "raylet.out.1.rotated.0001780000000000000.a1b2c3", // truncated ID + } + for _, fileName := range rejected { + if _, _, ok := parseCaptureFileName(fileName); ok { + t.Errorf("parseCaptureFileName(%q) accepted a malformed name", fileName) + } + } +} + +func TestObjectKeyIsFlatAndOwnerAware(t *testing.T) { + const id = "0001780000000000000.a1b2c3d4e5f60718" + + tests := []struct { + name string + identity clusterIdentity + relDir string + original string + want string + }{ + { + name: "raycluster keeps session/node/logs ordering", + identity: clusterIdentity{RootDir: "root", Namespace: "default", ClusterName: "my-cluster"}, + original: "worker-abc.out.1", + want: "root/cluster-history/raycluster/default/my-cluster/session-1/node-1/logs/worker-abc.out.1.rotated." + id, + }, + { + name: "nested relative directory is preserved", + identity: clusterIdentity{RootDir: "root", Namespace: "default", ClusterName: "my-cluster"}, + relDir: "events", + original: "event_EXPORT_TASK.log.2", + want: "root/cluster-history/raycluster/default/my-cluster/session-1/node-1/logs/events/event_EXPORT_TASK.log.2.rotated." + id, + }, + { + name: "rayjob nests under the owner name", + identity: clusterIdentity{ + RootDir: "root", OwnerKind: "rayjob", OwnerName: "job-1", + Namespace: "default", ClusterName: "my-cluster", + }, + original: "raylet.out.1", + want: "root/cluster-history/rayjob/default/job-1/my-cluster/session-1/node-1/logs/raylet.out.1.rotated." + id, + }, + { + name: "rayservice nests under the owner name", + identity: clusterIdentity{ + RootDir: "root", OwnerKind: "rayservice", OwnerName: "svc-1", + Namespace: "default", ClusterName: "my-cluster", + }, + original: "raylet.out.1", + want: "root/cluster-history/rayservice/default/svc-1/my-cluster/session-1/node-1/logs/raylet.out.1.rotated." + id, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.identity.objectKey("session-1", "node-1", tt.relDir, tt.original, id) + if got != tt.want { + t.Errorf("objectKey() = %q, want %q", got, tt.want) + } + // The captured object must sit in the node's own logs directory, so the + // History Server's non-recursive listing can see it. + if strings.Contains(got, "/rotated/") { + t.Errorf("objectKey() = %q, want no extra directory level", got) + } + }) + } +} + +func TestStagedPathRoundTrip(t *testing.T) { + root := filepath.Join("/tmp", "ray", "rotated-staging") + + tests := []struct { + name string + entry stagedEntry + want string + }{ + { + name: "pending at the top level", + entry: stagedEntry{ + State: statePending, SessionName: "session-1", NodeName: "node-1", + OriginalName: "raylet.out.1", CaptureID: "0001780000000000000.a1b2c3d4e5f60718", + }, + want: root + "/session-1/node-1/pending/raylet.out.1.rotated.0001780000000000000.a1b2c3d4e5f60718", + }, + { + name: "uploaded in a nested directory", + entry: stagedEntry{ + State: stateUploaded, SessionName: "session-1", NodeName: "node-1", RelDir: "events/subdir", + OriginalName: "event.log.3", CaptureID: "0001780000000000001.00ff00ff00ff00ff", + }, + want: root + "/session-1/node-1/uploaded/events/subdir/event.log.3.rotated.0001780000000000001.00ff00ff00ff00ff", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.entry.path(root) + if got != filepath.FromSlash(tt.want) { + t.Fatalf("path() = %q, want %q", got, tt.want) + } + parsed, err := parseStagedPath(root, got) + if err != nil { + t.Fatalf("parseStagedPath(%q) error: %v", got, err) + } + if parsed != tt.entry { + t.Errorf("parseStagedPath() = %+v, want %+v", parsed, tt.entry) + } + }) + } +} + +func TestStagedPathTransitionPreservesIdentity(t *testing.T) { + root := filepath.Join("/tmp", "ray", "rotated-staging") + pending := stagedEntry{ + State: statePending, SessionName: "session-1", NodeName: "node-1", RelDir: "events", + OriginalName: "event.log.1", CaptureID: "0001780000000000000.a1b2c3d4e5f60718", + } + + uploaded := pending.withState(stateUploaded) + if uploaded.CaptureID != pending.CaptureID || uploaded.OriginalName != pending.OriginalName || + uploaded.RelDir != pending.RelDir || uploaded.SessionName != pending.SessionName || + uploaded.NodeName != pending.NodeName { + t.Fatalf("withState() changed capture identity: %+v -> %+v", pending, uploaded) + } + if pending.State != statePending { + t.Errorf("withState() mutated the receiver: %+v", pending) + } + + // Only the state segment of the path differs, so the object key cannot drift. + identity := clusterIdentity{RootDir: "root", Namespace: "default", ClusterName: "my-cluster"} + if pending.objectKey(identity) != uploaded.objectKey(identity) { + t.Errorf("object key changed across states: %q vs %q", pending.objectKey(identity), uploaded.objectKey(identity)) + } + if strings.Replace(pending.path(root), string(statePending), string(stateUploaded), 1) != uploaded.path(root) { + t.Errorf("staging path changed by more than its state segment: %q vs %q", pending.path(root), uploaded.path(root)) + } +} + +func TestParseStagedPathRejectsMalformed(t *testing.T) { + root := filepath.Join("/tmp", "ray", "rotated-staging") + const leaf = "raylet.out.1.rotated.0001780000000000000.a1b2c3d4e5f60718" + + tests := []struct { + name string + path string + }{ + {name: "too shallow", path: filepath.Join(root, "session-1", leaf)}, + {name: "unknown state", path: filepath.Join(root, "session-1", "node-1", "draft", leaf)}, + {name: "missing capture ID", path: filepath.Join(root, "session-1", "node-1", "pending", "raylet.out.1")}, + {name: "outside staging root", path: filepath.Join("/tmp", "ray", "prev-logs", "session-1", "node-1", "pending", leaf)}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if entry, err := parseStagedPath(root, tt.path); err == nil { + t.Errorf("parseStagedPath(%q) = %+v, want error", tt.path, entry) + } + }) + } +} + +func TestRelDirFor(t *testing.T) { + logsDir := filepath.Join("/tmp", "ray", "session_2026-07-31", "logs") + + tests := []struct { + name string + file string + want string + wantErr bool + }{ + {name: "top level", file: filepath.Join(logsDir, "raylet.out.1"), want: ""}, + {name: "nested", file: filepath.Join(logsDir, "events", "event.log.1"), want: "events"}, + {name: "deeply nested", file: filepath.Join(logsDir, "serve", "replica", "r.log.1"), want: "serve/replica"}, + {name: "escapes logs dir", file: filepath.Join("/tmp", "ray", "elsewhere", "raylet.out.1"), wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := relDirFor(logsDir, tt.file) + if tt.wantErr { + if err == nil { + t.Fatalf("relDirFor(%q) = %q, want error", tt.file, got) + } + return + } + if err != nil { + t.Fatalf("relDirFor(%q) error: %v", tt.file, err) + } + if got != tt.want { + t.Errorf("relDirFor(%q) = %q, want %q", tt.file, got, tt.want) + } + }) + } +} + +func TestNewStagedEntryRejectsUnsafeComponents(t *testing.T) { + const goodID = "0001780000000000000.a1b2c3d4e5f60718" + + tests := []struct { + name string + state stagingState + sessionName string + nodeName string + relDir string + originalName string + captureID string + wantErr bool + }{ + {name: "valid", state: statePending, sessionName: "session-1", nodeName: "node-1", originalName: "raylet.out.1", captureID: goodID}, + {name: "valid nested", state: stateUploaded, sessionName: "session-1", nodeName: "node-1", relDir: "events/subdir", originalName: "e.log.1", captureID: goodID}, + {name: "unknown state", state: "draft", sessionName: "session-1", nodeName: "node-1", originalName: "raylet.out.1", captureID: goodID, wantErr: true}, + {name: "empty session", sessionName: "", nodeName: "node-1", state: statePending, originalName: "raylet.out.1", captureID: goodID, wantErr: true}, + {name: "session traversal", sessionName: "..", nodeName: "node-1", state: statePending, originalName: "raylet.out.1", captureID: goodID, wantErr: true}, + {name: "session with separator", sessionName: "a/b", nodeName: "node-1", state: statePending, originalName: "raylet.out.1", captureID: goodID, wantErr: true}, + {name: "node traversal", sessionName: "session-1", nodeName: "../../etc", state: statePending, originalName: "raylet.out.1", captureID: goodID, wantErr: true}, + {name: "name traversal", sessionName: "session-1", nodeName: "node-1", state: statePending, originalName: "../raylet.out.1", captureID: goodID, wantErr: true}, + {name: "empty name", sessionName: "session-1", nodeName: "node-1", state: statePending, originalName: "", captureID: goodID, wantErr: true}, + {name: "absolute relDir", sessionName: "session-1", nodeName: "node-1", relDir: "/etc", state: statePending, originalName: "raylet.out.1", captureID: goodID, wantErr: true}, + {name: "relDir traversal", sessionName: "session-1", nodeName: "node-1", relDir: "../../..", state: statePending, originalName: "raylet.out.1", captureID: goodID, wantErr: true}, + {name: "relDir hidden traversal", sessionName: "session-1", nodeName: "node-1", relDir: "events/../../..", state: statePending, originalName: "raylet.out.1", captureID: goodID, wantErr: true}, + {name: "relDir unclean", sessionName: "session-1", nodeName: "node-1", relDir: "./events", state: statePending, originalName: "raylet.out.1", captureID: goodID, wantErr: true}, + {name: "relDir trailing slash", sessionName: "session-1", nodeName: "node-1", relDir: "events/", state: statePending, originalName: "raylet.out.1", captureID: goodID, wantErr: true}, + // A capture ID that was never generated must not reach the staging volume. + {name: "empty capture ID", sessionName: "session-1", nodeName: "node-1", state: statePending, originalName: "raylet.out.1", captureID: "", wantErr: true}, + {name: "malformed capture ID", sessionName: "session-1", nodeName: "node-1", state: statePending, originalName: "raylet.out.1", captureID: "rot7", wantErr: true}, + {name: "capture ID with traversal", sessionName: "session-1", nodeName: "node-1", state: statePending, originalName: "raylet.out.1", captureID: "../../etc/passwd", wantErr: true}, + } + + stagingRoot := filepath.Join("/tmp", "ray", "rotated-staging") + identity := clusterIdentity{RootDir: "root", Namespace: "default", ClusterName: "my-cluster"} + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + entry, err := newStagedEntry(tt.state, tt.sessionName, tt.nodeName, tt.relDir, tt.originalName, tt.captureID) + if tt.wantErr { + if err == nil { + t.Fatalf("newStagedEntry() = %+v, want error", entry) + } + return + } + if err != nil { + t.Fatalf("newStagedEntry() error: %v", err) + } + // Every accepted entry must stay inside both roots it is joined into. + if got := entry.path(stagingRoot); !strings.HasPrefix(got, stagingRoot+string(filepath.Separator)) { + t.Errorf("path() = %q, want it under %q", got, stagingRoot) + } + if got := entry.objectKey(identity); !strings.HasPrefix(got, "root/cluster-history/") { + t.Errorf("objectKey() = %q, want it under the cluster prefix", got) + } + }) + } +} + +func TestCaptureFileNameRoundTripsNestedSeparator(t *testing.T) { + // A previously captured name fed back through capture must still split at the + // last separator, so identity cannot drift. + const innerID = "0001780000000000000.a1b2c3d4e5f60718" + const outerID = "0001780000000000001.00ff00ff00ff00ff" + + original := captureFileName("raylet.out.1", innerID) + fileName := captureFileName(original, outerID) + + gotName, gotID, ok := parseCaptureFileName(fileName) + if !ok { + t.Fatalf("parseCaptureFileName(%q) not recognized", fileName) + } + if gotID != outerID { + t.Errorf("capture ID = %q, want %q", gotID, outerID) + } + if gotName != original { + t.Errorf("original name = %q, want %q", gotName, original) + } +} diff --git a/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_runtime.go b/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_runtime.go new file mode 100644 index 00000000000..a2ab9b999a3 --- /dev/null +++ b/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_runtime.go @@ -0,0 +1,891 @@ +package logcollector + +import ( + "errors" + "fmt" + "os" + "sync" + "syscall" + "time" + + "github.com/sirupsen/logrus" +) + +// A rotatedCollector is scoped to exactly one Ray session on one node, because both +// values are baked into every staging path and every object key it writes. Ray +// restarts a session by pointing session_latest at a new directory, and the node ID +// can be rediscovered at any time, so production needs something that owns the +// lifetime of one collector and replaces it when either half of that identity +// changes. That is rotatedSupervisor. +// +// Everything below runs on the RayLogHandler's goroutines — Run and +// PollActiveSessionChanges — never on a collector's owner goroutine, and it reaches a +// collector only through Run, Stop, reconcileNow and stats. + +const ( + // defaultRotatedDrainBudget bounds how long a retiring collector is given to get + // already-captured work into storage. + // + // It has to be a budget rather than a wait. storage.StorageWriter.WriteFile takes + // no context, so an upload that has already entered the object client cannot be + // canceled; an unreachable object store would otherwise hold shutdown open for as + // long as its own timeouts last, and a capture whose uploads keep failing backs + // off into the minutes. + // + // The size is bounded from above by the pod's termination grace period, which + // KubeRay never sets for the collector sidecar and which therefore defaults to + // Kubernetes' 30s. Everything this drain spends is taken from the legacy shutdown + // walk that runs after it, and that walk is the pre-existing data path for every + // log file still in the live tree — so the drain must stay a small fraction of the + // window. Five seconds leaves roughly 25s for the legacy walk and the final + // endpoint poll. + // + // Anything unfinished when the budget expires stays pinned on the staging volume + // as pending. That is recoverable only where the staging volume outlives the + // process — a collector restart inside a surviving pod. When the pod itself is + // going away and /tmp/ray is an emptyDir, this drain is the last chance those + // captures get, which is why it is not zero either. + defaultRotatedDrainBudget = 5 * time.Second + + // defaultRotatedDrainPoll is how often the drain re-reads the collector's stats. + // Each read is one round trip through the owner goroutine, so this is deliberately + // coarse enough not to add load to a collector that is trying to finish. + defaultRotatedDrainPoll = 20 * time.Millisecond + + // rotatedRetryBase and rotatedRetryMax schedule the retry of an identity whose + // collector failed for a condition that can clear on its own. The session poller + // calls ensure every five seconds, so without a schedule a session whose logs + // directory does not exist yet would rebuild a collector twice a minute forever. + rotatedRetryBase = 15 * time.Second + rotatedRetryMax = 5 * time.Minute + + // maxRotatedFailureRecords caps the failure map. Records for sessions other than + // the one being ensured are dropped first — Ray session names are timestamps and + // never come back — and this is the backstop for anything that pruning misses, so + // a process that outlives thousands of session changes cannot grow this map. + maxRotatedFailureRecords = 16 + + // defaultStagingHighWaterBytes and defaultStagingLowWaterBytes bound the disk this + // feature keeps allocated. + // + // They are measured against retained bytes, not logical staged bytes: a capture + // counts only once the collector holds the last link to its inode. A segment Ray + // still has in its own backup ring costs nothing extra — the hard link shares Ray's + // blocks — so healthy rotation never approaches these marks. What does approach them + // is an object store that has stopped accepting writes, where captures Ray has since + // rolled off accumulate with nothing able to release them. + // + // Reaching the high mark pauses new capture only: nothing already captured is + // evicted, uploads, promotions and releases keep running, and a segment larger than + // the whole budget is still captured, because the limit is applied after the capture + // that crossed it. The gap to the low mark is hysteresis, so a total sitting near the + // limit does not flap. + defaultStagingHighWaterBytes int64 = 1 << 30 // 1 GiB + defaultStagingLowWaterBytes int64 = 1 << 29 // 512 MiB +) + +// rotatedKey is the identity a collector is built for. A change to either half means +// a different collector, never a mutated one: the old session's captureIndex, capture +// IDs, staging subtree and object prefix all belong to the old session. +type rotatedKey struct { + session string + node string +} + +func (k rotatedKey) String() string { return k.session + "/" + k.node } + +// rotatedRun is one running collector together with the identity it was built for. +type rotatedRun struct { + rc *rotatedCollector + done chan struct{} + logsDir string + key rotatedKey + + // err is the collector's exit status. It is written before done is closed and + // read only after, so it needs no lock of its own. + err error + + // ready records that this run completed startup. It is written and read under the + // supervisor's mu. + ready bool + + // noted makes the failure of this run recorded exactly once, no matter whether the + // collector's own goroutine or the retirement that joined it gets there first. + noted sync.Once +} + +// finished reports whether the collector's goroutine has already exited, which is how +// a failed run is prevented from being treated as active. +func (r *rotatedRun) finished() bool { + select { + case <-r.done: + return true + default: + return false + } +} + +// errRotatedRetryable marks a failure whose condition can clear on its own. +// +// Retryability is a property of the operation that failed, never of the errno it +// carries, and only the layer that performed the operation knows which it was. A logs +// directory that does not exist yet and a durable staging link that has disappeared +// both surface as fs.ErrNotExist, and they are opposites: the first is the session +// poller running ahead of Ray's own directory creation, the second is the staging +// volume contradicting the index, which is fatal by design and which no retry can +// repair. Classifying on the errno would restart the second every fifteen seconds +// forever. +// +// So nothing is retryable unless it was explicitly marked at the point of failure by +// retryableRotated, and the default for everything else — including every error that +// merely happens to wrap ENOENT — is durable. +var errRotatedRetryable = errors.New("rotated log collection may be retried for this identity") + +// retryableRotated marks err as a condition that can clear on its own. +func retryableRotated(err error) error { + if err == nil { + return nil + } + return fmt.Errorf("%w [%w]", err, errRotatedRetryable) +} + +// isRetryableRotatedFailure reports whether a failure was explicitly marked retryable. +func isRetryableRotatedFailure(err error) bool { + return errors.Is(err, errRotatedRetryable) +} + +// rotatedFailure is why an identity has no collector, and when it may be tried again. +// +// durable separates the two kinds of failure that matter. A staging volume that +// contradicts itself, a watcher the collector may not install, a filesystem that +// cannot hard-link: retrying those in fifteen seconds produces the identical failure +// and a log line per attempt for the life of the session. A logs directory that does +// not exist yet is the opposite — the session poller can observe a new session +// directory before Ray has created logs/ inside it — and refusing to try again would +// leave that entire session without rotated protection. +type rotatedFailure struct { + err error + notBefore time.Time + attempts int + seq int64 + durable bool +} + +// rotatedSupervisor owns at most one rotatedCollector at a time and mediates every +// question the rest of the runtime asks about it. +// +// It never exposes the collector itself. Callers get idempotent lifecycle operations, +// so nothing outside can hold a pointer to a collector that has since been retired. +// +// Two locks, in one order only: +// +// - lifecycle serializes the operations that start and stop collectors. It is what +// guarantees that exactly one collector owns the staging root at a time, and it is +// held across the slow parts of retirement — the final reconcile, the drain, Stop +// and the goroutine join. +// - mu guards the fields, and is never held across anything that waits: not a +// collector round trip, not a construction, not a storage call, not the tune hook. +// +// A holder of lifecycle may take mu. Nothing ever takes lifecycle while holding mu, so +// the two cannot invert. The consequence that matters is that a collector goroutine +// reporting its own failure, and every observer of the current run, take only mu and so +// never wait behind a drain. +// +// Neither lock is ever held while calling into RayLogHandler, and the handler never +// calls in while holding its own lock, so there is no inversion with the node-name +// lock either. +type rotatedSupervisor struct { + // tune adjusts a collector's configuration just before it is built. It is nil in + // production and exists so tests can substitute the watcher, the clock and the + // storage writer without a second construction path. + tune func(*rotatedCollectorConfig) + + // beforeFailurePublish runs at the top of runFailed, before the failed run is + // either recorded or detached. It is nil in production and exists so a test can + // hold a collector's failure in exactly the window where a non-atomic publication + // would let ensure restart the identity that just failed. + beforeFailurePublish func() + + // probeCreate, probeClose and probeRemove replace the three fallible steps of the + // staging writability probe. They are nil in production and exist so a test can + // fail a create, a close or an unlink without a filesystem that behaves that way. + probeCreate func(string) (*os.File, error) + probeClose func(*os.File) error + probeRemove func(string) error + + now func() time.Time + + // failures records identities whose collector could not be started or did not + // survive, together with why and when they may be retried. Keeping the error, + // rather than only logging it, is what lets the reason be reported rather than + // inferred. + failures map[rotatedKey]*rotatedFailure + + current *rotatedRun + + writer objectWriter + stagingRoot string + cluster clusterIdentity + + drainBudget time.Duration + drainPoll time.Duration + + failureSeq int64 + + // lifecycle serializes starting and stopping collectors. See the type comment. + lifecycle sync.Mutex + + mu sync.Mutex + // frozen stops new collectors being started once shutdown has begun, so a session + // change observed by the poller cannot resurrect the subsystem behind shutdown's + // back. It is set before shutdown takes lifecycle, so an ensure that is already + // running loses the race to start a replacement rather than winning it and having + // the replacement immediately retired. + frozen bool +} + +func newRotatedSupervisor(cluster clusterIdentity, writer objectWriter, stagingRoot string) *rotatedSupervisor { + return &rotatedSupervisor{ + cluster: cluster, + writer: writer, + stagingRoot: stagingRoot, + failures: make(map[rotatedKey]*rotatedFailure), + drainBudget: defaultRotatedDrainBudget, + drainPoll: defaultRotatedDrainPoll, + now: time.Now, + } +} + +// ensure points rotated-log protection at the session and node that are active now. +// +// It is idempotent: called with the identity that is already running, it does +// nothing, which is what lets the session poller call it on every tick. Called with a +// different identity it retires the old collector completely — final reconciliation, +// bounded drain, watcher and owner stopped, goroutine joined — before the new one is +// constructed, so no event from the old session can reach the new collector, no state +// is shared between them, and the staging root never has two owners. +// +// A nil supervisor is a working no-op: that is how the handler behaves when rotated +// collection was never started, and it keeps every legacy test path untouched. +func (s *rotatedSupervisor) ensure(session, node, logsDir string) { + if s == nil { + return + } + if session == "" || node == "" || logsDir == "" { + // Identity is not established yet — most often a node ID that has not been + // discovered. A later tick retries; starting with a placeholder would write + // objects under a key no reader looks at. + return + } + key := rotatedKey{session: session, node: node} + + s.lifecycle.Lock() + defer s.lifecycle.Unlock() + + if s.isFrozen() { + return + } + + if cur := s.activeRun(); cur != nil { + if cur.key == key && cur.logsDir == logsDir && !cur.finished() { + return + } + // Retiring the predecessor before anything else is what makes the handover + // safe: it is detached, reconciled one last time over a logs tree that still + // exists, drained, stopped and joined here, so by the time the replacement is + // constructed the previous owner of the staging root is gone. + s.stopRun(cur) + } + + if !s.startable(key) { + return + } + if err := s.preflight(key, logsDir); err != nil { + s.noteFailure(key, err) + return + } + + cfg := rotatedCollectorConfig{ + LogsDir: logsDir, + StagingRoot: s.stagingRoot, + SessionName: session, + NodeName: node, + Cluster: s.cluster, + Writer: s.writer, + // Bound what capture may pin, so an object store that stops accepting writes + // cannot turn every rotated segment into a permanent hold on the volume Ray is + // still logging to. ENOSPC handling is separate and unaffected: it comes from + // the filesystem refusing a link, which catches a volume another process filled. + // + // Everything else — reconcile interval, upload backoff, worker stop grace, + // capture IDs — takes the package defaults. + HighWaterBytes: defaultStagingHighWaterBytes, + LowWaterBytes: defaultStagingLowWaterBytes, + } + if s.tune != nil { + s.tune(&cfg) + } + + rc, err := newRotatedCollector(cfg) + if err != nil { + s.noteFailure(key, fmt.Errorf("rotated log collection cannot be configured for %s: %w", key, err)) + return + } + + run := &rotatedRun{rc: rc, key: key, logsDir: logsDir, done: make(chan struct{})} + // Run has not started, so nothing reads cfg yet and this is the last moment the + // callback can be given the run it belongs to. + rc.cfg.OnReady = func() { s.markReady(run) } + + if !s.attach(run) { + // Shutdown froze the subsystem while the predecessor was being retired. The + // collector has been constructed but never run: it holds no watcher, no + // goroutine and no staging link, so dropping it here is complete. + logrus.Debugf("Rotated log collection was not started for %s because shutdown had begun", key) + return + } + go func() { + err := rc.Run() + run.err = err + // done is closed before the supervisor is told, so a retirement that is + // already waiting on this run is released before this goroutine competes for + // the lock. + close(run.done) + if err != nil { + s.runFailed(run, err) + } + }() + logrus.Infof("Rotated log collection started for %s: logs=%s staging=%s", key, logsDir, s.stagingRoot) +} + +// preflight rejects, before a collector is built, deployment conditions that the +// collector itself could otherwise only discover one segment at a time. +// +// What it establishes, and nothing more: +// +// - the logs directory exists and is a directory; +// - the staging root exists, or can be created, and the collector can create a file +// in it; +// - inode identity is available on this platform, without which there are neither +// hard links nor a way to tell two captures apart; +// - the staging root and the logs directory are on the same device, since a hard +// link cannot cross one. +// +// What it does not establish is whether this collector may hard-link Ray's files. That +// depends on the ownership and mode of each source file — a custom image whose Ray user +// differs from the collector's UID gives EPERM on os.Link even though every check here +// passes — and probing it would mean linking a real Ray log during startup. Those +// deployments still degrade the way they did before: isUnsupportedLinkError reports the +// segment and capture continues. +// +// A logs directory that is not there yet is reported as the transient condition it is: +// the session poller can see a new session directory before Ray has created logs/ +// inside it, and the collector's own startup treats an unwatchable root as fatal. +func (s *rotatedSupervisor) preflight(key rotatedKey, logsDir string) error { + logsInfo, err := os.Stat(logsDir) + if err != nil { + // Only an absent path is the "not yet" case. A permission the collector does + // not have, or an I/O error, is a property of the deployment and reads exactly + // the same way on every retry, so marking those retryable would turn a + // misconfiguration into a rebuild every few minutes for the life of the pod. + if isVanished(err) { + return retryableRotated(fmt.Errorf("rotated log collection is not started for %s yet: %w", key, err)) + } + return fmt.Errorf("rotated log collection is disabled for %s: %w", key, err) + } + if !logsInfo.IsDir() { + return fmt.Errorf("rotated log collection is disabled for %s: %s is not a directory", key, logsDir) + } + if err := os.MkdirAll(s.stagingRoot, stagingDirPerm); err != nil { + err = fmt.Errorf("create staging root %s: %w", s.stagingRoot, err) + // A full volume is the one staging-root failure that can clear on its own, and + // this subsystem exists to survive one. Permissions and read-only mounts cannot. + if errors.Is(err, syscall.ENOSPC) { + return retryableRotated(fmt.Errorf("rotated log collection is not started for %s yet: %w", key, err)) + } + return fmt.Errorf("rotated log collection is disabled for %s: %w", key, err) + } + stagingInfo, err := os.Stat(s.stagingRoot) + if err != nil { + return fmt.Errorf("rotated log collection is disabled for %s: %w", key, err) + } + if err := s.probeWritable(s.stagingRoot); err != nil { + // MkdirAll succeeds on a staging root that already exists, whatever its mode + // or owner, so this is the only thing that answers "can we actually stage + // here?" — a read-only remount or a root left behind by another UID reaches + // exactly this line. + return fmt.Errorf("rotated log collection is disabled for %s: %w", key, err) + } + + logsDev, _, logsErr := inodeFromFileInfo(logsInfo) + stagingDev, _, stagingErr := inodeFromFileInfo(stagingInfo) + if logsErr != nil || stagingErr != nil { + // No inode identity means no hard links and no way to tell one captured + // segment from another. Saying so once is the graceful outcome; the + // alternative is a collector that fails every capture individually. + return fmt.Errorf("rotated log collection is disabled for %s: %w", + key, errors.Join(logsErr, stagingErr)) + } + if logsDev.Dev != stagingDev.Dev { + return fmt.Errorf("rotated log collection is disabled for %s: the staging root %s and the Ray logs directory %s are on different filesystems, so segments cannot be captured by hard link: %w", + key, s.stagingRoot, logsDir, syscall.EXDEV) + } + return nil +} + +// probeWritable proves the collector can create a file in dir *and remove it again*. +// +// Both halves are the point. This staging tree is not somewhere the collector only +// writes: promotion renames an entry from pending/ to uploaded/ and release unlinks it, +// so a directory that accepts new entries but will not give them up would let capture +// pin inodes that nothing could ever free. A probe that could not be cleaned up is +// therefore a failure, not a warning — and leaving it behind would also litter the +// staging root with files reconstruction has to skip on every start. +// +// Every path attempts both the close and the removal, and both results reach the +// caller, so a failure to clean up is never hidden by a failure to close. +// +// The probe name is prefixed so that a crash between create and remove leaves +// something an operator can recognize, and so that parseStagedPath ignores it if +// reconstruction ever sees it. +func (s *rotatedSupervisor) probeWritable(dir string) error { + f, err := s.probeCreateFile(dir) + if err != nil { + err = fmt.Errorf("staging root %s cannot be written to: %w", dir, err) + // A full volume is the one probe failure that is about the moment rather than + // the deployment, and this subsystem is built to survive one: the intake gate + // already pauses capture on ENOSPC and resumes when space returns. Marking it + // here is what lets a start that lost the race with a full disk be retried at + // all. Permissions and read-only mounts read the same way on every attempt and + // stay durable, as do the close and removal failures below — those say the + // directory cannot be used, not that it is momentarily full. + if errors.Is(err, syscall.ENOSPC) { + return retryableRotated(err) + } + return err + } + name := f.Name() + + closeErr := s.probeCloseFile(f) + removeErr := s.probeRemoveFile(name) + if removeErr != nil && isVanished(removeErr) { + removeErr = nil // something else cleaned it up; the entry is gone either way + } + if closeErr == nil && removeErr == nil { + return nil + } + return fmt.Errorf("staging root %s cannot be used: %w", dir, errors.Join(closeErr, removeErr)) +} + +// probeCreateFile, probeCloseFile and probeRemoveFile are the probe's fallible steps, +// isolated so a test can fail any of them without needing a filesystem that behaves +// that way. +func (s *rotatedSupervisor) probeCreateFile(dir string) (*os.File, error) { + if s.probeCreate != nil { + return s.probeCreate(dir) + } + return os.CreateTemp(dir, ".rotated-preflight-*") +} + +func (s *rotatedSupervisor) probeCloseFile(f *os.File) error { + if s.probeClose != nil { + return s.probeClose(f) + } + return f.Close() +} + +func (s *rotatedSupervisor) probeRemoveFile(name string) error { + if s.probeRemove != nil { + return s.probeRemove(name) + } + return os.Remove(name) +} + +// runFailed reacts to a collector whose Run returned an error. +// +// The error is reported rather than swallowed, the identity is recorded so nothing +// restarts it in a loop, and the active pointer is cleared so a dead goroutine is +// never mistaken for live protection. Legacy collection is deliberately untouched: +// rotated capture stopping is a loss of the extra protection this subsystem adds, not +// a reason to take down the collector that is still uploading everything else. +// +// Recording and detaching are one state transition under one lock. Doing them in two +// steps leaves a window in which the failed run is already unreachable but its failure +// is not yet recorded, and an ensure landing in that window sees an identity with no +// collector and no reason not to build one — which is precisely the restart of a +// just-failed identity this is here to prevent. +func (s *rotatedSupervisor) runFailed(run *rotatedRun, err error) { + if s.beforeFailurePublish != nil { + s.beforeFailurePublish() + } + run.noted.Do(func() { + s.publishFailure(run, err) + }) +} + +// publishFailure records a run's failure and retires its pointer atomically. +func (s *rotatedSupervisor) publishFailure(run *rotatedRun, err error) { + wrapped := fmt.Errorf("rotated log collection stopped for %s; legacy log collection continues: %w", run.key, err) + + s.mu.Lock() + f := s.recordFailureLocked(run.key, wrapped) + if s.current == run { + s.current = nil + } + s.mu.Unlock() + + logFailure(f) +} + +// noteFailure records why an identity has no collector and when it may be tried again. +// It is for failures with no run behind them — anything that went wrong before a +// collector existed. +func (s *rotatedSupervisor) noteFailure(key rotatedKey, err error) { + s.mu.Lock() + f := s.recordFailureLocked(key, err) + s.mu.Unlock() + + logFailure(f) +} + +// recordFailureLocked writes the failure record and returns it for logging, which the +// caller does after releasing the lock. +func (s *rotatedSupervisor) recordFailureLocked(key rotatedKey, err error) rotatedFailure { + durable := !isRetryableRotatedFailure(err) + + s.failureSeq++ + attempts := 1 + if prev := s.failures[key]; prev != nil { + attempts = prev.attempts + 1 + } + f := &rotatedFailure{err: err, durable: durable, attempts: attempts, seq: s.failureSeq} + if !durable { + f.notBefore = s.now().Add(retryDelay(attempts)) + } + s.failures[key] = f + s.pruneFailuresLocked(key) + return *f +} + +func logFailure(f rotatedFailure) { + switch { + case f.durable: + logrus.Errorf("%v", f.err) + case f.attempts == 1: + logrus.Warnf("%v (retrying in %s)", f.err, retryDelay(f.attempts)) + default: + // A condition that has not cleared after several minutes is worth one line + // per retry at most, and by then the retry interval is measured in minutes. + logrus.Debugf("%v (attempt %d, retrying in %s)", f.err, f.attempts, retryDelay(f.attempts)) + } +} + +// retryDelay backs a retryable identity off from one poll tick to one every few +// minutes, so a condition that never clears costs a rebuild an hour rather than one +// every five seconds. +func retryDelay(attempts int) time.Duration { + d := rotatedRetryBase + for i := 1; i < attempts; i++ { + d *= 2 + if d >= rotatedRetryMax { + return rotatedRetryMax + } + } + return d +} + +// pruneFailuresLocked keeps the failure map bounded. +// +// Records for other sessions go first: a Ray session name is a timestamp, so a +// session the runtime has moved past is never ensured again and its record can only +// accumulate. The size cap is the backstop for identities within one session. +func (s *rotatedSupervisor) pruneFailuresLocked(keep rotatedKey) { + for k := range s.failures { + if k.session != keep.session { + delete(s.failures, k) + } + } + for len(s.failures) > maxRotatedFailureRecords { + var oldest rotatedKey + var oldestSeq int64 + first := true + for k, f := range s.failures { + if k == keep { + continue + } + if first || f.seq < oldestSeq { + oldest, oldestSeq, first = k, f.seq, false + } + } + if first { + return + } + delete(s.failures, oldest) + } +} + +// startable reports whether a collector may be built for this identity now. +func (s *rotatedSupervisor) startable(key rotatedKey) bool { + s.mu.Lock() + defer s.mu.Unlock() + + f, known := s.failures[key] + if !known { + return true + } + if f.durable { + logrus.Debugf("Rotated log collection stays disabled for %s after an earlier failure: %v", key, f.err) + return false + } + if s.now().Before(f.notBefore) { + return false + } + return true +} + +// retireUnless stops the current collector unless it belongs to session. +// +// It exists for the one changeover that cannot start a replacement: the session has +// changed but the new session's node ID is not known yet, so there is no identity to +// build a collector for. The collector of the session that is going away still has to +// be retired — this is its last chance to reconcile a tree that is about to be +// relocated — and the new session goes unprotected until a later tick can name its +// node. +// +// The exemption is what makes it safe to call on every relocation retry. Once a +// collector for the current session exists, a later tick that merely cannot reach the +// dashboard must leave it alone: rediscovery failing says nothing about the collector, +// and retiring it would cost a drain, a watcher and a staging reconstruction for +// nothing. Passing an empty session retires whatever is running, which is the honest +// answer when the caller cannot name the current session at all. +func (s *rotatedSupervisor) retireUnless(session string) { + if s == nil { + return + } + s.lifecycle.Lock() + defer s.lifecycle.Unlock() + + run := s.activeRun() + if run == nil || (session != "" && run.key.session == session) { + return + } + s.stopRun(run) +} + +// shutdown freezes the subsystem and retires the active collector. +// +// After it returns no new collector can be started and no collector goroutine is +// running. Freezing happens before the lifecycle lock is taken, so an ensure racing +// with shutdown cannot start a replacement that shutdown would then have to stop. +// Calling it again is a no-op, so a repeated shutdown neither blocks nor undoes +// anything. +func (s *rotatedSupervisor) shutdown() { + if s == nil { + return + } + s.mu.Lock() + s.frozen = true + s.mu.Unlock() + + s.lifecycle.Lock() + defer s.lifecycle.Unlock() + + if run := s.activeRun(); run != nil { + s.stopRun(run) + } +} + +// stopRun performs the bounded shutdown of one collector, in the only order that is +// safe. It must be called with lifecycle held and mu not held. +// +// 1. detach it, so nothing can hand it more work or observe it as active; +// 2. one final reconciliation, while the logs tree still exists — this is the last +// intake pass by design, because stopping capture first would abandon every +// segment Ray rotates away during shutdown; +// 3. a bounded drain, giving already-captured work its chance to reach storage; +// 4. Stop, which ends intake for good and closes the watcher, waiting only the +// worker's own grace for an upload the storage interface cannot cancel; +// 5. join the goroutine, so no goroutine can touch collector state afterwards, and +// record its exit status if it failed. +// +// Anything the drain did not finish stays pending on the staging volume. Nothing here +// deletes a staging link. +func (s *rotatedSupervisor) stopRun(run *rotatedRun) { + s.detach(run) + + var last collectorStats + if !run.finished() { + run.rc.reconcileNow() + last = s.drain(run) + } + + run.rc.Stop() + <-run.done + + if run.err != nil { + run.noted.Do(func() { + s.noteFailure(run.key, fmt.Errorf("rotated log collection stopped for %s; legacy log collection continues: %w", run.key, run.err)) + }) + } + logrus.Infof("Rotated log collection stopped for %s (%d capture(s) still pending on the staging volume)", run.key, last.Pending) +} + +// drain waits for the collector to have nothing outstanding, but never longer than +// the budget. It returns the last stats it read. +// +// stats is answered by the owner goroutine, which is never inside a storage call, so +// this loop keeps making progress even while an uncancelable WriteFile is running — +// and a collector that has already exited answers with zeroes, so the loop ends at +// once rather than waiting for a goroutine that is gone. +func (s *rotatedSupervisor) drain(run *rotatedRun) collectorStats { + if s.drainBudget <= 0 { + return run.rc.stats() + } + deadline := time.Now().Add(s.drainBudget) + for { + st := run.rc.stats() + if st.Pending == 0 && st.QueuedUploads == 0 && st.InFlightUploads == 0 && st.AwaitingPromotion == 0 { + return st + } + if !time.Now().Before(deadline) { + logrus.Warnf("Rotated log collection for %s did not finish within %s: %d capture(s) stay pending on the staging volume and the next run will retry them (queued=%d in flight=%d awaiting promotion=%d)", + run.key, s.drainBudget, st.Pending, st.QueuedUploads, st.InFlightUploads, st.AwaitingPromotion) + return st + } + time.Sleep(s.drainPoll) + } +} + +// attach publishes a run as the active one, unless shutdown has frozen the subsystem +// in the meantime. It reports whether the run may be started. +// +// It deliberately does not touch the identity's failure record. Attaching says only +// that a collector was constructed; every startup step that can fail is still ahead of +// it, and most of the retryable failures this subsystem has — a logs tree that +// disappears mid-startup, an exhausted watch limit — happen there. Clearing the record +// here would reset the attempt count before each of those, so a condition that never +// clears would retry at the base delay forever instead of backing off. markReady is +// what clears it. +func (s *rotatedSupervisor) attach(run *rotatedRun) bool { + s.mu.Lock() + defer s.mu.Unlock() + if s.frozen { + return false + } + s.current = run + return true +} + +// markReady records that a run finished startup and is protecting its tree. +// +// It is the only thing that clears an identity's failure history, because it is the +// only evidence that the condition behind that history has actually cleared. A signal +// from a run that has since been retired or replaced clears nothing: the identity it +// would clear may belong to a different collector by now, and a stale clear would hand +// the next failure a fresh backoff. +func (s *rotatedSupervisor) markReady(run *rotatedRun) { + s.mu.Lock() + defer s.mu.Unlock() + if s.current != run { + return + } + run.ready = true + delete(s.failures, run.key) +} + +// detach clears the active pointer if it still refers to this run. +func (s *rotatedSupervisor) detach(run *rotatedRun) { + s.mu.Lock() + defer s.mu.Unlock() + if s.current == run { + s.current = nil + } +} + +func (s *rotatedSupervisor) activeRun() *rotatedRun { + s.mu.Lock() + defer s.mu.Unlock() + return s.current +} + +func (s *rotatedSupervisor) isFrozen() bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.frozen +} + +// runStatus is what the supervisor holds for one exact identity. +// +// "Exact" is the whole point: session, node and logs directory together. A collector +// for a different identity tells the caller nothing about the one it asked about, and +// answering "yes, something is running" is what let a failed handover look like a +// completed one. +type runStatus int + +const ( + // runAbsent: nothing is attached for this identity. Either it was never started, + // or it failed and was detached. + runAbsent runStatus = iota + // runStarting: attached, but its startup has not completed. It may still fail. + runStarting + // runReady: attached and past every startup step, so it is protecting its tree. + runReady + // runFinished: still attached, but its goroutine has exited. A failure that has + // not been published yet looks like this. + runFinished +) + +func (s runStatus) String() string { + switch s { + case runAbsent: + return "absent" + case runStarting: + return "starting" + case runReady: + return "ready" + case runFinished: + return "finished" + default: + return fmt.Sprintf("unknown status %d", int(s)) + } +} + +// statusFor reports what the supervisor holds for one exact identity. +// +// It is the only honest basis for "has the handover happened?". Recording that from +// the fact that ensure was called records an attempt, not an outcome: ensure returns +// without a collector when the identity is backing off, when preflight rejects the +// deployment, when construction fails and when shutdown has frozen the subsystem — and +// a collector that did attach can still fail on any startup step afterwards. +func (s *rotatedSupervisor) statusFor(session, node, logsDir string) runStatus { + if s == nil { + return runAbsent + } + s.mu.Lock() + defer s.mu.Unlock() + + run := s.current + if run == nil || run.key != (rotatedKey{session: session, node: node}) || run.logsDir != logsDir { + return runAbsent + } + if run.finished() { + return runFinished + } + if run.ready { + return runReady + } + return runStarting +} + +// activeKey reports the identity of the running collector, if there is one. It exists +// for diagnostics and tests; production lifecycle decisions go through ensure. +func (s *rotatedSupervisor) activeKey() (rotatedKey, bool) { + if s == nil { + return rotatedKey{}, false + } + run := s.activeRun() + if run == nil { + return rotatedKey{}, false + } + return run.key, true +} diff --git a/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_runtime_test.go b/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_runtime_test.go new file mode 100644 index 00000000000..27e8425177f --- /dev/null +++ b/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_runtime_test.go @@ -0,0 +1,3095 @@ +package logcollector + +import ( + "errors" + "fmt" + "io" + "os" + "path" + "path/filepath" + "runtime" + "sort" + "strings" + "sync" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/ray-project/kuberay/historyserver/pkg/storage/clusterlogs" + "github.com/ray-project/kuberay/historyserver/pkg/utils" +) + +// These tests exercise the production wiring: the RayLogHandler builds, replaces and +// retires rotatedCollectors, and the legacy shutdown walk keeps behaving exactly as it +// did. Everything below runs against real directories, real files and real hard links; +// only the fsnotify watcher, the reconcile ticker, the supervisor's clock and the +// object store are substituted. + +// disabledReason and durablyDisabled read the supervisor's failure record. Production +// never asks either question — it decides through startable — so they live here rather +// than adding permanently unused accessors to the supervisor. They take mu the same way +// the production readers do, so they are safe to call while a collector is running. + +// disabledReason returns why rotated collection has no collector for an identity, +// whether that condition is durable or merely not due for a retry yet. +func (s *rotatedSupervisor) disabledReason(key rotatedKey) error { + if s == nil { + return nil + } + s.mu.Lock() + defer s.mu.Unlock() + if f, ok := s.failures[key]; ok { + return f.err + } + return nil +} + +// durablyDisabled reports whether an identity was switched off for a condition that +// will not be retried. +func (s *rotatedSupervisor) durablyDisabled(key rotatedKey) bool { + if s == nil { + return false + } + s.mu.Lock() + defer s.mu.Unlock() + f, ok := s.failures[key] + return ok && f.durable +} + +// runtimeWriter is the one storage writer both halves of the runtime share, exactly as +// production does: the rotated uploader and the legacy walk both write through it, so a +// single recording shows which objects production produced. +type runtimeWriter struct { + written map[string]string + gate chan struct{} + entered chan string + dirs []string + attempts []string + mu sync.Mutex + failAll bool +} + +func newRuntimeWriter() *runtimeWriter { + return &runtimeWriter{written: make(map[string]string), entered: make(chan string, 64)} +} + +func (w *runtimeWriter) CreateDirectory(p string) error { + w.mu.Lock() + defer w.mu.Unlock() + w.dirs = append(w.dirs, p) + return nil +} + +func (w *runtimeWriter) WriteFile(file string, r io.ReadSeeker) error { + w.mu.Lock() + w.attempts = append(w.attempts, file) + gate, fail := w.gate, w.failAll + w.mu.Unlock() + + select { + case w.entered <- file: + default: + } + if gate != nil { + <-gate + } + if fail { + return errors.New("object store is unavailable") + } + + content, err := io.ReadAll(r) + if err != nil { + return err + } + w.mu.Lock() + defer w.mu.Unlock() + w.written[file] = string(content) + return nil +} + +// block makes every write wait until the returned function is called. It is how a test +// holds an upload inside the uncancelable storage call. +func (w *runtimeWriter) block(t *testing.T) func() { + t.Helper() + w.mu.Lock() + w.gate = make(chan struct{}) + gate := w.gate + w.mu.Unlock() + + var once sync.Once + release := func() { + once.Do(func() { + w.mu.Lock() + w.gate = nil + w.mu.Unlock() + close(gate) + }) + } + t.Cleanup(release) + return release +} + +func (w *runtimeWriter) setFailAll(v bool) { + w.mu.Lock() + defer w.mu.Unlock() + w.failAll = v +} + +func (w *runtimeWriter) keys() []string { + w.mu.Lock() + defer w.mu.Unlock() + out := make([]string, 0, len(w.written)) + for k := range w.written { + out = append(out, k) + } + sort.Strings(out) + return out +} + +func (w *runtimeWriter) has(key string) bool { + w.mu.Lock() + defer w.mu.Unlock() + _, ok := w.written[key] + return ok +} + +func (w *runtimeWriter) content(key string) string { + w.mu.Lock() + defer w.mu.Unlock() + return w.written[key] +} + +func (w *runtimeWriter) attemptCount() int { + w.mu.Lock() + defer w.mu.Unlock() + return len(w.attempts) +} + +// watcherFactory hands out one fakeWatcher per collector the supervisor builds and +// remembers them in order, so a test can tell one session's watcher from the next. +type watcherFactory struct { + mu sync.Mutex + made []*fakeWatcher +} + +func (f *watcherFactory) next() (fsWatcher, error) { + f.mu.Lock() + defer f.mu.Unlock() + w := newFakeWatcher() + f.made = append(f.made, w) + return w, nil +} + +func (f *watcherFactory) count() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.made) +} + +func (f *watcherFactory) at(i int) *fakeWatcher { + f.mu.Lock() + defer f.mu.Unlock() + if i >= len(f.made) { + return nil + } + return f.made[i] +} + +// testCollector reaches the running collector. Production never needs this — the +// supervisor deliberately does not expose it — but a test has to be able to round-trip +// the owner goroutine and read the configuration production built. +func (s *rotatedSupervisor) testCollector() *rotatedCollector { + run := s.activeRun() + if run == nil { + return nil + } + return run.rc +} + +func (s *rotatedSupervisor) failureCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.failures) +} + +func (s *rotatedSupervisor) failureRecord(key rotatedKey) (rotatedFailure, bool) { + s.mu.Lock() + defer s.mu.Unlock() + f, ok := s.failures[key] + if !ok { + return rotatedFailure{}, false + } + return *f, true +} + +const ( + testSessionA = "session_2026-07-31_10-00-00_000001" + testSessionB = "session_2026-07-31_11-00-00_000002" + // A session change is normally also a node change, though nothing in this + // repository guarantees it (see currentNodeID). The two are deliberately different + // values here because a test that reuses one node ID cannot see a collector built + // under the wrong one. + testNodeID = "0a1b2c3d4e5f60718293a4b5c6d7e8f9" + testNodeIDB = "fedcba98765432100123456789abcdef" +) + +// runtimeHarness is one RayLogHandler wired the way NewCollector wires it, with its +// rotated subsystem pointed at real temp directories. +type runtimeHarness struct { + t *testing.T + handler *RayLogHandler + writer *runtimeWriter + watchers *watcherFactory + root string + ticks chan time.Time + + mu sync.Mutex + // clock is what the supervisor reads for retry scheduling, so a test can make a + // backoff expire without sleeping through it. + clock time.Time + // nodeID is what the handler's node discovery returns, and nodeErr makes that + // discovery fail. Production reaches the dashboard over HTTP; a test moves the + // answer instead. + nodeID string + nodeErr bool + nodeCalls int + // livePredecessor records that a collector was constructed while the supervisor was + // still attached to its predecessor — which would mean two owners of one staging + // root. Ordering cannot be observed from outside ensure, so it is sampled from + // inside, in the moment between the retirement and the construction. + livePredecessor bool + // builds counts collector constructions, sampled at the same point, and builtKeys + // records the identity each one was constructed for. A collector built under the + // wrong node is corrected by the next one, so only the full history shows it. + builds int + builtKeys []rotatedKey +} + +func (h *runtimeHarness) built() []rotatedKey { + h.mu.Lock() + defer h.mu.Unlock() + return append([]rotatedKey(nil), h.builtKeys...) +} + +func (h *runtimeHarness) sawLivePredecessor() bool { + h.mu.Lock() + defer h.mu.Unlock() + return h.livePredecessor +} + +func (h *runtimeHarness) buildCount() int { + h.mu.Lock() + defer h.mu.Unlock() + return h.builds +} + +func (h *runtimeHarness) advanceClock(d time.Duration) { + h.mu.Lock() + defer h.mu.Unlock() + h.clock = h.clock.Add(d) +} + +func (h *runtimeHarness) now() time.Time { + h.mu.Lock() + defer h.mu.Unlock() + return h.clock +} + +// discoverNodeB makes the handler's node discovery answer with the node ID a restarted +// raylet would report, which is how a session change carries a node change. +func (h *runtimeHarness) discoverNodeB() { + h.mu.Lock() + defer h.mu.Unlock() + h.nodeID, h.nodeErr = testNodeIDB, false +} + +// failNodeDiscovery makes discovery fail, as it does while the dashboard is still +// coming up after a session restart. +func (h *runtimeHarness) failNodeDiscovery() { + h.mu.Lock() + defer h.mu.Unlock() + h.nodeErr = true +} + +func (h *runtimeHarness) discoverNode() (string, bool) { + h.mu.Lock() + defer h.mu.Unlock() + h.nodeCalls++ + if h.nodeErr { + return "", false + } + return h.nodeID, true +} + +func newRuntimeHarness(t *testing.T) *runtimeHarness { + t.Helper() + // The temp root is resolved once, because production resolves the session symlink + // and macOS puts /var behind a symlink to /private/var. Without this the harness + // would be comparing two spellings of the same directory. + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatalf("resolve temp root: %v", err) + } + t.Setenv("RAY_TMP_ROOT", root) + + h := &runtimeHarness{ + t: t, + root: root, + writer: newRuntimeWriter(), + watchers: &watcherFactory{}, + ticks: make(chan time.Time), + clock: time.Date(2026, 7, 31, 12, 0, 0, 0, time.UTC), + nodeID: testNodeID, + } + h.makeSession(testSessionA) + h.pointSessionLatest(testSessionA) + + h.handler = &RayLogHandler{ + Writer: h.writer, + RootDir: "/history", + RayClusterName: "raycluster-sample", + RayClusterNamespace: "ray-system", + OwnerKind: "RayJob", + OwnerName: "rayjob-sample", + RayNodeName: testNodeID, + SessionDir: h.sessionDir(testSessionA), + LogDir: h.logsDir(testSessionA), + ShutdownChan: make(chan struct{}), + discoverNodeID: h.discoverNode, + // The production tick is five seconds; a test that waits for several polling + // cycles would otherwise spend most of its time asleep. + sessionPollInterval: 20 * time.Millisecond, + } + h.handler.rotated = h.newSupervisor() + t.Cleanup(func() { h.handler.rotatedCollection().shutdown() }) + return h +} + +// newSupervisor builds the supervisor with exactly the arguments startRotatedCollection +// uses, so the identity, writer and staging root under test are the production ones. +// tune only substitutes the watcher and the reconcile ticker. +func (h *runtimeHarness) newSupervisor() *rotatedSupervisor { + sup := newRotatedSupervisor(h.handler.clusterIdentity(), h.handler.Writer, utils.GetRayRotatedStagingPath()) + sup.now = h.now + sup.tune = func(cfg *rotatedCollectorConfig) { + // tune runs inside ensure, in the moment between retiring the predecessor and + // constructing the replacement. A predecessor still attached here would mean + // the new collector is being built while the old one may still own the staging + // tree. + live := sup.activeRun() != nil + h.mu.Lock() + h.builds++ + h.builtKeys = append(h.builtKeys, rotatedKey{session: cfg.SessionName, node: cfg.NodeName}) + if live { + h.livePredecessor = true + } + h.mu.Unlock() + cfg.NewWatcher = h.watchers.next + cfg.NewTicker = func(time.Duration) (<-chan time.Time, func()) { return h.ticks, func() {} } + } + // Short enough that a test which deliberately leaves work pending does not pay the + // production budget on every cleanup; tests that care about the budget set their own. + sup.drainBudget = 200 * time.Millisecond + sup.drainPoll = time.Millisecond + return sup +} + +func (h *runtimeHarness) sessionDir(name string) string { return filepath.Join(h.root, name) } + +func (h *runtimeHarness) logsDir(name string) string { + return filepath.Join(h.sessionDir(name), utils.RAY_SESSIONDIR_LOGDIR_NAME) +} + +func (h *runtimeHarness) makeSession(name string) { + h.t.Helper() + if err := os.MkdirAll(h.logsDir(name), 0o750); err != nil { + h.t.Fatalf("create session %s: %v", name, err) + } +} + +func (h *runtimeHarness) pointSessionLatest(name string) { + h.t.Helper() + link := utils.GetRaySessionLatestPath() + _ = os.Remove(link) + if err := os.Symlink(h.sessionDir(name), link); err != nil { + h.t.Fatalf("point session_latest at %s: %v", name, err) + } +} + +// write puts a file in a session's logs directory and returns its path. +func (h *runtimeHarness) write(session, name, content string) string { + h.t.Helper() + p := filepath.Join(h.logsDir(session), filepath.FromSlash(name)) + writeFile(h.t, p, content) + return p +} + +// start brings the rotated subsystem up exactly as Run does, and waits until the +// collector's owner goroutine has finished startup. +func (h *runtimeHarness) start() *rotatedCollector { + h.t.Helper() + h.handler.startRotatedCollection() + return h.awaitCollector() +} + +func (h *runtimeHarness) awaitCollector() *rotatedCollector { + h.t.Helper() + rc := h.handler.rotatedCollection().testCollector() + if rc == nil { + h.t.Fatal("no rotated collector is active") + } + rc.snapshot() // round-trips the owner goroutine, so startup has completed + return rc +} + +// capture creates the active raylet.out log plus one rotation backup in the given +// session, and waits for the collector to have pinned it. +func (h *runtimeHarness) captureIn(session, backup, content string) stagedEntry { + h.t.Helper() + h.write(session, "raylet.out", "active") + p := h.write(session, backup, content) + + rc := h.handler.rotatedCollection().testCollector() + if rc == nil { + h.t.Fatal("no rotated collector is active") + } + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + rc.reconcileNow() + for _, e := range rc.snapshot() { + if e.OriginalName == backup { + return e + } + } + time.Sleep(2 * time.Millisecond) + } + h.t.Fatalf("%s was never captured", p) + return stagedEntry{} +} + +func (h *runtimeHarness) capture(content string) stagedEntry { + h.t.Helper() + return h.captureIn(testSessionA, "raylet.out.1", content) +} + +func (h *runtimeHarness) waitForOneUploaded(rc *rotatedCollector) { + h.t.Helper() + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + for _, e := range rc.snapshot() { + if e.State == stateUploaded { + return + } + } + time.Sleep(2 * time.Millisecond) + } + h.t.Fatalf("timed out waiting for an uploaded capture: %+v", rc.snapshot()) +} + +// legacyLogsPrefix is where the legacy walk puts a session's logs for this node — +// computed from clusterlogs, not from the collector. Nothing in this tranche may +// change it. +func (h *runtimeHarness) logsPrefixFor(session, node string) string { + return clusterlogs.LogsDir( + h.handler.RootDir, + h.handler.OwnerKind, + h.handler.OwnerName, + h.handler.RayClusterNamespace, + h.handler.RayClusterName, + session, + node, + ) +} + +func (h *runtimeHarness) legacyLogsPrefixFor(session string) string { + return h.logsPrefixFor(session, h.handler.GetRayNodeName()) +} + +func (h *runtimeHarness) legacyLogsPrefix() string { return h.legacyLogsPrefixFor(testSessionA) } + +func (h *runtimeHarness) stagingFiles() []string { + h.t.Helper() + root := utils.GetRayRotatedStagingPath() + var out []string + err := filepath.WalkDir(root, func(p string, d os.DirEntry, err error) error { + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + if !d.IsDir() { + rel, relErr := filepath.Rel(root, p) + if relErr != nil { + return relErr + } + out = append(out, filepath.ToSlash(rel)) + } + return nil + }) + if err != nil && !os.IsNotExist(err) { + h.t.Fatalf("walk staging root: %v", err) + } + sort.Strings(out) + return out +} + +func eventually(t *testing.T, what string, cond func() bool) { + t.Helper() + eventuallyWithin(t, 10*time.Second, what, cond) +} + +// eventuallyWithin is for the rare wait that needs a budget other than the default. +func eventuallyWithin(t *testing.T, d time.Duration, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(d) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(2 * time.Millisecond) + } + t.Fatalf("timed out waiting for %s", what) +} + +// --------------------------------------------------------------------------- +// Startup and session identity +// --------------------------------------------------------------------------- + +// 1. The handler starts a collector for the session it was configured with. +func TestRuntimeStartsRotatedCollectorForActiveSession(t *testing.T) { + h := newRuntimeHarness(t) + h.start() + + key, ok := h.handler.rotatedCollection().activeKey() + if !ok { + t.Fatal("no rotated collector is active after startRotatedCollection") + } + if key.session != testSessionA || key.node != testNodeID { + t.Errorf("active collector is for %+v, want session %s on node %s", key, testSessionA, testNodeID) + } + if h.watchers.count() != 1 { + t.Errorf("watchers created = %d, want exactly 1", h.watchers.count()) + } +} + +// 2. It receives the real storage writer and the complete owner-aware identity. +func TestRuntimePassesRealWriterAndClusterIdentity(t *testing.T) { + h := newRuntimeHarness(t) + rc := h.start() + + if rc.cfg.Writer != objectWriter(h.writer) { + t.Errorf("collector writer = %#v, want the handler's storage writer", rc.cfg.Writer) + } + want := clusterIdentity{ + RootDir: "/history", + OwnerKind: "RayJob", + OwnerName: "rayjob-sample", + Namespace: "ray-system", + ClusterName: "raycluster-sample", + } + if rc.cfg.Cluster != want { + t.Errorf("collector cluster identity = %+v, want %+v", rc.cfg.Cluster, want) + } +} + +// 3. It receives the active session's logs directory, the shared staging root, and the +// session and node the handler is actually running for. +func TestRuntimePassesSessionNodeAndPaths(t *testing.T) { + h := newRuntimeHarness(t) + rc := h.start() + + if rc.cfg.LogsDir != h.logsDir(testSessionA) { + t.Errorf("LogsDir = %s, want %s", rc.cfg.LogsDir, h.logsDir(testSessionA)) + } + if rc.cfg.StagingRoot != utils.GetRayRotatedStagingPath() { + t.Errorf("StagingRoot = %s, want %s", rc.cfg.StagingRoot, utils.GetRayRotatedStagingPath()) + } + if rc.cfg.SessionName != testSessionA { + t.Errorf("SessionName = %s, want %s", rc.cfg.SessionName, testSessionA) + } + if rc.cfg.NodeName != testNodeID { + t.Errorf("NodeName = %s, want %s", rc.cfg.NodeName, testNodeID) + } +} + +// TestRuntimeConfiguresBoundedIntake is the wiring regression test for the watermarks. +// +// The collector-level tests already prove what the intake gate does once it has a +// limit; what they cannot show is whether production ever gives it one. A collector +// built with zero watermarks captures without any staging bound, so an object store +// that stops accepting writes would let capture pin every byte Ray logs from then on — +// a hard link keeps the blocks alive after Ray unlinks its own name. +func TestRuntimeConfiguresBoundedIntake(t *testing.T) { + h := newRuntimeHarness(t) + rc := h.start() + + if rc.cfg.HighWaterBytes <= 0 { + t.Errorf("HighWaterBytes = %d, want a positive bound: capture is otherwise unbounded while uploads fail", + rc.cfg.HighWaterBytes) + } + if rc.cfg.LowWaterBytes <= 0 { + t.Errorf("LowWaterBytes = %d, want a positive resume threshold", rc.cfg.LowWaterBytes) + } + if rc.cfg.LowWaterBytes >= rc.cfg.HighWaterBytes { + t.Errorf("watermarks = %d/%d, want LowWaterBytes below HighWaterBytes so the gate has hysteresis", + rc.cfg.HighWaterBytes, rc.cfg.LowWaterBytes) + } +} + +// 4. A handler configured with the session_latest symlink — not a resolved directory — +// still runs under the real session ID, in staging and in storage. +// +// This is the difference between writing beside the legacy objects and writing under a +// "session_latest" prefix nothing reads. +func TestRuntimeResolvesSessionLatestToTheRealSessionID(t *testing.T) { + h := newRuntimeHarness(t) + h.handler.SessionDir = utils.GetRaySessionLatestPath() + + rc := h.start() + if rc.cfg.SessionName != testSessionA { + t.Errorf("SessionName = %s, want the resolved session %s", rc.cfg.SessionName, testSessionA) + } + if rc.cfg.LogsDir != h.logsDir(testSessionA) { + t.Errorf("LogsDir = %s, want the resolved %s", rc.cfg.LogsDir, h.logsDir(testSessionA)) + } + key, _ := h.handler.rotatedCollection().activeKey() + if key.session != testSessionA { + t.Errorf("collector identity = %+v, want session %s", key, testSessionA) + } + + // The object key it would write has to sit under the same node prefix the legacy + // walk uses, which is only true when the session name is the real one. + entry := h.capture("rotated segment") + if got := entry.SessionName; got != testSessionA { + t.Errorf("staged entry session = %s, want %s", got, testSessionA) + } + if k := entry.objectKey(rc.cfg.Cluster); !strings.HasPrefix(k, h.legacyLogsPrefix()+"/") { + t.Errorf("object key %s is not under the legacy node prefix %s", k, h.legacyLogsPrefix()) + } +} + +// 5. A session directory that cannot be resolved at all is skipped, not started under a +// made-up name, and the handler survives it. +func TestRuntimeUnresolvableSessionDirStartsNothing(t *testing.T) { + h := newRuntimeHarness(t) + h.handler.SessionDir = filepath.Join(h.root, "session_that_does_not_exist") + + h.handler.startRotatedCollection() + + if _, ok := h.handler.rotatedCollection().activeKey(); ok { + t.Error("a collector was started for a session directory that does not exist") + } + if h.watchers.count() != 0 { + t.Errorf("watchers created = %d, want none", h.watchers.count()) + } + if got := h.handler.rotatedCollection().failureCount(); got != 0 { + t.Errorf("failures recorded = %d, want none: an unresolved path is not a failure of any identity", got) + } +} + +// 6. An empty node ID starts nothing: objects would land under a key no reader looks at. +func TestRuntimeUnknownNodeStartsNothing(t *testing.T) { + h := newRuntimeHarness(t) + h.handler.SetRayNodeName("") + + h.handler.startRotatedCollection() + if _, ok := h.handler.rotatedCollection().activeKey(); ok { + t.Error("a collector was started before the node ID was known") + } + + // And it starts normally once the node ID is discovered. + h.handler.SetRayNodeName(testNodeID) + h.handler.ensureRotatedCollection(h.handler.SessionDir) + if key, ok := h.handler.rotatedCollection().activeKey(); !ok || key.node != testNodeID { + t.Errorf("collector after node discovery = %+v (active=%v), want %s", key, ok, testNodeID) + } +} + +// --------------------------------------------------------------------------- +// Error policy: transient conditions recover, durable ones do not loop +// --------------------------------------------------------------------------- + +// 7. A logs directory that does not exist yet is a transient condition. The session +// poller can see a new session directory before Ray has created logs/ inside it, and +// that must not cost the whole session its rotated protection. +func TestRuntimeMissingLogsDirRecoversWithoutPermanentDisable(t *testing.T) { + h := newRuntimeHarness(t) + sessionDir := h.sessionDir(testSessionB) + if err := os.MkdirAll(sessionDir, 0o750); err != nil { + t.Fatalf("create session dir: %v", err) + } + key := rotatedKey{session: testSessionB, node: testNodeID} + + h.handler.ensureRotatedCollection(sessionDir) + if _, ok := h.handler.rotatedCollection().activeKey(); ok { + t.Fatal("a collector was started for a session with no logs directory") + } + reason := h.handler.rotatedCollection().disabledReason(key) + if reason == nil { + t.Fatal("the missing logs directory was not recorded at all") + } + if h.handler.rotatedCollection().durablyDisabled(key) { + t.Fatalf("a missing logs directory was classified as permanent: %v", reason) + } + + // Ray creates it a moment later. The retry is scheduled, so nothing happens until + // it comes due — and then it starts normally. + h.makeSession(testSessionB) + h.handler.ensureRotatedCollection(sessionDir) + if _, ok := h.handler.rotatedCollection().activeKey(); ok { + t.Error("the retry ran before its backoff expired") + } + + h.advanceClock(rotatedRetryBase + time.Second) + h.handler.ensureRotatedCollection(sessionDir) + got, ok := h.handler.rotatedCollection().activeKey() + if !ok || got != key { + t.Fatalf("collector after the logs directory appeared = %+v (active=%v), want %+v", got, ok, key) + } + h.awaitCollector() +} + +// 7b. A recovered identity keeps no trace of the condition that stopped it: the record +// is gone, diagnostics agree with the running collector, and a later, unrelated +// failure starts its backoff from the beginning. +func TestRuntimeSuccessfulStartClearsTheFailureRecord(t *testing.T) { + h := newRuntimeHarness(t) + sup := h.handler.rotatedCollection() + sessionDir := h.sessionDir(testSessionB) + if err := os.MkdirAll(sessionDir, 0o750); err != nil { + t.Fatalf("create session dir: %v", err) + } + key := rotatedKey{session: testSessionB, node: testNodeID} + + h.handler.ensureRotatedCollection(sessionDir) // fails: no logs/ yet + first, ok := sup.failureRecord(key) + if !ok || first.attempts != 1 { + t.Fatalf("first failure record = %+v (present=%v), want attempt 1", first, ok) + } + + h.makeSession(testSessionB) + h.advanceClock(rotatedRetryBase + time.Second) + h.handler.ensureRotatedCollection(sessionDir) + h.awaitCollector() + + if reason := sup.disabledReason(key); reason != nil { + t.Errorf("disabledReason after recovery = %v, want nil while the collector runs", reason) + } + if _, still := sup.failureRecord(key); still { + t.Error("the recovered identity still has a failure record") + } + if got := sup.failureCount(); got != 0 { + t.Errorf("failure records after recovery = %d, want 0", got) + } + + // A later transient failure for the same identity is charged from attempt 1, not + // from the history of the one that recovered. + sup.shutdown() + h.handler.rotated = h.newSupervisor() + sup = h.handler.rotatedCollection() + if err := os.RemoveAll(h.logsDir(testSessionB)); err != nil { + t.Fatalf("remove logs dir: %v", err) + } + h.handler.ensureRotatedCollection(sessionDir) + again, ok := sup.failureRecord(key) + if !ok { + t.Fatal("the later failure was not recorded") + } + if again.attempts != 1 { + t.Errorf("later failure attempts = %d, want 1", again.attempts) + } + if want := h.now().Add(rotatedRetryBase); !again.notBefore.Equal(want) { + t.Errorf("later failure retries at %s, want the base backoff %s", again.notBefore, want) + } +} + +// 7c. Retryability is carried by the error, not guessed from its errno. A durable +// staging inconsistency reported by the uploader wraps fs.ErrNotExist exactly as a +// missing logs directory does, and it must never be retried. +func TestRuntimeMissingStagedUploadPathIsDurable(t *testing.T) { + h := newRuntimeHarness(t) + sup := h.handler.rotatedCollection() + base := sup.tune + sup.tune = func(cfg *rotatedCollectorConfig) { + base(cfg) + // A short retry so the collector reaches its second upload attempt — the one + // that has to open a staged file that is no longer there — without waiting. + cfg.UploadBackoff = []time.Duration{time.Millisecond} + } + h.writer.setFailAll(true) // the first attempt fails remotely, which is retryable + + h.start() + entry := h.capture("segment") + run := sup.activeRun() + eventually(t, "the first upload attempt", func() bool { return h.writer.attemptCount() > 0 }) + + // The staging volume now contradicts the index: the capture is still pending and + // indexed, but its durable link is gone. + staged := entry.path(utils.GetRayRotatedStagingPath()) + if err := os.Remove(staged); err != nil { + t.Fatalf("remove staged capture: %v", err) + } + + eventually(t, "the collector to stop on the staging inconsistency", func() bool { + return run.finished() + }) + if run.err == nil || !strings.Contains(run.err.Error(), "staging volume contradicts") { + t.Fatalf("collector exited with %v, want the staging inconsistency", run.err) + } + + key := rotatedKey{session: testSessionA, node: testNodeID} + eventually(t, "the failure to be published", func() bool { return sup.disabledReason(key) != nil }) + if !sup.durablyDisabled(key) { + t.Fatalf("a missing staged upload path was classified as retryable: %v", sup.disabledReason(key)) + } + + // No amount of waiting brings it back. + builds := h.buildCount() + for range 5 { + h.advanceClock(time.Hour) + h.handler.ensureRotatedCollection(h.handler.SessionDir) + } + if got := h.buildCount(); got != builds { + t.Errorf("collectors built after a durable staging failure = %d, want none", got-builds) + } +} + +// 7d. A logs directory the collector may not read is a property of the deployment, not +// of the moment, so it is durable however many times it is looked at. +func TestRuntimeUnreadableLogsDirIsDurable(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("running as root: mode bits do not deny access") + } + h := newRuntimeHarness(t) + // A session directory that cannot be traversed makes the stat of logs/ fail with + // EACCES rather than ENOENT. + sessionDir := h.sessionDir(testSessionB) + if err := os.MkdirAll(filepath.Join(sessionDir, utils.RAY_SESSIONDIR_LOGDIR_NAME), 0o750); err != nil { + t.Fatalf("create session: %v", err) + } + if err := os.Chmod(sessionDir, 0o000); err != nil { + t.Fatalf("chmod session dir: %v", err) + } + t.Cleanup(func() { _ = os.Chmod(sessionDir, 0o750) }) + + // resolveSessionIdentity resolves the session directory itself, which still works; + // the stat of logs/ inside it is what is denied. + h.handler.ensureRotatedCollectionForNode(sessionDir, testNodeID) + + key := rotatedKey{session: testSessionB, node: testNodeID} + reason := h.handler.rotatedCollection().disabledReason(key) + if reason == nil { + t.Fatal("a permission-denied logs directory was not recorded") + } + if !h.handler.rotatedCollection().durablyDisabled(key) { + t.Fatalf("a permission-denied logs directory was classified as retryable: %v", reason) + } + builds := h.buildCount() + for range 5 { + h.advanceClock(time.Hour) + h.handler.ensureRotatedCollectionForNode(sessionDir, testNodeID) + } + if got := h.buildCount(); got != builds { + t.Errorf("collectors built for a permission-denied logs directory = %d, want none", got-builds) + } +} + +// 7e. Watcher construction that fails because the kernel is out of watch resources is +// retryable at that boundary; any other construction failure is durable. +func TestRuntimeWatcherConstructionResourceFailureIsRetryable(t *testing.T) { + h := newRuntimeHarness(t) + sup := h.handler.rotatedCollection() + key := rotatedKey{session: testSessionA, node: testNodeID} + + var fail atomic.Bool + fail.Store(true) + base := sup.tune + sup.tune = func(cfg *rotatedCollectorConfig) { + base(cfg) + inner := cfg.NewWatcher + cfg.NewWatcher = func() (fsWatcher, error) { + if fail.Load() { + return nil, fmt.Errorf("inotify_init: %w", syscall.EMFILE) + } + return inner() + } + } + + h.handler.startRotatedCollection() + eventually(t, "the exhausted watch limit to be recorded", func() bool { + return sup.disabledReason(key) != nil + }) + if sup.durablyDisabled(key) { + t.Fatalf("EMFILE from watcher construction was classified as durable: %v", sup.disabledReason(key)) + } + + // The limit clears and the collector starts. + fail.Store(false) + h.advanceClock(rotatedRetryBase + time.Second) + h.handler.ensureRotatedCollection(h.handler.SessionDir) + h.awaitCollector() + if key2, ok := sup.activeKey(); !ok || key2 != key { + t.Errorf("collector after the limit cleared = %+v (active=%v), want %+v", key2, ok, key) + } +} + +func TestRuntimeWatcherConstructionPermissionFailureIsDurable(t *testing.T) { + h := newRuntimeHarness(t) + sup := h.handler.rotatedCollection() + base := sup.tune + sup.tune = func(cfg *rotatedCollectorConfig) { + base(cfg) + cfg.NewWatcher = func() (fsWatcher, error) { + return nil, fmt.Errorf("create fsnotify watcher: %w", os.ErrPermission) + } + } + + h.handler.startRotatedCollection() + key := rotatedKey{session: testSessionA, node: testNodeID} + eventually(t, "the watcher failure to be recorded", func() bool { + return sup.disabledReason(key) != nil + }) + if !sup.durablyDisabled(key) { + t.Errorf("a permission failure from watcher construction was classified as retryable: %v", sup.disabledReason(key)) + } +} + +// 7f. Retry history grows across repeated startup failures, because being constructed is +// not the same as having started. Only a collector that completed startup clears it. +func TestRuntimeBackoffGrowsUntilStartupActuallySucceeds(t *testing.T) { + h := newRuntimeHarness(t) + sup := h.handler.rotatedCollection() + key := rotatedKey{session: testSessionA, node: testNodeID} + + var fail atomic.Bool + fail.Store(true) + base := sup.tune + sup.tune = func(cfg *rotatedCollectorConfig) { + base(cfg) + if fail.Load() { + // Fatal to the collector, and reached only after attach: the failure that + // a record cleared at attach time would keep resetting to attempt 1. + cfg.NewWatcher = func() (fsWatcher, error) { + return nil, fmt.Errorf("inotify_init: %w", syscall.ENFILE) + } + } + } + + var delays []time.Duration + for i := range 3 { + if i > 0 { + h.advanceClock(rotatedRetryMax) // whatever the backoff is, it is due + } + h.handler.ensureRotatedCollection(h.handler.SessionDir) + eventually(t, "the failure to be recorded", func() bool { + f, ok := sup.failureRecord(key) + return ok && f.attempts == i+1 + }) + f, _ := sup.failureRecord(key) + delays = append(delays, f.notBefore.Sub(h.now())) + } + + if delays[0] != rotatedRetryBase { + t.Errorf("first retry delay = %s, want the base %s", delays[0], rotatedRetryBase) + } + for i := 1; i < len(delays); i++ { + if delays[i] <= delays[i-1] { + t.Errorf("retry delay %d = %s, want longer than the previous %s (backoff reset to attempt 1)", + i, delays[i], delays[i-1]) + } + } + + // A startup that actually completes clears the history. + fail.Store(false) + h.advanceClock(rotatedRetryMax) + h.handler.ensureRotatedCollection(h.handler.SessionDir) + h.awaitCollector() + eventually(t, "the recovered identity's history to be cleared", func() bool { + return sup.disabledReason(key) == nil + }) + if _, still := sup.failureRecord(key); still { + t.Error("a healthy startup left the failure record in place") + } +} + +// 7g. A ready signal from a run that is no longer current clears nothing — including, +// and especially, the failure record of its own identity, which by then describes a +// later attempt rather than the one that is signaling. +func TestRuntimeStaleReadySignalClearsNothing(t *testing.T) { + h := newRuntimeHarness(t) + sup := h.handler.rotatedCollection() + h.start() + stale := sup.activeRun() + key := stale.key + + // The run is retired, and a later attempt at the same identity fails. + sup.retireUnless("") + sup.noteFailure(key, errors.New("a later attempt failed durably")) + + stale.rc.cfg.OnReady() // the retired run's startup callback, arriving late + + if sup.disabledReason(key) == nil { + t.Error("a retired run's ready signal cleared the failure of a later attempt") + } + if !sup.durablyDisabled(key) { + t.Error("a retired run's ready signal downgraded a durable failure") + } + if _, ok := sup.activeKey(); ok { + t.Error("a retired run's ready signal re-published it as active") + } +} + +// 7h. The handover uses the node verified for the session being handed over, not +// whatever the handler's mutable node field happens to hold. Those two agree today only +// because step 1 writes the field just before step 2 reads it, and a handover that +// depends on that ordering is one edit away from addressing a session to the wrong node. +func TestAdvanceSessionHandsOffUnderTheVerifiedNode(t *testing.T) { + h := newRuntimeHarness(t) + h.makeSession(testSessionB) + + // The verified identity for session B, with the handler's field deliberately + // holding something else and discovery unable to correct it. + h.handler.SetRayNodeName("stale-node-from-a-previous-session") + h.failNodeDiscovery() + + st := sessionTransition{dir: h.sessionDir(testSessionB), node: testNodeIDB} + h.handler.advanceSession(&st, h.sessionDir(testSessionB)) + + key, ok := h.handler.rotatedCollection().activeKey() + if !ok { + t.Fatal("no collector was started for the verified identity") + } + if key.node != testNodeIDB { + t.Errorf("collector was built on node %s, want the verified %s", key.node, testNodeIDB) + } + if !st.handedOff { + t.Error("the handover was not recorded") + } +} + +// 8. A durable failure is reported once and never restarted, and it does not take the +// legacy collector down with it. +func TestRuntimeRotatedStartupFailureLeavesLegacyCollectionWorking(t *testing.T) { + h := newRuntimeHarness(t) + // Incomplete watch coverage is fatal to the collector: a segment created and + // deleted in the unwatched gap would vanish unseen. A permission the collector does + // not have will not appear later, so this is durable. + h.handler.rotated.tune = func(cfg *rotatedCollectorConfig) { + cfg.NewWatcher = func() (fsWatcher, error) { + w := newFakeWatcher() + w.failAdd = map[string]error{cfg.LogsDir: os.ErrPermission} + return w, nil + } + cfg.NewTicker = func(time.Duration) (<-chan time.Time, func()) { return h.ticks, func() {} } + } + + h.write(testSessionA, "raylet.out", "active") + h.handler.startRotatedCollection() + + key := rotatedKey{session: testSessionA, node: testNodeID} + eventually(t, "the failed collector to be reported", func() bool { + return h.handler.rotatedCollection().disabledReason(key) != nil + }) + if _, ok := h.handler.rotatedCollection().activeKey(); ok { + t.Error("a failed collector is still being treated as active") + } + if !h.handler.rotatedCollection().durablyDisabled(key) { + t.Error("a permission failure was classified as retryable") + } + if reason := h.handler.rotatedCollection().disabledReason(key); !strings.Contains(reason.Error(), "watch") { + t.Errorf("disabled reason = %v, want it to name the watch failure", reason) + } + + // The failure must not be restarted in a loop for the same session, however long + // the process runs. + before := h.buildCount() + for range 5 { + h.advanceClock(time.Hour) + h.handler.ensureRotatedCollection(h.sessionDir(testSessionA)) + } + if got := h.buildCount(); got != before { + t.Errorf("collectors built after a durable failure = %d, want none", got-before) + } + + // A different session is a different collector over a different tree, so it starts. + h.makeSession(testSessionB) + h.handler.rotated.tune = h.newSupervisor().tune + h.handler.ensureRotatedCollection(h.sessionDir(testSessionB)) + if k, ok := h.handler.rotatedCollection().activeKey(); !ok || k.session != testSessionB { + t.Errorf("a new session did not start after an earlier session's failure: %+v (active=%v)", k, ok) + } + + // Legacy collection keeps working throughout, and nothing is suppressed. + h.handler.rotatedCollection().shutdown() + h.handler.processSessionLatestLogs() + if !h.writer.has(path.Join(h.legacyLogsPrefix(), "raylet.out")) { + t.Errorf("legacy shutdown upload did not run; wrote %v", h.writer.keys()) + } +} + +// 9. Failure records cannot grow without bound across many sessions. +func TestRuntimeFailureRecordsStayBounded(t *testing.T) { + h := newRuntimeHarness(t) + sup := h.handler.rotatedCollection() + + for i := range 200 { + session := fmt.Sprintf("session_2026-07-31_%02d-%02d-%02d_000000", i/3600, (i/60)%60, i%60) + dir := h.sessionDir(session) + if err := os.MkdirAll(dir, 0o750); err != nil { + t.Fatalf("create session dir: %v", err) + } + // No logs/ directory, so every one of them fails. + h.handler.ensureRotatedCollection(dir) + } + if got := sup.failureCount(); got > maxRotatedFailureRecords { + t.Errorf("failure records = %d, want at most %d", got, maxRotatedFailureRecords) + } + // Records for sessions the runtime has moved past are pruned, not merely capped. + if got := sup.failureCount(); got != 1 { + t.Errorf("failure records = %d, want only the session last ensured", got) + } +} + +// 10. A collector that dies leaves no stale active pointer behind, and shutdown does not +// wait on it. +func TestRuntimeFailedCollectorLeavesNoStaleActivePointer(t *testing.T) { + h := newRuntimeHarness(t) + h.start() + + // The watcher's event channel closing is fatal to the collector: discovery would + // silently stop. + close(h.watchers.at(0).events) + + key := rotatedKey{session: testSessionA, node: testNodeID} + eventually(t, "the collector to be recorded as failed", func() bool { + return h.handler.rotatedCollection().disabledReason(key) != nil + }) + if _, ok := h.handler.rotatedCollection().activeKey(); ok { + t.Error("the supervisor still points at a collector whose goroutine has exited") + } + if h.handler.rotatedCollection().activeRun() != nil { + t.Error("the failed run was not detached") + } + + done := make(chan struct{}) + go func() { h.handler.rotatedCollection().shutdown(); close(done) }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("shutdown blocked on a collector that had already failed") + } +} + +// 10b. A failed collector is never restarted by an ensure that lands between the +// failure and its publication. +// +// The seam holds the collector's failure at the top of runFailed, before either the +// record or the active pointer moves. An ensure arriving here must not be able to +// observe "no collector, and no reason not to build one". +func TestRuntimeFailedRunIsNotRestartedDuringPublication(t *testing.T) { + h := newRuntimeHarness(t) + sup := h.handler.rotatedCollection() + entered := make(chan struct{}) + release := make(chan struct{}) + var once sync.Once + sup.beforeFailurePublish = func() { + once.Do(func() { + close(entered) + <-release + }) + } + + h.start() + watchers := h.watchers.count() + close(h.watchers.at(0).events) // fatal to the collector + + select { + case <-entered: + case <-time.After(10 * time.Second): + t.Fatal("the collector never reached failure publication") + } + + // Exactly the window the old ordering left open. + ensured := make(chan struct{}) + go func() { + defer close(ensured) + h.handler.ensureRotatedCollection(h.handler.SessionDir) + }() + + select { + case <-ensured: + case <-time.After(10 * time.Second): + close(release) + t.Fatal("ensure blocked indefinitely during failure publication") + } + if got := h.watchers.count(); got != watchers { + t.Errorf("collectors started while the failure was being published = %d, want none", got-watchers) + } + + close(release) + + key := rotatedKey{session: testSessionA, node: testNodeID} + eventually(t, "the identity to be durably disabled", func() bool { return sup.durablyDisabled(key) }) + if _, ok := sup.activeKey(); ok { + t.Error("a collector is active after the failure was published") + } + if got := h.watchers.count(); got != watchers { + t.Errorf("collectors started after the failure = %d, want none", got-watchers) + } +} + +// 11. A Run failure that happens exactly while shutdown is retiring the collector is +// recorded once and deadlocks nothing. +func TestRuntimeRunFailureConcurrentWithShutdownDoesNotDeadlock(t *testing.T) { + h := newRuntimeHarness(t) + h.start() + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + close(h.watchers.at(0).events) // fatal to the collector + }() + go func() { + defer wg.Done() + h.handler.rotatedCollection().shutdown() + }() + + done := make(chan struct{}) + go func() { wg.Wait(); close(done) }() + select { + case <-done: + case <-time.After(15 * time.Second): + t.Fatal("a Run failure concurrent with shutdown deadlocked") + } + + if _, ok := h.handler.rotatedCollection().activeKey(); ok { + t.Error("a collector is still active after shutdown") + } + // However the race went, the failure is recorded at most once per run. + if got := h.handler.rotatedCollection().failureCount(); got > 1 { + t.Errorf("failure records = %d, want at most one", got) + } +} + +// --------------------------------------------------------------------------- +// Session transition +// --------------------------------------------------------------------------- + +// 12. A new session retires the old collector and builds a genuinely separate one. +// +// "Separate" is about in-memory state and live intake, not about the staging volume: +// the new collector still adopts the previous session's durable records at startup, +// because the staging root is per-node and nothing else would ever drain them. What it +// must not do is share an index, a generator, a watcher or an identity. +func TestRuntimeSessionChangeReplacesTheCollector(t *testing.T) { + h := newRuntimeHarness(t) + first := h.start() + oldEntry := h.capture("old segment") + firstRun := h.handler.rotatedCollection().activeRun() + + h.makeSession(testSessionB) + h.pointSessionLatest(testSessionB) + h.handler.ensureRotatedCollection(h.sessionDir(testSessionB)) + + second := h.awaitCollector() + if first == second { + t.Fatal("the old collector was reused for the new session") + } + if first.ix == second.ix { + t.Error("the new session reuses the old captureIndex") + } + if first.cfg.CaptureIDs == second.cfg.CaptureIDs { + t.Error("the new session reuses the old capture ID generator") + } + if second.cfg.SessionName != testSessionB || second.cfg.LogsDir != h.logsDir(testSessionB) { + t.Errorf("new collector is for %s at %s, want %s at %s", + second.cfg.SessionName, second.cfg.LogsDir, testSessionB, h.logsDir(testSessionB)) + } + + // The old collector is fully stopped before the new one exists. + if !firstRun.finished() { + t.Error("the old collector's goroutine was still running when the new one was built") + } + if h.sawLivePredecessor() { + t.Error("the new collector was constructed while the supervisor was still attached to the old one") + } + if !h.watchers.at(0).isClosed() { + t.Error("the old session's watcher was not closed") + } + // Its capture stays on the staging volume, under the old session's own subtree. + staged := h.stagingFiles() + if len(staged) != 1 || !strings.HasPrefix(staged[0], testSessionA+"/") { + t.Fatalf("staged files = %v, want the old session's capture preserved under its own subtree", staged) + } + // Whatever the new collector adopted is the identical durable record: same session, + // same capture ID. Nothing was re-minted and nothing was rewritten into session B. + adopted := second.snapshot() + if len(adopted) != 1 { + t.Fatalf("new collector holds %+v, want the one adopted record", adopted) + } + // State is deliberately excluded: the new collector may already have uploaded and + // promoted the adopted capture, which is the point of adopting it. + if adopted[0].withState(statePending) != oldEntry.withState(statePending) { + t.Errorf("adopted record = %+v, want the old session's record with only its state advanced (%+v)", + adopted[0], oldEntry) + } +} + +// 13. Repeating the same identity builds nothing new. The session poller calls ensure on +// every tick, and a relocation that keeps failing makes it repeat the identity of a +// session change on every tick, so idempotence is what keeps a failing relocation from +// churning collectors. +func TestRuntimeEnsureIsIdempotentForTheSameIdentity(t *testing.T) { + h := newRuntimeHarness(t) + h.start() + h.makeSession(testSessionB) + h.handler.ensureRotatedCollection(h.sessionDir(testSessionB)) + h.awaitCollector() + + builds := h.buildCount() + run := h.handler.rotatedCollection().activeRun() + for range 10 { + h.handler.ensureRotatedCollection(h.sessionDir(testSessionB)) + } + if got := h.buildCount(); got != builds { + t.Errorf("collectors built by repeated ensure = %d, want none", got-builds) + } + if h.handler.rotatedCollection().activeRun() != run { + t.Error("repeated ensure replaced the running collector") + } + if run.finished() { + t.Error("repeated ensure retired the running collector") + } +} + +// 14. Nothing from the old session's live tree, and no event from its watcher, can reach +// the new collector. +func TestRuntimeOldSessionCannotMutateTheNewCollector(t *testing.T) { + h := newRuntimeHarness(t) + h.start() + h.capture("old segment") + oldRun := h.handler.rotatedCollection().activeRun() + oldWatcher := h.watchers.at(0) + + h.makeSession(testSessionB) + h.handler.ensureRotatedCollection(h.sessionDir(testSessionB)) + second := h.awaitCollector() + before := len(second.snapshot()) + + // The old collector was isolated before the new one was built, so a segment that + // rotates in the old tree afterwards reaches neither of them. + if !oldRun.finished() || !oldWatcher.isClosed() || h.sawLivePredecessor() { + t.Fatal("the old collector was not isolated before the new one started") + } + h.write(testSessionA, "raylet.out.2", "after the changeover") + second.reconcileNow() + + for _, e := range second.snapshot() { + if e.OriginalName == "raylet.out.2" { + t.Errorf("new collector captured a file from the old session's live tree: %+v", e) + } + } + if got := len(second.snapshot()); got != before { + t.Errorf("new collector's capture count moved from %d to %d because of the old session's tree", before, got) + } +} + +// 15. Repeated, concurrent session changes never let two collectors own the staging root, +// and leave exactly one running. +func TestRuntimeConcurrentSessionChangesKeepOneOwner(t *testing.T) { + h := newRuntimeHarness(t) + h.handler.rotatedCollection().drainBudget = 50 * time.Millisecond + h.start() + h.makeSession(testSessionB) + + var wg sync.WaitGroup + for i := range 8 { + wg.Add(1) + go func(i int) { + defer wg.Done() + session := testSessionA + if i%2 == 1 { + session = testSessionB + } + h.handler.ensureRotatedCollection(h.sessionDir(session)) + }(i) + } + wg.Wait() + + if h.sawLivePredecessor() { + t.Error("a collector was constructed while another was still attached") + } + sup := h.handler.rotatedCollection() + run := sup.activeRun() + if run == nil { + t.Fatal("no collector survived the concurrent session changes") + } + if run.finished() { + t.Error("the surviving collector is not running") + } + // ensure returns as soon as the survivor's goroutine is spawned, so wait for that + // goroutine to reach its loop before inspecting watchers — it installs its watcher + // on the way there. This is the same round trip awaitCollector uses. + run.rc.snapshot() + // Every retired collector is stopped, so its watcher is closed. Exactly one — the + // survivor's — is open. + open := 0 + for i := range h.watchers.count() { + if !h.watchers.at(i).isClosed() { + open++ + } + } + if open != 1 { + t.Errorf("open watchers = %d, want exactly 1", open) + } +} + +// 16. A session change while an upload is in flight preserves the old session's pending +// capture, under the old session's own subtree. +func TestRuntimeSessionChangeDuringUploadPreservesPendingState(t *testing.T) { + h := newRuntimeHarness(t) + h.handler.rotatedCollection().drainBudget = 50 * time.Millisecond + release := h.writer.block(t) + h.start() + h.capture("old segment") + select { + case <-h.writer.entered: + case <-time.After(10 * time.Second): + t.Fatal("the upload never reached the storage writer") + } + + h.makeSession(testSessionB) + h.handler.ensureRotatedCollection(h.sessionDir(testSessionB)) + h.awaitCollector() + release() + + staged := h.stagingFiles() + if len(staged) != 1 { + t.Fatalf("staged files = %v, want the old session's single capture preserved", staged) + } + if !strings.HasPrefix(staged[0], testSessionA+"/") || !strings.Contains(staged[0], "/"+string(statePending)+"/") { + t.Errorf("staged file = %s, want a pending capture under %s", staged[0], testSessionA) + } +} + +// 17. A capture the previous session left pending is uploaded by the next session's +// collector to the key it was always destined for: the old session's prefix, the old +// capture ID. +func TestRuntimeCrossSessionAdoptionPreservesTheObjectKey(t *testing.T) { + h := newRuntimeHarness(t) + h.handler.rotatedCollection().drainBudget = 10 * time.Millisecond + release := h.writer.block(t) + h.start() + old := h.capture("old segment") + select { + case <-h.writer.entered: + case <-time.After(10 * time.Second): + t.Fatal("the upload never reached the storage writer") + } + + // The old session goes away with its capture still pending: the upload was in the + // uncancelable storage call, so its result is discarded. + h.makeSession(testSessionB) + h.handler.ensureRotatedCollection(h.sessionDir(testSessionB)) + second := h.awaitCollector() + release() + + want := old.objectKey(second.cfg.Cluster) + eventually(t, "the adopted capture to be uploaded under its original key", func() bool { + second.reconcileNow() + return h.writer.has(want) + }) + if !strings.HasPrefix(want, h.legacyLogsPrefixFor(testSessionA)+"/") { + t.Errorf("adopted object key %s is not under the old session's prefix", want) + } + if h.writer.content(want) != "old segment" { + t.Errorf("adopted object content = %q, want the old session's bytes", h.writer.content(want)) + } + // Nothing was rewritten into the new session. + for _, k := range h.writer.keys() { + if strings.Contains(k, testSessionB) && strings.Contains(k, captureIDSeparator) { + t.Errorf("adopted capture was re-keyed into the new session: %s", k) + } + } + for _, e := range second.snapshot() { + if e.SessionName != testSessionA { + t.Errorf("adopted record was rewritten to session %s, want %s", e.SessionName, testSessionA) + } + } +} + +// --------------------------------------------------------------------------- +// Shutdown +// --------------------------------------------------------------------------- + +// 18. Shutdown reconciles the tree one last time before intake stops, so a segment that +// rotated during shutdown is still captured. +func TestRuntimeShutdownReconcilesBeforeStoppingIntake(t *testing.T) { + h := newRuntimeHarness(t) + h.write(testSessionA, "raylet.out", "active") + h.start() + + // Created after startup and never announced through the watcher: only the final + // reconciliation can find it. + h.write(testSessionA, "raylet.out.1", "rotated during shutdown") + h.handler.rotatedCollection().shutdown() + + staged := h.stagingFiles() + found := false + for _, p := range staged { + if strings.Contains(p, "raylet.out.1"+captureIDSeparator) { + found = true + } + } + if !found { + t.Errorf("staged files = %v, want the segment reconciled at shutdown to be captured", staged) + } +} + +// 19. When the drain budget expires, captured work stays pinned on disk as pending. +func TestRuntimeShutdownTimeoutPreservesPendingCaptures(t *testing.T) { + h := newRuntimeHarness(t) + h.handler.rotatedCollection().drainBudget = 50 * time.Millisecond + release := h.writer.block(t) + h.start() + h.capture("segment") + + h.handler.rotatedCollection().shutdown() + release() + + staged := h.stagingFiles() + if len(staged) != 1 || !strings.Contains(staged[0], "/"+string(statePending)+"/") { + t.Errorf("staged files = %v, want exactly one pending capture preserved", staged) + } +} + +// 20. An upload already inside the uncancelable storage call does not hold shutdown open. +func TestRuntimeShutdownIsBoundedByAnUncancelableUpload(t *testing.T) { + h := newRuntimeHarness(t) + h.handler.rotatedCollection().drainBudget = 100 * time.Millisecond + release := h.writer.block(t) + h.start() + h.capture("segment") + + select { + case <-h.writer.entered: + case <-time.After(10 * time.Second): + t.Fatal("the upload never reached the storage writer") + } + + start := time.Now() + h.handler.rotatedCollection().shutdown() + elapsed := time.Since(start) + release() + + // The budget plus the worker's own stop grace, with generous slack for a loaded + // test machine. What matters is that it is bounded at all: WriteFile is still + // running and cannot be canceled. + if elapsed > 5*time.Second { + t.Errorf("shutdown took %s while an uncancelable upload was in flight", elapsed) + } +} + +// 21. Shutting the subsystem down twice changes nothing and does not block, and nothing +// can be started afterwards. +func TestRuntimeShutdownIsIdempotentAndFreezes(t *testing.T) { + h := newRuntimeHarness(t) + rc := h.start() + h.capture("segment") + h.waitForOneUploaded(rc) + + h.handler.rotatedCollection().shutdown() + builds := h.buildCount() + h.handler.rotatedCollection().shutdown() + h.handler.rotatedCollection().shutdown() + + h.makeSession(testSessionB) + h.handler.ensureRotatedCollection(h.sessionDir(testSessionB)) + h.handler.ensureRotatedCollection(h.sessionDir(testSessionA)) + if _, ok := h.handler.rotatedCollection().activeKey(); ok { + t.Error("a collector was started after shutdown") + } + if got := h.buildCount(); got != builds { + t.Errorf("collectors built after shutdown = %d, want none", got-builds) + } +} + +// 22. A session change that arrives while shutdown is running never leaves a collector +// behind it. +func TestRuntimeSessionChangeConcurrentWithShutdownStartsNoReplacement(t *testing.T) { + h := newRuntimeHarness(t) + h.handler.rotatedCollection().drainBudget = 50 * time.Millisecond + h.start() + h.makeSession(testSessionB) + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + h.handler.rotatedCollection().shutdown() + }() + go func() { + defer wg.Done() + h.handler.ensureRotatedCollection(h.sessionDir(testSessionB)) + }() + wg.Wait() + + if _, ok := h.handler.rotatedCollection().activeKey(); ok { + t.Error("a collector is active after shutdown returned") + } + for i := range h.watchers.count() { + if !h.watchers.at(i).isClosed() { + t.Errorf("watcher %d is still open after shutdown", i) + } + } +} + +// 23. A replacement that is already in flight when shutdown freezes the subsystem is +// abandoned rather than started. +// +// This is the window the early frozen check cannot cover: ensure has already decided to +// build a replacement and is retiring the predecessor, which takes as long as the drain +// budget. Shutdown freezes during that gap, and the replacement must never run — +// constructing it and then having to retire it again would mean a collector taking +// ownership of the staging root after shutdown had declared the subsystem down. +// +// A constructed-but-never-started collector is inert: it holds no watcher, no goroutine +// and no staging link, so the observable requirement is that no watcher is ever created +// for it. +func TestRuntimeShutdownFreezesAReplacementAlreadyInFlight(t *testing.T) { + h := newRuntimeHarness(t) + sup := h.handler.rotatedCollection() + sup.drainBudget = 2 * time.Second + release := h.writer.block(t) + h.start() + h.capture("segment") // stays pending, so the retirement spends its whole budget + select { + case <-h.writer.entered: + case <-time.After(10 * time.Second): + t.Fatal("the upload never reached the storage writer") + } + h.makeSession(testSessionB) + watchers := h.watchers.count() + + ensured := make(chan struct{}) + go func() { + defer close(ensured) + h.handler.ensureRotatedCollection(h.sessionDir(testSessionB)) + }() + + // Detaching is the first thing a retirement does, so a nil active run means ensure + // is inside the drain: the exact window this test is about. + eventually(t, "the retirement of the old collector to begin", func() bool { + return sup.activeRun() == nil + }) + + done := make(chan struct{}) + go func() { + defer close(done) + sup.shutdown() + }() + + <-ensured + <-done + release() + + if got := h.watchers.count(); got != watchers { + t.Errorf("collectors started while shutdown was freezing = %d, want none", got-watchers) + } + if _, ok := sup.activeKey(); ok { + t.Error("a collector is active after shutdown returned") + } +} + +// 24. Watcher, owner and worker goroutines all go away. +func TestRuntimeLeavesNoGoroutinesBehind(t *testing.T) { + before := runtime.NumGoroutine() + + h := newRuntimeHarness(t) + rc := h.start() + h.capture("segment") + h.waitForOneUploaded(rc) + + h.makeSession(testSessionB) + h.handler.ensureRotatedCollection(h.sessionDir(testSessionB)) + h.awaitCollector() + h.handler.rotatedCollection().shutdown() + + for range 60 { + if runtime.NumGoroutine() <= before { + return + } + time.Sleep(50 * time.Millisecond) + } + t.Errorf("goroutines leaked: %d before, %d after", before, runtime.NumGoroutine()) +} + +// 23b. Once shutdown has closed the transition gate, a polling tick performs none of a +// transition's side effects: no node discovery, no node mutation, no handover, no +// retirement, no relocation. +func TestRuntimeShutdownGateStopsSessionTransitions(t *testing.T) { + h := newRuntimeHarness(t) + h.start() + h.write(testSessionA, "raylet.out", "active") + run := h.handler.rotatedCollection().activeRun() + + h.handler.transitions.close() + + h.makeSession(testSessionB) + h.write(testSessionB, "raylet.out", "active") + h.pointSessionLatest(testSessionB) + h.discoverNodeB() + + st := sessionTransition{dir: h.sessionDir(testSessionA), node: testNodeID, handedOff: true} + builds := h.buildCount() + h.handler.advanceSession(&st, h.sessionDir(testSessionB)) + + h.mu.Lock() + calls := h.nodeCalls + h.mu.Unlock() + if calls != 0 { + t.Errorf("node discovery ran %d time(s) after the gate closed, want none", calls) + } + if got := h.handler.GetRayNodeName(); got != testNodeID { + t.Errorf("the handler's node ID changed to %s after the gate closed", got) + } + if got := h.buildCount(); got != builds { + t.Errorf("collectors built after the gate closed = %d, want none", got-builds) + } + if h.handler.rotatedCollection().activeRun() != run || run.finished() { + t.Error("the running collector was retired after the gate closed") + } + if _, err := os.Stat(h.logsDir(testSessionA)); err != nil { + t.Errorf("the live tree was relocated after the gate closed: %v", err) + } + if st.dir != h.sessionDir(testSessionA) { + t.Errorf("transition state advanced to %s after the gate closed", st.dir) + } +} + +// 23c. Shutdown does not begin — not the rotated retirement, not the legacy walk — +// while an admitted transition is still able to change the identity those steps depend +// on. Each of a transition's three steps is held in turn. +// +// Abandoning the wait would not stop any of them: there is nothing to cancel, so the +// only thing an early return buys is running the walk concurrently with a relocation of +// the tree it is walking. +func TestRuntimeShutdownWaitsForATransitionInFlight(t *testing.T) { + for _, tc := range []struct { + name string + // hold blocks the transition at one of its steps and returns a channel that is + // closed once it is inside, plus the release. + hold func(h *runtimeHarness) (entered <-chan struct{}, release func()) + }{ + { + name: "inside node discovery", + hold: func(h *runtimeHarness) (<-chan struct{}, func()) { + entered, release := make(chan struct{}), make(chan struct{}) + h.handler.discoverNodeID = func() (string, bool) { + close(entered) + <-release + return testNodeIDB, true + } + return entered, func() { close(release) } + }, + }, + { + name: "inside the collector handoff", + hold: func(h *runtimeHarness) (<-chan struct{}, func()) { + entered, release := make(chan struct{}), make(chan struct{}) + var once sync.Once + base := h.handler.rotatedCollection().tune + // tune runs inside ensure with the supervisor's lifecycle lock held, + // which is also the lock the rotated retirement needs. + h.handler.rotatedCollection().tune = func(cfg *rotatedCollectorConfig) { + base(cfg) + once.Do(func() { + close(entered) + <-release + }) + } + h.discoverNodeB() + return entered, func() { close(release) } + }, + }, + { + name: "immediately before relocation", + hold: func(h *runtimeHarness) (<-chan struct{}, func()) { + entered, release := make(chan struct{}), make(chan struct{}) + h.discoverNodeB() + h.handler.beforeRelocation = func() { + close(entered) + <-release + } + return entered, func() { close(release) } + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + h := newRuntimeHarness(t) + h.start() + h.write(testSessionA, "raylet.out", "active") + h.makeSession(testSessionB) + h.write(testSessionB, "raylet.out", "active") + h.pointSessionLatest(testSessionB) + + entered, release := tc.hold(h) + + st := sessionTransition{dir: h.sessionDir(testSessionA), node: testNodeID} + go h.handler.advanceSession(&st, h.sessionDir(testSessionB)) + <-entered + + done := make(chan struct{}) + go func() { defer close(done); h.handler.shutdownLogCollection() }() + + select { + case <-done: + t.Fatal("shutdown completed while a transition was still running") + case <-time.After(250 * time.Millisecond): + } + // Neither later step may have begun. Freezing is the first thing the + // rotated retirement does, and it is what the handoff case needs: by then + // the transition has already retired the outgoing collector itself, so an + // absent collector proves nothing, but a frozen supervisor would. + if h.handler.rotatedCollection().isFrozen() { + t.Error("the rotated retirement began while a transition was still running") + } + for _, k := range h.writer.keys() { + if strings.Contains(k, "/logs/") { + t.Errorf("the legacy walk wrote %s while a transition was still running", k) + } + } + + release() + select { + case <-done: + case <-time.After(30 * time.Second): + t.Fatal("shutdown never completed after the transition finished") + } + + // Afterwards: nothing is admitted, and the walk has run exactly once. + if h.handler.transitions.enter() { + t.Error("a transition was admitted after shutdown") + } + if _, ok := h.handler.rotatedCollection().activeKey(); ok { + t.Error("a collector is still active after shutdown") + } + }) + } +} + +// 23d. Shutdown stays bounded when the object store never returns, which is the one +// thing that genuinely cannot be waited on. That bound is the rotated drain budget and +// has nothing to do with session transitions. +func TestRuntimeShutdownIsBoundedWhenWriteFileNeverReturns(t *testing.T) { + h := newRuntimeHarness(t) + h.handler.rotatedCollection().drainBudget = 100 * time.Millisecond + release := h.writer.block(t) + h.start() + h.capture("segment") + + select { + case <-h.writer.entered: + case <-time.After(10 * time.Second): + t.Fatal("the upload never reached the storage writer") + } + + done := make(chan struct{}) + go func() { + defer close(done) + h.handler.transitions.close() + h.handler.rotatedCollection().shutdown() + }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("shutdown blocked on an uncancelable upload") + } + release() +} + +// --------------------------------------------------------------------------- +// The legacy walk is untouched +// --------------------------------------------------------------------------- + +// 24. The two halves address disjoint object keys. This is the invariant the whole +// "no suppression" decision rests on: a captured segment is written as +// ".rotated.", the legacy walk writes "", and no capture ID can +// ever be empty. +func TestRotatedAndLegacyObjectKeysAreDisjoint(t *testing.T) { + h := newRuntimeHarness(t) + rc := h.start() + entry := h.capture("rotated segment") + + rotatedKey := entry.objectKey(rc.cfg.Cluster) + legacyKey := path.Join(h.legacyLogsPrefix(), entry.OriginalName) + if rotatedKey == legacyKey { + t.Fatalf("rotated and legacy object keys collide at %s", rotatedKey) + } + if !strings.HasPrefix(rotatedKey, legacyKey+captureIDSeparator) { + t.Errorf("rotated key %s is not the legacy key %s plus a capture ID", rotatedKey, legacyKey) + } +} + +// 25. Every regular file in the live tree is uploaded by the shutdown walk, including +// one the rotated subsystem has already put in storage under its own key. +// +// Suppressing that write would not avoid a duplicate — the keys differ — it would +// delete a key the legacy walk has always produced. +func TestLegacyShutdownUploadsEveryLiveFileEvenWhenAlreadyCaptured(t *testing.T) { + h := newRuntimeHarness(t) + rc := h.start() + entry := h.capture("rotated segment") + h.waitForOneUploaded(rc) + rotatedKey := entry.objectKey(rc.cfg.Cluster) + + h.handler.rotatedCollection().shutdown() + h.handler.processSessionLatestLogs() + + prefix := h.legacyLogsPrefix() + for _, want := range []string{ + path.Join(prefix, "raylet.out"), + path.Join(prefix, "raylet.out.1"), + rotatedKey, + } { + if !h.writer.has(want) { + t.Errorf("object %s was never written; wrote %v", want, h.writer.keys()) + } + } + if got := h.writer.content(path.Join(prefix, "raylet.out.1")); got != "rotated segment" { + t.Errorf("legacy object content = %q, want the segment's bytes", got) + } +} + +// 26. Ray's rotation renames a segment through raylet.out.1, .2, .3 without changing its +// inode, so at shutdown the very same physical file the collector captured as ".1" is +// sitting at ".2" — a name whose object no other writer produces. +// +// This is why physical identity cannot stand in for a remote key: an inode-based skip +// would have dropped the ".2" object entirely. +func TestLegacyShutdownUploadsARenamedCapturedSegment(t *testing.T) { + h := newRuntimeHarness(t) + rc := h.start() + h.capture("the segment") + h.waitForOneUploaded(rc) + + logs := h.logsDir(testSessionA) + first := filepath.Join(logs, "raylet.out.1") + second := filepath.Join(logs, "raylet.out.2") + before, _, err := statInode(first) + if err != nil { + t.Fatalf("statInode(%s): %v", first, err) + } + if err := os.Rename(first, second); err != nil { + t.Fatalf("rotate %s to %s: %v", first, second, err) + } + after, _, err := statInode(second) + if err != nil { + t.Fatalf("statInode(%s): %v", second, err) + } + if before != after { + t.Fatalf("the rename changed the inode (%s -> %s); the test cannot show what it means to", before, after) + } + + h.handler.rotatedCollection().shutdown() + h.handler.processSessionLatestLogs() + + key := path.Join(h.legacyLogsPrefix(), "raylet.out.2") + if !h.writer.has(key) { + t.Errorf("the renamed segment was not uploaded under its current name; wrote %v", h.writer.keys()) + } + if got := h.writer.content(key); got != "the segment" { + t.Errorf("uploaded content = %q, want the segment's bytes", got) + } +} + +// 27. A capture whose upload failed, and one that never got to storage at all, are +// uploaded by the legacy walk exactly as they would have been without this subsystem. +func TestLegacyShutdownUploadsPendingAndFailedCaptures(t *testing.T) { + h := newRuntimeHarness(t) + h.handler.rotatedCollection().drainBudget = 100 * time.Millisecond + h.writer.setFailAll(true) + rc := h.start() + h.capture("rotated segment") + + eventually(t, "the upload to have been attempted and failed", func() bool { + if h.writer.attemptCount() == 0 { + return false + } + for _, e := range rc.snapshot() { + if e.OriginalName == "raylet.out.1" && e.State == statePending { + return true + } + } + return false + }) + + h.handler.rotatedCollection().shutdown() + h.writer.setFailAll(false) + h.handler.processSessionLatestLogs() + + if !h.writer.has(path.Join(h.legacyLogsPrefix(), "raylet.out.1")) { + t.Errorf("a capture whose upload failed was skipped by the legacy walk; wrote %v", h.writer.keys()) + } +} + +// 27b. The shutdown walk works from an immutable snapshot: a session change during the +// walk cannot move it to another session's tree or split one shutdown across two +// identities. +func TestLegacyShutdownWalksTheSnapshottedSession(t *testing.T) { + h := newRuntimeHarness(t) + h.write(testSessionA, "raylet.out", "session A active") + h.write(testSessionA, "events/event_GCS.log", "session A nested") + + snap, ok := h.handler.takeShutdownSnapshot() + if !ok { + t.Fatal("no shutdown snapshot was taken") + } + if snap.sessionID != testSessionA { + t.Fatalf("snapshot session = %s, want %s", snap.sessionID, testSessionA) + } + if snap.logsDir != h.logsDir(testSessionA) { + t.Errorf("snapshot logs directory = %s, want the real %s", snap.logsDir, h.logsDir(testSessionA)) + } + + // Ray restarts the session, and the node ID moves with it, after the snapshot. + h.makeSession(testSessionB) + h.write(testSessionB, "raylet.out", "session B active") + h.pointSessionLatest(testSessionB) + h.handler.SetRayNodeName(testNodeIDB) + + h.handler.processSessionLogs(snap) + + prefixA := h.logsPrefixFor(testSessionA, testNodeID) + prefixB := h.logsPrefixFor(testSessionB, testNodeIDB) + for _, want := range []string{ + path.Join(prefixA, "raylet.out"), + path.Join(prefixA, "events/event_GCS.log"), + } { + if !h.writer.has(want) { + t.Errorf("object %s was not written; wrote %v", want, h.writer.keys()) + } + } + if got := h.writer.content(path.Join(prefixA, "raylet.out")); got != "session A active" { + t.Errorf("object content = %q, want session A's bytes", got) + } + for _, k := range h.writer.keys() { + if strings.HasPrefix(k, prefixB) { + t.Errorf("the walk wrote %s, which belongs to the session that replaced the snapshot", k) + } + if strings.HasPrefix(k, h.logsPrefixFor(testSessionA, testNodeIDB)) { + t.Errorf("the walk wrote %s, mixing session A with the new node ID", k) + } + if strings.Contains(k, "session B active") { + t.Errorf("the walk uploaded session B's content: %s", k) + } + } + if h.writer.has(path.Join(prefixA, "raylet.out")) && h.writer.content(path.Join(prefixA, "raylet.out")) == "session B active" { + t.Error("the walk followed session_latest to the new session's file") + } +} + +// 28. With no rotated subsystem at all, the legacy shutdown walk behaves exactly as +// before: same files, same owner-aware keys, nothing suppressed. +func TestLegacyShutdownUnchangedWithoutRotatedCollection(t *testing.T) { + h := newRuntimeHarness(t) + h.handler.rotated = nil + + h.write(testSessionA, "raylet.out", "active") + h.write(testSessionA, "raylet.out.1", "rotated") + h.write(testSessionA, "events/event_GCS.log", "nested") + + h.handler.processSessionLatestLogs() + + prefix := h.legacyLogsPrefix() + want := []string{ + path.Join(prefix, "events/event_GCS.log"), + path.Join(prefix, "raylet.out"), + path.Join(prefix, "raylet.out.1"), + } + var logs []string + for _, k := range h.writer.keys() { + // The head node also writes the session metadata marker; drop it. + if strings.HasPrefix(k, prefix) { + logs = append(logs, k) + } + } + if strings.Join(logs, ",") != strings.Join(want, ",") { + t.Errorf("legacy shutdown wrote %v, want %v", logs, want) + } +} + +// 29. Owner-aware object keys are unchanged by the production wiring: a captured segment +// lands beside the node's other logs, under the RayJob-nested prefix. +func TestRuntimeObjectKeysAreOwnerAwareAndUnchanged(t *testing.T) { + h := newRuntimeHarness(t) + rc := h.start() + h.capture("rotated segment") + h.waitForOneUploaded(rc) + + var rotatedKeys []string + for _, e := range rc.snapshot() { + rotatedKeys = append(rotatedKeys, e.objectKey(rc.cfg.Cluster)) + } + h.handler.rotatedCollection().shutdown() + h.handler.processSessionLatestLogs() + + prefix := h.legacyLogsPrefix() + if !strings.HasPrefix(prefix, "/history/cluster-history/rayjob/ray-system/rayjob-sample/raycluster-sample/") { + t.Fatalf("legacy prefix %s is not the owner-aware RayJob prefix", prefix) + } + for _, k := range rotatedKeys { + if !strings.HasPrefix(k, prefix+"/") { + t.Errorf("rotated object key %s escapes the node's log prefix %s", k, prefix) + } + if !h.writer.has(k) { + t.Errorf("rotated object key %s was never written; wrote %v", k, h.writer.keys()) + } + } + if !h.writer.has(path.Join(prefix, "raylet.out")) { + t.Errorf("legacy key for the active log changed; wrote %v", h.writer.keys()) + } +} + +// --------------------------------------------------------------------------- +// Partial initialization +// --------------------------------------------------------------------------- + +// 30. A handler that was never fully built keeps its legacy behavior and panics at +// nothing. +func TestRuntimeNilAndPartialHandlersArePassive(t *testing.T) { + root := t.TempDir() + t.Setenv("RAY_TMP_ROOT", root) + + var bare RayLogHandler + bare.ensureRotatedCollection("") + bare.ensureRotatedCollection(filepath.Join(root, "session_nowhere")) + bare.rotatedCollection().shutdown() + bare.rotatedCollection().shutdown() + if _, ok := bare.rotatedCollection().activeKey(); ok { + t.Error("a zero-value handler reports an active rotated collector") + } + if bare.rotatedCollection().disabledReason(rotatedKey{}) != nil { + t.Error("a zero-value handler reports a disabled reason") + } + + // A handler with no storage writer still captures; it simply has nowhere to send + // what it captured, which is what keeps segments pinned until a writer exists. + h := newRuntimeHarness(t) + h.handler.Writer = nil + h.handler.rotated = h.newSupervisor() + h.handler.rotated.writer = nil + + rc := h.start() + if rc.up.enabled() { + t.Error("the upload pipeline is enabled without a storage writer") + } + h.capture("segment") + staged := h.stagingFiles() + if len(staged) != 1 || !strings.Contains(staged[0], "/"+string(statePending)+"/") { + t.Errorf("staged files = %v, want the capture pinned and pending", staged) + } + h.handler.rotatedCollection().shutdown() +} + +// 30b. A staging root that exists but cannot be written to is a durable +// misconfiguration. Creating the root succeeds when it is already there, whatever its +// mode, so only actually creating a file in it answers the question. +func TestRuntimePreflightRejectsAnUnwritableStagingRoot(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("running as root: mode bits do not deny access") + } + h := newRuntimeHarness(t) + staging := utils.GetRayRotatedStagingPath() + if err := os.MkdirAll(staging, 0o500); err != nil { + t.Fatalf("create staging root: %v", err) + } + t.Cleanup(func() { _ = os.Chmod(staging, 0o750) }) + + h.handler.startRotatedCollection() + + key := rotatedKey{session: testSessionA, node: testNodeID} + if !h.handler.rotatedCollection().durablyDisabled(key) { + t.Errorf("an unwritable staging root was not durably disabled: %v", + h.handler.rotatedCollection().disabledReason(key)) + } + if _, ok := h.handler.rotatedCollection().activeKey(); ok { + t.Error("a collector was started against an unwritable staging root") + } + // The probe leaves nothing behind. + if entries, err := os.ReadDir(staging); err == nil && len(entries) != 0 { + t.Errorf("staging root holds %d leftover entries after preflight", len(entries)) + } +} + +// 30c. The preflight probe cleans up after a successful start too. +func TestRuntimePreflightProbeLeavesNothingBehind(t *testing.T) { + h := newRuntimeHarness(t) + h.start() + + entries, err := os.ReadDir(utils.GetRayRotatedStagingPath()) + if err != nil { + t.Fatalf("read staging root: %v", err) + } + for _, e := range entries { + if strings.HasPrefix(e.Name(), ".rotated-preflight-") { + t.Errorf("preflight probe %s was left behind", e.Name()) + } + } +} + +// 30d. A staging root that accepts a file but will not give it up is not usable. +// Promotion renames and release unlinks, so a directory whose entries cannot be removed +// would let capture pin inodes nothing could ever free. +func TestRuntimePreflightProbeRequiresCleanRemoval(t *testing.T) { + errClose := errors.New("close refused") + errRemove := errors.New("unlink refused") + + for _, tc := range []struct { + name string + close func(*os.File) error + remove func(string) error + wantErrs []error + }{ + { + name: "removal fails", + remove: func(string) error { return errRemove }, + wantErrs: []error{errRemove}, + }, + { + name: "close fails but removal is still attempted", + close: func(*os.File) error { return errClose }, + wantErrs: []error{errClose}, + }, + { + name: "both fail and both causes survive", + close: func(*os.File) error { return errClose }, + remove: func(string) error { return errRemove }, + wantErrs: []error{errClose, errRemove}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + h := newRuntimeHarness(t) + sup := h.handler.rotatedCollection() + + var removed []string + sup.probeClose = tc.close + sup.probeRemove = func(name string) error { + removed = append(removed, name) + if tc.remove != nil { + return tc.remove(name) + } + return os.Remove(name) + } + + h.handler.startRotatedCollection() + + key := rotatedKey{session: testSessionA, node: testNodeID} + err := sup.disabledReason(key) + if err == nil { + t.Fatal("an unusable staging root was accepted") + } + for _, want := range tc.wantErrs { + if !errors.Is(err, want) { + t.Errorf("preflight error %v does not carry %v", err, want) + } + } + if !sup.durablyDisabled(key) { + t.Errorf("an unusable staging root was classified as retryable: %v", err) + } + if _, ok := sup.activeKey(); ok { + t.Error("a collector was started against an unusable staging root") + } + // Removal is attempted on every path, including the one where close failed. + if len(removed) != 1 { + t.Errorf("probe removal attempts = %d, want exactly 1", len(removed)) + } + // A close that this test faked leaves the real descriptor open; clean it up + // so the temp directory can be removed on Windows-like platforms. + if tc.close != nil { + _ = os.Remove(removed[0]) + } + }) + } +} + +// 30e. A staging root that is momentarily full is retryable — that is a condition this +// subsystem is built to survive — while permission and cleanup failures are not. +func TestRuntimePreflightProbeENOSPCIsRetryable(t *testing.T) { + h := newRuntimeHarness(t) + sup := h.handler.rotatedCollection() + key := rotatedKey{session: testSessionA, node: testNodeID} + + var full atomic.Bool + full.Store(true) + sup.probeCreate = func(dir string) (*os.File, error) { + if full.Load() { + return nil, &os.PathError{Op: "open", Path: dir, Err: syscall.ENOSPC} + } + return os.CreateTemp(dir, ".rotated-preflight-*") + } + + h.handler.startRotatedCollection() + if _, ok := sup.activeKey(); ok { + t.Fatal("a collector started against a full staging root") + } + if sup.disabledReason(key) == nil { + t.Fatal("the full staging root was not recorded") + } + if sup.durablyDisabled(key) { + t.Fatalf("a full staging root was classified as durable: %v", sup.disabledReason(key)) + } + + // Capacity comes back. + full.Store(false) + h.advanceClock(rotatedRetryBase + time.Second) + h.handler.ensureRotatedCollection(h.handler.SessionDir) + h.awaitCollector() + if got, ok := sup.activeKey(); !ok || got != key { + t.Errorf("collector after capacity recovery = %+v (active=%v), want %+v", got, ok, key) + } + if sup.disabledReason(key) != nil { + t.Errorf("the recovered identity still reports %v", sup.disabledReason(key)) + } +} + +func TestRuntimePreflightProbePermissionFailureIsDurable(t *testing.T) { + h := newRuntimeHarness(t) + sup := h.handler.rotatedCollection() + sup.probeCreate = func(dir string) (*os.File, error) { + return nil, &os.PathError{Op: "open", Path: dir, Err: syscall.EACCES} + } + + h.handler.startRotatedCollection() + key := rotatedKey{session: testSessionA, node: testNodeID} + if !sup.durablyDisabled(key) { + t.Errorf("a permission failure creating the probe was classified as retryable: %v", sup.disabledReason(key)) + } +} + +// 31. A logs path that is not a directory is a durable misconfiguration, not something +// to retry forever. +func TestRuntimeLogsPathThatIsNotADirectoryIsDurable(t *testing.T) { + h := newRuntimeHarness(t) + sessionDir := h.sessionDir(testSessionB) + writeFile(t, filepath.Join(sessionDir, utils.RAY_SESSIONDIR_LOGDIR_NAME), "not a directory") + + h.handler.ensureRotatedCollection(sessionDir) + + key := rotatedKey{session: testSessionB, node: testNodeID} + if !h.handler.rotatedCollection().durablyDisabled(key) { + t.Errorf("a logs path that is a regular file was not durably disabled: %v", + h.handler.rotatedCollection().disabledReason(key)) + } + if _, ok := h.handler.rotatedCollection().activeKey(); ok { + t.Error("a collector was started for a logs path that is not a directory") + } +} + +// --------------------------------------------------------------------------- +// The production hook +// --------------------------------------------------------------------------- + +// 33. A session change carries a node change, and the first collector built for the new +// session must already be on the new node. +// +// There is no second chance at this. The node ID is written into the staged entry, the +// staging path and the object key at capture time, and a later collector on the correct +// node adopts those records without rewriting them, so anything captured under the +// previous session's node stays addressed to it permanently. +func TestPollActiveSessionChangesUsesTheNewSessionsNodeID(t *testing.T) { + h := newRuntimeHarness(t) + h.start() + h.write(testSessionA, "raylet.out", "active") + + h.makeSession(testSessionB) + h.write(testSessionB, "raylet.out", "active") + h.pointSessionLatest(testSessionB) + h.discoverNodeB() // the raylet restarted, so the node ID moved with it + + go h.handler.PollActiveSessionChanges() + t.Cleanup(func() { close(h.handler.ShutdownChan) }) + + wantKey := rotatedKey{session: testSessionB, node: testNodeIDB} + eventually(t, "the poller to move the collector to the new session and node", func() bool { + key, ok := h.handler.rotatedCollection().activeKey() + return ok && key == wantKey + }) + + // Not one collector was ever built for session B under node A — not even a + // short-lived one that was replaced a moment later. + for _, k := range h.built() { + if k.session == testSessionB && k.node != testNodeIDB { + t.Errorf("a collector for session B was built on node %s, want only %s", k.node, testNodeIDB) + } + } + if got := h.handler.GetRayNodeName(); got != testNodeIDB { + t.Errorf("handler node ID = %s, want %s", got, testNodeIDB) + } + + // Everything the new session captures is addressed to the new node. + entry := h.captureIn(testSessionB, "raylet.out.1", "new session segment") + if entry.NodeName != testNodeIDB || entry.SessionName != testSessionB { + t.Errorf("staged entry = %s/%s, want %s/%s", entry.SessionName, entry.NodeName, testSessionB, testNodeIDB) + } + rc := h.handler.rotatedCollection().testCollector() + wantPrefix := h.logsPrefixFor(testSessionB, testNodeIDB) + "/" + if k := entry.objectKey(rc.cfg.Cluster); !strings.HasPrefix(k, wantPrefix) { + t.Errorf("object key %s is not under the new node's prefix %s", k, wantPrefix) + } + for _, p := range h.stagingFiles() { + if strings.HasPrefix(p, testSessionB+"/") && !strings.HasPrefix(p, testSessionB+"/"+testNodeIDB+"/") { + t.Errorf("session B staged a capture outside its own node's subtree: %s", p) + } + } + + // The old session's logs were relocated under the node they were written on. + oldPrevLogs := filepath.Join(utils.GetRayPrevLogsPath(), testSessionA, testNodeID, "logs") + if _, err := os.Stat(oldPrevLogs); err != nil { + t.Errorf("old session logs were not relocated to %s: %v", oldPrevLogs, err) + } + if _, err := os.Stat(filepath.Join(utils.GetRayPrevLogsPath(), testSessionA, testNodeIDB)); err == nil { + t.Error("the old session's logs were relocated under the new session's node ID") + } +} + +// 34. A session change whose node ID cannot be discovered starts no collector at all +// rather than one under the previous session's node. +// +// The old collector is still retired first — that is its final reconciliation, and it +// happens before anything touches its tree — and the relocation still runs, because +// prev-logs is the legacy path for those logs and a dashboard that is briefly +// unreachable must not strand a session's logs outside it. What must not happen is a +// collector for the new session addressed to the old session's node: that would be +// baked into its staged entries and object keys permanently. +func TestPollActiveSessionChangesWaitsForTheNewNodeID(t *testing.T) { + h := newRuntimeHarness(t) + h.start() + first := h.handler.rotatedCollection().activeRun() + + h.makeSession(testSessionB) + h.write(testSessionB, "raylet.out", "active") + h.pointSessionLatest(testSessionB) + h.failNodeDiscovery() // the dashboard is still coming up after the restart + + go h.handler.PollActiveSessionChanges() + t.Cleanup(func() { close(h.handler.ShutdownChan) }) + + // The relocation is the observable end of the changeover. + oldPrevLogs := filepath.Join(utils.GetRayPrevLogsPath(), testSessionA, testNodeID, "logs") + eventually(t, "the old session's logs to be relocated", func() bool { + _, err := os.Stat(oldPrevLogs) + return err == nil + }) + + if _, ok := h.handler.rotatedCollection().activeKey(); ok { + t.Error("a collector is active even though the new session's node ID is unknown") + } + if !first.finished() { + t.Error("the old collector was not retired before its tree was relocated") + } + for _, k := range h.built() { + if k.session == testSessionB { + t.Errorf("a collector for session B was built before its node ID was known: %+v", k) + } + } + if _, err := os.Stat(filepath.Join(utils.GetRayPrevLogsPath(), testSessionA, testNodeIDB)); err == nil { + t.Error("the old session's logs were relocated under a node it never ran on") + } + + // Once the dashboard answers, protection resumes under the correct node. + h.discoverNodeB() + eventually(t, "rotated protection to resume on the new node", func() bool { + key, ok := h.handler.rotatedCollection().activeKey() + return ok && key == rotatedKey{session: testSessionB, node: testNodeIDB} + }) + for _, k := range h.built() { + if k.session == testSessionB && k.node != testNodeIDB { + t.Errorf("session B was eventually built on node %s, want only %s", k.node, testNodeIDB) + } + } +} + +// 35. A relocation that keeps failing re-runs the move without disturbing the collector +// that is already correct. +func TestPollActiveSessionChangesDoesNotChurnWhenRelocationFails(t *testing.T) { + h := newRuntimeHarness(t) + h.start() + h.write(testSessionA, "raylet.out", "active") + + // prev-logs is a regular file, so MoveSessionLogsToPrevLogs cannot create its + // destination and every relocation attempt fails. + writeFile(t, utils.GetRayPrevLogsPath(), "not a directory") + + h.makeSession(testSessionB) + h.write(testSessionB, "raylet.out", "active") + h.pointSessionLatest(testSessionB) + h.discoverNodeB() + + go h.handler.PollActiveSessionChanges() + t.Cleanup(func() { close(h.handler.ShutdownChan) }) + + wantKey := rotatedKey{session: testSessionB, node: testNodeIDB} + eventually(t, "the collector to move to the new session", func() bool { + key, ok := h.handler.rotatedCollection().activeKey() + return ok && key == wantKey + }) + run := h.handler.rotatedCollection().activeRun() + builds := h.buildCount() + + // The relocation is retried on every tick because the session change is never + // recorded as complete. The collector must sit still through all of it. + eventually(t, "at least two further relocation attempts", func() bool { + h.mu.Lock() + defer h.mu.Unlock() + return h.nodeCalls >= 3 + }) + + if got := h.handler.rotatedCollection().activeRun(); got != run { + t.Error("a failing relocation replaced the collector that was already correct") + } + if run.finished() { + t.Error("a failing relocation retired the running collector") + } + if got := h.buildCount(); got != builds { + t.Errorf("collectors built by repeated relocation failures = %d, want none", got-builds) + } + if key, _ := h.handler.rotatedCollection().activeKey(); key != wantKey { + t.Errorf("identity drifted to %+v across relocation retries, want %+v", key, wantKey) + } +} + +// 35b. Node discovery that keeps failing never falls back to the previous session's +// node. +// +// The tick after a session change no longer looks like a change, so a runtime that +// treats "no verified node" as "use whatever the handler holds" starts the new session +// under the old session's node on that tick instead of the first one. Nothing later +// repairs it: the node is durable in the staged entry and the object key from the first +// capture onwards. +func TestPollActiveSessionChangesNeverFallsBackToThePreviousNode(t *testing.T) { + h := newRuntimeHarness(t) + h.start() + h.write(testSessionA, "raylet.out", "active") + + h.makeSession(testSessionB) + h.write(testSessionB, "raylet.out", "active") + h.pointSessionLatest(testSessionB) + h.failNodeDiscovery() + + go h.handler.PollActiveSessionChanges() + t.Cleanup(func() { close(h.handler.ShutdownChan) }) + + // Three full polling cycles with no node ID available. + eventually(t, "three failed discovery cycles", func() bool { + h.mu.Lock() + defer h.mu.Unlock() + return h.nodeCalls >= 3 + }) + + if key, ok := h.handler.rotatedCollection().activeKey(); ok { + t.Errorf("a collector is active for %+v while no node ID has been verified", key) + } + for _, k := range h.built() { + if k.session == testSessionB { + t.Errorf("session B was built under node %s before any node was verified", k.node) + } + } + for _, p := range h.stagingFiles() { + if strings.HasPrefix(p, testSessionB+"/"+testNodeID+"/") { + t.Errorf("session B staged a capture under the previous session's node: %s", p) + } + } + + // And when the node finally is discovered, exactly one collector starts for it. + h.discoverNodeB() + wantKey := rotatedKey{session: testSessionB, node: testNodeIDB} + eventually(t, "the collector to start under the verified node", func() bool { + key, ok := h.handler.rotatedCollection().activeKey() + return ok && key == wantKey + }) + starts := 0 + for _, k := range h.built() { + if k.session == testSessionB { + if k.node != testNodeIDB { + t.Errorf("session B was built under node %s, want only %s", k.node, testNodeIDB) + } + starts++ + } + } + if starts != 1 { + t.Errorf("session B collectors built = %d, want exactly 1", starts) + } +} + +// 35c. A relocation retry is only a relocation retry. A node rediscovery that fails +// during one must not disturb the collector the previous tick got right. +func TestPollActiveSessionChangesRelocationRetryKeepsTheCollector(t *testing.T) { + h := newRuntimeHarness(t) + h.start() + h.write(testSessionA, "raylet.out", "active") + + // prev-logs is a regular file, so every relocation attempt fails. + writeFile(t, utils.GetRayPrevLogsPath(), "not a directory") + + h.makeSession(testSessionB) + h.write(testSessionB, "raylet.out", "active") + h.pointSessionLatest(testSessionB) + h.discoverNodeB() + + go h.handler.PollActiveSessionChanges() + t.Cleanup(func() { close(h.handler.ShutdownChan) }) + + wantKey := rotatedKey{session: testSessionB, node: testNodeIDB} + eventually(t, "the collector to move to the new session and node", func() bool { + key, ok := h.handler.rotatedCollection().activeKey() + return ok && key == wantKey + }) + run := h.handler.rotatedCollection().activeRun() + rc := h.handler.rotatedCollection().testCollector() + watchers := h.watchers.count() + builds := h.buildCount() + callsBefore := func() int { + h.mu.Lock() + defer h.mu.Unlock() + return h.nodeCalls + }() + + // The dashboard goes away while the relocation is still being retried. + h.failNodeDiscovery() + eventually(t, "two further ticks with discovery failing", func() bool { + h.mu.Lock() + defer h.mu.Unlock() + return h.nodeCalls >= callsBefore+2 + }) + + // A discovery that fails leaves the verified identity exactly as it was. Treating + // "no answer" as an answer would blank the node the rest of the runtime writes + // under — including the legacy walk, which has no other source for it. + if got := h.handler.GetRayNodeName(); got != testNodeIDB { + t.Errorf("handler node ID = %q after failed rediscoveries, want it left at %s", got, testNodeIDB) + } + + if got := h.handler.rotatedCollection().activeRun(); got != run { + t.Error("a relocation retry replaced the collector that was already correct") + } + if h.handler.rotatedCollection().testCollector() != rc { + t.Error("a relocation retry rebuilt the collector") + } + if run.finished() { + t.Error("a relocation retry retired the running collector") + } + if h.watchers.at(watchers - 1).isClosed() { + t.Error("a relocation retry closed the running collector's watcher") + } + if got := h.watchers.count(); got != watchers { + t.Errorf("watchers created during relocation retries = %d, want none", got-watchers) + } + if got := h.buildCount(); got != builds { + t.Errorf("collectors built during relocation retries = %d, want none", got-builds) + } + + // When relocation finally works, it is still the same collector. + if err := os.Remove(utils.GetRayPrevLogsPath()); err != nil { + t.Fatalf("clear prev-logs: %v", err) + } + oldPrevLogs := filepath.Join(utils.GetRayPrevLogsPath(), testSessionA, testNodeID, "logs") + eventually(t, "the relocation to succeed", func() bool { + _, err := os.Stat(oldPrevLogs) + return err == nil + }) + if got := h.handler.rotatedCollection().activeRun(); got != run { + t.Error("the collector changed once relocation succeeded") + } +} + +// 35d. Handover state is read from the supervisor, not remembered from having called +// ensure. Run's initial start can fail retryably — the session directory exists but +// logs/ does not yet — and a poller that seeds "handed off" from the fact that +// startRotatedCollection ran would never try again. +func TestPollActiveSessionChangesRecoversFromAFailedInitialStart(t *testing.T) { + h := newRuntimeHarness(t) + // The handler is configured for a session whose logs/ has not appeared yet. + if err := os.RemoveAll(h.logsDir(testSessionA)); err != nil { + t.Fatalf("remove logs dir: %v", err) + } + h.handler.startRotatedCollection() + + key := rotatedKey{session: testSessionA, node: testNodeID} + sup := h.handler.rotatedCollection() + if _, ok := sup.activeKey(); ok { + t.Fatal("a collector started without a logs directory") + } + if sup.durablyDisabled(key) { + t.Fatalf("a missing logs directory was recorded as durable: %v", sup.disabledReason(key)) + } + if h.handler.rotatedHandedOff(h.sessionDir(testSessionA), testNodeID) { + t.Fatal("a failed start counts as a handover") + } + + go h.handler.PollActiveSessionChanges() + t.Cleanup(func() { close(h.handler.ShutdownChan) }) + + // Ray creates the directory, and the retry deadline passes. + h.makeSession(testSessionA) + h.advanceClock(rotatedRetryBase + time.Second) + + eventually(t, "the collector to start once its logs directory exists", func() bool { + return sup.statusFor(testSessionA, testNodeID, h.logsDir(testSessionA)) == runReady + }) + starts := 0 + for _, k := range h.built() { + if k == key { + starts++ + } + } + if starts != 1 { + t.Errorf("collectors built for %+v = %d, want exactly 1", key, starts) + } +} + +// 35e. A collector that fails asynchronously after it was attached is noticed, and the +// next polls try again — subject to the supervisor's backoff, not to a poller that +// believes the handover already happened. +func TestPollActiveSessionChangesRetriesAfterAsynchronousStartupFailure(t *testing.T) { + h := newRuntimeHarness(t) + sup := h.handler.rotatedCollection() + key := rotatedKey{session: testSessionA, node: testNodeID} + + var fail atomic.Bool + fail.Store(true) + base := sup.tune + sup.tune = func(cfg *rotatedCollectorConfig) { + base(cfg) + if fail.Load() { + // Attaches, then fails during startup: retryable, and only observable + // through the supervisor. + cfg.NewWatcher = func() (fsWatcher, error) { + return nil, fmt.Errorf("inotify_init: %w", syscall.EMFILE) + } + } + } + + h.handler.startRotatedCollection() + eventually(t, "the asynchronous failure to be published", func() bool { + return sup.disabledReason(key) != nil + }) + if sup.durablyDisabled(key) { + t.Fatal("an exhausted watch limit was recorded as durable") + } + if h.handler.rotatedHandedOff(h.sessionDir(testSessionA), testNodeID) { + t.Fatal("a collector that failed after attaching still counts as a handover") + } + + go h.handler.PollActiveSessionChanges() + t.Cleanup(func() { close(h.handler.ShutdownChan) }) + + // Backoff is respected: nothing is rebuilt until the deadline passes. + builds := h.buildCount() + time.Sleep(150 * time.Millisecond) // several polling cycles at the test interval + if got := h.buildCount(); got != builds { + t.Errorf("collectors built before the retry deadline = %d, want none", got-builds) + } + + fail.Store(false) + h.advanceClock(rotatedRetryBase + time.Second) + eventually(t, "the retry to start and reach ready", func() bool { + return sup.statusFor(testSessionA, testNodeID, h.logsDir(testSessionA)) == runReady + }) + if sup.disabledReason(key) != nil { + t.Errorf("the recovered identity still reports %v", sup.disabledReason(key)) + } +} + +// 35e2. A run whose goroutine has exited is not a handover, even in the window before +// its failure has been published and its pointer cleared. +// +// That window is short but it is the only time runFinished is observable, and it is +// exactly when a poller that counted it as a handover would stop retrying. +func TestPollActiveSessionChangesTreatsAFinishedRunAsNotHandedOff(t *testing.T) { + h := newRuntimeHarness(t) + sup := h.handler.rotatedCollection() + + entered, release := make(chan struct{}), make(chan struct{}) + var once sync.Once + sup.beforeFailurePublish = func() { + once.Do(func() { + close(entered) + <-release + }) + } + + h.start() + close(h.watchers.at(0).events) // fatal to the collector + select { + case <-entered: + case <-time.After(10 * time.Second): + t.Fatal("the collector never reached failure publication") + } + defer close(release) + + // Attached, but its goroutine has exited. + if got := sup.statusFor(testSessionA, testNodeID, h.logsDir(testSessionA)); got != runFinished { + t.Fatalf("status = %v, want %v", got, runFinished) + } + if h.handler.rotatedHandedOff(h.sessionDir(testSessionA), testNodeID) { + t.Error("a run whose goroutine has exited counts as a completed handover") + } +} + +// 35f. A durable failure is not retried by the poller either, however many ticks pass. +func TestPollActiveSessionChangesDoesNotHotLoopADurableFailure(t *testing.T) { + h := newRuntimeHarness(t) + sup := h.handler.rotatedCollection() + base := sup.tune + sup.tune = func(cfg *rotatedCollectorConfig) { + base(cfg) + cfg.NewWatcher = func() (fsWatcher, error) { + return nil, fmt.Errorf("create fsnotify watcher: %w", os.ErrPermission) + } + } + + h.handler.startRotatedCollection() + key := rotatedKey{session: testSessionA, node: testNodeID} + eventually(t, "the durable failure to be published", func() bool { + return sup.durablyDisabled(key) + }) + builds := h.buildCount() + + go h.handler.PollActiveSessionChanges() + t.Cleanup(func() { close(h.handler.ShutdownChan) }) + + h.advanceClock(24 * time.Hour) + time.Sleep(200 * time.Millisecond) // many polling cycles at the test interval + if got := h.buildCount(); got != builds { + t.Errorf("collectors built for a durable failure = %d, want none", got-builds) + } + if _, ok := sup.activeKey(); ok { + t.Error("a durably failed identity became active") + } +} + +// 35g. Every observed session is relocated under the node it actually ran on, however +// many sessions pass while one of them is stuck. +func TestPollActiveSessionChangesFilesEachSessionUnderItsOwnNode(t *testing.T) { + const testSessionC = "session_2026-07-31_12-00-00_000003" + const testNodeIDC = "00112233445566778899aabbccddeeff" + + h := newRuntimeHarness(t) + h.start() + h.write(testSessionA, "raylet.out", "session A") + + // prev-logs is a regular file, so relocating A fails. + writeFile(t, utils.GetRayPrevLogsPath(), "not a directory") + + h.makeSession(testSessionB) + h.write(testSessionB, "raylet.out", "session B") + h.pointSessionLatest(testSessionB) + h.discoverNodeB() + + go h.handler.PollActiveSessionChanges() + t.Cleanup(func() { close(h.handler.ShutdownChan) }) + + eventually(t, "the collector to move to session B", func() bool { + key, ok := h.handler.rotatedCollection().activeKey() + return ok && key == rotatedKey{session: testSessionB, node: testNodeIDB} + }) + + // A third session arrives while A is still stuck. + h.makeSession(testSessionC) + h.write(testSessionC, "raylet.out", "session C") + h.pointSessionLatest(testSessionC) + h.mu.Lock() + h.nodeID = testNodeIDC + h.mu.Unlock() + + wantC := rotatedKey{session: testSessionC, node: testNodeIDC} + eventually(t, "the collector to move to session C", func() bool { + key, ok := h.handler.rotatedCollection().activeKey() + return ok && key == wantC + }) + runC := h.handler.rotatedCollection().activeRun() + + // Relocation starts working. + if err := os.Remove(utils.GetRayPrevLogsPath()); err != nil { + t.Fatalf("clear prev-logs: %v", err) + } + prev := utils.GetRayPrevLogsPath() + wantA := filepath.Join(prev, testSessionA, testNodeID, "logs", "raylet.out") + wantB := filepath.Join(prev, testSessionB, testNodeIDB, "logs", "raylet.out") + eventually(t, "both stuck sessions to be relocated", func() bool { + _, errA := os.Stat(wantA) + _, errB := os.Stat(wantB) + return errA == nil && errB == nil + }) + + // Neither landed under the other's node, or under the current one. + for _, wrong := range []string{ + filepath.Join(prev, testSessionA, testNodeIDB), + filepath.Join(prev, testSessionA, testNodeIDC), + filepath.Join(prev, testSessionB, testNodeID), + filepath.Join(prev, testSessionB, testNodeIDC), + } { + if _, err := os.Stat(wrong); err == nil { + t.Errorf("logs were filed under the wrong node: %s", wrong) + } + } + // And session C's collector was never disturbed by any of it. + if got := h.handler.rotatedCollection().activeRun(); got != runC || runC.finished() { + t.Error("the current session's collector was replaced or retired by relocation retries") + } +} + +// 35h. The broad sweep waits while a known session's own relocation is stuck. +// +// The sweep files every inactive session directory under one node, so running it while +// session A is still waiting would put A's logs under whatever node is current now. +// Blocking only A's exact destination is what separates the two: A's own move fails +// while the sweep would succeed. +func TestRelocationSweepWaitsForAStuckKnownSession(t *testing.T) { + h := newRuntimeHarness(t) + h.write(testSessionA, "raylet.out", "session A") + + // prev-logs// is a regular file, so only A's exact move can fail. + blocked := filepath.Join(utils.GetRayPrevLogsPath(), testSessionA, testNodeID) + writeFile(t, blocked, "not a directory") + + h.makeSession(testSessionB) + h.write(testSessionB, "raylet.out", "session B") + h.discoverNodeB() + + st := sessionTransition{dir: h.sessionDir(testSessionA), node: testNodeID, handedOff: true} + h.handler.advanceSession(&st, h.sessionDir(testSessionB)) + + if len(st.pending) != 1 || st.pending[0].node != testNodeID { + t.Fatalf("pending relocations = %+v, want session A still waiting under its own node", st.pending) + } + if _, err := os.Stat(h.logsDir(testSessionA)); err != nil { + t.Errorf("session A's logs were moved even though its own destination is blocked: %v", err) + } + wrong := filepath.Join(utils.GetRayPrevLogsPath(), testSessionA, testNodeIDB) + if _, err := os.Stat(wrong); err == nil { + t.Errorf("the broad sweep filed session A under %s while its own relocation was still waiting", testNodeIDB) + } +} + +// 36. The session poller is the production hook for a session change, so it is exercised +// end to end rather than only through ensureRotatedCollection. +func TestPollActiveSessionChangesReplacesTheRotatedCollector(t *testing.T) { + h := newRuntimeHarness(t) + h.start() + first := h.handler.rotatedCollection().activeRun() + + h.makeSession(testSessionB) + h.write(testSessionB, "raylet.out", "active") + h.pointSessionLatest(testSessionB) + + go h.handler.PollActiveSessionChanges() + t.Cleanup(func() { close(h.handler.ShutdownChan) }) + + eventually(t, "the poller to move the collector to the new session", func() bool { + key, ok := h.handler.rotatedCollection().activeKey() + return ok && key.session == testSessionB + }) + if !first.finished() { + t.Error("the old session's collector is still running") + } + // The handover happened before the old tree was relocated, which is the only order + // in which the old collector's final reconciliation can see anything. + if h.sawLivePredecessor() { + t.Error("the poller let two collectors overlap") + } +} + +// 35h. A transient session_latest resolution failure at startup must not be mistaken for +// a session change. +// +// The poller seeds its transition state with the *resolved* form of the configured +// SessionDir, because every later observation is a resolved symlink target and the two +// have to be comparable. The fallback taken when session_latest cannot be resolved has to +// use that same resolved value. Falling back to the raw configured path makes two +// spellings of one directory look like two directories, and advanceSession compares them +// by string: the live session is then queued for relocation into prev-logs, its verified +// node identity is cleared, and the collector is left running over a logs tree that has +// been moved out from under it. +// +// The two spellings are not hypothetical. utils.GetSessionDir falls back to os.Readlink +// when EvalSymlinks fails, and that path resolves only the session_latest link itself, so +// any symlinked component above it — a symlinked RAY_TMP_ROOT, or macOS putting /private +// in front of /var — survives into the configured value. +func TestPollActiveSessionChangesFallbackKeepsTheResolvedSessionPath(t *testing.T) { + h := newRuntimeHarness(t) + h.start() + h.write(testSessionA, "raylet.out", "active") + + // A second spelling of the temp root, so the configured SessionDir and its resolved + // form differ while naming one directory. + alias := filepath.Join(filepath.Dir(h.root), "alias-"+filepath.Base(h.root)) + if err := os.Symlink(h.root, alias); err != nil { + t.Fatalf("create root alias symlink: %v", err) + } + aliasSessionDir := filepath.Join(alias, testSessionA) + resolved, err := filepath.EvalSymlinks(aliasSessionDir) + if err != nil { + t.Fatalf("resolve %s: %v", aliasSessionDir, err) + } + if resolved == aliasSessionDir { + t.Skipf("this filesystem does not produce two spellings for %s", aliasSessionDir) + } + h.handler.SessionDir = aliasSessionDir + + // session_latest cannot be resolved, which is what sends the poller down its + // fallback. A dangling symlink is what Ray leaves behind mid-restart. + latest := utils.GetRaySessionLatestPath() + if err := os.Remove(latest); err != nil { + t.Fatalf("remove session_latest: %v", err) + } + if err := os.Symlink(filepath.Join(h.root, "session_does_not_exist"), latest); err != nil { + t.Fatalf("point session_latest at a missing target: %v", err) + } + + relocating := make(chan struct{}, 4) + h.handler.beforeRelocation = func() { + select { + case relocating <- struct{}{}: + default: + } + } + + liveLogs := h.logsDir(testSessionA) + go h.handler.PollActiveSessionChanges() + t.Cleanup(func() { close(h.handler.ShutdownChan) }) + + // The observation has happened once node discovery has been consulted; relocation, + // if the poller decided to do any, follows synchronously in the same call. + eventually(t, "the poller's first observation", func() bool { + h.mu.Lock() + defer h.mu.Unlock() + return h.nodeCalls >= 1 + }) + select { + case <-relocating: + t.Fatal("the poller treated the unresolved fallback path as a session change and began relocating the live session") + case <-time.After(500 * time.Millisecond): + } + + if _, err := os.Stat(liveLogs); err != nil { + t.Fatalf("the live session's logs directory was moved: %v", err) + } + if entries, err := os.ReadDir(utils.GetRayPrevLogsPath()); err == nil && len(entries) > 0 { + names := make([]string, 0, len(entries)) + for _, e := range entries { + names = append(names, e.Name()) + } + t.Errorf("prev-logs holds %v, want nothing: the active session was relocated", names) + } + if got := strings.TrimSpace(h.handler.GetRayNodeName()); got != testNodeID { + t.Errorf("node identity = %q, want %q unchanged: no session change occurred", got, testNodeID) + } + if key, ok := h.handler.rotatedCollection().activeKey(); !ok || key != (rotatedKey{session: testSessionA, node: testNodeID}) { + t.Errorf("collector identity = %+v (present=%v), want the session it started with", key, ok) + } +} diff --git a/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_stat_other.go b/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_stat_other.go new file mode 100644 index 00000000000..ca9ba1dc762 --- /dev/null +++ b/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_stat_other.go @@ -0,0 +1,15 @@ +//go:build !unix + +package logcollector + +import ( + "fmt" + "io/fs" +) + +// inodeFromFileInfo has no portable implementation: rotated-log capture relies on +// hard links and link counts, which the collector only ever runs against a unix +// filesystem shared with the Ray container. +func inodeFromFileInfo(fi fs.FileInfo) (inodeKey, uint64, error) { + return inodeKey{}, 0, fmt.Errorf("rotated log capture is unsupported on this platform: cannot read inode of %s", fi.Name()) +} diff --git a/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_stat_unix.go b/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_stat_unix.go new file mode 100644 index 00000000000..6c58d14d3c1 --- /dev/null +++ b/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_stat_unix.go @@ -0,0 +1,26 @@ +//go:build unix + +package logcollector + +import ( + "fmt" + "io/fs" + "syscall" +) + +// inodeFromFileInfo extracts the device/inode pair and current link count from a +// stat result. +func inodeFromFileInfo(fi fs.FileInfo) (inodeKey, uint64, error) { + st, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return inodeKey{}, 0, fmt.Errorf("unexpected stat type %T for %s", fi.Sys(), fi.Name()) + } + return inodeKey{Dev: statNumber(st.Dev), Ino: statNumber(st.Ino)}, statNumber(st.Nlink), nil +} + +// statNumber widens the platform-specific integers in syscall.Stat_t: the device, +// inode and link-count fields differ in both width and signedness across the unix +// targets this collector is built and tested on. +func statNumber[T ~int16 | ~uint16 | ~int32 | ~uint32 | ~int64 | ~uint64](v T) uint64 { + return uint64(v) +} diff --git a/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_state.go b/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_state.go new file mode 100644 index 00000000000..d5f565d58e9 --- /dev/null +++ b/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_state.go @@ -0,0 +1,145 @@ +package logcollector + +import ( + "fmt" + "sort" +) + +// inodeKey correlates a staged hard link with the file Ray rotated. +// +// It is only meaningful while the collector's own link keeps the inode alive: once +// the last link is dropped the kernel may hand the same number to an unrelated +// file. So inodeKey answers "have I already pinned this exact file?" and nothing +// else — permanent identity is the capture ID. +type inodeKey struct { + Dev uint64 + Ino uint64 +} + +func (k inodeKey) String() string { + return fmt.Sprintf("dev=%d,ino=%d", k.Dev, k.Ino) +} + +// capture is one rotated segment the collector has pinned with a hard link. +type capture struct { + Inode inodeKey + Entry stagedEntry +} + +// releasable reports whether the staged link may be unlinked. Both conditions +// matter: the bytes must already be in storage, and nlink == 1 proves ours is the +// last remaining link, so Ray has finished with the segment and no writer can +// reach it by name any more. +func (c *capture) releasable(nlink uint64) bool { + return c.Entry.State == stateUploaded && nlink == 1 +} + +// captureIndex tracks every pinned segment for one collector. +// +// It is deliberately free of locks: a single owner goroutine is the only caller +// that reads or mutates it, so "look up the inode, create the link, register the +// entry" is one indivisible step from the collector's point of view. Two events for +// the same pinned inode therefore cannot mint two capture IDs. +type captureIndex struct { + byInode map[inodeKey]*capture + + // bases remembers active log file names per directory. A rotation cascade + // briefly unlinks the active name, so a backup can legitimately appear while + // its base is missing; without this memory such a segment would look like an + // unrelated file ending in "." and be skipped. + bases map[string]map[string]struct{} +} + +func newCaptureIndex() *captureIndex { + return &captureIndex{ + byInode: make(map[inodeKey]*capture), + bases: make(map[string]map[string]struct{}), + } +} + +func (ix *captureIndex) len() int { + return len(ix.byInode) +} + +func (ix *captureIndex) lookup(key inodeKey) (*capture, bool) { + c, ok := ix.byInode[key] + return c, ok +} + +// add registers a freshly pinned segment. It reports added=false and returns the +// existing capture when the inode is already pinned, which is how a ".1" event and +// a later ".2" event for the same physical file collapse to one capture. +func (ix *captureIndex) add(key inodeKey, e stagedEntry) (*capture, bool, error) { + if e.State != statePending { + return nil, false, fmt.Errorf("add capture %s: new captures start in %q, got %q", key, statePending, e.State) + } + if existing, ok := ix.byInode[key]; ok { + return existing, false, nil + } + c := &capture{Inode: key, Entry: e} + ix.byInode[key] = c + return c, true, nil +} + +// restore re-registers an entry discovered on the staging volume at startup, +// accepting either durable state. Reconstruction must finish before any live event +// is handled, otherwise a capture from the previous run could be duplicated. +func (ix *captureIndex) restore(key inodeKey, e stagedEntry) (*capture, error) { + if !validStagingState(e.State) { + return nil, fmt.Errorf("restore capture %s: unknown staging state %q", key, e.State) + } + if existing, ok := ix.byInode[key]; ok { + // Every field has to agree, not just the ID. One inode found under two + // staging records — say a leftover pending link and an uploaded one from the + // same capture — is a corrupt staging tree, and which record won would + // otherwise depend on the order the filesystem walk happened to return. + if existing.Entry != e { + return nil, fmt.Errorf("restore capture %s: inode already staged as %+v, cannot also be %+v", + key, existing.Entry, e) + } + return existing, nil + } + c := &capture{Inode: key, Entry: e} + ix.byInode[key] = c + return c, nil +} + +// remove drops a capture from the index once its staged link is gone. +func (ix *captureIndex) remove(key inodeKey) { + delete(ix.byInode, key) +} + +// entries returns every tracked entry, sorted by capture ID so that iteration is +// deterministic rather than in Go's randomized map order. Nothing depends on that +// order for correctness. +func (ix *captureIndex) entries() []stagedEntry { + out := make([]stagedEntry, 0, len(ix.byInode)) + for _, c := range ix.byInode { + out = append(out, c.Entry) + } + sort.Slice(out, func(i, j int) bool { return out[i].CaptureID < out[j].CaptureID }) + return out +} + +// observeBase records that name is an active (non-backup) log file in dir. +func (ix *captureIndex) observeBase(dir, name string) { + d, ok := ix.bases[dir] + if !ok { + d = make(map[string]struct{}) + ix.bases[dir] = d + } + d[name] = struct{}{} +} + +// baseObserved reports whether name was previously seen as an active log file in dir. +func (ix *captureIndex) baseObserved(dir, name string) bool { + _, ok := ix.bases[dir][name] + return ok +} + +// validTransition allows only the one durable state change a capture can make. +// Nothing moves back to pending: re-uploading a changed file rewrites the same +// object key and leaves the capture uploaded. +func validTransition(from, to stagingState) bool { + return from == statePending && to == stateUploaded +} diff --git a/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_state_test.go b/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_state_test.go new file mode 100644 index 00000000000..fb204b89520 --- /dev/null +++ b/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_state_test.go @@ -0,0 +1,226 @@ +package logcollector + +import ( + "testing" +) + +func testEntry(originalName, captureID string) stagedEntry { + return stagedEntry{ + State: statePending, + SessionName: "session-1", + NodeName: "node-1", + OriginalName: originalName, + CaptureID: captureID, + } +} + +func TestCaptureIndexAddIsIdempotentPerInode(t *testing.T) { + ix := newCaptureIndex() + key := inodeKey{Dev: 1, Ino: 42} + + first, added, err := ix.add(key, testEntry("raylet.out.1", "0001780000000000000.a1b2c3d4e5f60718")) + if err != nil { + t.Fatalf("add() error: %v", err) + } + if !added { + t.Fatal("add() reported the first capture as already present") + } + + // The same physical file reappears as ".2" after the next rotation. It is the + // same pinned inode, so it must stay one capture with one ID and one object. + second, added, err := ix.add(key, testEntry("raylet.out.2", "0001780000000000001.ffffffffffffffff")) + if err != nil { + t.Fatalf("add() error on second reference: %v", err) + } + if added { + t.Error("add() created a second capture for an already pinned inode") + } + if second != first { + t.Error("add() returned a different capture for an already pinned inode") + } + if second.Entry.CaptureID != first.Entry.CaptureID { + t.Errorf("capture ID changed on re-discovery: %q -> %q", first.Entry.CaptureID, second.Entry.CaptureID) + } + if second.Entry.OriginalName != "raylet.out.1" { + t.Errorf("original name changed on re-discovery: %q", second.Entry.OriginalName) + } + if ix.len() != 1 { + t.Errorf("index holds %d captures, want 1", ix.len()) + } +} + +func TestCaptureIndexDistinctInodesAreDistinctCaptures(t *testing.T) { + ix := newCaptureIndex() + + // Two segments that successively occupy the same rotation filename. + if _, _, err := ix.add(inodeKey{Dev: 1, Ino: 42}, testEntry("raylet.out.1", "0001780000000000000.aaaaaaaaaaaaaaaa")); err != nil { + t.Fatalf("add() error: %v", err) + } + if _, _, err := ix.add(inodeKey{Dev: 1, Ino: 43}, testEntry("raylet.out.1", "0001780000000000001.bbbbbbbbbbbbbbbb")); err != nil { + t.Fatalf("add() error: %v", err) + } + + if ix.len() != 2 { + t.Fatalf("index holds %d captures, want 2", ix.len()) + } + entries := ix.entries() + if entries[0].CaptureID == entries[1].CaptureID { + t.Error("two segments sharing a rotation filename were given the same capture ID") + } + + identity := clusterIdentity{RootDir: "root", Namespace: "default", ClusterName: "my-cluster"} + if entries[0].objectKey(identity) == entries[1].objectKey(identity) { + t.Errorf("two segments sharing a rotation filename map to one object key: %q", entries[0].objectKey(identity)) + } +} + +func TestCaptureIndexAddRejectsNonPendingState(t *testing.T) { + ix := newCaptureIndex() + entry := testEntry("raylet.out.1", "0001780000000000000.a1b2c3d4e5f60718").withState(stateUploaded) + + if _, _, err := ix.add(inodeKey{Dev: 1, Ino: 42}, entry); err == nil { + t.Error("add() accepted a capture that did not start as pending") + } +} + +func TestValidTransition(t *testing.T) { + tests := []struct { + from, to stagingState + want bool + }{ + {from: statePending, to: stateUploaded, want: true}, + {from: statePending, to: statePending, want: false}, + {from: stateUploaded, to: statePending, want: false}, + {from: stateUploaded, to: stateUploaded, want: false}, + } + for _, tt := range tests { + if got := validTransition(tt.from, tt.to); got != tt.want { + t.Errorf("validTransition(%q, %q) = %v, want %v", tt.from, tt.to, got, tt.want) + } + } +} + +func TestCaptureReleasable(t *testing.T) { + tests := []struct { + name string + state stagingState + nlink uint64 + want bool + }{ + {name: "uploaded and last link", state: stateUploaded, nlink: 1, want: true}, + {name: "uploaded but Ray still holds a link", state: stateUploaded, nlink: 2, want: false}, + // Releasing pending data would lose it: nothing has reached storage yet. + {name: "pending and last link", state: statePending, nlink: 1, want: false}, + {name: "pending with Ray's link", state: statePending, nlink: 2, want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &capture{Inode: inodeKey{Dev: 1, Ino: 42}, Entry: testEntry("raylet.out.1", "id").withState(tt.state)} + if got := c.releasable(tt.nlink); got != tt.want { + t.Errorf("releasable(nlink=%d) with state %q = %v, want %v", tt.nlink, tt.state, got, tt.want) + } + }) + } +} + +func TestCaptureIndexRestore(t *testing.T) { + ix := newCaptureIndex() + key := inodeKey{Dev: 1, Ino: 42} + entry := testEntry("raylet.out.1", "0001780000000000000.a1b2c3d4e5f60718").withState(stateUploaded) + + restored, err := ix.restore(key, entry) + if err != nil { + t.Fatalf("restore() error: %v", err) + } + if restored.Entry.State != stateUploaded { + t.Errorf("restore() lost the durable state: %q", restored.Entry.State) + } + + // Restoring the identical record twice is harmless. Anything that differs is + // not: one inode staged under two records is a corrupt staging tree, and + // accepting either would make the result depend on walk order. + if _, err := ix.restore(key, entry); err != nil { + t.Errorf("restore() rejected an identical entry: %v", err) + } + conflicts := map[string]stagedEntry{ + "different capture ID": testEntry("raylet.out.1", "0001780000000000009.cccccccccccccccc").withState(stateUploaded), + // Same capture ID, but a leftover pending record from the same capture. + "different state": entry.withState(statePending), + "different original name": testEntry("raylet.out.2", entry.CaptureID).withState(stateUploaded), + } + for name, conflicting := range conflicts { + if _, err := ix.restore(key, conflicting); err == nil { + t.Errorf("restore() accepted a conflicting record (%s) for one inode", name) + } + } + if c, _ := ix.lookup(key); c.Entry != entry { + t.Errorf("a rejected restore changed the tracked entry to %+v", c.Entry) + } + if _, err := ix.restore(inodeKey{Dev: 1, Ino: 43}, stagedEntry{State: "draft"}); err == nil { + t.Error("restore() accepted an unknown staging state") + } +} + +func TestCaptureIndexRemove(t *testing.T) { + ix := newCaptureIndex() + key := inodeKey{Dev: 1, Ino: 42} + if _, _, err := ix.add(key, testEntry("raylet.out.1", "0001780000000000000.a1b2c3d4e5f60718")); err != nil { + t.Fatalf("add() error: %v", err) + } + + ix.remove(key) + if _, ok := ix.lookup(key); ok { + t.Error("lookup() found a removed capture") + } + if ix.len() != 0 { + t.Errorf("index holds %d captures after remove, want 0", ix.len()) + } + ix.remove(key) // removing twice must not panic +} + +func TestCaptureIndexObservedBases(t *testing.T) { + ix := newCaptureIndex() + const dirA, dirB = "/tmp/ray/session-1/logs", "/tmp/ray/session-1/logs/events" + + if ix.baseObserved(dirA, "raylet.out") { + t.Error("baseObserved() reported an unseen base") + } + ix.observeBase(dirA, "raylet.out") + if !ix.baseObserved(dirA, "raylet.out") { + t.Error("baseObserved() forgot a recorded base") + } + // Bases are scoped per directory: Ray reuses names across subdirectories. + if ix.baseObserved(dirB, "raylet.out") { + t.Error("baseObserved() leaked a base across directories") + } + ix.observeBase(dirB, "raylet.out") + if !ix.baseObserved(dirB, "raylet.out") { + t.Error("baseObserved() forgot a base in a second directory") + } +} + +func TestCaptureIndexEntriesOrderedByCaptureID(t *testing.T) { + ix := newCaptureIndex() + ids := []string{ + "0001780000000000002.cccccccccccccccc", + "0001780000000000000.aaaaaaaaaaaaaaaa", + "0001780000000000001.bbbbbbbbbbbbbbbb", + } + inodes := []uint64{10, 11, 12} + for i, id := range ids { + if _, _, err := ix.add(inodeKey{Dev: 1, Ino: inodes[i]}, testEntry("raylet.out.1", id)); err != nil { + t.Fatalf("add() error: %v", err) + } + } + + entries := ix.entries() + if len(entries) != len(ids) { + t.Fatalf("entries() returned %d entries, want %d", len(entries), len(ids)) + } + for i := 1; i < len(entries); i++ { + if entries[i-1].CaptureID > entries[i].CaptureID { + t.Errorf("entries() not ordered by capture ID: %q before %q", entries[i-1].CaptureID, entries[i].CaptureID) + } + } +} diff --git a/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_uploader.go b/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_uploader.go new file mode 100644 index 00000000000..7f8101db5ee --- /dev/null +++ b/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_uploader.go @@ -0,0 +1,1155 @@ +package logcollector + +import ( + "errors" + "fmt" + "io" + "os" + "slices" + "sort" + "strings" + "time" + + "github.com/sirupsen/logrus" +) + +// objectWriter is the slice of storage.StorageWriter the uploader needs. Keeping it +// narrow means the collector can be tested against a fake that blocks, fails or +// counts calls without constructing a cloud client, and it documents that a capture +// upload is exactly one object write and nothing else. +type objectWriter interface { + WriteFile(file string, reader io.ReadSeeker) error +} + +// defaultUploadBackoff is the delay before each successive retry, capped at the last +// entry, which repeats for as long as the failure lasts. No jitter: a node runs one +// collector uploading one capture at a time, so there is no herd to spread out, and +// a fixed sequence is exactly assertable in tests. +var defaultUploadBackoff = []time.Duration{ + 1 * time.Second, + 2 * time.Second, + 5 * time.Second, + 15 * time.Second, + 30 * time.Second, + 1 * time.Minute, + 2 * time.Minute, + 5 * time.Minute, +} + +// defaultWorkerStopGrace bounds how long Stop waits for the upload worker. An idle +// worker exits at once; one inside a storage write cannot be interrupted, because +// storage.StorageWriter.WriteFile takes no context, so the wait has to be bounded. +const defaultWorkerStopGrace = 100 * time.Millisecond + +var ( + // errUploadInFlightAtStop reports the one case Stop cannot resolve: an + // uncancelable storage write outliving the collector. The worker cannot touch + // collector state afterwards, so the only consequence is that the write's result + // is discarded and the capture stays pending for the next run to retry. + errUploadInFlightAtStop = errors.New("rotated log upload was still running at stop: storage.StorageWriter.WriteFile cannot be canceled, so its result is discarded and the capture stays pending") + + // errUploadStale marks a result that no longer describes anything the collector + // is tracking, which is how a result that outlived its capture is discarded + // instead of applied to whatever now occupies the same inode. + errUploadStale = errors.New("upload result no longer matches the capture it was submitted for") + + // errStagingInconsistent marks the staging volume contradicting the index. It is + // fatal by design: retrying cannot make a missing or replaced staged file into + // the capture the index says it is, so the collector stops with the staging tree + // untouched rather than scheduling work that can never succeed. + errStagingInconsistent = errors.New("staging volume contradicts the capture index") +) + +// uploadIdentity is everything that has to still be true for an upload result to be +// applied. It is compared as a whole, so a capture that was replaced, promoted or +// re-staged between submission and completion cannot be mutated by the old result. +// +// Every field is comparable, which is what makes that check one ==. +type uploadIdentity struct { + inode inodeKey + entry stagedEntry + localPath string + objectKey string +} + +// uploadJob is one immutable unit of work handed to the worker. attempt is carried +// for diagnostics only and is deliberately outside uploadIdentity: a retry of the +// same capture is the same object write to the same key. +type uploadJob struct { + uploadIdentity + attempt int +} + +// uploadResult is what the worker returns. It repeats the submitted job so the owner +// can decide, without consulting any worker-owned state, whether the result still +// applies. +// +// local separates "the staged file is not what we were asked to upload" from "the +// object store rejected the write". Only the second is a transport problem; the first +// means the staging volume changed under us and the retry will fail the same way +// until reconciliation re-establishes the truth. +type uploadResult struct { + err error + job uploadJob + local bool +} + +// uploadWorker performs the blocking object write and nothing else. +// +// It never touches captureIndex, capture state, retry state or byte accounting: it +// opens a file, validates that the descriptor is the capture it was given, writes it, +// and returns an immutable result. Every decision that follows belongs to the owner +// goroutine, which is why object-store latency cannot delay event handling, +// reconciliation, snapshots or stop. +type uploadWorker struct { + writer objectWriter + jobs <-chan uploadJob + results chan<- uploadResult + quit <-chan struct{} + // beforeWrite runs after local validation and before the shutdown check that + // guards the remote write. It is nil in production and exists so a test can hold + // the worker in exactly that window. + beforeWrite func() + stagingRoot string +} + +// stopping reports whether shutdown has begun. A nil quit channel never fires, so a +// worker constructed without one — as tests that call execute directly do — simply +// never sees a stop. +func (w *uploadWorker) stopping() bool { + select { + case <-w.quit: + return true + default: + return false + } +} + +// run processes jobs until quit is closed. Both the receive and the send are guarded +// by quit, so the worker can never block forever on an owner that has stopped, and it +// never sends on a closed channel because quit is closed by the owner and only ever +// received from here. +func (w *uploadWorker) run(done chan<- struct{}) { + defer close(done) + for { + select { + case <-w.quit: + return + case job := <-w.jobs: + // A job can already be buffered when quit closes, and select chooses + // at random between two ready cases. So the guard has to be re-checked + // here: an upload that has not started must never start after Stop + // began. Only a WriteFile already under way outlives the owner, and + // that is the one case the storage interface gives no way to cancel. + if w.stopping() { + return + } + + res, ok := w.execute(job) + if !ok { + // Shutdown overtook the job during local validation, so there is no + // result to report: nothing was sent to the object store. + return + } + select { + case w.results <- res: + case <-w.quit: + return + } + } + } +} + +// execute validates the staged file and writes it to storage. +// +// Validation is done on the open descriptor, not on the path: fstat after open +// answers "what did I actually open?", so a staging path that was replaced between +// the check and the read cannot be uploaded under another capture's key. A path-based +// stat would leave exactly that window open. +// +// The descriptor is handed to the writer directly. The staged file is a hard link to +// an inode Ray has already rotated away, so its bytes cannot change under the read, +// and streaming it avoids holding a whole segment in memory. +// +// ok is false when shutdown began before the remote write started. Nothing was sent, +// so there is no result for the owner to apply. +func (w *uploadWorker) execute(job uploadJob) (res uploadResult, ok bool) { + if want := job.entry.path(w.stagingRoot); want != job.localPath { + return uploadResult{job: job, local: true, err: fmt.Errorf( + "upload job for capture %s names %s but its entry stages at %s", job.entry.CaptureID, job.localPath, want)}, true + } + + f, err := os.Open(job.localPath) + if err != nil { + return uploadResult{job: job, local: true, err: fmt.Errorf("open staged capture %s: %w", job.localPath, err)}, true + } + defer func() { + if err := f.Close(); err != nil { + logrus.Debugf("Rotated log uploader: close %s: %v", job.localPath, err) + } + }() + + fi, err := f.Stat() + if err != nil { + return uploadResult{job: job, local: true, err: fmt.Errorf("stat staged capture %s: %w", job.localPath, err)}, true + } + if !fi.Mode().IsRegular() { + return uploadResult{job: job, local: true, err: fmt.Errorf( + "staged capture %s is not a regular file (%s)", job.localPath, fi.Mode())}, true + } + opened, _, err := inodeFromFileInfo(fi) + if err != nil { + return uploadResult{job: job, local: true, err: fmt.Errorf("read inode of staged capture %s: %w", job.localPath, err)}, true + } + if opened != job.inode { + return uploadResult{job: job, local: true, err: fmt.Errorf( + "staged capture %s holds %s, not the captured %s", job.localPath, opened, job.inode)}, true + } + + if w.beforeWrite != nil { + w.beforeWrite() + } + // Local validation can take a while on a loaded node, and Stop may have begun + // during it. A remote write that has not started must not start now. + if w.stopping() { + return uploadResult{}, false + } + + if err := w.writer.WriteFile(job.objectKey, f); err != nil { + return uploadResult{job: job, err: fmt.Errorf("write object %s: %w", job.objectKey, err)}, true + } + return uploadResult{job: job}, true +} + +// uploadPhase is where one capture currently sits in the upload pipeline. A capture +// with no phase at all is one the owner has nothing outstanding for. +type uploadPhase int + +const ( + // phaseQueued: waiting for the worker to be free. + phaseQueued uploadPhase = iota + // phaseInFlight: handed to the worker; no second job may be created for it. + phaseInFlight + // phaseBackoff: the upload failed and a retry is due at dueAt. + phaseBackoff + // phaseAwaitingPromotion: the object write succeeded but the local pending -> + // uploaded promotion did not. Re-uploading would be pointless, so only the + // promotion is retried. + phaseAwaitingPromotion +) + +func (p uploadPhase) String() string { + switch p { + case phaseQueued: + return "queued" + case phaseInFlight: + return "in flight" + case phaseBackoff: + return "backing off" + case phaseAwaitingPromotion: + return "awaiting promotion" + default: + return fmt.Sprintf("unknown phase %d", int(p)) + } +} + +// uploadState is the owner's record of one capture's progress. Only the owner +// goroutine reads or writes it, so it needs no synchronization. +type uploadState struct { + dueAt time.Time + job uploadJob + attempts int + phase uploadPhase +} + +// queuedUpload keeps the capture ID beside the key so the queue stays ordered without +// consulting the index. +type queuedUpload struct { + captureID string + key inodeKey +} + +// uploadScheduler is the owner's side of the pipeline: the queue, the per-capture +// state, the retry timer and the channels to the single worker. Like captureIndex it +// is lock-free because one goroutine owns it. +type uploadScheduler struct { + writer objectWriter + states map[inodeKey]*uploadState + jobs chan uploadJob + results chan uploadResult + quit chan struct{} + workerDone chan struct{} + retryC <-chan time.Time + stopTimer func() + backoff []time.Duration + queue []queuedUpload + armedFor time.Time + inFlight int +} + +func newUploadScheduler(writer objectWriter, backoff []time.Duration) *uploadScheduler { + return &uploadScheduler{ + writer: writer, + states: make(map[inodeKey]*uploadState), + backoff: backoff, + } +} + +// enabled reports whether uploads happen at all. Production always supplies a writer; +// without one the collector still captures, tracks bytes and reconstructs staging, but +// has nowhere to send the bytes. +func (u *uploadScheduler) enabled() bool { return u.writer != nil } + +// start launches the worker. It is called once, from the owner goroutine, after +// startup reconstruction. +func (u *uploadScheduler) start(stagingRoot string) { + if !u.enabled() { + return + } + // One job at a time. The owner only sends when nothing is in flight, so this + // buffer is never full when it sends, which is what keeps submission from + // blocking the owner loop. + u.jobs = make(chan uploadJob, 1) + u.results = make(chan uploadResult, 1) + u.quit = make(chan struct{}) + u.workerDone = make(chan struct{}) + + w := &uploadWorker{ + writer: u.writer, + jobs: u.jobs, + results: u.results, + quit: u.quit, + stagingRoot: stagingRoot, + } + go w.run(u.workerDone) +} + +// stop shuts the worker down and waits for it, but only for grace. +// +// An idle worker is parked on a select and leaves immediately. One inside +// WriteFile cannot be interrupted — the storage interface takes no context — so +// waiting for it would let object-store latency delay Stop, which is exactly what +// this design forbids. After quit is closed the worker can no longer deliver a +// result, so outliving the collector costs correctness nothing. +func (u *uploadScheduler) stop(grace time.Duration, report func(error)) { + if u.quit == nil { + return + } + close(u.quit) + u.disarm() + + t := time.NewTimer(grace) + defer t.Stop() + select { + case <-u.workerDone: + case <-t.C: + report(errUploadInFlightAtStop) + } +} + +func (u *uploadScheduler) disarm() { + if u.stopTimer != nil { + u.stopTimer() + u.stopTimer = nil + } + u.retryC = nil + u.armedFor = time.Time{} +} + +// queued reports whether key is already in the queue. The state map is the real +// guard against duplicates; this exists so the queue can be kept consistent with it. +func (u *uploadScheduler) queuedAt(key inodeKey) int { + return slices.IndexFunc(u.queue, func(q queuedUpload) bool { return q.key == key }) +} + +// enqueue inserts a capture in capture-ID order, so the oldest capture is uploaded +// first and the order does not depend on Go's map iteration. +func (u *uploadScheduler) enqueue(key inodeKey, captureID string) { + if u.queuedAt(key) >= 0 { + return + } + i := sort.Search(len(u.queue), func(i int) bool { return u.queue[i].captureID > captureID }) + u.queue = slices.Insert(u.queue, i, queuedUpload{key: key, captureID: captureID}) +} + +func (u *uploadScheduler) dequeue() (queuedUpload, bool) { + if len(u.queue) == 0 { + return queuedUpload{}, false + } + head := u.queue[0] + u.queue = u.queue[1:] + return head, true +} + +// forget drops every trace of a capture from the pipeline. It is only safe once the +// capture can no longer produce a result the owner would act on. +func (u *uploadScheduler) forget(key inodeKey) { + delete(u.states, key) + if i := u.queuedAt(key); i >= 0 { + u.queue = slices.Delete(u.queue, i, i+1) + } +} + +// delay returns the wait before attempt n (1-based), capped at the final entry. +func (u *uploadScheduler) delay(attempt int) time.Duration { + if len(u.backoff) == 0 { + return 0 + } + if attempt < 1 { + attempt = 1 + } + if attempt > len(u.backoff) { + attempt = len(u.backoff) + } + return u.backoff[attempt-1] +} + +// stagedBytes is the owner's accounting of the staging volume, kept as two separate +// totals because they answer two different questions. +// +// - total is how many logical bytes the collector has pinned. One inode is counted +// once, at the size of its staged regular file. It is a diagnostic. +// - retained is how many of those bytes exist *because of* the collector: the ones +// the filesystem would have reclaimed already if this feature were not running. +// It is what backpressure is measured against. +// +// They differ because a capture is a hard link, not a copy. While Ray still holds its +// own link to a rotated segment the blocks belong to Ray's backup ring, and releasing +// the capture would free nothing, so the capture contributes zero to retained. Only +// once Ray rolls the segment off its ring — leaving the staging link as the last +// reference, nlink == 1 — is the collector keeping those blocks alive, and only then +// does the capture count. Gating on total instead would charge the collector for Ray's +// entire backup ring and pause capture during perfectly healthy rotation. +type stagedBytes struct { + sizes map[inodeKey]int64 + // retainedSizes holds an entry only for captures the collector is solely + // responsible for. Presence is the record that nlink was last seen at 1. + retainedSizes map[inodeKey]int64 + total int64 + retained int64 + // stale records that an incremental update did not add up. Rather than let the + // totals drift, the next maintenance sweep recomputes them from the index, which + // is the authoritative list of what the collector is holding. + stale bool +} + +func newStagedBytes() *stagedBytes { + return &stagedBytes{ + sizes: make(map[inodeKey]int64), + retainedSizes: make(map[inodeKey]int64), + } +} + +// observe records a capture's size and whether the collector is now its only owner. +// nlink comes from the same stat as size, so the two describe one moment. +func (b *stagedBytes) observe(key inodeKey, size int64, nlink uint64) { + if prev, ok := b.sizes[key]; ok { + b.total += size - prev + } else { + b.total += size + } + b.sizes[key] = size + + prev, wasRetained := b.retainedSizes[key] + if nlink == 1 { + if wasRetained { + b.retained += size - prev + } else { + b.retained += size + } + b.retainedSizes[key] = size + return + } + // Someone else still holds a link, so these blocks are not this feature's doing. + if wasRetained { + b.retained -= prev + delete(b.retainedSizes, key) + } +} + +func (b *stagedBytes) forget(key inodeKey) { + if size, ok := b.retainedSizes[key]; ok { + b.retained -= size + delete(b.retainedSizes, key) + } + size, ok := b.sizes[key] + if !ok { + // The caller released something accounting never saw, so the totals are no + // longer trustworthy. + b.stale = true + return + } + b.total -= size + delete(b.sizes, key) +} + +func (b *stagedBytes) markStale() { b.stale = true } + +// recompute rebuilds both totals from the captures the index actually holds. +// +// It is also how a capture's ownership is re-read. Ray rolling a segment off its +// backup ring drops nlink from 2 to 1 without touching the staging path, so no +// filesystem event on the staging tree announces it; this sweep is what notices. +// +// Every entry must be proven: a regular file at the entry's own staging path holding +// exactly the inode the index pinned, with size and link count taken from that same +// stat. An entry that cannot be proven keeps its last known size and is counted as +// retained, rather than dropping to zero, and leaves accounting stale. Counting an +// unreadable capture as zero would make retained bytes fall, and a falling total is +// what releases backpressure; the collector would conclude the pressure was over +// precisely because it had lost track of what it was holding. +// +// Only a sweep in which every indexed capture verified may clear stale. +func (b *stagedBytes) recompute(stagingRoot string, ix *captureIndex, report func(error)) { + sizes := make(map[inodeKey]int64, ix.len()) + retainedSizes := make(map[inodeKey]int64, len(b.retainedSizes)) + var total, retained int64 + verified := true + + for key, c := range ix.byInode { + size, nlink, err := verifiedStagedSize(c, stagingRoot) + if err != nil { + verified = false + report(fmt.Errorf("recompute staged bytes: %w", err)) + // Retain what was last known about this capture, and charge it as + // retained: an entry the collector cannot inspect is one it must assume + // it is holding. If nothing was ever known it contributes nothing — but + // accounting stays stale, so that gap can never be mistaken for relieved + // pressure. + if prev, known := b.sizes[key]; known { + sizes[key] = prev + total += prev + retainedSizes[key] = prev + retained += prev + } + continue + } + sizes[key] = size + total += size + if nlink == 1 { + retainedSizes[key] = size + retained += size + } + } + + b.sizes = sizes + b.retainedSizes = retainedSizes + b.total = total + b.retained = retained + b.stale = !verified +} + +// trusted reports whether the totals are currently believed to describe the volume. +func (b *stagedBytes) trusted() bool { return !b.stale } + +// verifiedStagedSize returns the size and link count of a capture's staged file, but +// only once that path has been proven to still be a regular file holding the pinned +// inode. Size, link count and identity come from one stat, so the numbers returned +// describe the file that was checked rather than whatever the path names a moment +// later. +func verifiedStagedSize(c *capture, stagingRoot string) (int64, uint64, error) { + p := c.Entry.path(stagingRoot) + fi, err := os.Lstat(p) + if err != nil { + return 0, 0, fmt.Errorf("stat capture %s at %s: %w", c.Entry.CaptureID, p, err) + } + if !fi.Mode().IsRegular() { + return 0, 0, fmt.Errorf("capture %s at %s is not a regular file (%s)", c.Entry.CaptureID, p, fi.Mode()) + } + staged, nlink, err := inodeFromFileInfo(fi) + if err != nil { + return 0, 0, fmt.Errorf("read inode of capture %s at %s: %w", c.Entry.CaptureID, p, err) + } + if staged != c.Inode { + return 0, 0, fmt.Errorf("capture %s at %s now holds %s, not the pinned %s", c.Entry.CaptureID, p, staged, c.Inode) + } + return fi.Size(), nlink, nil +} + +// intakeGate decides whether new captures may be created. +// +// It is a fail-safe against filling the shared volume Ray is still writing its own +// logs to, not a promise of lossless capture: while intake is paused, a backup that +// Ray rotates away is gone. Nothing already captured is ever evicted to make room — +// deleting pending data to accept newer pending data would trade one loss for +// another and lose the older segment for certain. +type intakeGate struct { + high int64 + low int64 + // watermark is set once staged bytes reach high and cleared once they fall to + // low. The gap is what stops a volume hovering at the threshold from flapping. + watermark bool + // diskFull is set when the filesystem itself refused a link. Watermarks cannot + // predict this: another writer can fill the volume regardless of what the + // collector is holding. + diskFull bool + // spaceObserved records that a filesystem operation proved the volume has room + // again — a release that freed blocks, or a probe link that succeeded. It is what + // clears diskFull, and it is a request rather than the clearing itself so that + // evaluate stays the only place the paused state changes and no transition can be + // lost by a caller flipping the flag behind its back. + // + // Nothing here is time-based. Whoever filled the volume is under no obligation to + // empty it, so only a successful operation may lift the condition. + spaceObserved bool + // probe allows exactly one real capture attempt through a disk-full pause during + // one reconciliation sweep. Without it the pause would be self-latching: capture + // returns before it ever tries to link, so if the volume was filled by another + // process and this collector holds nothing it can release, nothing would ever + // discover that space came back. + probe bool +} + +func (g *intakeGate) enabled() bool { return g.high > 0 } + +func (g *intakeGate) paused() bool { return g.watermark || g.diskFull } + +// applyHighWater engages the watermark the instant tracked bytes reach it, and +// reports whether that is the transition into a paused state. +// +// This exists separately from evaluate because it has to be safe to call in the +// middle of a scan. A scan can register hundreds of captures without returning to +// the event loop, so waiting for the loop's own settling pass would let every +// remaining backup through after the limit was already breached — overshooting the +// high-water mark by an unbounded amount, which defeats the point of having one. +// +// It only ever engages the watermark. It never clears one, never touches diskFull +// and never consumes a capacity observation, so it cannot erase a resume transition +// that the event loop has not seen yet. +func (g *intakeGate) applyHighWater(total int64) (paused bool) { + if !g.enabled() || g.watermark || total < g.high { + return false + } + was := g.paused() + g.watermark = true + // A watermark pause is this collector's own limit, so an outstanding disk-full + // probe must not carry a capture through it. + g.probe = false + return !was +} + +// observedSpace records that a filesystem operation demonstrated the volume has room +// — a release that freed blocks, or a link that succeeded. +func (g *intakeGate) observedSpace() { g.spaceObserved = true } + +// observedFull records that the volume refused a link, and reports whether that is +// the transition into the disk-full state. +// +// It discards any earlier capacity observation. Within one scan an early link can +// succeed and a later one hit ENOSPC; keeping the stale success would let the gate +// clear diskFull on the strength of an observation the filesystem has since +// contradicted. Only an operation that succeeds after this point may lift the +// condition. +func (g *intakeGate) observedFull() (first bool) { + g.spaceObserved = false + first = !g.diskFull + g.diskFull = true + return first +} + +// armProbe permits one capacity probe for the sweep that is starting. +// +// Only a disk-full pause is probed. A watermark pause is this collector's own +// accounting saying it already holds too much, and letting a probe through it would +// be bypassing the very limit it asked for. +func (g *intakeGate) armProbe() { + if g.diskFull && !g.watermark { + g.probe = true + } +} + +func (g *intakeGate) disarmProbe() { g.probe = false } + +// takeProbe consumes the sweep's single allowance, if there is one. +func (g *intakeGate) takeProbe() bool { + if !g.probe { + return false + } + g.probe = false + return true +} + +// evaluate folds the current total into the gate and reports which transition, if +// any, happened. Reporting only on change is what keeps a paused volume from filling +// the log with the same line on every event. +// +// trusted says whether the byte total can be believed. Untrusted accounting may still +// pause intake — erring towards holding back is safe — but it may never resume it: a +// total that fell because the collector lost track of a capture must not be mistaken +// for pressure that genuinely eased. +func (g *intakeGate) evaluate(total int64, trusted bool) (paused, resumed bool) { + was := g.paused() + if g.spaceObserved { + // The volume has room again. If it does not, the next link attempt says so + // and pauses intake once more. + g.diskFull = false + g.spaceObserved = false + } + if g.enabled() { + switch { + case !g.watermark && total >= g.high: + g.watermark = true + case g.watermark && trusted && total <= g.low: + g.watermark = false + } + } + now := g.paused() + return now && !was, !now && was +} + +// validateWatermarks rejects a configuration that could never resume, or that would +// resume the moment it paused. +func validateWatermarks(high, low int64) error { + if high < 0 || low < 0 { + return fmt.Errorf("rotated collector: watermarks must not be negative (high=%d, low=%d)", high, low) + } + if high == 0 { + if low != 0 { + return fmt.Errorf("rotated collector: LowWaterBytes=%d needs a HighWaterBytes to sit below", low) + } + return nil + } + if low >= high { + return fmt.Errorf("rotated collector: LowWaterBytes=%d must be below HighWaterBytes=%d", low, high) + } + return nil +} + +// --------------------------------------------------------------------------- +// Owner-goroutine scheduling. Everything below runs on the collector's own +// goroutine and is the only code allowed to mutate captureIndex, capture state, +// retry state, byte accounting and the intake gate. +// --------------------------------------------------------------------------- + +func (rc *rotatedCollector) now() time.Time { return rc.cfg.Now() } + +// uploadResults is the owner's receive side. A disabled uploader leaves it nil, and a +// nil channel simply never fires. +func (rc *rotatedCollector) uploadResults() <-chan uploadResult { return rc.up.results } + +func (rc *rotatedCollector) retryTimer() <-chan time.Time { return rc.up.retryC } + +// intakePaused reports whether capture may create new staging links. +func (rc *rotatedCollector) intakePaused() bool { return rc.gate.paused() } + +// pump is the owner's between-events housekeeping: settle the intake gate, hand work +// to the worker, and arm the retry timer. It runs at the top of every loop iteration, +// before any request is served, so a caller that observes a snapshot is observing +// state the scheduler has already acted on. +// +// It never blocks. Dispatch only sends when nothing is in flight, and the job channel +// has room for exactly that one job. +// +// It returns an error only for a condition no retry can fix, which stops the +// collector rather than letting it schedule impossible work forever. +func (rc *rotatedCollector) pump() error { + if rc.settleIntake() { + // Resuming means the volume has room again, and backups that were skipped + // while it did not may still be on disk. Reconcile at once rather than wait + // for the tick, because rotation is still deleting them. + rc.maintain() + rc.settleIntake() + } + if err := rc.dispatchUploads(); err != nil { + return err + } + rc.armRetryTimer() + return nil +} + +// enforceHighWater settles the watermark against the retained total right where that +// total changed, so a sweep cannot keep capturing past the limit. +func (rc *rotatedCollector) enforceHighWater() { + if rc.gate.applyHighWater(rc.bytes.retained) { + rc.reportIntakePaused() + } +} + +func (rc *rotatedCollector) reportIntakePaused() { + rc.intakePauses++ + rc.report(fmt.Errorf("%w: %d retained byte(s) reached the high-water mark of %d; captured data is kept and uploads continue, but backups rotated away from now on cannot be preserved", + errIntakePaused, rc.bytes.retained, rc.gate.high)) +} + +// settleIntake applies the retained total to the gate and reports transitions. +// +// Retained rather than logical bytes: the gate exists to bound the disk this feature +// keeps allocated, and a capture Ray still has its own link to keeps none. +// +// The transition counters matter beyond diagnostics: a resume followed immediately by +// a fresh pause leaves the gate looking untouched, so the count is the only evidence +// that intake was ever wrongly reopened. +func (rc *rotatedCollector) settleIntake() (resumed bool) { + paused, resumed := rc.gate.evaluate(rc.bytes.retained, rc.bytes.trusted()) + if paused { + rc.reportIntakePaused() + } + if resumed { + rc.intakeResumes++ + logrus.Infof("Rotated log collector: retained staging fell to %d byte(s); resuming intake", rc.bytes.retained) + } + return resumed +} + +// maintain is the backstop sweep: rediscover the tree, queue anything pending that is +// not already scheduled, retry releases, re-establish accounting and act on anything +// whose retry has come due. +// +// Accounting is recomputed on every sweep rather than only when it is marked stale, +// because a capture's ownership changes without anything touching the staging tree: +// when Ray rolls a segment off its backup ring the staging link's nlink falls from 2 +// to 1, and that is the moment the collector starts retaining blocks the filesystem +// would otherwise have reclaimed. Nothing reports that, so the sweep re-reads it. It +// costs one Lstat per tracked capture and reuses the pass that already existed. +// +// It runs after the releases so the totals describe what is still held once this +// sweep's releases are done. +// +// The sweep is also the only place a disk-full pause can be tested, so it opens with +// a single probe allowance and closes by withdrawing whatever is left of it. That +// bound is what keeps a full volume from being retried once per filesystem event. +func (rc *rotatedCollector) maintain() { + rc.gate.armProbe() + rc.scanTree() + rc.gate.disarmProbe() + + rc.sweepUploads() + rc.sweepReleases() + rc.bytes.recompute(rc.cfg.StagingRoot, rc.ix, rc.report) + rc.processDue() +} + +// sweepUploads queues every pending capture the pipeline is not already handling. +// It is idempotent: a capture that is queued, in flight, backing off or awaiting +// promotion already has state, and state is what stops a second job being created. +func (rc *rotatedCollector) sweepUploads() { + if !rc.up.enabled() { + return + } + for key, c := range rc.ix.byInode { + if c.Entry.State != statePending { + continue + } + rc.enqueueUpload(key) + } +} + +// enqueueUpload schedules one pending capture for upload. +func (rc *rotatedCollector) enqueueUpload(key inodeKey) { + if !rc.up.enabled() { + return + } + c, ok := rc.ix.lookup(key) + if !ok || c.Entry.State != statePending { + return + } + if _, busy := rc.up.states[key]; busy { + return + } + rc.up.states[key] = &uploadState{phase: phaseQueued} + rc.up.enqueue(key, c.Entry.CaptureID) +} + +// dispatchUploads hands queued work to the worker while it is free. +// +// The loop condition is the deduplication that matters: one job may be outstanding, +// and inFlight is only cleared when its result is applied, so a capture cannot be +// uploaded twice concurrently no matter how many events name it. +func (rc *rotatedCollector) dispatchUploads() error { + if !rc.up.enabled() { + return nil + } + for rc.up.inFlight == 0 { + head, ok := rc.up.dequeue() + if !ok { + return nil + } + st, tracked := rc.up.states[head.key] + if !tracked || st.phase != phaseQueued { + // The queue entry lost its state, or the state moved on without it. + // Whatever the state says is authoritative. + continue + } + c, present := rc.ix.lookup(head.key) + if !present || c.Entry.State != statePending { + // Released, or already uploaded by an earlier result. Nothing to send. + rc.up.forget(head.key) + continue + } + + job, err := rc.newUploadJob(c, st.attempts+1) + if err != nil { + // The key is a pure function of validated capture identity and static + // configuration, so if it cannot be built now it will never be built. + // Retrying it on the transport schedule would spin forever, so this + // stops the collector with the pending staging link untouched. + rc.up.forget(head.key) + return fmt.Errorf("%w: %w", errStagingInconsistent, err) + } + + st.phase = phaseInFlight + st.job = job + st.dueAt = time.Time{} + rc.up.inFlight++ + // Cannot block: inFlight was zero, so the worker holds no job and the + // capacity-1 channel is empty. + rc.up.jobs <- job + } + return nil +} + +// newUploadJob freezes everything the worker and the result check need. +// +// The object key comes from the capture's own identity — session, node, relative +// directory, original name, capture ID — and never from its staging state, so a +// retry, a restart and a promotion all address the same object. +func (rc *rotatedCollector) newUploadJob(c *capture, attempt int) (uploadJob, error) { + key := c.Entry.objectKey(rc.cfg.Cluster) + prefix := rc.cfg.Cluster.logsPrefix(c.Entry.SessionName, c.Entry.NodeName) + if !strings.HasPrefix(key, prefix+"/") { + return uploadJob{}, fmt.Errorf("object key %q for capture %s escapes the cluster log prefix %q", + key, c.Entry.CaptureID, prefix) + } + return uploadJob{ + uploadIdentity: uploadIdentity{ + inode: c.Inode, + entry: c.Entry, + localPath: c.Entry.path(rc.cfg.StagingRoot), + objectKey: key, + }, + attempt: attempt, + }, nil +} + +// applyUploadResult is the only place a result may change anything. +// +// Nothing is trusted from the worker except the job it was given: the result is +// matched against the state the owner recorded when it submitted, and then against +// the capture the index currently holds. Only a result that still describes exactly +// that capture is acted on. +// +// It returns an error only when the staging volume contradicted the index. That is +// not an outage and no amount of retrying resolves it, so it stops the collector. +func (rc *rotatedCollector) applyUploadResult(res uploadResult) error { + if rc.up.inFlight > 0 { + rc.up.inFlight-- + } + + st, tracked := rc.up.states[res.job.inode] + if !tracked || st.phase != phaseInFlight || st.job.uploadIdentity != res.job.uploadIdentity { + rc.report(fmt.Errorf("%w: capture %s (%s)", errUploadStale, res.job.entry.CaptureID, res.job.inode)) + return nil + } + c, present := rc.ix.lookup(res.job.inode) + if !present || c.Entry != res.job.entry { + rc.up.forget(res.job.inode) + rc.report(fmt.Errorf("%w: capture %s (%s) is no longer staged as submitted", errUploadStale, res.job.entry.CaptureID, res.job.inode)) + return nil + } + + if res.err != nil { + // A local validation failure is categorically different from an object-store + // one. The store being unreachable is temporary and the same bytes will go + // out later; a staged path that is missing, non-regular, or holding a + // different inode than the one the index pinned is a durable contradiction + // between memory and disk. Reconciliation cannot repair it — the index + // entry keeps naming a file that is not there — and no storage call will + // ever succeed, so retrying on the transport schedule would mean retrying + // forever. Fail closed instead, leaving the pending staging state exactly as + // it is for an operator, or the next run's reconstruction, to resolve. + if res.local { + rc.up.forget(res.job.inode) + return fmt.Errorf("%w: capture %s staged at %s: %w", + errStagingInconsistent, res.job.entry.CaptureID, res.job.localPath, res.err) + } + rc.failUpload(st, res.job.entry.CaptureID, res.err) + return nil + } + + // The bytes are in storage. From here the capture is only ever promoted, never + // re-uploaded, even if the promotion itself has to be retried. + st.phase = phaseAwaitingPromotion + st.attempts = 0 + rc.completeUpload(res.job.inode) + return nil +} + +// failUpload keeps the capture pending and schedules a retry. The capture ID, the +// staged link and the object key are all untouched, so the retry is the identical +// write to the identical key. +func (rc *rotatedCollector) failUpload(st *uploadState, captureID string, err error) { + st.phase = phaseBackoff + st.attempts++ + delay := rc.up.delay(st.attempts) + st.dueAt = rc.now().Add(delay) + rc.report(fmt.Errorf("upload of capture %s failed (attempt %d), retrying in %s: %w", + captureID, st.attempts, delay, err)) +} + +// completeUpload promotes a capture whose object write has succeeded. +// +// Promotion is the durable record that the bytes are safe, so it happens only after +// the write returns, and the pending staging link is kept until it does. If the +// rename fails the capture stays pending on disk and in the index — consistent, just +// not yet promoted — and only the promotion is retried. Re-uploading would be a +// second write of bytes storage already has. +func (rc *rotatedCollector) completeUpload(key inodeKey) { + st, tracked := rc.up.states[key] + if !tracked || st.phase != phaseAwaitingPromotion { + return + } + c, present := rc.ix.lookup(key) + if !present || c.Entry != st.job.entry || c.Entry.State != statePending { + rc.up.forget(key) + rc.report(fmt.Errorf("%w: capture %s cannot be promoted after upload", errUploadStale, st.job.entry.CaptureID)) + return + } + + if _, err := promoteCapture(rc.cfg.StagingRoot, rc.ix, key); err != nil { + st.attempts++ + delay := rc.up.delay(st.attempts) + st.dueAt = rc.now().Add(delay) + rc.report(fmt.Errorf("capture %s is uploaded to %s but could not be promoted locally (attempt %d); the pending staging link is kept and promotion retries in %s: %w", + st.job.entry.CaptureID, st.job.objectKey, st.attempts, delay, err)) + return + } + + // Promotion rewrote the entry, so the submitted identity no longer matches + // anything: the pipeline is done with this capture. + rc.up.forget(key) + rc.releaseIfUnheld(key) +} + +// processDue acts on every capture whose retry time has arrived. +func (rc *rotatedCollector) processDue() { + if !rc.up.enabled() { + return + } + now := rc.now() + + var promotions []queuedUpload + for key, st := range rc.up.states { + if st.dueAt.IsZero() || now.Before(st.dueAt) { + continue + } + switch st.phase { + case phaseBackoff: + // Queue order comes from the index rather than the submitted job: a + // capture whose very first job could not be built has no job to read an + // ID from, and the index is authoritative in any case. + c, present := rc.ix.lookup(key) + if !present { + delete(rc.up.states, key) + continue + } + st.phase = phaseQueued + st.dueAt = time.Time{} + rc.up.enqueue(key, c.Entry.CaptureID) + case phaseAwaitingPromotion: + promotions = append(promotions, queuedUpload{key: key, captureID: st.job.entry.CaptureID}) + case phaseQueued, phaseInFlight: + } + } + + // completeUpload mutates the state map, so it runs outside the range, and in + // capture-ID order so a sweep is not at the mercy of map iteration. + sort.Slice(promotions, func(i, j int) bool { return promotions[i].captureID < promotions[j].captureID }) + for _, p := range promotions { + rc.completeUpload(p.key) + } +} + +// armRetryTimer keeps one timer armed for the earliest outstanding retry. Re-arming +// for a time already armed is skipped so that an idle loop does not churn timers. +func (rc *rotatedCollector) armRetryTimer() { + if !rc.up.enabled() { + return + } + var earliest time.Time + for _, st := range rc.up.states { + if st.dueAt.IsZero() { + continue + } + if earliest.IsZero() || st.dueAt.Before(earliest) { + earliest = st.dueAt + } + } + if earliest.IsZero() { + rc.up.disarm() + return + } + if earliest.Equal(rc.up.armedFor) { + return + } + if rc.up.stopTimer != nil { + rc.up.stopTimer() + } + wait := max(earliest.Sub(rc.now()), 0) + rc.up.retryC, rc.up.stopTimer = rc.cfg.NewTimer(wait) + rc.up.armedFor = earliest +} + +// sweepReleases retries the release of every uploaded capture, in capture-ID order. +// Reconstructed uploaded captures come through here too: they are never re-uploaded, +// but they are always candidates for release. +func (rc *rotatedCollector) sweepReleases() { + var uploaded []queuedUpload + for key, c := range rc.ix.byInode { + if c.Entry.State == stateUploaded { + uploaded = append(uploaded, queuedUpload{key: key, captureID: c.Entry.CaptureID}) + } + } + sort.Slice(uploaded, func(i, j int) bool { return uploaded[i].captureID < uploaded[j].captureID }) + for _, u := range uploaded { + rc.releaseIfUnheld(u.key) + } +} + +// releaseIfUnheld drops an uploaded capture's staging link once Ray has let go of its +// own. +// +// A remaining link is the expected steady state, not a failure: Ray keeps a rotated +// segment until its backup count rolls it off, and the collector's whole purpose is to +// hold the inode until then. That case is logged at debug and retried by the next +// sweep. A missing file or a different inode is different in kind — the index and the +// volume disagree — and is reported, with byte accounting marked for recomputation. +// +// The link count is checked here only to tell the expected case from the surprising +// one. releaseCapture re-reads it immediately before unlinking, so the decision that +// actually removes the file rests on its own fresh evidence. +func (rc *rotatedCollector) releaseIfUnheld(key inodeKey) { + c, present := rc.ix.lookup(key) + if !present || c.Entry.State != stateUploaded { + return + } + + p := c.Entry.path(rc.cfg.StagingRoot) + staged, nlink, err := statInode(p) + if err != nil { + rc.bytes.markStale() + rc.report(fmt.Errorf("release capture %s: index and staging volume disagree about %s: %w", c.Entry.CaptureID, p, err)) + return + } + if staged != c.Inode { + rc.bytes.markStale() + rc.report(fmt.Errorf("release capture %s: %s now holds %s, not the pinned %s", c.Entry.CaptureID, p, staged, c.Inode)) + return + } + if nlink != 1 { + logrus.Debugf("Rotated log collector: capture %s still has %d link(s); Ray has not finished with the segment", + c.Entry.CaptureID, nlink) + return + } + + if err := releaseCapture(rc.cfg.StagingRoot, rc.ix, key); err != nil { + rc.report(fmt.Errorf("release capture %s: %w", c.Entry.CaptureID, err)) + return + } + + rc.bytes.forget(key) + // Space came back, so a filesystem that refused a link may accept one now. The + // gate decides what that means: clearing the flag here would erase the paused -> + // resumed transition before anything could act on it. The watermark is untouched + // either way, because only crossing the low-water mark clears that. + rc.gate.spaceObserved = true +} diff --git a/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_uploader_test.go b/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_uploader_test.go new file mode 100644 index 00000000000..56d92c6dedb --- /dev/null +++ b/historyserver/pkg/collector/logcollector/runtime/logcollector/rotated_uploader_test.go @@ -0,0 +1,2322 @@ +package logcollector + +import ( + "errors" + "io" + "os" + "path" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" + "syscall" + "testing" + "time" + + "github.com/ray-project/kuberay/historyserver/pkg/storage" +) + +// The uploader is written against the storage writer the rest of the collector +// already uses; nothing in this tranche widens that interface. +var _ objectWriter = (storage.StorageWriter)(nil) + +// errUploadRejected is what the fake object store returns when a test wants a write +// to fail. +var errUploadRejected = errors.New("object store rejected the write") + +// testCluster is a plain RayCluster, whose logs are not nested under an owner. +var testCluster = clusterIdentity{ + RootDir: "root", + OwnerKind: "RayCluster", + Namespace: "ns", + ClusterName: "cluster-a", +} + +// testBackoff is short and obvious so the retry schedule can be asserted exactly. +var testBackoff = []time.Duration{time.Second, 2 * time.Second, 4 * time.Second} + +// writeCall records one attempted object write. +type writeCall struct { + key string + content string + err error +} + +// fakeWriter is an object store that can block, fail and count. It never touches the +// filesystem, so a test can prove the worker read the staged descriptor it was given +// by comparing the bytes that arrived. +type fakeWriter struct { + failures map[string]int + release chan struct{} + entered chan string + calls []writeCall + mu sync.Mutex + failAll bool + active int + maxActive int +} + +func newFakeWriter() *fakeWriter { + return &fakeWriter{ + failures: make(map[string]int), + entered: make(chan string, 64), + } +} + +func (w *fakeWriter) WriteFile(file string, reader io.ReadSeeker) error { + w.mu.Lock() + w.active++ + if w.active > w.maxActive { + w.maxActive = w.active + } + fail := w.failAll + if n := w.failures[file]; n > 0 { + w.failures[file] = n - 1 + fail = true + } + release := w.release + w.mu.Unlock() + + select { + case w.entered <- file: + default: + } + if release != nil { + <-release + } + + body, err := io.ReadAll(reader) + + w.mu.Lock() + defer w.mu.Unlock() + w.active-- + switch { + case err != nil: + w.calls = append(w.calls, writeCall{key: file, err: err}) + return err + case fail: + w.calls = append(w.calls, writeCall{key: file, err: errUploadRejected}) + return errUploadRejected + default: + w.calls = append(w.calls, writeCall{key: file, content: string(body)}) + return nil + } +} + +// blockWrites makes every write wait until the returned function is called. +func (w *fakeWriter) blockWrites() func() { + release := make(chan struct{}) + w.mu.Lock() + w.release = release + w.mu.Unlock() + + var once sync.Once + return func() { + once.Do(func() { + w.mu.Lock() + w.release = nil + w.mu.Unlock() + close(release) + }) + } +} + +func (w *fakeWriter) setFailAll(v bool) { + w.mu.Lock() + defer w.mu.Unlock() + w.failAll = v +} + +func (w *fakeWriter) attempts() []writeCall { + w.mu.Lock() + defer w.mu.Unlock() + return append([]writeCall(nil), w.calls...) +} + +func (w *fakeWriter) attemptCount() int { + w.mu.Lock() + defer w.mu.Unlock() + return len(w.calls) +} + +func (w *fakeWriter) concurrentPeak() int { + w.mu.Lock() + defer w.mu.Unlock() + return w.maxActive +} + +// stored returns the content of the last successful write for key. +func (w *fakeWriter) stored(key string) (string, bool) { + w.mu.Lock() + defer w.mu.Unlock() + for i := len(w.calls) - 1; i >= 0; i-- { + if w.calls[i].key == key && w.calls[i].err == nil { + return w.calls[i].content, true + } + } + return "", false +} + +// fakeClock is the collector's clock. Retry deadlines are computed from it, so a test +// can move time instead of waiting for it. +type fakeClock struct { + t time.Time + mu sync.Mutex +} + +func newFakeClock() *fakeClock { + return &fakeClock{t: time.Date(2026, 7, 31, 12, 0, 0, 0, time.UTC)} +} + +func (c *fakeClock) now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.t +} + +func (c *fakeClock) advance(d time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.t = c.t.Add(d) +} + +// fakeTimers records every retry delay the collector asks for and fires on demand. +// The channel is unbuffered, so firing it also proves the owner loop received it. +type fakeTimers struct { + c chan time.Time + durs []time.Duration + mu sync.Mutex + stops int +} + +func newFakeTimers() *fakeTimers { + return &fakeTimers{c: make(chan time.Time)} +} + +func (f *fakeTimers) newTimer(d time.Duration) (<-chan time.Time, func()) { + f.mu.Lock() + f.durs = append(f.durs, d) + f.mu.Unlock() + return f.c, func() { + f.mu.Lock() + f.stops++ + f.mu.Unlock() + } +} + +func (f *fakeTimers) delays() []time.Duration { + f.mu.Lock() + defer f.mu.Unlock() + return append([]time.Duration(nil), f.durs...) +} + +// fire delivers one retry tick and waits for the loop to take it. +func (f *fakeTimers) fire(t *testing.T) { + t.Helper() + select { + case f.c <- time.Now(): + case <-time.After(5 * time.Second): + t.Fatal("timed out firing the retry timer: nothing was armed") + } +} + +// upHarness is a running collector with an uploader attached. +type upHarness struct { + *harness + writer *fakeWriter + clock *fakeClock + timers *fakeTimers +} + +func startUploading(t *testing.T, dir string, tweak func(*rotatedCollectorConfig)) *upHarness { + t.Helper() + u := &upHarness{ + writer: newFakeWriter(), + clock: newFakeClock(), + timers: newFakeTimers(), + } + u.harness = startWith(t, dir, func(cfg *rotatedCollectorConfig) { + cfg.Writer = u.writer + cfg.Cluster = testCluster + cfg.Now = u.clock.now + cfg.NewTimer = u.timers.newTimer + cfg.UploadBackoff = testBackoff + cfg.WorkerStopGrace = 50 * time.Millisecond + if tweak != nil { + tweak(cfg) + } + }) + return u +} + +// waitFor round-trips the owner loop until cond holds. The round-trip is the +// synchronization point: anything the scheduler was going to do has been done by the +// time a request is served. +func (u *upHarness) waitFor(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + u.rc.stats() + if cond() { + return + } + time.Sleep(time.Millisecond) + } + t.Fatalf("timed out waiting for %s (attempts=%d, stats=%+v)", what, u.writer.attemptCount(), u.rc.stats()) +} + +func (u *upHarness) waitForAttempts(t *testing.T, n int) { + t.Helper() + u.waitFor(t, "storage attempt "+strconv.Itoa(n), func() bool { return u.writer.attemptCount() >= n }) +} + +// waitForUploaded waits until exactly n captures have been promoted. +func (u *upHarness) waitForUploaded(t *testing.T, n int) { + t.Helper() + u.waitFor(t, strconv.Itoa(n)+" uploaded capture(s)", func() bool { + count := 0 + for _, e := range u.rc.snapshot() { + if e.State == stateUploaded { + count++ + } + } + return count == n + }) +} + +// captureOne writes the active raylet.out log and one rotation backup, then delivers +// the event and waits for the capture to be registered. +func (u *upHarness) captureOne(t *testing.T, backup, content string) stagedEntry { + t.Helper() + u.writeLog(t, "raylet.out", "active") + p := u.writeLog(t, backup, content) + u.sendEvent(t, p) + for _, e := range u.rc.snapshot() { + if e.OriginalName == filepath.Base(backup) { + return e + } + } + t.Fatalf("%s was not captured: %+v", backup, u.rc.snapshot()) + return stagedEntry{} +} + +// stageManually pins a backup exactly as a previous collector run would have, without +// a collector running. It returns the entry and the inode it pinned. +func stageManually(t *testing.T, logsDir, stagingRoot, name, content string, promote bool) (stagedEntry, inodeKey) { + t.Helper() + src := filepath.Join(logsDir, name) + writeFile(t, src, content) + + id, err := newCaptureIDGenerator().next() + if err != nil { + t.Fatalf("next() error: %v", err) + } + entry, err := newStagedEntry(statePending, "session-1", "node-1", "", name, id) + if err != nil { + t.Fatalf("newStagedEntry() error: %v", err) + } + if err := captureLink(src, entry.path(stagingRoot)); err != nil { + t.Fatalf("captureLink() error: %v", err) + } + key, _, err := statInode(entry.path(stagingRoot)) + if err != nil { + t.Fatalf("statInode() error: %v", err) + } + if !promote { + return entry, key + } + + ix := newCaptureIndex() + if _, _, err := ix.add(key, entry); err != nil { + t.Fatalf("add() error: %v", err) + } + promoted, err := promoteCapture(stagingRoot, ix, key) + if err != nil { + t.Fatalf("promoteCapture() error: %v", err) + } + return promoted, key +} + +// 1. A newly captured pending entry is scheduled for upload. +func TestUploaderSchedulesNewlyCapturedPendingEntry(t *testing.T) { + dir := t.TempDir() + u := startUploading(t, dir, nil) + + const content = "rotated bytes" + entry := u.captureOne(t, "raylet.out.1", content) + wantKey := entry.objectKey(testCluster) + + u.waitForAttempts(t, 1) + got, ok := u.writer.stored(wantKey) + if !ok { + t.Fatalf("no successful write to %s, attempts = %+v", wantKey, u.writer.attempts()) + } + if got != content { + t.Errorf("uploaded %q, want %q", got, content) + } +} + +// 2. A blocked upload does not block fsnotify handling or snapshots. +func TestUploaderBlockedUploadDoesNotBlockTheOwnerLoop(t *testing.T) { + dir := t.TempDir() + u := startUploading(t, dir, nil) + release := u.writer.blockWrites() + defer release() + + u.captureOne(t, "raylet.out.1", "first") + select { + case <-u.writer.entered: + case <-time.After(5 * time.Second): + t.Fatal("the first upload never started") + } + + // The worker is now parked inside the object store. Every owner-loop duty must + // still be prompt. + start := time.Now() + for i := 2; i <= 6; i++ { + p := u.writeLog(t, "raylet.out."+strconv.Itoa(i), "segment") + u.sendEvent(t, p) + } + u.rc.reconcileNow() + entries := u.rc.snapshot() + elapsed := time.Since(start) + + if len(entries) != 6 { + t.Errorf("captured %d segments while an upload was blocked, want 6", len(entries)) + } + if elapsed > 5*time.Second { + t.Errorf("event handling took %v while an upload was blocked, which means uploads run on the owner loop", elapsed) + } +} + +// 3. Only one upload is in flight for one capture, and only one at a time overall. +func TestUploaderKeepsOneUploadInFlight(t *testing.T) { + dir := t.TempDir() + u := startUploading(t, dir, nil) + release := u.writer.blockWrites() + + u.writeLog(t, "raylet.out", "active") + for i := 1; i <= 4; i++ { + p := u.writeLog(t, "raylet.out."+strconv.Itoa(i), "segment "+strconv.Itoa(i)) + u.sendEvent(t, p) + } + select { + case <-u.writer.entered: + case <-time.After(5 * time.Second): + t.Fatal("no upload started") + } + + s := u.rc.stats() + if s.InFlightUploads != 1 { + t.Errorf("InFlightUploads = %d, want exactly 1", s.InFlightUploads) + } + if got := u.writer.attemptCount(); got != 0 { + t.Errorf("%d writes completed while the store was blocked, want 0", got) + } + + release() + u.waitForUploaded(t, 4) + if peak := u.writer.concurrentPeak(); peak != 1 { + t.Errorf("peak concurrent writes = %d, want 1", peak) + } + if got := u.writer.attemptCount(); got != 4 { + t.Errorf("%d storage attempts for 4 captures, want 4: %+v", got, u.writer.attempts()) + } +} + +// 4. Upload success promotes pending -> uploaded, on disk and in the index. +func TestUploadSuccessPromotesPendingToUploaded(t *testing.T) { + dir := t.TempDir() + u := startUploading(t, dir, nil) + + entry := u.captureOne(t, "raylet.out.1", "segment") + u.waitForUploaded(t, 1) + + got := u.rc.snapshot()[0] + if got.CaptureID != entry.CaptureID { + t.Errorf("capture ID changed on promotion: %s -> %s", entry.CaptureID, got.CaptureID) + } + if _, err := os.Lstat(got.path(u.stagingRoot)); err != nil { + t.Errorf("uploaded staging link missing: %v", err) + } + if _, err := os.Lstat(entry.path(u.stagingRoot)); !os.IsNotExist(err) { + t.Errorf("pending staging link still present after promotion: %v", err) + } +} + +// 5. Upload failure leaves the entry pending, with its link and identity intact. +func TestUploadFailureLeavesCapturePending(t *testing.T) { + dir := t.TempDir() + u := startUploading(t, dir, nil) + u.writer.setFailAll(true) + + entry := u.captureOne(t, "raylet.out.1", "segment") + u.waitForAttempts(t, 1) + + entries := u.rc.snapshot() + if len(entries) != 1 || entries[0] != entry { + t.Fatalf("capture changed after a failed upload: %+v, want %+v", entries, entry) + } + if _, err := os.Lstat(entry.path(u.stagingRoot)); err != nil { + t.Errorf("pending staging link was not kept after a failed upload: %v", err) + } + if s := u.rc.stats(); s.Uploaded != 0 || s.Pending != 1 { + t.Errorf("stats = %+v, want one pending and nothing uploaded", s) + } +} + +// 6. Failed uploads retry on the injected backoff schedule, capped at its last entry. +// 7. Every retry writes the identical object key. +func TestUploadRetriesFollowBackoffAndReuseTheSameKey(t *testing.T) { + dir := t.TempDir() + u := startUploading(t, dir, nil) + u.writer.setFailAll(true) + + entry := u.captureOne(t, "raylet.out.1", "segment") + wantKey := entry.objectKey(testCluster) + + // testBackoff is 1s, 2s, 4s, and 4s repeats once the sequence is exhausted. + want := []time.Duration{time.Second, 2 * time.Second, 4 * time.Second, 4 * time.Second} + for i, delay := range want { + u.waitForAttempts(t, i+1) + u.waitFor(t, "retry timer armed", func() bool { return len(u.timers.delays()) >= i+1 }) + + if got := u.timers.delays()[i]; got != delay { + t.Errorf("retry %d armed for %v, want %v (all delays: %v)", i+1, got, delay, u.timers.delays()) + } + u.clock.advance(delay) + u.timers.fire(t) + } + u.waitForAttempts(t, len(want)+1) + + for i, call := range u.writer.attempts() { + if call.key != wantKey { + t.Errorf("attempt %d wrote key %q, want the original %q", i+1, call.key, wantKey) + } + } + if entries := u.rc.snapshot(); len(entries) != 1 || entries[0] != entry { + t.Errorf("capture identity changed across retries: %+v", entries) + } +} + +// 8. A restart uploads reconstructed pending entries under their original identity. +func TestRestartUploadsPendingWithoutMintingNewIDs(t *testing.T) { + dir := t.TempDir() + logsDir := filepath.Join(dir, "session", "logs") + stagingRoot := filepath.Join(dir, "rotated-staging") + writeFile(t, filepath.Join(logsDir, "raylet.out"), "active") + entry, _ := stageManually(t, logsDir, stagingRoot, "raylet.out.1", "left over", false) + + u := startUploading(t, dir, nil) + u.waitForAttempts(t, 1) + + if got := u.writer.attempts()[0].key; got != entry.objectKey(testCluster) { + t.Errorf("restart uploaded key %q, want the original %q", got, entry.objectKey(testCluster)) + } + u.waitForUploaded(t, 1) + if got := u.rc.snapshot()[0].CaptureID; got != entry.CaptureID { + t.Errorf("restart minted capture ID %s for an existing staged capture %s", got, entry.CaptureID) + } +} + +// 9. A restart never re-uploads an entry reconstructed as uploaded. +func TestRestartDoesNotReuploadUploadedEntries(t *testing.T) { + dir := t.TempDir() + logsDir := filepath.Join(dir, "session", "logs") + stagingRoot := filepath.Join(dir, "rotated-staging") + writeFile(t, filepath.Join(logsDir, "raylet.out"), "active") + uploaded, _ := stageManually(t, logsDir, stagingRoot, "raylet.out.1", "already sent", true) + + u := startUploading(t, dir, nil) + // Give every path that could schedule an upload a chance to run. + u.rc.reconcileNow() + u.fireTick(t) + u.rc.reconcileNow() + + if got := u.writer.attemptCount(); got != 0 { + t.Errorf("an already-uploaded capture was re-uploaded %d time(s): %+v", got, u.writer.attempts()) + } + entries := u.rc.snapshot() + if len(entries) != 1 || entries[0] != uploaded { + t.Errorf("reconstructed uploaded capture = %+v, want %+v", entries, uploaded) + } +} + +// 10. A restart retries release of reconstructed uploaded entries. +func TestRestartReleasesReconstructedUploadedEntries(t *testing.T) { + dir := t.TempDir() + logsDir := filepath.Join(dir, "session", "logs") + stagingRoot := filepath.Join(dir, "rotated-staging") + writeFile(t, filepath.Join(logsDir, "raylet.out"), "active") + uploaded, _ := stageManually(t, logsDir, stagingRoot, "raylet.out.1", "already sent", true) + + // Ray has since dropped its own link, so the staged link is the last one. + if err := os.Remove(filepath.Join(logsDir, "raylet.out.1")); err != nil { + t.Fatalf("remove Ray's link: %v", err) + } + + u := startUploading(t, dir, nil) + + if entries := u.rc.snapshot(); len(entries) != 0 { + t.Errorf("startup did not release the uploaded capture: %+v", entries) + } + if _, err := os.Lstat(uploaded.path(stagingRoot)); !os.IsNotExist(err) { + t.Errorf("uploaded staging link was not unlinked: %v", err) + } + if got := u.writer.attemptCount(); got != 0 { + t.Errorf("released capture was also uploaded %d time(s)", got) + } +} + +// 11. A successful upload while Ray still holds a link leaves the entry uploaded. +// 12. Once Ray drops its link, maintenance releases the staged link. +// 27. A successful release decreases accounting exactly once. +func TestUploadedCaptureIsReleasedOnlyAfterRayLetsGo(t *testing.T) { + dir := t.TempDir() + u := startUploading(t, dir, nil) + + const content = "segment bytes" + entry := u.captureOne(t, "raylet.out.1", content) + u.waitForUploaded(t, 1) + + // Ray still holds raylet.out.1, so the segment must stay pinned. + u.fireTick(t) + if entries := u.rc.snapshot(); len(entries) != 1 || entries[0].State != stateUploaded { + t.Fatalf("capture was released while Ray still held a link: %+v", entries) + } + before := u.rc.stats() + if before.StagedBytes != int64(len(content)) { + t.Fatalf("StagedBytes = %d, want %d", before.StagedBytes, len(content)) + } + + if err := os.Remove(filepath.Join(u.logsDir, "raylet.out.1")); err != nil { + t.Fatalf("remove Ray's link: %v", err) + } + u.fireTick(t) + + if entries := u.rc.snapshot(); len(entries) != 0 { + t.Errorf("capture was not released after Ray dropped its link: %+v", entries) + } + if _, err := os.Lstat(entry.withState(stateUploaded).path(u.stagingRoot)); !os.IsNotExist(err) { + t.Errorf("staged link still present after release: %v", err) + } + after := u.rc.stats() + if after.StagedBytes != 0 { + t.Errorf("StagedBytes = %d after release, want 0", after.StagedBytes) + } + + // A second sweep must not double-count. + u.fireTick(t) + if got := u.rc.stats().StagedBytes; got != 0 { + t.Errorf("StagedBytes = %d after a second sweep, want 0", got) + } +} + +// 13. A release that cannot proceed retains the index entry and the byte accounting. +func TestReleaseFailureRetainsEntryAndAccounting(t *testing.T) { + dir := t.TempDir() + u := startUploading(t, dir, nil) + + const content = "still held by ray" + u.captureOne(t, "raylet.out.1", content) + u.waitForUploaded(t, 1) + + for range 3 { + u.fireTick(t) + } + s := u.rc.stats() + if s.Captures != 1 || s.Uploaded != 1 { + t.Errorf("stats = %+v, want the uploaded capture retained", s) + } + if s.StagedBytes != int64(len(content)) { + t.Errorf("StagedBytes = %d, want %d retained across failed releases", s.StagedBytes, len(content)) + } +} + +// 14. A stale success result cannot promote a replaced or untracked capture. +func TestStaleUploadResultCannotPromote(t *testing.T) { + dir := t.TempDir() + logsDir := filepath.Join(dir, "session", "logs") + stagingRoot := filepath.Join(dir, "rotated-staging") + writeFile(t, filepath.Join(logsDir, "raylet.out"), "active") + entry, key := stageManually(t, logsDir, stagingRoot, "raylet.out.1", "segment", false) + + // A collector that is not running: this test is the only goroutine, so it may + // drive the owner-side functions directly. + issues := &issueLog{} + rc, err := newRotatedCollector(rotatedCollectorConfig{ + LogsDir: logsDir, + StagingRoot: stagingRoot, + SessionName: "session-1", + NodeName: "node-1", + Writer: newFakeWriter(), + Cluster: testCluster, + OnIssue: issues.add, + }) + if err != nil { + t.Fatalf("newRotatedCollector() error: %v", err) + } + if _, err := rc.ix.restore(key, entry); err != nil { + t.Fatalf("restore() error: %v", err) + } + + job := uploadJob{uploadIdentity: uploadIdentity{ + inode: key, + entry: entry, + localPath: entry.path(stagingRoot), + objectKey: entry.objectKey(testCluster), + }} + + // (a) No state was ever recorded for this job, so it belongs to nothing. + if err := rc.applyUploadResult(uploadResult{job: job}); err != nil { + t.Fatalf("applyUploadResult() on a stale result returned %v, want nil: a stale result is discarded, not fatal", err) + } + if c, _ := rc.ix.lookup(key); c.Entry.State != statePending { + t.Errorf("an untracked result promoted the capture to %q", c.Entry.State) + } + + // (b) The capture at that inode was replaced by a different one between + // submission and completion. + rc.up.states[key] = &uploadState{phase: phaseInFlight, job: job} + replacement, err := newStagedEntry(statePending, "session-1", "node-1", "", "raylet.out.1", "1234567890123456789.abcdefabcdef0123") + if err != nil { + t.Fatalf("newStagedEntry() error: %v", err) + } + rc.ix.byInode[key].Entry = replacement + + if err := rc.applyUploadResult(uploadResult{job: job}); err != nil { + t.Fatalf("applyUploadResult() on a replaced capture returned %v, want nil", err) + } + if c, _ := rc.ix.lookup(key); c.Entry != replacement { + t.Errorf("a stale result mutated the replacement capture: %+v", c.Entry) + } + if len(issues.matching("no longer matches the capture")) == 0 { + t.Errorf("stale results were not reported: %v", issues.all()) + } +} + +// stallFirstUpload blocks the first upload inside the object store and returns once +// it is parked there, plus the function that releases it. While it is held, the +// collector is guaranteed not to dispatch anything else, which is what lets a test +// change a queued capture's staged file underneath the worker deterministically. +func (u *upHarness) stallFirstUpload(t *testing.T) func() { + t.Helper() + release := u.writer.blockWrites() + u.captureOne(t, "raylet.out.9", "the upload that stalls") + select { + case <-u.writer.entered: + case <-time.After(5 * time.Second): + release() + t.Fatal("the first upload never started") + } + return release +} + +// A local validation failure is a durable contradiction between the index and the +// staging volume. Retrying it on the transport schedule would retry forever, so the +// collector fails closed and leaves the staging state for the next run to reconstruct. +func TestLocalValidationFailureStopsTheCollector(t *testing.T) { + tests := []struct { + name string + corrupt func(t *testing.T, staged string) + wantIn string + }{ + { + name: "staged path holds a different inode", + corrupt: func(t *testing.T, staged string) { + t.Helper() + if err := os.Remove(staged); err != nil { + t.Fatalf("remove staged link: %v", err) + } + writeFile(t, staged, "an entirely different file") + }, + wantIn: "not the captured", + }, + { + name: "staged path is gone", + corrupt: func(t *testing.T, staged string) { + t.Helper() + if err := os.Remove(staged); err != nil { + t.Fatalf("remove staged link: %v", err) + } + }, + wantIn: "open staged capture", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + u := startUploading(t, dir, nil) + release := u.stallFirstUpload(t) + + // A second capture, queued behind the stalled one. + doomed := u.captureOne(t, "raylet.out.1", "segment") + staged := doomed.path(u.stagingRoot) + if _, err := os.Lstat(staged); err != nil { + t.Fatalf("second capture was not staged: %v", err) + } + tc.corrupt(t, staged) + + release() + + var runErr error + select { + case runErr = <-u.runErr: + case <-time.After(10 * time.Second): + t.Fatal("Run() did not return after a local consistency failure") + } + if !errors.Is(runErr, errStagingInconsistent) { + t.Fatalf("Run() returned %v, want an error wrapping errStagingInconsistent", runErr) + } + for _, want := range []string{doomed.CaptureID, staged, tc.wantIn} { + if !strings.Contains(runErr.Error(), want) { + t.Errorf("Run() error %q does not name %q", runErr, want) + } + } + + // Only the healthy capture reached storage: a locally rejected file is + // never sent, and nothing is retried. + if got := u.writer.attemptCount(); got != 1 { + t.Errorf("%d storage attempts, want only the first capture's: %+v", got, u.writer.attempts()) + } + if delays := u.timers.delays(); len(delays) != 0 { + t.Errorf("a retry timer was armed for a local failure: %v", delays) + } + + // The staging volume is left exactly as it was found. + if _, err := os.Lstat(doomed.withState(stateUploaded).path(u.stagingRoot)); !os.IsNotExist(err) { + t.Errorf("the rejected capture was promoted: %v", err) + } + if tc.name == "staged path holds a different inode" { + if _, err := os.Lstat(staged); err != nil { + t.Errorf("the pending staging path was not left in place: %v", err) + } + } + + // Nothing further happens once the collector has stopped. + time.Sleep(100 * time.Millisecond) + if got := u.writer.attemptCount(); got != 1 { + t.Errorf("%d storage attempts after Run returned, want 1", got) + } + }) + } +} + +// The policy itself, asserted directly: a transport error retries, a local one does +// not, and neither promotes. +func TestUploadResultFailurePolicySeparatesLocalFromTransport(t *testing.T) { + newCollector := func(t *testing.T, dir string) (*rotatedCollector, uploadJob, inodeKey) { + t.Helper() + logsDir := filepath.Join(dir, "session", "logs") + stagingRoot := filepath.Join(dir, "rotated-staging") + writeFile(t, filepath.Join(logsDir, "raylet.out"), "active") + entry, key := stageManually(t, logsDir, stagingRoot, "raylet.out.1", "segment", false) + + rc, err := newRotatedCollector(rotatedCollectorConfig{ + LogsDir: logsDir, + StagingRoot: stagingRoot, + SessionName: "session-1", + NodeName: "node-1", + Writer: newFakeWriter(), + Cluster: testCluster, + UploadBackoff: testBackoff, + OnIssue: func(error) {}, + }) + if err != nil { + t.Fatalf("newRotatedCollector() error: %v", err) + } + if _, err := rc.ix.restore(key, entry); err != nil { + t.Fatalf("restore() error: %v", err) + } + job := uploadJob{uploadIdentity: uploadIdentity{ + inode: key, + entry: entry, + localPath: entry.path(stagingRoot), + objectKey: entry.objectKey(testCluster), + }} + rc.up.states[key] = &uploadState{phase: phaseInFlight, job: job} + rc.up.inFlight = 1 + return rc, job, key + } + + t.Run("transport failure retries", func(t *testing.T) { + rc, job, key := newCollector(t, t.TempDir()) + err := rc.applyUploadResult(uploadResult{job: job, err: errUploadRejected}) + if err != nil { + t.Fatalf("a transport failure returned %v, want nil so the retry can happen", err) + } + st, tracked := rc.up.states[key] + if !tracked || st.phase != phaseBackoff { + t.Fatalf("state = %+v, want the capture backing off", st) + } + if st.dueAt.IsZero() { + t.Error("no retry deadline was scheduled for a transport failure") + } + }) + + t.Run("local failure fails closed", func(t *testing.T) { + rc, job, key := newCollector(t, t.TempDir()) + err := rc.applyUploadResult(uploadResult{job: job, local: true, err: errors.New("inode mismatch")}) + if !errors.Is(err, errStagingInconsistent) { + t.Fatalf("a local failure returned %v, want an error wrapping errStagingInconsistent", err) + } + if st, tracked := rc.up.states[key]; tracked { + t.Errorf("upload state %+v was left behind, so a retry could still be scheduled", st) + } + if len(rc.up.queue) != 0 { + t.Errorf("queue = %+v, want nothing requeued after a local failure", rc.up.queue) + } + if c, present := rc.ix.lookup(key); !present || c.Entry.State != statePending { + t.Errorf("capture = %+v, want it left pending and tracked", c) + } + }) +} + +// 15. The worker refuses a staged path whose inode no longer matches its job. +func TestWorkerRejectsMismatchedInode(t *testing.T) { + dir := t.TempDir() + stagingRoot := filepath.Join(dir, "staging") + entry, err := newStagedEntry(statePending, "session-1", "node-1", "", "raylet.out.1", "1234567890123456789.abcdefabcdef0123") + if err != nil { + t.Fatalf("newStagedEntry() error: %v", err) + } + staged := entry.path(stagingRoot) + writeFile(t, staged, "the real capture") + + key, _, err := statInode(staged) + if err != nil { + t.Fatalf("statInode() error: %v", err) + } + writer := newFakeWriter() + w := &uploadWorker{writer: writer, stagingRoot: stagingRoot} + job := uploadJob{uploadIdentity: uploadIdentity{ + inode: key, + entry: entry, + localPath: staged, + objectKey: entry.objectKey(testCluster), + }} + + // The happy path first, so the rejection below is known to be about the inode. + if res, ok := w.execute(job); !ok || res.err != nil { + t.Fatalf("execute() on a matching file returned (%+v, ok=%v)", res, ok) + } + + // Rotation replaced the staged path with a different file. + if err := os.Remove(staged); err != nil { + t.Fatalf("remove staged file: %v", err) + } + writeFile(t, staged, "a different file entirely") + + before := writer.attemptCount() + res, ok := w.execute(job) + if !ok { + t.Fatal("execute() reported a shutdown that was not requested") + } + if res.err == nil { + t.Fatal("execute() accepted a staged path holding a different inode") + } + if !res.local { + t.Errorf("inode mismatch reported as a transport failure: %v", res.err) + } + if got := writer.attemptCount(); got != before { + t.Errorf("the store was called %d extra time(s) for a rejected file", got-before) + } + if res.job.uploadIdentity != job.uploadIdentity { + t.Errorf("result identity = %+v, want the submitted %+v", res.job.uploadIdentity, job.uploadIdentity) + } + + // A vanished file is also a local failure, not a transport one. + if err := os.Remove(staged); err != nil { + t.Fatalf("remove staged file: %v", err) + } + if res, ok := w.execute(job); !ok || res.err == nil || !res.local { + t.Errorf("execute() on a missing file = (%+v, ok=%v), want a local failure", res, ok) + } +} + +// 16. Repeated events and repeated reconciliation create no duplicate work. +func TestNoDuplicateQueueOrInFlightJobs(t *testing.T) { + dir := t.TempDir() + u := startUploading(t, dir, nil) + release := u.writer.blockWrites() + + backupPath := filepath.Join(u.logsDir, "raylet.out.1") + u.captureOne(t, "raylet.out.1", "segment") + select { + case <-u.writer.entered: + case <-time.After(5 * time.Second): + t.Fatal("no upload started") + } + + // The same file, seen again and again: fsnotify repeats, and every sweep looks + // at the whole tree. + for range 5 { + u.sendEvent(t, backupPath) + u.rc.reconcileNow() + } + + s := u.rc.stats() + if s.Captures != 1 { + t.Fatalf("Captures = %d, want 1", s.Captures) + } + if s.InFlightUploads != 1 || s.QueuedUploads != 0 { + t.Errorf("stats = %+v, want exactly one in-flight upload and an empty queue", s) + } + + release() + u.waitForUploaded(t, 1) + u.rc.reconcileNow() + if got := u.writer.attemptCount(); got != 1 { + t.Errorf("%d storage attempts for one capture, want 1: %+v", got, u.writer.attempts()) + } +} + +// blockPromotion makes promoteCapture's MkdirAll fail by occupying the uploaded +// state directory with a regular file. Removing that file makes promotion possible +// again. +func blockPromotion(t *testing.T, stagingRoot string) string { + t.Helper() + p := filepath.Join(stagingRoot, "session-1", "node-1", string(stateUploaded)) + if err := os.MkdirAll(filepath.Dir(p), 0o750); err != nil { + t.Fatalf("create staging parent: %v", err) + } + writeFile(t, p, "not a directory") + return p +} + +// 17. A remote success whose local promotion fails is not uploaded again. +// 18. The promotion retry eventually succeeds without another storage call. +func TestRemoteSuccessWithLocalPromotionFailureRetriesPromotionOnly(t *testing.T) { + dir := t.TempDir() + blocker := blockPromotion(t, filepath.Join(dir, "rotated-staging")) + u := startUploading(t, dir, nil) + + entry := u.captureOne(t, "raylet.out.1", "segment") + u.waitForAttempts(t, 1) + u.waitFor(t, "the capture to be awaiting promotion", func() bool { + return u.rc.stats().AwaitingPromotion == 1 + }) + + // The bytes are in storage but the capture is still pending on disk. + if entries := u.rc.snapshot(); len(entries) != 1 || entries[0] != entry { + t.Fatalf("capture = %+v, want it left exactly as submitted", entries) + } + if _, err := os.Lstat(entry.path(u.stagingRoot)); err != nil { + t.Errorf("pending staging link was not preserved: %v", err) + } + if len(u.issues.matching("could not be promoted locally")) == 0 { + t.Errorf("the promotion failure was not reported: %v", u.issues.all()) + } + + // Retrying must not send the bytes a second time. + u.clock.advance(10 * time.Second) + u.timers.fire(t) + u.rc.reconcileNow() + if got := u.writer.attemptCount(); got != 1 { + t.Errorf("%d storage attempts while promotion was failing, want 1", got) + } + + // Once promotion can succeed, the retry finishes the job with no further write. + if err := os.Remove(blocker); err != nil { + t.Fatalf("unblock promotion: %v", err) + } + u.clock.advance(10 * time.Second) + u.timers.fire(t) + u.waitForUploaded(t, 1) + + if got := u.writer.attemptCount(); got != 1 { + t.Errorf("%d storage attempts in total, want 1: %+v", got, u.writer.attempts()) + } + if got := u.rc.snapshot()[0].CaptureID; got != entry.CaptureID { + t.Errorf("capture ID changed during promotion retry: %s -> %s", entry.CaptureID, got) + } + if s := u.rc.stats(); s.AwaitingPromotion != 0 { + t.Errorf("stats = %+v, want nothing awaiting promotion", s) + } +} + +// segment writes a rotation backup of an exact size and delivers its event. +func (u *upHarness) segment(t *testing.T, name string, size int) { + t.Helper() + u.sendEvent(t, u.writeLog(t, name, strings.Repeat("x", size))) +} + +// hasCapturedName reports whether any uploaded object key is a capture of originalName, +// whose key carries a capture ID the test cannot predict. +func hasCapturedName(uploaded map[string]bool, originalName string) bool { + for name := range uploaded { + if got, _, ok := parseCaptureFileName(name); ok && got == originalName { + return true + } + } + return false +} + +// rolledOffSegment captures a rotation backup and then takes Ray's own link away, which +// is what Ray does when the segment falls off the end of its backup ring. +// +// Until that happens the capture shares Ray's blocks and retains nothing, so this is +// the only way a test can put real pressure on the intake watermark. The reconcile is +// what re-reads the link count: nothing touches the staging path when Ray unlinks its +// own name, so no event announces it. +func (u *upHarness) rolledOffSegment(t *testing.T, name string, size int) { + t.Helper() + u.segment(t, name, size) + if err := os.Remove(filepath.Join(u.logsDir, name)); err != nil { + t.Fatalf("remove Ray's link to %s: %v", name, err) + } + u.rc.reconcileNow() +} + +// 19. Reaching the high-water mark pauses intake. +// 20. Uploads and releases keep running while intake is paused. +// 21. Nothing already staged is evicted at high water. +// 22. Falling to the low-water mark resumes intake. +// 23. Resuming reconciles immediately. +func TestBackpressurePausesIntakeWithoutEviction(t *testing.T) { + dir := t.TempDir() + u := startUploading(t, dir, func(cfg *rotatedCollectorConfig) { + cfg.HighWaterBytes = 200 + cfg.LowWaterBytes = 100 + }) + // Hold the uploads so the captures are still pending while the volume is over + // its high-water mark: pending data is precisely what must never be evicted. + release := u.writer.blockWrites() + + u.writeLog(t, "raylet.out", "active") + // Ray has rolled both of these off its backup ring, so the collector is now the + // only thing keeping their blocks allocated. That — not the logical size — is what + // the watermark measures. + u.rolledOffSegment(t, "raylet.out.1", 120) + u.rolledOffSegment(t, "raylet.out.2", 120) + + // 19: 240 retained bytes is past the high-water mark of 200. + s := u.rc.stats() + if !s.IntakePaused { + t.Fatalf("stats = %+v, want intake paused at 240 retained bytes", s) + } + if s.Captures != 2 || s.Pending != 2 || s.StagedBytes != 240 || s.RetainedBytes != 240 { + t.Fatalf("stats = %+v, want 2 pending captures totalling 240 bytes, all retained", s) + } + if len(u.issues.matching("intake paused")) != 1 { + t.Errorf("the pause was reported %d time(s), want exactly 1: %v", + len(u.issues.matching("intake paused")), u.issues.all()) + } + + // 21: a new backup arriving under pressure is skipped, and no capture the + // collector already holds is evicted to make room for it. + u.segment(t, "raylet.out.3", 120) + if s := u.rc.stats(); s.Captures != 2 || s.Pending != 2 || s.StagedBytes != 240 { + t.Errorf("stats = %+v, want the two pending captures kept and the new one skipped", s) + } + staged := u.stagedPaths(t) + if len(staged) != 2 { + t.Errorf("staging holds %v, want the two captures made before the pause", staged) + } + + // The pause is a transition, not a per-event message, and repeated pressure must + // not start evicting either. + for range 3 { + u.segment(t, "raylet.out.4", 10) + } + if got := len(u.issues.matching("intake paused")); got != 1 { + t.Errorf("the pause was reported %d times, want 1", got) + } + if s := u.rc.stats(); s.Captures != 2 || s.Pending != 2 { + t.Errorf("stats = %+v after sustained pressure, want both captures still held", s) + } + + // 20: the uploader is unaffected by the gate — it is working on a capture right now, + // with intake shut, and finishing that work is what relieves the pressure. + if s := u.rc.stats(); s.InFlightUploads != 1 || !s.IntakePaused { + t.Errorf("stats = %+v, want an upload in flight while intake stays paused", s) + } + release() + + // 22 + 23: each capture that reaches storage is released — the collector holds the + // only link, so unlinking actually frees the blocks — retained bytes fall to the + // low-water mark, and the resume rescans the tree in the same pass, so the backup + // skipped while paused is captured with no further event. + u.waitFor(t, "intake to resume", func() bool { return !u.rc.stats().IntakePaused }) + + s = u.rc.stats() + if s.RetainedBytes != 0 { + t.Errorf("stats = %+v, want every retained capture released once uploaded", s) + } + uploaded := map[string]bool{} + for _, c := range u.writer.attempts() { + uploaded[path.Base(c.key)] = true + } + for _, want := range []string{"raylet.out.1", "raylet.out.2"} { + if !hasCapturedName(uploaded, want) { + t.Errorf("%s never reached storage while intake was paused: %v", want, uploaded) + } + } + names := map[string]bool{} + for _, e := range u.rc.snapshot() { + names[e.OriginalName] = true + } + if !names["raylet.out.3"] { + t.Errorf("resuming did not reconcile: raylet.out.3 was never captured, got %v", names) + } +} + +// The high-water mark has to bite inside the scan that breaches it. A scan registers +// every backup it finds without returning to the event loop, so a limit that is only +// checked afterwards can be overshot by however many files happen to be on disk. +func TestHighWaterStopsCaptureWithinOneScan(t *testing.T) { + dir := t.TempDir() + logsDir := filepath.Join(dir, "session", "logs") + writeFile(t, filepath.Join(logsDir, "raylet.out"), "active") + // Four backups, 100 bytes each, all present before the collector starts. The + // second one takes the total to 200, which is the limit. + for i := 1; i <= 4; i++ { + writeFile(t, filepath.Join(logsDir, "raylet.out."+strconv.Itoa(i)), strings.Repeat("x", 100)) + } + + u := startUploading(t, dir, func(cfg *rotatedCollectorConfig) { + cfg.HighWaterBytes = 200 + cfg.LowWaterBytes = 100 + // Ray rolls each segment off its ring the instant it is captured, so every + // capture is retained the moment it is made. Without that the scan could not + // breach the limit at all: a capture Ray still has a link to shares Ray's + // blocks and retains nothing. + cfg.Link = func(src, dst string) error { + if err := captureLink(src, dst); err != nil { + return err + } + return os.Remove(src) + } + }) + + s := u.rc.stats() + if !s.IntakePaused { + t.Fatalf("stats = %+v, want intake paused", s) + } + if s.Captures != 2 || s.StagedBytes != 200 || s.RetainedBytes != 200 { + t.Fatalf("stats = %+v, want the scan stopped at the two captures that reach the limit", s) + } + + captured := map[string]bool{} + for _, e := range u.rc.snapshot() { + captured[e.OriginalName] = true + } + for _, want := range []string{"raylet.out.1", "raylet.out.2"} { + if !captured[want] { + t.Errorf("%s was not captured: %v", want, captured) + } + } + for _, notWant := range []string{"raylet.out.3", "raylet.out.4"} { + if captured[notWant] { + t.Errorf("%s was captured after the limit was already reached: %v", notWant, captured) + } + } + if got := len(u.issues.matching("intake paused")); got != 1 { + t.Errorf("the pause was reported %d time(s), want exactly 1: %v", got, u.issues.all()) + } + + // A further sweep must not quietly take the rest either. + u.fireTick(t) + if s := u.rc.stats(); s.Captures != 2 { + t.Errorf("stats = %+v after another sweep, want still 2 captures", s) + } + + // And the resume path still works: once Ray drops its links the sweep releases + // both captures, the total falls to the low-water mark, intake resumes and the + // same pass reconciles — picking up the backups that were skipped earlier. Those + // are another 200 bytes, so the limit engages again, which is the loop working + // exactly as intended rather than a failure. + // Ray's links are already gone — the Link hook above dropped them at capture time — + // so uploading is all that stands between these captures and release. Each release + // frees real blocks, retained bytes fall to the low-water mark, intake resumes and + // the same pass reconciles, which is what finally picks up the skipped backups. + // They breach the limit again on the way through, and that loop is the design + // working rather than a failure, so what is asserted is the outcome: everything + // reaches storage and nothing is left retained. + uploaded := map[string]bool{} + u.waitFor(t, "the skipped backups to be captured and uploaded", func() bool { + u.fireTick(t) + for _, c := range u.writer.attempts() { + uploaded[path.Base(c.key)] = true + } + return hasCapturedName(uploaded, "raylet.out.3") && hasCapturedName(uploaded, "raylet.out.4") + }) + + for _, want := range []string{"raylet.out.1", "raylet.out.2"} { + if !hasCapturedName(uploaded, want) { + t.Errorf("%s never reached storage: %v", want, uploaded) + } + } + if s := u.rc.stats(); s.RetainedBytes != 0 || s.IntakePaused { + t.Errorf("stats = %+v, want everything drained and intake open once the tree is exhausted", s) + } +} + +// A restart that adopts a volume already over its limit must not capture its way +// further past it during the startup scan. +func TestStartupPausesBeforeScanningWhenReconstructionIsAboveHighWater(t *testing.T) { + dir := t.TempDir() + logsDir := filepath.Join(dir, "session", "logs") + stagingRoot := filepath.Join(dir, "rotated-staging") + writeFile(t, filepath.Join(logsDir, "raylet.out"), "active") + + // The previous run left 250 staged bytes behind, and Ray has since rolled that + // segment off its backup ring, so the staging link is the only thing keeping those + // blocks allocated — which is what makes them count against the limit. + reconstructed, _ := stageManually(t, logsDir, stagingRoot, "raylet.out.5", strings.Repeat("r", 250), false) + if err := os.Remove(filepath.Join(logsDir, "raylet.out.5")); err != nil { + t.Fatalf("remove Ray's link to raylet.out.5: %v", err) + } + // ...and an eligible backup is sitting in the live tree. + writeFile(t, filepath.Join(logsDir, "raylet.out.1"), strings.Repeat("x", 10)) + + u := startUploading(t, dir, func(cfg *rotatedCollectorConfig) { + cfg.HighWaterBytes = 200 + cfg.LowWaterBytes = 100 + cfg.Writer = nil // uploads off: this test is only about the startup gate + }) + + s := u.rc.stats() + if !s.IntakePaused { + t.Fatalf("stats = %+v, want intake paused before the startup live scan", s) + } + if s.Captures != 1 || s.StagedBytes != 250 { + t.Fatalf("stats = %+v, want only the reconstructed capture", s) + } + + entries := u.rc.snapshot() + if len(entries) != 1 || entries[0].CaptureID != reconstructed.CaptureID { + t.Fatalf("snapshot = %+v, want just the reconstructed capture %s", entries, reconstructed.CaptureID) + } + if _, err := os.Lstat(reconstructed.path(stagingRoot)); err != nil { + t.Errorf("the reconstructed capture was not retained: %v", err) + } + if got := u.stagedPaths(t); len(got) != 1 { + t.Errorf("staging holds %v, want only the reconstructed capture", got) + } +} + +// Within one sweep an early link can succeed and a later one hit ENOSPC. The stale +// success must not be what clears the newer failure. +func TestLaterENOSPCIsNotClearedByAnEarlierSuccess(t *testing.T) { + dir := t.TempDir() + logsDir := filepath.Join(dir, "session", "logs") + writeFile(t, filepath.Join(logsDir, "raylet.out"), "active") + writeFile(t, filepath.Join(logsDir, "raylet.out.1"), "first, links fine") + writeFile(t, filepath.Join(logsDir, "raylet.out.2"), "second, no space left") + + var mu sync.Mutex + calls := 0 + full := true + u := startUploading(t, dir, func(cfg *rotatedCollectorConfig) { + cfg.Link = func(src, dst string) error { + mu.Lock() + calls++ + // The very first link of the run succeeds; while the volume is full + // every later one fails. + fail := full && calls > 1 + mu.Unlock() + if fail { + return syscall.ENOSPC + } + return captureLink(src, dst) + } + }) + + // The startup scan captured the first and hit ENOSPC on the second, in that + // order. The gate must reflect the last thing the filesystem said. + s := u.rc.stats() + if !s.IntakePaused { + t.Fatalf("stats = %+v, want intake paused: an earlier success cleared a later ENOSPC", s) + } + if s.Captures != 1 { + t.Fatalf("stats = %+v, want the first capture kept and the second refused", s) + } + if len(u.issues.matching("intake paused")) == 0 { + t.Errorf("the ENOSPC pause was not reported: %v", u.issues.all()) + } + // The count, not the flag, is the real evidence. A gate wrongly reopened by the + // stale success would be shut again by the very next failing scan, leaving + // IntakePaused looking correct while intake had in fact been reopened. + if s.IntakeResumes != 0 { + t.Errorf("intake was resumed %d time(s) while the volume was full: a stale success cleared a later ENOSPC", s.IntakeResumes) + } + + // It must stay paused across further evaluations, not just the first. + for range 3 { + u.fireTick(t) + s := u.rc.stats() + if !s.IntakePaused { + t.Fatalf("stats = %+v, want intake still paused while the volume is full", s) + } + if s.IntakeResumes != 0 { + t.Fatalf("intake was resumed %d time(s) across sweeps while the volume was full", s.IntakeResumes) + } + } + + // Once capacity genuinely comes back, the next bounded probe succeeds and lifts + // the pause. + mu.Lock() + full = false + mu.Unlock() + u.fireTick(t) + + s = u.rc.stats() + if s.IntakePaused { + t.Errorf("stats = %+v, want intake resumed after a probe succeeded", s) + } + if s.IntakeResumes != 1 { + t.Errorf("intake resumed %d time(s), want exactly the one that followed real capacity recovery", s.IntakeResumes) + } + names := map[string]bool{} + for _, e := range u.rc.snapshot() { + names[e.OriginalName] = true + } + if !names["raylet.out.2"] { + t.Errorf("the refused backup was not captured after recovery: %v", names) + } +} + +// 24. ENOSPC pauses intake, and a later successful release lets it resume. +func TestENOSPCPausesIntakeUntilSpaceIsReleased(t *testing.T) { + dir := t.TempDir() + + var mu sync.Mutex + full := false + setFull := func(v bool) { + mu.Lock() + defer mu.Unlock() + full = v + } + u := startUploading(t, dir, func(cfg *rotatedCollectorConfig) { + cfg.Link = func(src, dst string) error { + mu.Lock() + defer mu.Unlock() + if full { + return syscall.ENOSPC + } + return captureLink(src, dst) + } + }) + + // One capture the collector owns and can eventually release: that release is + // what will free space later. + held := u.captureOne(t, "raylet.out.9", "old segment") + u.waitForUploaded(t, 1) + if err := os.Remove(filepath.Join(u.logsDir, "raylet.out.9")); err != nil { + t.Fatalf("remove Ray's link: %v", err) + } + + // The volume fills up — not because of what the collector holds, which is why + // watermarks cannot see this coming. + setFull(true) + backup := u.writeLog(t, "raylet.out.1", "new segment") + u.sendEvent(t, backup) + + s := u.rc.stats() + if !s.IntakePaused { + t.Fatalf("stats = %+v, want intake paused after ENOSPC", s) + } + if s.Captures != 1 { + t.Errorf("stats = %+v, want nothing new captured while the volume was full", s) + } + if len(u.issues.matching("intake paused")) == 0 { + t.Errorf("ENOSPC was not reported as an intake pause: %v", u.issues.all()) + } + + // Space comes back, and the sweep that releases the held capture is what lets + // intake resume and rescan. + setFull(false) + u.fireTick(t) + + if s := u.rc.stats(); s.IntakePaused { + t.Errorf("stats = %+v, want intake resumed after a successful release", s) + } + if _, err := os.Lstat(held.withState(stateUploaded).path(u.stagingRoot)); !os.IsNotExist(err) { + t.Errorf("the releasable capture was not released: %v", err) + } + names := map[string]bool{} + for _, e := range u.rc.snapshot() { + names[e.OriginalName] = true + } + if !names["raylet.out.1"] { + t.Errorf("the backup skipped during ENOSPC was not captured after resuming: %v", names) + } +} + +// countingLink wraps captureLink with an ENOSPC switch and an attempt counter, so a +// test can prove how often a paused collector actually touches the filesystem. +type countingLink struct { + mu sync.Mutex + attempts int + full bool +} + +func (l *countingLink) link(src, dst string) error { + l.mu.Lock() + l.attempts++ + full := l.full + l.mu.Unlock() + if full { + return syscall.ENOSPC + } + return captureLink(src, dst) +} + +func (l *countingLink) setFull(v bool) { + l.mu.Lock() + defer l.mu.Unlock() + l.full = v +} + +func (l *countingLink) count() int { + l.mu.Lock() + defer l.mu.Unlock() + return l.attempts +} + +// A disk-full pause must not latch. The volume can be filled and then emptied by +// something that is not this collector, and with nothing of its own to release the +// collector would otherwise never find out. +func TestENOSPCRecoversWhenSpaceIsFreedExternally(t *testing.T) { + dir := t.TempDir() + link := &countingLink{full: true} + u := startUploading(t, dir, func(cfg *rotatedCollectorConfig) { cfg.Link = link.link }) + + // Nothing has ever been captured, so there is nothing this collector could + // release to free space. + u.writeLog(t, "raylet.out", "active") + backup := u.writeLog(t, "raylet.out.1", "the surviving backup") + u.sendEvent(t, backup) + + s := u.rc.stats() + if !s.IntakePaused { + t.Fatalf("stats = %+v, want intake paused after ENOSPC", s) + } + if s.Captures != 0 { + t.Fatalf("stats = %+v, want nothing captured", s) + } + if got := len(u.issues.matching("intake paused")); got != 1 { + t.Fatalf("the pause was reported %d time(s), want exactly 1: %v", got, u.issues.all()) + } + afterPause := link.count() + + // Further sweeps while the volume is still full: exactly one probe each, no + // captures, no queued work, and no repeat of the pause report. + for i := range 3 { + u.fireTick(t) + if got := link.count() - afterPause; got != i+1 { + t.Errorf("after %d sweep(s) the collector made %d link attempts, want one probe per sweep", i+1, got) + } + s := u.rc.stats() + if !s.IntakePaused || s.Captures != 0 || s.QueuedUploads != 0 || s.InFlightUploads != 0 { + t.Fatalf("stats = %+v after a failed probe, want still paused with no work", s) + } + } + if got := len(u.issues.matching("intake paused")); got != 1 { + t.Errorf("failed probes reported the pause %d times, want 1: %v", got, u.issues.all()) + } + if got := u.stagedPaths(t); len(got) != 0 { + t.Errorf("failed probes created staging links: %v", got) + } + + // Something else frees the volume. The next sweep's probe succeeds, which is the + // only evidence the collector will accept, and intake resumes without this + // collector having released anything of its own. + link.setFull(false) + u.fireTick(t) + + s = u.rc.stats() + if s.IntakePaused { + t.Fatalf("stats = %+v, want intake resumed once a probe proved there was space", s) + } + names := map[string]bool{} + for _, e := range u.rc.snapshot() { + names[e.OriginalName] = true + } + if !names["raylet.out.1"] { + t.Errorf("the surviving backup was not captured after recovery: %v", names) + } + u.waitForUploaded(t, 1) +} + +// A watermark pause is the collector's own limit, so the disk-full probe must not +// punch through it. +func TestCapacityProbeDoesNotBypassTheWatermark(t *testing.T) { + dir := t.TempDir() + u := startUploading(t, dir, func(cfg *rotatedCollectorConfig) { + cfg.HighWaterBytes = 100 + cfg.LowWaterBytes = 50 + }) + release := u.writer.blockWrites() + defer release() + + u.writeLog(t, "raylet.out", "active") + // Ray has rolled this one off, so the collector alone retains its 150 bytes and the + // watermark engages. + u.rolledOffSegment(t, "raylet.out.1", 150) + if s := u.rc.stats(); !s.IntakePaused || s.Captures != 1 { + t.Fatalf("stats = %+v, want intake paused at the high-water mark with one capture", s) + } + + u.writeLog(t, "raylet.out.2", "would be captured if the probe leaked through") + for range 3 { + u.fireTick(t) + } + if s := u.rc.stats(); s.Captures != 1 { + t.Errorf("stats = %+v, want the watermark pause to hold: a probe bypassed it", s) + } +} + +// 25. Startup accounting counts pending and uploaded entries exactly once each. +func TestStartupAccountingCountsEveryStagedInodeOnce(t *testing.T) { + dir := t.TempDir() + logsDir := filepath.Join(dir, "session", "logs") + stagingRoot := filepath.Join(dir, "rotated-staging") + writeFile(t, filepath.Join(logsDir, "raylet.out"), "active") + + const pendingSize, uploadedSize = 40, 25 + stageManually(t, logsDir, stagingRoot, "raylet.out.1", strings.Repeat("p", pendingSize), false) + stageManually(t, logsDir, stagingRoot, "raylet.out.2", strings.Repeat("u", uploadedSize), true) + + u := startUploading(t, dir, nil) + u.writer.setFailAll(true) // keep the pending entry pending + + s := u.rc.stats() + if s.Captures != 2 || s.Pending != 1 || s.Uploaded != 1 { + t.Fatalf("stats = %+v, want one pending and one uploaded capture", s) + } + if want := int64(pendingSize + uploadedSize); s.StagedBytes != want { + t.Errorf("StagedBytes = %d, want %d", s.StagedBytes, want) + } + + // Repeated sweeps re-see the same staged files and must not count them again. + u.fireTick(t) + u.rc.reconcileNow() + if got := u.rc.stats().StagedBytes; got != int64(pendingSize+uploadedSize) { + t.Errorf("StagedBytes = %d after further sweeps, want %d", got, pendingSize+uploadedSize) + } +} + +// 26. Promotion does not alter byte accounting. +func TestPromotionDoesNotChangeAccounting(t *testing.T) { + dir := t.TempDir() + u := startUploading(t, dir, nil) + release := u.writer.blockWrites() + + const size = 64 + u.writeLog(t, "raylet.out", "active") + u.segment(t, "raylet.out.1", size) + + before := u.rc.stats() + if before.StagedBytes != size || before.Pending != 1 { + t.Fatalf("stats before promotion = %+v, want %d pending bytes", before, size) + } + + release() + u.waitForUploaded(t, 1) + + after := u.rc.stats() + if after.StagedBytes != before.StagedBytes { + t.Errorf("StagedBytes changed on promotion: %d -> %d", before.StagedBytes, after.StagedBytes) + } + if after.Uploaded != 1 { + t.Errorf("stats after promotion = %+v, want one uploaded capture", after) + } +} + +// 28. Storage calls use the owner-aware key beneath the node's logs directory. +func TestObjectKeysAreOwnerAwareAndUnderTheClusterPrefix(t *testing.T) { + dir := t.TempDir() + owned := clusterIdentity{ + RootDir: "root", + OwnerKind: "RayJob", + OwnerName: "job-1", + Namespace: "ns", + ClusterName: "cluster-a", + } + u := startUploading(t, dir, func(cfg *rotatedCollectorConfig) { cfg.Cluster = owned }) + + u.writeLog(t, "raylet.out", "active") + u.writeLog(t, "events/event.log", "active nested") + flat := u.captureOne(t, "raylet.out.1", "flat segment") + + nested := u.writeLog(t, "events/event.log.1", "nested segment") + u.sendEvent(t, nested) + u.waitForUploaded(t, 2) + + prefix := "root/cluster-history/rayjob/ns/job-1/cluster-a/session-1/node-1/logs" + wantFlat := path.Join(prefix, "raylet.out.1"+captureIDSeparator+flat.CaptureID) + + var nestedEntry stagedEntry + for _, e := range u.rc.snapshot() { + if e.OriginalName == "event.log.1" { + nestedEntry = e + } + } + wantNested := path.Join(prefix, "events", "event.log.1"+captureIDSeparator+nestedEntry.CaptureID) + + keys := map[string]bool{} + for _, c := range u.writer.attempts() { + keys[c.key] = true + if !strings.HasPrefix(c.key, prefix+"/") { + t.Errorf("key %q escapes the cluster log prefix %q", c.key, prefix) + } + } + for _, want := range []string{wantFlat, wantNested} { + if !keys[want] { + t.Errorf("no write to %q, got %v", want, keys) + } + } +} + +// 29. Successive generations at one X.N path upload under distinct keys. +func TestSuccessiveGenerationsProduceDistinctKeys(t *testing.T) { + dir := t.TempDir() + u := startUploading(t, dir, nil) + u.writeLog(t, "raylet.out", "active") + + backup := filepath.Join(u.logsDir, "raylet.out.1") + writeFile(t, backup, "first generation") + u.sendEvent(t, backup) + u.waitForAttempts(t, 1) + + // Rotation replaces the same name with a completely different file. Removing the + // old one also lets the first capture be released, so the proof that both + // generations were preserved is in the object keys, not in the index. + if err := os.Remove(backup); err != nil { + t.Fatalf("remove first generation: %v", err) + } + writeFile(t, backup, "second generation") + u.sendEvent(t, backup) + u.waitForAttempts(t, 2) + + keys := map[string]string{} + for _, c := range u.writer.attempts() { + if c.err == nil { + keys[c.key] = c.content + } + } + if len(keys) != 2 { + t.Fatalf("two generations produced %d distinct keys: %v", len(keys), keys) + } + if u.writer.attemptCount() != 2 { + t.Errorf("%d storage attempts for two generations, want 2", u.writer.attemptCount()) + } + contents := map[string]bool{} + for _, v := range keys { + contents[v] = true + } + if !contents["first generation"] || !contents["second generation"] { + t.Errorf("both generations were not uploaded: %v", keys) + } +} + +// 30. Worker and owner exit cleanly, with no goroutine left behind. +func TestUploaderStopsWithoutLeakingGoroutines(t *testing.T) { + before := runtime.NumGoroutine() + dir := t.TempDir() + u := startUploading(t, dir, nil) + + u.captureOne(t, "raylet.out.1", "segment") + u.waitForUploaded(t, 1) + + u.rc.Stop() + u.rc.Stop() // idempotent + + select { + case err := <-u.runErr: + if err != nil { + t.Errorf("Run() returned %v, want nil on a deliberate stop", err) + } + case <-time.After(5 * time.Second): + t.Fatal("Run() did not return after Stop()") + } + + for range 40 { + if runtime.NumGoroutine() <= before { + return + } + time.Sleep(50 * time.Millisecond) + } + t.Errorf("goroutines leaked: %d before, %d after", before, runtime.NumGoroutine()) +} + +// 31. Stop is prompt even mid-upload, and the late result changes nothing. +func TestLateUploadResultAfterStopCannotMutateState(t *testing.T) { + dir := t.TempDir() + u := startUploading(t, dir, nil) + release := u.writer.blockWrites() + + entry := u.captureOne(t, "raylet.out.1", "segment") + select { + case <-u.writer.entered: + case <-time.After(5 * time.Second): + t.Fatal("no upload started") + } + + // The worker is inside an object write that cannot be canceled. Stop must not + // wait for it. + start := time.Now() + u.rc.Stop() + if elapsed := time.Since(start); elapsed > 3*time.Second { + t.Errorf("Stop() took %v while an upload was blocked", elapsed) + } + select { + case err := <-u.runErr: + if err != nil { + t.Errorf("Run() returned %v, want nil", err) + } + case <-time.After(5 * time.Second): + t.Fatal("Run() did not return while an upload was blocked") + } + if len(u.issues.matching("still running at stop")) == 0 { + t.Errorf("the uncancelable upload was not reported at stop: %v", u.issues.all()) + } + + // The upload now completes with nobody listening. + release() + u.waitFor(t, "the abandoned upload to finish", func() bool { return u.writer.attemptCount() == 1 }) + time.Sleep(100 * time.Millisecond) + + // Disk still says pending, which is what lets the next run retry it. + if _, err := os.Lstat(entry.path(u.stagingRoot)); err != nil { + t.Errorf("pending staging link was mutated after stop: %v", err) + } + if _, err := os.Lstat(entry.withState(stateUploaded).path(u.stagingRoot)); !os.IsNotExist(err) { + t.Errorf("a result delivered after stop promoted the capture: %v", err) + } + if entries := u.rc.snapshot(); entries != nil { + t.Errorf("snapshot() after Stop() = %+v, want nil", entries) + } +} + +// 32. Everything above runs under -race; this exercises the request seams against a +// live uploader so that concurrent readers are covered too. +func TestUploaderStateIsOnlyTouchedByTheOwnerLoop(t *testing.T) { + dir := t.TempDir() + u := startUploading(t, dir, nil) + u.writeLog(t, "raylet.out", "active") + + var wg sync.WaitGroup + for range 8 { + wg.Go(func() { + for range 25 { + u.rc.snapshot() + u.rc.stats() + } + }) + } + for i := range 15 { + u.writeLog(t, "raylet.out."+strconv.Itoa(i+1), "segment "+strconv.Itoa(i)) + } + u.rc.reconcileNow() + wg.Wait() + + u.waitForUploaded(t, 15) + if got := u.writer.attemptCount(); got != 15 { + t.Errorf("%d storage attempts for 15 captures, want 15", got) + } +} + +// The watermark configuration has to be self-consistent or backpressure could pause +// and never resume. +func TestWatermarkConfigurationIsValidated(t *testing.T) { + tests := []struct { + name string + high int64 + low int64 + wantErr bool + }{ + {name: "disabled", high: 0, low: 0}, + {name: "valid", high: 100, low: 50}, + {name: "zero low water", high: 100, low: 0}, + {name: "low equals high", high: 100, low: 100, wantErr: true}, + {name: "low above high", high: 100, low: 200, wantErr: true}, + {name: "low without high", high: 0, low: 50, wantErr: true}, + {name: "negative", high: -1, low: 0, wantErr: true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := validateWatermarks(tc.high, tc.low) + if (err != nil) != tc.wantErr { + t.Errorf("validateWatermarks(%d, %d) = %v, wantErr %v", tc.high, tc.low, err, tc.wantErr) + } + }) + } +} + +// A non-positive retry delay would make a failed upload due the moment it failed, +// turning the backoff schedule into a spin against the object store. +func TestUploadBackoffMustBePositive(t *testing.T) { + base := func(dir string) rotatedCollectorConfig { + return rotatedCollectorConfig{ + LogsDir: filepath.Join(dir, "logs"), + StagingRoot: filepath.Join(dir, "staging"), + SessionName: "session-1", + NodeName: "node-1", + } + } + tests := []struct { + name string + backoff []time.Duration + wantErr bool + }{ + {name: "default", backoff: nil}, + {name: "positive", backoff: []time.Duration{time.Second, time.Minute}}, + {name: "contains zero", backoff: []time.Duration{time.Second, 0}, wantErr: true}, + {name: "contains negative", backoff: []time.Duration{-time.Second}, wantErr: true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cfg := base(t.TempDir()) + cfg.UploadBackoff = tc.backoff + _, err := newRotatedCollector(cfg) + if (err != nil) != tc.wantErr { + t.Errorf("newRotatedCollector(UploadBackoff=%v) error = %v, wantErr %v", tc.backoff, err, tc.wantErr) + } + }) + } +} + +// recomputeFixture is one indexed capture with a staged file, ready to be corrupted. +type recomputeFixture struct { + b *stagedBytes + ix *captureIndex + entry stagedEntry + stagingRoot string + key inodeKey +} + +func newRecomputeFixture(t *testing.T, size int) *recomputeFixture { + t.Helper() + stagingRoot := filepath.Join(t.TempDir(), "staging") + entry, err := newStagedEntry(stateUploaded, "session-1", "node-1", "", "raylet.out.1", "1234567890123456789.abcdefabcdef0123") + if err != nil { + t.Fatalf("newStagedEntry() error: %v", err) + } + writeFile(t, entry.path(stagingRoot), strings.Repeat("z", size)) + key, _, err := statInode(entry.path(stagingRoot)) + if err != nil { + t.Fatalf("statInode() error: %v", err) + } + ix := newCaptureIndex() + if _, err := ix.restore(key, entry); err != nil { + t.Fatalf("restore() error: %v", err) + } + b := newStagedBytes() + // The fixture stages a standalone file, so the collector is its only owner and it + // is retained as well as staged. + b.observe(key, int64(size), 1) + return &recomputeFixture{b: b, ix: ix, entry: entry, stagingRoot: stagingRoot, key: key} +} + +func (f *recomputeFixture) staged() string { return f.entry.path(f.stagingRoot) } + +func (f *recomputeFixture) recompute(t *testing.T) []error { + t.Helper() + var issues []error + f.b.recompute(f.stagingRoot, f.ix, func(err error) { issues = append(issues, err) }) + return issues +} + +// Byte accounting that loses track of itself must recompute from the index rather +// than let the total drift. +func TestStagedBytesRecomputesWhenIncrementalAccountingIsUncertain(t *testing.T) { + f := newRecomputeFixture(t, 32) + + // A total that was never told about a capture, and a forget for one it never + // knew, are both ways of ending up wrong. + f.b.total = 999 + f.b.forget(inodeKey{Dev: 1, Ino: 2}) + if !f.b.stale { + t.Fatal("forgetting an untracked capture did not mark accounting stale") + } + + if issues := f.recompute(t); len(issues) != 0 { + t.Fatalf("recompute reported %v on a healthy staging volume", issues) + } + if f.b.total != 32 { + t.Errorf("total = %d after recompute, want 32", f.b.total) + } + if !f.b.trusted() { + t.Error("accounting is still stale after a fully verified recompute") + } +} + +// A capture the collector cannot understand must not be silently written down to +// zero: a falling total is what releases backpressure, so an unverifiable entry would +// otherwise look exactly like pressure that had eased. +func TestStagedBytesRecomputeIsConservativeAboutUnverifiableCaptures(t *testing.T) { + tests := []struct { + name string + corrupt func(t *testing.T, f *recomputeFixture) + wantIn string + }{ + { + name: "staged file is missing", + corrupt: func(t *testing.T, f *recomputeFixture) { + t.Helper() + if err := os.Remove(f.staged()); err != nil { + t.Fatalf("remove staged file: %v", err) + } + }, + wantIn: "stat capture", + }, + { + name: "staged path holds another inode", + corrupt: func(t *testing.T, f *recomputeFixture) { + t.Helper() + if err := os.Remove(f.staged()); err != nil { + t.Fatalf("remove staged file: %v", err) + } + // A much larger replacement: counting it would corrupt the total in + // the other direction. + writeFile(t, f.staged(), strings.Repeat("q", 500)) + }, + wantIn: "not the pinned", + }, + { + name: "staged path is no longer a regular file", + corrupt: func(t *testing.T, f *recomputeFixture) { + t.Helper() + if err := os.Remove(f.staged()); err != nil { + t.Fatalf("remove staged file: %v", err) + } + if err := os.Mkdir(f.staged(), 0o750); err != nil { + t.Fatalf("replace staged file with a directory: %v", err) + } + }, + wantIn: "not a regular file", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + f := newRecomputeFixture(t, 32) + tc.corrupt(t, f) + + issues := f.recompute(t) + if len(issues) != 1 || !strings.Contains(issues[0].Error(), tc.wantIn) { + t.Fatalf("recompute reported %v, want one issue mentioning %q", issues, tc.wantIn) + } + if f.b.total != 32 { + t.Errorf("total = %d, want the last known 32 retained rather than replaced", f.b.total) + } + if f.b.trusted() { + t.Error("accounting was marked trusted despite an unverifiable capture") + } + + // Untrusted accounting may pause intake but must never resume it. + g := &intakeGate{high: 100, low: 50, watermark: true} + if _, resumed := g.evaluate(f.b.total, f.b.trusted()); resumed || !g.paused() { + t.Errorf("gate resumed on an unverified total (resumed=%v, paused=%v)", resumed, g.paused()) + } + + // Re-stage the capture so the path and the index agree again. The next + // sweep must then be able to trust itself. + if err := os.RemoveAll(f.staged()); err != nil { + t.Fatalf("clear staged path: %v", err) + } + writeFile(t, f.staged(), strings.Repeat("z", 32)) + restored, _, err := statInode(f.staged()) + if err != nil { + t.Fatalf("statInode() error: %v", err) + } + delete(f.ix.byInode, f.key) + if _, err := f.ix.restore(restored, f.entry); err != nil { + t.Fatalf("restore() error: %v", err) + } + + if issues := f.recompute(t); len(issues) != 0 { + t.Fatalf("recompute still reported %v after the capture was restored", issues) + } + if !f.b.trusted() { + t.Error("accounting is still stale after every capture verified") + } + if f.b.total != 32 { + t.Errorf("total = %d after recovery, want 32", f.b.total) + } + }) + } +} + +// uploadableJob stages a real file and returns a job that would upload cleanly. +func uploadableJob(t *testing.T) (uploadJob, string) { + t.Helper() + stagingRoot := filepath.Join(t.TempDir(), "staging") + entry, err := newStagedEntry(statePending, "session-1", "node-1", "", "raylet.out.1", "1234567890123456789.abcdefabcdef0123") + if err != nil { + t.Fatalf("newStagedEntry() error: %v", err) + } + writeFile(t, entry.path(stagingRoot), "a perfectly uploadable capture") + key, _, err := statInode(entry.path(stagingRoot)) + if err != nil { + t.Fatalf("statInode() error: %v", err) + } + return uploadJob{uploadIdentity: uploadIdentity{ + inode: key, + entry: entry, + localPath: entry.path(stagingRoot), + objectKey: entry.objectKey(testCluster), + }}, stagingRoot +} + +// Local validation takes real syscalls, and Stop can begin while they run. A job that +// has passed validation but has not reached the object store must not reach it. +func TestWorkerDoesNotStartTheRemoteWriteAfterQuit(t *testing.T) { + job, stagingRoot := uploadableJob(t) + writer := newFakeWriter() + + reached := make(chan struct{}) + release := make(chan struct{}) + quit := make(chan struct{}) + jobs := make(chan uploadJob, 1) + results := make(chan uploadResult, 1) + done := make(chan struct{}) + + w := &uploadWorker{ + writer: writer, + jobs: jobs, + results: results, + quit: quit, + stagingRoot: stagingRoot, + // Hold the worker in the window between validating the staged file and + // deciding whether to write it. + beforeWrite: func() { + close(reached) + <-release + }, + } + jobs <- job + go w.run(done) + + select { + case <-reached: + case <-time.After(5 * time.Second): + t.Fatal("the worker never reached the pre-write checkpoint") + } + + // Stop begins while the worker sits between validation and the write. + close(quit) + close(release) + + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("the worker did not exit after quit closed during validation") + } + if got := writer.attemptCount(); got != 0 { + t.Errorf("%d storage call(s) started after shutdown began, want 0", got) + } + if len(results) != 0 { + t.Errorf("a discarded job produced a result: %+v", <-results) + } +} + +// An upload that has not begun must not begin after Stop. A buffered job and a closed +// quit channel are both ready, and select chooses between ready cases at random, so +// the guard has to be re-checked after the job is taken. +func TestWorkerDoesNotStartAQueuedJobAfterQuit(t *testing.T) { + job, stagingRoot := uploadableJob(t) + + // Repeated because select is random when both cases are ready: one run proves + // nothing, many runs cover both orderings. + const rounds = 200 + writer := newFakeWriter() + for range rounds { + jobs := make(chan uploadJob, 1) + results := make(chan uploadResult, 1) + quit := make(chan struct{}) + done := make(chan struct{}) + + jobs <- job // queued, not started + close(quit) // Stop begins before the worker ever looks at the job + + w := &uploadWorker{writer: writer, jobs: jobs, results: results, quit: quit, stagingRoot: stagingRoot} + go w.run(done) + + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("the worker did not exit after quit closed") + } + if len(results) != 0 { + t.Fatalf("the worker produced a result for a job it should not have started: %+v", <-results) + } + } + + if got := writer.attemptCount(); got != 0 { + t.Errorf("%d storage call(s) were made for jobs queued before quit, want 0", got) + } +} + +// --------------------------------------------------------------------------- +// Retained-byte accounting. A capture is a hard link, so it costs no additional +// blocks while Ray still has its own link to the segment. Only once Ray rolls the +// segment off its backup ring is the collector keeping those blocks alive, and only +// that is what the intake watermark may measure. +// --------------------------------------------------------------------------- + +// A capture Ray still owns must not count against the watermark. Gating on logical +// staged bytes charged the collector for Ray's entire backup ring, which pauses intake +// during ordinary healthy rotation and silently stops the feature doing its job. +func TestRetainedBytesExcludeBlocksRayStillOwns(t *testing.T) { + dir := t.TempDir() + // A watermark far below the segment: logical accounting would pause immediately. + u := startUploading(t, dir, func(cfg *rotatedCollectorConfig) { + cfg.HighWaterBytes = 10 + cfg.LowWaterBytes = 5 + cfg.Writer = nil // uploads off: this is only about accounting + }) + + u.writeLog(t, "raylet.out", "active") + u.segment(t, "raylet.out.1", 500) + + s := u.rc.stats() + if s.Captures != 1 { + t.Fatalf("stats = %+v, want the backup captured", s) + } + if s.StagedBytes != 500 { + t.Errorf("StagedBytes = %d, want the full logical size 500", s.StagedBytes) + } + if s.RetainedBytes != 0 { + t.Errorf("RetainedBytes = %d, want 0 while Ray still holds its own link", s.RetainedBytes) + } + if s.IntakePaused { + t.Errorf("stats = %+v, want intake open: the capture shares Ray's blocks and retains nothing", s) + } + + // Sweeps must not drift into charging for it either. + u.fireTick(t) + if s := u.rc.stats(); s.RetainedBytes != 0 || s.IntakePaused { + t.Errorf("stats = %+v after a sweep, want the capture still retaining nothing", s) + } +} + +// Once Ray rolls the segment off its ring the collector's link is the last one, so the +// blocks exist only because of this feature and must be charged for. Nothing touches +// the staging path when Ray unlinks its own name, so the reconcile sweep is what has +// to notice. +func TestRetainedBytesCountCapturesRayHasRolledOff(t *testing.T) { + dir := t.TempDir() + u := startUploading(t, dir, func(cfg *rotatedCollectorConfig) { + cfg.Writer = nil + }) + + u.writeLog(t, "raylet.out", "active") + u.segment(t, "raylet.out.1", 500) + if s := u.rc.stats(); s.RetainedBytes != 0 { + t.Fatalf("stats = %+v, want nothing retained while Ray holds its link", s) + } + + // Ray rolls the segment off the end of its backup ring. + if err := os.Remove(filepath.Join(u.logsDir, "raylet.out.1")); err != nil { + t.Fatalf("remove Ray's link: %v", err) + } + // No event fires for a path outside the staging tree, so only the sweep can see it. + u.fireTick(t) + + s := u.rc.stats() + if s.StagedBytes != 500 { + t.Errorf("StagedBytes = %d, want the logical size unchanged at 500", s.StagedBytes) + } + if s.RetainedBytes != 500 { + t.Errorf("RetainedBytes = %d, want 500 now that the collector holds the only link", s.RetainedBytes) + } +} + +// The property B1 exists for: a storage outage cannot grow local disk without bound, +// and recovery restores capture on its own. +func TestOutageRetainsBoundedDiskThenResumes(t *testing.T) { + dir := t.TempDir() + u := startUploading(t, dir, func(cfg *rotatedCollectorConfig) { + cfg.HighWaterBytes = 200 + cfg.LowWaterBytes = 100 + }) + // The object store is refusing writes, so nothing can be released. + u.writer.setFailAll(true) + + u.writeLog(t, "raylet.out", "active") + u.rolledOffSegment(t, "raylet.out.1", 120) + u.rolledOffSegment(t, "raylet.out.2", 120) + + s := u.rc.stats() + if s.RetainedBytes != 240 || !s.IntakePaused { + t.Fatalf("stats = %+v, want 240 retained bytes and intake paused", s) + } + if s.Captures != 2 || s.Pending != 2 { + t.Fatalf("stats = %+v, want both captures held as pending", s) + } + + // Nothing already captured is evicted to make room, and new backups are skipped + // rather than displacing what is already held. + u.rolledOffSegment(t, "raylet.out.3", 120) + s = u.rc.stats() + if s.Captures != 2 || s.Pending != 2 || s.RetainedBytes != 240 { + t.Fatalf("stats = %+v, want the held captures kept and the new backup skipped", s) + } + if got := u.stagedPaths(t); len(got) != 2 { + t.Errorf("staging holds %v, want only the two captures made before the pause", got) + } + + // Retry work continues while intake is shut — that is what eventually clears it. + before := u.writer.attemptCount() + u.clock.advance(time.Hour) + u.fireTick(t) + u.waitFor(t, "uploads to keep being retried while paused", func() bool { + u.clock.advance(time.Hour) + u.fireTick(t) + return u.writer.attemptCount() > before + }) + if s := u.rc.stats(); !s.IntakePaused { + t.Errorf("stats = %+v, want intake still paused while nothing has been released", s) + } + + // Storage recovers. The uploads succeed, the collector is the only link holder so + // each release frees real blocks, retained bytes fall past the low mark and intake + // reopens on its own. + u.writer.setFailAll(false) + u.waitFor(t, "intake to resume once the retained captures drain", func() bool { + u.clock.advance(time.Hour) + u.fireTick(t) + return !u.rc.stats().IntakePaused + }) + if s := u.rc.stats(); s.RetainedBytes > u.rc.gate.low { + t.Errorf("stats = %+v, want retained bytes at or below the low-water mark", s) + } +} + +// A restart has no persisted record of which captures the collector alone was holding, +// so retention has to be re-derived from the link counts on the staging volume itself. +func TestReconstructionComputesRetainedFromLinkCounts(t *testing.T) { + dir := t.TempDir() + logsDir := filepath.Join(dir, "session", "logs") + stagingRoot := filepath.Join(dir, "rotated-staging") + writeFile(t, filepath.Join(logsDir, "raylet.out"), "active") + + // One staged capture Ray still has a link to... + stageManually(t, logsDir, stagingRoot, "raylet.out.1", strings.Repeat("a", 300), false) + // ...and one Ray has already rolled off, leaving staging as the only reference. + stageManually(t, logsDir, stagingRoot, "raylet.out.2", strings.Repeat("b", 700), false) + if err := os.Remove(filepath.Join(logsDir, "raylet.out.2")); err != nil { + t.Fatalf("remove Ray's link to raylet.out.2: %v", err) + } + + u := startUploading(t, dir, func(cfg *rotatedCollectorConfig) { + cfg.Writer = nil // uploads off, so reconstruction is what is measured + }) + + s := u.rc.stats() + if s.Captures != 2 { + t.Fatalf("stats = %+v, want both staged captures adopted", s) + } + if s.StagedBytes != 1000 { + t.Errorf("StagedBytes = %d, want both logical sizes, 1000", s.StagedBytes) + } + if s.RetainedBytes != 700 { + t.Errorf("RetainedBytes = %d, want only the sole-owned capture, 700", s.RetainedBytes) + } +} diff --git a/historyserver/pkg/utils/constant.go b/historyserver/pkg/utils/constant.go index 2b8bc3a4ae2..85f3a63b7bc 100644 --- a/historyserver/pkg/utils/constant.go +++ b/historyserver/pkg/utils/constant.go @@ -33,3 +33,11 @@ func GetRayPersistCompletePath() string { func GetRaySessionLatestPath() string { return filepath.Join(GetTmpRayRoot(), "session_latest") } + +// GetRayRotatedStagingPath returns the directory where the collector stages rotated +// log segments it has pinned with hard links. It sits beside prev-logs rather than +// inside it so that the prev-logs walker, which uploads and then deletes whole node +// directories, never touches captures that are still draining. +func GetRayRotatedStagingPath() string { + return filepath.Join(GetTmpRayRoot(), "rotated-staging") +}