Merge nucleic/golden-jade-newt-ffvw into dev

Sources/NucleicCore/Claude/ClaudeCodeBackend.swift

@@ -309,6 +309,20 @@ public actor ClaudeCodeBackend: AgentBackend {
     /// How long lines are coalesced before one delivery, so a burst of output becomes a single turn
     /// rather than one turn per line (delivery is far costlier here than the native in-session notice).
     static let monitorCoalesceNanos: UInt64 = 400_000_000
+    /// Once a bounded monitor is past its `timeout_ms` age, how long the watched process must ALSO be
+    /// output-silent and resource-quiet (under ``ProcessStallConfig``'s CPU and disk-I/O thresholds)
+    /// before the watch is reaped. The age alone never stops a monitor: a build that simply takes longer
+    /// than the bound is still working, and killing its tree there is the bug this guards against.
+    /// Only used when the handle's pid is host-samplable; see ``beginMonitorStallWatch(id:ageNanos:)``.
+    static let monitorStallIdleNanos: UInt64 = 60 * 1_000_000_000
+    /// How often a `host_exec` command that outlived ``Configuration/hostCommandTimeoutNanos`` is
+    /// re-examined. Past the ceiling the command is spared for as long as it keeps working, so the
+    /// stall verdict is polled on this cadence rather than granting it another whole ceiling.
+    static let hostStallRecheckNanos: UInt64 = 5 * 60 * 1_000_000_000
+    /// How long an administrative monitor notice waits for a real event to carry it before it is
+    /// delivered on its own. Long enough that a live watch's next event usually collects it for free;
+    /// short enough that an agent waiting on a watch that already ended isn't left guessing.
+    static let monitorNoticeParkNanos: UInt64 = 5 * 60 * 1_000_000_000
 
     /// How to spawn a monitor's command so it runs in the SAME environment as the agent's Bash tool:
     /// inside the session's container when sandboxed, else on the host worktree. Captured at run
@@ -347,11 +361,20 @@ public actor ClaudeCodeBackend: AgentBackend {
         /// once on end.
         let pinnedContainerName: String?
         var task: Task<Void, Never>?
+        /// The bounded watch's age timer, and after it lapses the follow-on idle re-check (WebSocket
+        /// sources). Cancelled exactly once, when the monitor is claimed by an end path.
+        var deadlineTask: Task<Void, Never>?
+        /// Armed only once a bounded process monitor is past its age, to decide whether it is genuinely
+        /// wedged or merely slow. Nil until then, and for persistent / WebSocket monitors.
+        var stallMonitor: ProcessStallMonitor?
         var pending: [String] = []
         var flushScheduled = false
         var delivered = 0
         var stopped = false
         var containerReleased = false
+        /// When this monitor last produced an event. The liveness signal for a WebSocket source, which
+        /// has no process to sample; a process source uses its handle's own activity clock instead.
+        var lastEventNanos: UInt64 = DispatchTime.now().uptimeNanoseconds
         init(
             source: MonitorSource, isWebSocket: Bool, description: String,
             pinnedContainerName: String?
@@ -375,6 +398,13 @@ public actor ClaudeCodeBackend: AgentBackend {
     /// Armed monitors by id. Survives `teardownRun()`; cleared only on ``shutdown()``.
     private var activeMonitors: [String: MonitorRun] = [:]
     private var monitorSeq: UInt64 = 0
+    /// Administrative notices (a watch ended / was stopped) parked until a real monitor event can carry
+    /// them, so they don't each buy an agent turn to say nothing. See ``deliverMonitorNotice(_:label:)``.
+    private var pendingMonitorNotices: [String] = []
+    /// The watch label the parked notices are delivered under if the backstop has to send them alone.
+    private var pendingMonitorNoticeLabel: String?
+    /// Backstop that delivers parked notices when no real event turns up to carry them.
+    private var pendingNoticeTask: Task<Void, Never>?
 
     public init(
         configuration: Configuration = Configuration(),
@@ -1694,14 +1724,14 @@ public actor ClaudeCodeBackend: AgentBackend {
         let run = MonitorRun(
             source: .process(handle), isWebSocket: false, description: description,
             pinnedContainerName: pinnedContainerName)
-        run.task = Task { [weak self] in
-            await self?.pumpMonitor(
-                id: id, handle: handle, persistent: call.persistent, timeoutMs: call.timeoutMs)
-        }
+        run.task = Task { [weak self] in await self?.pumpMonitor(id: id, handle: handle) }
         activeMonitors[id] = run
+        armMonitorDeadline(id: id, persistent: call.persistent, timeoutMs: call.timeoutMs)
         let bound = call.persistent
             ? "It runs until the command exits or the session ends"
-            : "It runs until the command exits, the timeout lapses, or the session ends"
+            : "It runs until the command exits, the session ends, or — at the earliest once past the "
+                + "timeout — the command goes idle with near-zero CPU and disk I/O (a command that is "
+                + "still working past the timeout keeps running)"
         return "Monitor armed (\(id)): watching \"\(description)\". Each batch of output arrives as a "
             + "new turn in this chat — you don't need to wait, just end your turn and you'll be "
             + "re-invoked when it fires. \(bound)."
@@ -1734,13 +1764,14 @@ public actor ClaudeCodeBackend: AgentBackend {
         let run = MonitorRun(
             source: .webSocket(task), isWebSocket: true, description: description,
             pinnedContainerName: nil)
-        run.task = Task { [weak self] in
-            await self?.pumpMonitorWebSocket(id: id, persistent: persistent, timeoutMs: timeoutMs)
-        }
+        run.task = Task { [weak self] in await self?.pumpMonitorWebSocket(id: id) }
         activeMonitors[id] = run
+        armMonitorDeadline(id: id, persistent: persistent, timeoutMs: timeoutMs)
         let bound = persistent
             ? "It runs until the socket closes or the session ends"
-            : "It runs until the socket closes, the timeout lapses, or the session ends"
+            : "It runs until the socket closes, the session ends, or — at the earliest once past the "
+                + "timeout — the socket goes quiet for that same window (a socket still pushing frames "
+                + "past the timeout keeps running)"
         return "Monitor armed (\(id)): watching WebSocket \"\(description)\". Each text frame arrives "
             + "as a new turn in this chat — you don't need to wait, just end your turn and you'll be "
             + "re-invoked when it fires. \(bound)."
@@ -1801,36 +1832,104 @@ public actor ClaudeCodeBackend: AgentBackend {
         return try await processHost.launch(spec)
     }
 
-    /// Consume a monitor's stdout until the command exits (or the bounded timeout fires), coalescing
-    /// lines into batched deliveries. Emits a closing note when the watch ends so the agent isn't
-    /// left expecting more events from a monitor that's already done.
-    private func pumpMonitor(
-        id: String, handle: any ProcessHandle, persistent: Bool, timeoutMs: Int?
-    ) async {
-        let timeoutTask: Task<Void, Never>?
-        if persistent {
-            timeoutTask = nil
-        } else {
-            let ms = min(max(timeoutMs ?? 300_000, 1_000), 3_600_000)
-            timeoutTask = Task { [weak self] in
-                try? await Task.sleep(nanoseconds: UInt64(ms) * 1_000_000)
-                await self?.stopMonitor(id: id, reason: .timeout)
-            }
+    /// Arm a bounded monitor's age timer. The age NEVER stops the watch by itself — when it lapses,
+    /// ``beginMonitorStallWatch(id:ageNanos:)`` starts looking for evidence the source is actually
+    /// wedged, and only that evidence reaps it. A persistent monitor arms nothing.
+    ///
+    /// The timer is owned by the ``MonitorRun`` (not the pump) so every end path cancels it through the
+    /// same claim, and a cancelled timer can never race a natural exit into a second closing note.
+    private func armMonitorDeadline(id: String, persistent: Bool, timeoutMs: Int?) {
+        guard !persistent, let run = activeMonitors[id] else { return }
+        let ageNanos = UInt64(min(max(timeoutMs ?? 300_000, 1_000), 3_600_000)) * 1_000_000
+        run.deadlineTask = Task { [weak self] in
+            try? await Task.sleep(nanoseconds: ageNanos)
+            // `try?` swallows the cancellation error, so a cancelled timer would otherwise fall
+            // straight through and stop a monitor that already ended on its own.
+            if Task.isCancelled { return }
+            await self?.beginMonitorStallWatch(id: id, ageNanos: ageNanos)
         }
+    }
+
+    /// A bounded monitor is now past its age. Decide whether it is wedged or merely slow, and reap it
+    /// only in the first case — "past an age" alone is not evidence of a hang.
+    ///
+    /// For a process source this hands the verdict to ``ProcessStallMonitor``, which requires the tree
+    /// to be output-silent AND under both the CPU and disk-I/O thresholds continuously before it fires;
+    /// a single busy sample resets that window, so a slow build keeps running. When the pid is
+    /// host-samplable the confirmation window is ``monitorStallIdleNanos``; when it isn't (container /
+    /// VM handles, whose guest pid can't be sampled from here) output-idle is the only signal, so the
+    /// bar is the full age window of complete silence instead of a short one.
+    ///
+    /// A WebSocket source has no process to sample: its liveness is frame arrival, so it re-checks idle
+    /// time against the same window.
+    private func beginMonitorStallWatch(id: String, ageNanos: UInt64) {
+        guard let run = activeMonitors[id], !run.stopped else { return }
+        switch run.source {
+        case .process(let handle):
+            let samplable = handle.supportsHostCPUSampling
+            let cpuSampler: (@Sendable () -> UInt64)?
+            let ioSampler: (@Sendable () -> UInt64)?
+            if samplable {
+                let pid = handle.processID
+                cpuSampler = { hostProcessTreeCPUNanos(of: pid) }
+                ioSampler = { hostProcessTreeIOBytes(of: pid) }
+            } else {
+                cpuSampler = nil
+                ioSampler = nil
+            }
+            let monitor = ProcessStallMonitor(
+                handle: handle,
+                config: ProcessStallConfig(
+                    idleThresholdNanos: samplable ? Self.monitorStallIdleNanos : ageNanos),
+                cpuSampler: cpuSampler,
+                ioSampler: ioSampler,
+                onStall: { [weak self] _ in await self?.stopMonitor(id: id, reason: .stalled) },
+                // Recovery needs no action: the monitor re-arms itself and fires again if the
+                // command goes quiet for good.
+                onResolve: { _ in })
+            run.stallMonitor = monitor
+            monitor.start()
+        #if canImport(Darwin)
+        case .webSocket:
+            run.deadlineTask = Task { [weak self] in
+                await self?.watchWebSocketIdle(id: id, windowNanos: ageNanos)
+            }
+        #endif
+        }
+    }
+
+    #if canImport(Darwin)
+    /// Reap a past-its-age WebSocket monitor once — and only once — the socket has been silent for a
+    /// full window. A feed still pushing frames is live work and is left alone; each frame pushes the
+    /// decision out by another window.
+    private func watchWebSocketIdle(id: String, windowNanos: UInt64) async {
+        while true {
+            guard let run = activeMonitors[id], !run.stopped else { return }
+            let idle = DispatchTime.now().uptimeNanoseconds &- run.lastEventNanos
+            if idle >= windowNanos {
+                await stopMonitor(id: id, reason: .stalled)
+                return
+            }
+            try? await Task.sleep(nanoseconds: windowNanos &- idle)
+            if Task.isCancelled { return }
+        }
+    }
+    #endif
+
+    /// Consume a monitor's stdout until the command exits, coalescing lines into batched deliveries.
+    /// Emits a closing note when the watch ends so the agent isn't left expecting more events from a
+    /// monitor that's already done. The bounded-watch deadline is owned by the run, not this pump.
+    private func pumpMonitor(id: String, handle: any ProcessHandle) async {
         do {
             for try await line in handle.stdoutLines {
                 // Over the event cap → the monitor already stopped itself; stop pumping.
                 let keepGoing = await appendMonitorLine(
                     id: id, text: String(decoding: line, as: UTF8.self))
-                if !keepGoing {
-                    timeoutTask?.cancel()
-                    return
-                }
+                if !keepGoing { return }
             }
         } catch {
             // Stream ended abnormally (process died / container torn down) — fall through to finish.
         }
-        timeoutTask?.cancel()
         await finishMonitor(id: id, reason: .exited)
     }
 
@@ -1839,18 +1938,8 @@ public actor ClaudeCodeBackend: AgentBackend {
     /// delivering each text frame through the same coalesce/cap path as stdout lines. Binary frames
     /// become a placeholder line. The task is read from the monitor's state (never captured across
     /// the concurrency boundary — `URLSessionWebSocketTask` isn't `Sendable`).
-    private func pumpMonitorWebSocket(id: String, persistent: Bool, timeoutMs: Int?) async {
+    private func pumpMonitorWebSocket(id: String) async {
         guard let run = activeMonitors[id], case .webSocket(let task) = run.source else { return }
-        let timeoutTask: Task<Void, Never>?
-        if persistent {
-            timeoutTask = nil
-        } else {
-            let ms = min(max(timeoutMs ?? 300_000, 1_000), 3_600_000)
-            timeoutTask = Task { [weak self] in
-                try? await Task.sleep(nanoseconds: UInt64(ms) * 1_000_000)
-                await self?.stopMonitor(id: id, reason: .timeout)
-            }
-        }
         while true {
             do {
                 let message = try await task.receive()
@@ -1861,15 +1950,11 @@ public actor ClaudeCodeBackend: AgentBackend {
                 @unknown default: text = "[unknown frame]"
                 }
                 let keepGoing = await appendMonitorLine(id: id, text: text)
-                if !keepGoing {
-                    timeoutTask?.cancel()
-                    return
-                }
+                if !keepGoing { return }
             } catch {
                 break  // socket closed or errored — end the watch
             }
         }
-        timeoutTask?.cancel()
         await finishMonitor(id: id, reason: .exited)
     }
     #endif
@@ -1890,19 +1975,16 @@ public actor ClaudeCodeBackend: AgentBackend {
     private func appendMonitorLine(id: String, text: String) async -> Bool {
         guard let run = activeMonitors[id], !run.stopped else { return false }
         run.delivered += 1
+        run.lastEventNanos = DispatchTime.now().uptimeNanoseconds
         if run.delivered > Self.maxMonitorEvents {
-            run.stopped = true
-            activeMonitors[id] = nil
-            run.task?.cancel()
+            guard let run = claimMonitor(id: id) else { return false }
             await terminateMonitorSource(run)
             await releaseMonitorContainer(run)
-            if let sink = monitorEventSink {
-                await sink(
-                    run.description,
-                    "[monitor \(id) stopped: it produced more than \(Self.maxMonitorEvents) events. "
-                        + "If you still need this signal, re-arm nucleic_monitor with a tighter "
-                        + "filter so it emits only the lines you'd act on.]")
-            }
+            await deliverMonitorNotice(
+                "[monitor \(id) (\(run.description)) stopped: it produced more than "
+                    + "\(Self.maxMonitorEvents) events. If you still need this signal, re-arm "
+                    + "nucleic_monitor with a tighter filter so it emits only the lines you'd act on.]",
+                label: run.description)
             return false
         }
         run.pending.append(text)
@@ -1922,63 +2004,161 @@ public actor ClaudeCodeBackend: AgentBackend {
         run.flushScheduled = false
         let lines = run.pending
         run.pending.removeAll()
-        guard !lines.isEmpty, let sink = monitorEventSink else { return }
-        await sink(run.description, lines.joined(separator: "\n"))
+        guard !lines.isEmpty else { return }
+        await deliverMonitorTurn(label: run.description, lines: lines)
+    }
+
+    /// Send one monitor wake-up: the agent is re-invoked with `lines`, preceded by any administrative
+    /// notices that were parked waiting for a ride (see ``deliverMonitorNotice(_:label:)``). Every
+    /// monitor delivery goes through here, so a parked notice always leaves on the next real event
+    /// instead of buying a turn of its own.
+    private func deliverMonitorTurn(label: String, lines: [String]) async {
+        guard let sink = monitorEventSink else { return }
+        let parked = takePendingMonitorNotices()
+        let all = parked + lines
+        guard !all.isEmpty else { return }
+        await sink(label, all.joined(separator: "\n"))
+    }
+
+    /// Whether an administrative monitor notice can be parked to ride out with a later event, instead
+    /// of waking the agent by itself.
+    ///
+    /// Every monitor delivery re-invokes the agent as a whole turn, so a notice sent on its own buys a
+    /// full turn to say "nothing happened" — the "just the old monitor's teardown notices, nothing to
+    /// act on" turn, at the cost of a real inference pass. Parking is safe exactly while something else
+    /// is going to wake the agent anyway: another armed watch. With trailing output in hand the wake is
+    /// already happening, so the notice rides along with it and nothing needs parking. When neither
+    /// holds, the notice IS the signal the agent is parked waiting for and must go now.
+    static func monitorNoticeCanBeParked(hasTrailingOutput: Bool, otherMonitorsArmed: Bool) -> Bool {
+        !hasTrailingOutput && otherMonitorsArmed
+    }
+
+    /// Deliver an administrative notice — a watch ended, or was stopped for being over-chatty — with
+    /// any trailing output it should be carried out with. These notices carry no work of their own:
+    /// they exist so the agent doesn't sit waiting on a watch that is already dead, so they are parked
+    /// rather than spent on a turn whenever ``monitorNoticeCanBeParked(hasTrailingOutput:otherMonitorsArmed:)``
+    /// allows. ``monitorNoticeParkNanos`` bounds how long a parked notice waits for a ride.
+    private func deliverMonitorNotice(_ note: String, label: String, trailing: [String] = []) async {
+        guard monitorEventSink != nil else { return }
+        guard
+            !Self.monitorNoticeCanBeParked(
+                hasTrailingOutput: !trailing.isEmpty, otherMonitorsArmed: !activeMonitors.isEmpty)
+        else { return parkMonitorNotice(note, label: label) }
+        await deliverMonitorTurn(label: label, lines: trailing + [note])
+    }
+
+    /// Park a notice until a real event carries it, or the backstop timer gives up waiting for one.
+    private func parkMonitorNotice(_ note: String, label: String) {
+        pendingMonitorNotices.append(note)
+        pendingMonitorNoticeLabel = label
+        guard pendingNoticeTask == nil else { return }
+        pendingNoticeTask = Task { [weak self] in
+            try? await Task.sleep(nanoseconds: Self.monitorNoticeParkNanos)
+            if Task.isCancelled { return }
+            await self?.flushPendingMonitorNotices()
+        }
+    }
+
+    /// Take the parked notices, atomically, and disarm the backstop — the caller now owns delivering
+    /// them. No suspension between the read and the clear, so two concurrent deliveries can't both
+    /// carry the same notice.
+    private func takePendingMonitorNotices() -> [String] {
+        pendingNoticeTask?.cancel()
+        pendingNoticeTask = nil
+        let notices = pendingMonitorNotices
+        pendingMonitorNotices.removeAll()
+        return notices
+    }
+
+    /// No real event came along to carry the parked notices, so spend the turn on them now rather than
+    /// leave the agent waiting indefinitely on a watch that already ended.
+    private func flushPendingMonitorNotices() async {
+        pendingNoticeTask = nil
+        let label = pendingMonitorNoticeLabel ?? "monitor"
+        await deliverMonitorTurn(label: label, lines: [])
     }
 
     /// Reason a monitor ended, for the closing note wording.
-    private enum MonitorEndReason: Equatable { case exited, timeout, shutdown }
+    private enum MonitorEndReason: Equatable {
+        /// The source ended on its own — the command exited, or the WebSocket closed.
+        case exited
+        /// Past its age AND confirmed quiet: reaped as wedged (see ``beginMonitorStallWatch(id:ageNanos:)``).
+        case stalled
+        case shutdown
+    }
 
-    /// Stop an armed monitor early (bounded timeout, or teardown). Kills the process tree, cancels
-    /// the pump, balances the container ref, and — except on session shutdown — delivers a closing note.
-    private func stopMonitor(id: String, reason: MonitorEndReason) async {
-        guard let run = activeMonitors[id] else { return }
+    /// Claim a monitor for ending, atomically. Returns the run to exactly ONE caller — the first to
+    /// claim it — and nil to every other; the winner owns the teardown and the single closing note.
+    ///
+    /// This is the whole fix for the double-end race: it removes the run from ``activeMonitors``, marks
+    /// it stopped, and cancels its timers with NO suspension point in between, so a deadline firing
+    /// concurrently with a natural exit can no longer slip through a half-finished teardown and deliver
+    /// a second, contradictory note ("reached its timeout" on a command that had already exited).
+    private func claimMonitor(id: String) -> MonitorRun? {
+        guard let run = activeMonitors[id], !run.stopped else { return nil }
         activeMonitors[id] = nil
         run.stopped = true
         run.task?.cancel()
+        run.deadlineTask?.cancel()
+        run.stallMonitor?.cancel()
+        return run
+    }
+
+    /// Stop an armed monitor early (confirmed stall, or teardown). Kills the process tree, cancels the
+    /// pump, balances the container ref, and — except on session shutdown — delivers a closing note.
+    private func stopMonitor(id: String, reason: MonitorEndReason) async {
+        guard let run = claimMonitor(id: id) else { return }
         await terminateMonitorSource(run)
         await releaseMonitorContainer(run)
         if reason != .shutdown { await deliverMonitorEnd(run: run, id: id, reason: reason) }
     }
 
-    /// The pump saw the command exit on its own: flush any tail, balance the container ref, drop the
-    /// monitor, and note the end.
+    /// The pump saw the command exit on its own: claim the monitor, flush any tail, balance the
+    /// container ref, and note the end. Claiming BEFORE the flush is what keeps the flush's suspension
+    /// from being a window in which the deadline can end the same monitor a second time.
     private func finishMonitor(id: String, reason: MonitorEndReason) async {
-        guard let run = activeMonitors[id], !run.stopped else { return }
-        // Flush a final partial batch before the closing note so no output is lost.
+        guard let run = claimMonitor(id: id) else { return }
+        // Carry the final partial batch out WITH the closing note, in one turn — sending the tail and
+        // then "…and it ended" as two wake-ups costs two full turns to say one thing.
         let lines = run.pending
         run.pending.removeAll()
-        if !lines.isEmpty, let sink = monitorEventSink {
-            await sink(run.description, lines.joined(separator: "\n"))
-        }
-        activeMonitors[id] = nil
-        run.stopped = true
         await releaseMonitorContainer(run)
-        await deliverMonitorEnd(run: run, id: id, reason: reason)
+        await deliverMonitorEnd(run: run, id: id, reason: reason, trailing: lines)
     }
 
-    /// Tell the session a monitor stopped, so the agent doesn't keep waiting on a dead watch.
-    private func deliverMonitorEnd(run: MonitorRun, id: String, reason: MonitorEndReason) async {
-        guard let sink = monitorEventSink else { return }
+    /// Tell the session a monitor stopped, so the agent doesn't keep waiting on a dead watch. Called
+    /// exactly once per monitor, by whichever end path claimed it. `trailing` is the watch's last
+    /// unflushed output, which rides out in the same turn as the note.
+    private func deliverMonitorEnd(
+        run: MonitorRun, id: String, reason: MonitorEndReason, trailing: [String] = []
+    ) async {
+        guard monitorEventSink != nil else { return }
         let why: String
-        if reason == .timeout {
-            why = "reached its timeout"
+        if reason == .stalled {
+            why = run.isWebSocket
+                ? "it passed its timeout and the socket then went silent for that same window"
+                : "it passed its timeout and the command then went idle — no output and near-zero "
+                    + "CPU and disk I/O — so it was treated as wedged and its process tree was killed"
         } else if run.isWebSocket {
             why = "the WebSocket closed"
         } else {
             why = "the watched command exited"
         }
-        await sink(run.description, "[monitor \(id) ended: \(why).]")
+        // Name the watch inline: a parked notice can be carried out by a DIFFERENT monitor's delivery,
+        // whose label heads the turn, so the note has to identify itself.
+        let note = "[monitor \(id) (\(run.description)) ended: \(why).]"
+        await deliverMonitorNotice(note, label: run.description, trailing: trailing)
     }
 
     /// Kill every armed monitor (session ``shutdown()``). Silent — the session is going away, so a
     /// closing note would have nowhere to land. Still balances each container ref.
     private func teardownMonitors() async {
-        let runs = activeMonitors
-        activeMonitors.removeAll()
-        for (_, run) in runs {
-            run.stopped = true
-            run.task?.cancel()
+        // Parked notices die with the session too — there is no turn left to carry them.
+        pendingNoticeTask?.cancel()
+        pendingNoticeTask = nil
+        pendingMonitorNotices.removeAll()
+        for id in Array(activeMonitors.keys) {  // snapshot: each claim mutates `activeMonitors`
+            guard let run = claimMonitor(id: id) else { continue }
             await terminateMonitorSource(run)
             await releaseMonitorContainer(run)
         }
@@ -3391,14 +3571,32 @@ public actor ClaudeCodeBackend: AgentBackend {
             // human-in-the-loop calls, so nothing else caps this. On timeout, reap the WHOLE process
             // tree (the login shell forks make/swift/xctest/…; killing just the shell orphans them and
             // they keep the shared trunk's `.build` lock) and hand the agent a real result.
+            //
+            // But the wall clock alone is NOT evidence of a hang, and a long build that is still
+            // compiling must be allowed to finish. So the ceiling only reaps a command that `stallMonitor`
+            // has independently confirmed wedged — output-silent AND under both the CPU and disk-I/O
+            // thresholds, sustained. A command that is merely slow gets another window, re-checked on the
+            // same terms, for as long as it keeps doing work; the user still sees the stall alert (with
+            // its Kill button) and Stop, which are the interactive ways out.
             let timeout = configuration.hostCommandTimeoutNanos
-            guard await handle.waitForExit(within: timeout) else {
+            var waited: UInt64 = 0
+            var window = timeout
+            while !(await handle.waitForExit(within: window)) {
+                waited &+= window
+                // Past the ceiling, re-examine on a shorter cadence rather than granting another full
+                // ceiling — a command that wedges just after the 30m mark shouldn't hold the tool call
+                // for another 30. Each pass leaves one background waiter parked on `wait()` (harmless,
+                // and it resolves on exit), so the cadence is minutes, not seconds; the user already
+                // gets the interactive stall alert within ~6m of real silence either way.
+                window = Self.hostStallRecheckNanos
+                guard stallMonitor.isStalled else { continue }  // still working — let it run
                 await handle.terminateTree()
                 let (stdout, stderr) = await (out, err)
-                let minutes = timeout / 1_000_000_000 / 60
-                let note = "… host command exceeded \(minutes)m with no exit and was terminated "
-                    + "(its process tree was killed). Re-run with a narrower scope, or increase the "
-                    + "host command timeout if it legitimately needs longer."
+                let minutes = waited / 1_000_000_000 / 60
+                let note = "… host command ran \(minutes)m and then went idle — no output and "
+                    + "near-zero CPU and disk I/O — so it was treated as wedged and terminated (its "
+                    + "process tree was killed). Re-run with a narrower scope, or increase the host "
+                    + "command timeout if it legitimately needs longer."
                 return .ran(
                     exitCode: -1, stdout: stdout,
                     stderr: stderr.isEmpty ? note : stderr + "\n" + note)

Sources/NucleicCore/Claude/MCPApprovalServer.swift

@@ -2758,9 +2758,13 @@ public actor MCPApprovalServer {
                                 + "`fflush()`; never `| head`); prefer `ws` over "
                                 + "`command: 'websocat …'` when a WebSocket feed exists. Pass "
                                 + "`persistent: true` for a session-length watch, or `timeout_ms` to "
-                                + "auto-stop a bounded one (default 300000, max 3600000). The watch "
-                                + "runs outside your turn and survives until the source ends, the "
-                                + "timeout lapses, or the session ends. Returns an armed "
+                                + "auto-stop a bounded one (default 300000, max 3600000). "
+                                + "`timeout_ms` is the EARLIEST the watch may auto-stop, not a hard "
+                                + "kill: past it, the watch is reaped only once the source also goes "
+                                + "idle (no output and near-zero CPU/disk I/O), so a build that "
+                                + "simply takes longer than the timeout keeps running to completion. "
+                                + "The watch runs outside your turn and survives until the source "
+                                + "ends, it is reaped as idle, or the session ends. Returns an armed "
                                 + "confirmation with a monitor id."),
                         "inputSchema": .object([
                             "type": .string("object"),
@@ -2799,8 +2803,10 @@ public actor MCPApprovalServer {
                                 "timeout_ms": .object([
                                     "type": .string("integer"),
                                     "description": .string(
-                                        "Auto-stop after this many ms (default 300000, max "
-                                            + "3600000). Ignored when persistent."),
+                                        "Earliest auto-stop, in ms (default 300000, max 3600000). "
+                                            + "After this the watch stops only once the source is "
+                                            + "also idle; work still in progress is never cut off. "
+                                            + "Ignored when persistent."),
                                 ]),
                                 "persistent": .object([
                                     "type": .string("boolean"),

Sources/NucleicCore/ProcessStallMonitor.swift

@@ -113,6 +113,12 @@ public final class ProcessStallMonitor: @unchecked Sendable {
         }
     }
 
+    /// Whether the process currently looks wedged: `onStall` has fired for this stretch and nothing has
+    /// recovered it yet. Callers that hold a wall-clock ceiling consult this so they only reap a command
+    /// that is BOTH past its age AND genuinely quiet — a command that is merely slow (still burning CPU,
+    /// still moving I/O, or still emitting output) is never stalled here and must be allowed to continue.
+    public var isStalled: Bool { lock.withLock { alerted } }
+
     /// Stop watching now (the caller resolved the alert its own way — killed or dismissed it). Any
     /// pending self-resolution is suppressed.
     public func cancel() {

Tests/NucleicCoreTests/ProcessHostTests.swift

@@ -223,6 +223,31 @@ import Testing
         #expect(!ClaudeCodeBackend.terminalStageBuffersOutput("some-cmd | head -6"))
         #expect(!ClaudeCodeBackend.terminalStageBuffersOutput("swift build"))
     }
+
+    /// A monitor's closing note re-invokes the agent as a full turn, so it may only buy one when it is
+    /// the sole remaining thing that can wake it. The case that motivated this: several superseded
+    /// watches tear down while a replacement watch is running, and each teardown notice woke the agent
+    /// for a turn whose whole content was "nothing to act on".
+    @Test func parksMonitorNoticesThatWouldBuyAPointlessTurn() {
+        // Superseded watch ends quietly while a replacement is armed — the replacement will wake the
+        // agent, so the notice rides along with it instead of costing a turn.
+        #expect(
+            ClaudeCodeBackend.monitorNoticeCanBeParked(
+                hasTrailingOutput: false, otherMonitorsArmed: true))
+        // Last watch ends with nothing left armed: the note is the only remaining wake-up, so it must
+        // go now — otherwise the agent waits on a watch that is already dead.
+        #expect(
+            !ClaudeCodeBackend.monitorNoticeCanBeParked(
+                hasTrailingOutput: false, otherMonitorsArmed: false))
+        // Trailing output means the wake is already happening; the note rides out in that same turn
+        // rather than being parked for a later one.
+        #expect(
+            !ClaudeCodeBackend.monitorNoticeCanBeParked(
+                hasTrailingOutput: true, otherMonitorsArmed: true))
+        #expect(
+            !ClaudeCodeBackend.monitorNoticeCanBeParked(
+                hasTrailingOutput: true, otherMonitorsArmed: false))
+    }
 }
 
 /// A `ProcessHandle` whose `wait()` never resolves (unless told to exit on the first signal), used to

Tests/NucleicCoreTests/ProcessStallMonitorTests.swift

@@ -239,6 +239,64 @@ import Testing
         #expect(await fires(within: 2_000_000_000, resumed))
     }
 
+    /// The verdict `host_exec`'s wall-clock ceiling and a bounded monitor's age both consult before
+    /// reaping anything: it must read false while the tree is working, and only flip once the process
+    /// is confirmed wedged. A ceiling that fires on `isStalled == false` would be the old
+    /// kill-a-slow-build bug.
+    @Test func isStalledReportsTheLiveVerdict() async {
+        let handle = FakeHandle()
+        handle.setSilent(forNanos: 1_000_000_000)
+        let stalled = Latch()
+        let monitor = ProcessStallMonitor(
+            handle: handle, config: fastConfig, cpuSampler: nil,
+            onStall: { _ in stalled.fire() }, onResolve: { _ in })
+        #expect(monitor.isStalled == false)  // nothing sampled yet
+        monitor.start()
+        defer { monitor.cancel() }
+        #expect(await fires(within: 1_000_000_000, stalled))
+        #expect(monitor.isStalled)
+    }
+
+    /// A busy-but-silent tree is NOT stalled, however long it has been running — the case that used to
+    /// get killed at the timeout. CPU keeps climbing, so the ceiling must never see a stall verdict.
+    @Test func isStalledStaysFalseWhileTheTreeBurnsCPU() async {
+        let handle = FakeHandle()
+        handle.setSilent(forNanos: 1_000_000_000)
+        let cpu = ManagedAtomicNanos()
+        let monitor = ProcessStallMonitor(
+            handle: handle, config: fastConfig,
+            cpuSampler: { cpu.advance(by: 1_000_000_000) },
+            onStall: { _ in }, onResolve: { _ in })
+        monitor.start()
+        defer { monitor.cancel() }
+        // Well past several sample intervals and the sustain window.
+        try? await Task.sleep(nanoseconds: 300_000_000)
+        #expect(monitor.isStalled == false)
+    }
+
+    /// Recovery clears the verdict, so a command that goes quiet and then resumes work is spared by a
+    /// ceiling that happens to land in between.
+    @Test func isStalledClearsWhenOutputReturns() async {
+        let handle = FakeHandle()
+        handle.setSilent(forNanos: 5_000_000_000)
+        let stalled = Latch()
+        let resumed = Latch()
+        let config = ProcessStallConfig(
+            sampleIntervalNanos: 20_000_000, idleThresholdNanos: 300_000_000,
+            cpuFractionThreshold: 0.02, lowCPUSustainNanos: 40_000_000)
+        let monitor = ProcessStallMonitor(
+            handle: handle, config: config, cpuSampler: nil,
+            onStall: { _ in stalled.fire() },
+            onResolve: { resolution in if resolution == .resumed { resumed.fire() } })
+        monitor.start()
+        defer { monitor.cancel() }
+        #expect(await fires(within: 2_000_000_000, stalled))
+        #expect(monitor.isStalled)
+        handle.bump()
+        #expect(await fires(within: 2_000_000_000, resumed))
+        #expect(monitor.isStalled == false)
+    }
+
     /// Minimal monotonic counter for the CPU sampler.
     final class ManagedAtomicNanos: @unchecked Sendable {
         private let lock = NSLock()