Merge nucleic/upbeat-opal-viper-cc8l into main

Sources/RunnerCore/Config.swift

@@ -140,6 +140,19 @@ public struct RunnerConfig: Codable, Sendable, Equatable {
 
         /// Ceiling on boot + DHCP lease + SSH readiness before a slot is
         /// declared dead and recycled.
+        ///
+        /// - Important: This must exceed the guest's *worst-case* time to become
+        ///   SSH-ready, not its typical one. A clone that is still booting when
+        ///   this expires is destroyed and replaced by another clone that starts
+        ///   from zero — and because the replacement adds load to an already
+        ///   contended host, the next boot is slower still. Set too tight, this
+        ///   is not a timeout but a livelock: the daemon boots forever and no
+        ///   runner ever registers.
+        ///
+        ///   A guest sharing an Apple Silicon host with other Virtualization
+        ///   guests can take several minutes to reach `sshd`, so the default is
+        ///   deliberately generous. A genuinely wedged guest still gets caught;
+        ///   it just takes longer to notice, which is the cheaper mistake.
         public var bootTimeoutSeconds: Int
 
         public init(
@@ -147,7 +160,7 @@ public struct RunnerConfig: Codable, Sendable, Equatable {
             pollIntervalSeconds: Int = 5,
             reconcileIntervalSeconds: Int = 300,
             jobTimeoutMinutes: Int = 120,
-            bootTimeoutSeconds: Int = 300
+            bootTimeoutSeconds: Int = 900
         ) {
             self.maxConcurrentVMs = maxConcurrentVMs
             self.pollIntervalSeconds = pollIntervalSeconds

Sources/RunnerCore/SSHExec.swift

@@ -179,7 +179,7 @@ public final class SSHExecutor: GuestExecutor, @unchecked Sendable {
     ///     otherwise read from the terminal exits instead of hanging.
     ///   - timeout: Wall-clock ceiling on the whole exchange.
     /// - Throws: ``SSHTransportError`` for connect/auth problems (which
-    ///   ``waitForSSH(host:port:username:password:timeout:pollInterval:)`` needs
+    ///   ``waitForSSH(host:port:username:password:timeout:pollInterval:reportInterval:onAttemptFailure:)`` needs
     ///   to tell apart), or ``CoreError/timeout(_:)`` when the ceiling elapses.
     func execute(_ command: String, stdin: Data?, timeout: Duration) async throws -> SSHCommandResult {
         let group = MultiThreadedEventLoopGroup.singleton
@@ -189,7 +189,18 @@ public final class SSHExecutor: GuestExecutor, @unchecked Sendable {
         let host = self.host
         let port = self.port
 
+        // The `timeout` argument below cannot bound the connect: its watchdog is
+        // scheduled on the channel's event loop, which does not exist until the
+        // connect has already succeeded. A booting guest answers ARP long before
+        // it answers SYNs, so without an explicit ceiling here each probe hangs
+        // for the platform default — around 75 s — and `waitForSSH` gets a
+        // handful of attempts inside its budget instead of one every couple of
+        // seconds. Bounded by `timeout` so a caller asking for less than the
+        // default gets what it asked for.
+        let connectTimeout = min(timeout, Self.defaultConnectTimeout)
+
         let bootstrap = ClientBootstrap(group: group)
+            .connectTimeout(.nanoseconds(Self.nanoseconds(connectTimeout)))
             .channelOption(ChannelOptions.socketOption(.tcp_nodelay), value: 1)
             .channelInitializer { channel in
                 channel.eventLoop.makeCompletedFuture {
@@ -288,7 +299,14 @@ public final class SSHExecutor: GuestExecutor, @unchecked Sendable {
         "'" + value.replacingOccurrences(of: "'", with: "'\\''") + "'"
     }
 
-    private static func nanoseconds(_ duration: Duration) -> Int64 {
+    /// Ceiling on the TCP connect alone.
+    ///
+    /// Long enough that a loaded host's slow-but-working connect is not cut
+    /// short, short enough that a silently dropped SYN costs one poll interval
+    /// rather than the platform's ~75 s.
+    static let defaultConnectTimeout = Duration.seconds(10)
+
+    static func nanoseconds(_ duration: Duration) -> Int64 {
         let components = duration.components
         let seconds = components.seconds.multipliedReportingOverflow(by: 1_000_000_000)
         guard !seconds.overflow else { return .max }
@@ -302,7 +320,7 @@ public final class SSHExecutor: GuestExecutor, @unchecked Sendable {
 // MARK: - Transport failures
 
 /// Connection-level failures, kept distinct from ``CoreError`` so that
-/// ``waitForSSH(host:port:username:password:timeout:pollInterval:)`` can tell
+/// ``waitForSSH(host:port:username:password:timeout:pollInterval:reportInterval:onAttemptFailure:)`` can tell
 /// "sshd is not up yet" (retry) from "the password is wrong" (give up now).
 enum SSHTransportError: Error {
     /// No TCP connection could be established.
@@ -568,12 +586,42 @@ final class ExecChannelHandler: ChannelInboundHandler {
     }
 }
 
+/// One failed probe, handed to ``waitForSSH(host:port:username:password:timeout:pollInterval:reportInterval:onAttemptFailure:)``'s
+/// reporting callback.
+///
+/// Carries a rendered `error` rather than the `Error` itself so the whole value
+/// is `Sendable` and can cross into a logger on another isolation domain.
+public struct SSHWaitAttempt: Sendable {
+    /// 1-based probe count.
+    public let attempt: Int
+    /// Time since the wait began.
+    public let elapsed: Duration
+    /// The failure, rendered through `asCoreError` where applicable so the
+    /// Local Network privacy hint survives.
+    public let error: String
+
+    public init(attempt: Int, elapsed: Duration, error: String) {
+        self.attempt = attempt
+        self.elapsed = elapsed
+        self.error = error
+    }
+}
+
 /// Blocks until a guest accepts an authenticated SSH session, or the deadline
 /// passes.
 ///
 /// Called after a DHCP lease appears but before any provisioning: a fresh guest
 /// answers on port 22 only once `launchd` has started `sshd`, which lags the
-/// lease by tens of seconds.
+/// lease by tens of seconds — and by minutes on a host running several guests
+/// at once.
+///
+/// - Important: A caller whose own supervisor also enforces a deadline must pass
+///   a `timeout` strictly smaller than the supervisor's *remaining* budget.
+///   Otherwise the supervisor always fires first, this function is cancelled
+///   mid-`Task.sleep`, and the `CoreError.timeout` below — the only place the
+///   last error is ever rendered — is never thrown. That is why failures are
+///   also reported as they happen via `onAttemptFailure` rather than solely at
+///   the end.
 ///
 /// - Parameters:
 ///   - host: Guest IP.
@@ -582,6 +630,11 @@ final class ExecChannelHandler: ChannelInboundHandler {
 ///   - password: Guest password.
 ///   - timeout: Overall ceiling.
 ///   - pollInterval: Delay between attempts. Defaults to 2 s.
+///   - reportInterval: Floor on the gap between `onAttemptFailure` calls.
+///     Defaults to 30 s. The first failure is always reported.
+///   - onAttemptFailure: Called for the first failure and then no more often
+///     than `reportInterval`, so a boot that is merely slow is visible while it
+///     is happening instead of only in the post-mortem.
 /// - Throws: ``CoreError/timeout(_:)`` if the guest never answers.
 public func waitForSSH(
     host: String,
@@ -589,13 +642,18 @@ public func waitForSSH(
     username: String,
     password: String,
     timeout: Duration,
-    pollInterval: Duration = .seconds(2)
+    pollInterval: Duration = .seconds(2),
+    reportInterval: Duration = .seconds(30),
+    onAttemptFailure: (@Sendable (SSHWaitAttempt) -> Void)? = nil
 ) async throws {
     let executor = SSHExecutor(host: host, port: port, username: username, password: password)
     let started = ContinuousClock.now
     var lastError: Error?
+    var attempt = 0
+    var lastReport: ContinuousClock.Instant?
 
     while true {
+        attempt += 1
         do {
             // A real authenticated session running a trivial command, not a bare
             // TCP probe: sshd binds the port before it is ready to authenticate,
@@ -616,20 +674,36 @@ public func waitForSSH(
             lastError = error
         }
 
+        if let onAttemptFailure, let lastError {
+            let now = ContinuousClock.now
+            // Every probe of a guest that is still booting fails, so reporting
+            // each one would bury the log. First one, then a heartbeat.
+            if lastReport.map({ now - $0 >= reportInterval }) ?? true {
+                lastReport = now
+                onAttemptFailure(
+                    SSHWaitAttempt(
+                        attempt: attempt,
+                        elapsed: now - started,
+                        error: renderSSHWaitError(lastError)
+                    )
+                )
+            }
+        }
+
         guard ContinuousClock.now - started < timeout else { break }
         try await Task.sleep(for: pollInterval)
         guard ContinuousClock.now - started < timeout else { break }
     }
 
-    // Rendered through `asCoreError` rather than interpolated raw: a connect
-    // failure is where the Local Network privacy hint lives, and the timeout
-    // message is the *only* place most operators will ever see the last error.
-    let detail: String
-    if let lastError {
-        let rendered = (lastError as? SSHTransportError).map { "\($0.asCoreError)" } ?? "\(lastError)"
-        detail = "; last error: \(rendered)"
-    } else {
-        detail = ""
-    }
-    throw CoreError.timeout("ssh on \(host):\(port)\(detail)")
+    let detail = lastError.map { "; last error: \(renderSSHWaitError($0))" } ?? ""
+    throw CoreError.timeout("ssh on \(host):\(port) after \(attempt) attempts\(detail)")
+}
+
+/// Renders a probe failure for humans.
+///
+/// Goes through `asCoreError` rather than interpolating raw: a connect failure
+/// is where the Local Network privacy hint lives, and these strings are the only
+/// place most operators will ever see why a boot stalled.
+private func renderSSHWaitError(_ error: Error) -> String {
+    (error as? SSHTransportError).map { "\($0.asCoreError)" } ?? "\(error)"
 }