import Foundation import NIOCore import NIOPosix import NIOSSH /// The outcome of a command run inside a guest. public struct SSHCommandResult: Sendable, Equatable { /// The remote process's exit status. `0` on success. public let exitCode: Int32 /// Captured stdout, UTF-8 decoded with lossy replacement. public let stdout: String /// Captured stderr, UTF-8 decoded with lossy replacement. public let stderr: String public init(exitCode: Int32, stdout: String, stderr: String) { self.exitCode = exitCode self.stdout = stdout self.stderr = stderr } /// Whether the command exited zero. public var succeeded: Bool { exitCode == 0 } /// Throws ``CoreError/sshFailed(_:)`` unless the command exited zero. /// /// - Parameter command: Echoed into the error message for context. public func throwIfFailed(command: String) throws { guard exitCode != 0 else { return } let detail = stderr.isEmpty ? stdout : stderr let trimmed = detail.trimmingCharacters(in: .whitespacesAndNewlines) throw CoreError.sshFailed( "command failed (exit \(exitCode)): \(command)" + (trimmed.isEmpty ? "" : "\n\(trimmed)") ) } } /// The seam for talking to a guest. /// /// The orchestrator and the provisioner are written against this rather than /// against ``SSHExecutor`` so that provisioning logic can be unit-tested with a /// recording fake, and so a future vsock-based transport could be dropped in /// without touching callers. public protocol GuestExecutor: Sendable { /// Runs a shell command in the guest and waits for it to exit. /// /// - Parameters: /// - command: A `/bin/sh`-compatible command line. /// - timeout: Wall-clock ceiling; exceeding it throws /// ``CoreError/timeout(_:)`` and closes the channel. /// - Returns: Exit status and captured output. func run(_ command: String, timeout: Duration) async throws -> SSHCommandResult /// Copies a local file into the guest. /// /// - Parameters: /// - localPath: Source path on the host. /// - remotePath: Destination path in the guest. func upload(localPath: String, remotePath: String) async throws /// Writes bytes to a guest file with an explicit mode. /// /// Used for secrets — notably the registration token, which is written with /// mode `0600` and deleted immediately after `gitea-runner register` reads /// it, so it never appears in a process argument list. /// /// - Parameters: /// - data: File contents. /// - remotePath: Destination path in the guest. /// - mode: Octal mode string, e.g. `"0600"`. func uploadData(_ data: Data, remotePath: String, mode: String) async throws } extension GuestExecutor { /// ``run(_:timeout:)`` with a two-minute default ceiling. public func run(_ command: String) async throws -> SSHCommandResult { try await run(command, timeout: .seconds(120)) } /// Runs a command and throws unless it exits zero. /// /// - Returns: The successful result. @discardableResult public func runChecked(_ command: String, timeout: Duration = .seconds(120)) async throws -> SSHCommandResult { let result = try await run(command, timeout: timeout) try result.throwIfFailed(command: command) return result } } /// SSH client over swift-nio-ssh using password authentication. /// /// Password auth (rather than keys) is deliberate: the guest is a throwaway VM /// on a host-private NAT network whose credentials come from the same config /// that created it, and injecting a key would mean another provisioning step /// during the window before SSH is up. /// /// - Important: Host keys are **not** verified. The peer is a VM this process /// just booted, on a link no other host shares; there is no trust-on-first-use /// story that would add security here, and pinning would break on every clone. public final class SSHExecutor: GuestExecutor, @unchecked Sendable { /// Guest IP, as learned from ``DHCPLeaseParser``. public let host: String /// SSH port; `22` for a stock guest with Remote Login enabled. public let port: Int /// Guest account name. public let username: String /// Guest account password. public let password: String /// Creates an executor. No connection is made until the first command. /// /// - Parameters: /// - host: Guest IP address. /// - port: SSH port. Defaults to `22`. /// - username: Guest account. /// - password: Guest password. public init(host: String, port: Int = 22, username: String, password: String) { self.host = host self.port = port self.username = username self.password = password } public func run(_ command: String, timeout: Duration) async throws -> SSHCommandResult { do { return try await execute(command, stdin: nil, timeout: timeout) } catch let error as SSHTransportError { throw error.asCoreError } } public func upload(localPath: String, remotePath: String) async throws { let url = URL(fileURLWithPath: localPath) guard let data = try? Data(contentsOf: url) else { throw CoreError.notFound("local file for upload: \(localPath)") } try await uploadData(data, remotePath: remotePath, mode: "0644") } public func uploadData(_ data: Data, remotePath: String, mode: String) async throws { // Deliberately not SFTP or SCP: a stock macOS guest runs an sshd whose // subsystem set we do not control at this point in provisioning, and an // exec channel with the payload as stdin needs nothing beyond what we // already use for every other command. let quotedPath = Self.shellQuote(remotePath) guard mode.allSatisfy(\.isNumber), !mode.isEmpty else { throw CoreError.sshFailed("invalid file mode \(mode.debugDescription) for \(remotePath)") } let command = """ mkdir -p "$(dirname \(quotedPath))" && cat > \(quotedPath) && chmod \(mode) \(quotedPath) """ let result: SSHCommandResult do { result = try await execute(command, stdin: data, timeout: .seconds(300)) } catch let error as SSHTransportError { throw error.asCoreError } try result.throwIfFailed(command: "upload to \(remotePath)") } /// Releases any pooled connection and event loop resources. /// /// Connections are not pooled — each command opens and closes its own — and /// the event loop group is NIO's process-wide singleton, so there is nothing /// to release. Kept so callers can be written against a lifecycle that a /// future pooling or vsock transport may need. public func close() async {} // MARK: - Transport /// Opens a connection, runs one exec channel, and tears both down. /// /// - Parameters: /// - command: The `/bin/sh` command line to exec. /// - stdin: Bytes to stream as the command's standard input. Standard /// input is closed (channel EOF) either way, so a command that would /// otherwise read from the terminal exits instead of hanging. /// - timeout: Wall-clock ceiling on the whole exchange. /// - Throws: ``SSHTransportError`` for connect/auth problems (which /// ``waitForSSH(host:port:username:password:timeout:pollInterval:reportInterval:onAttemptFailure:)`` needs /// to tell apart), or ``CoreError/timeout(_:)`` when the ceiling elapses. func execute(_ command: String, stdin: Data?, timeout: Duration) async throws -> SSHCommandResult { let group = MultiThreadedEventLoopGroup.singleton let auth = SSHAuthOutcome() let username = self.username let password = self.password let host = self.host let port = self.port // The `timeout` argument below cannot bound the connect: its watchdog is // scheduled on the channel's event loop, which does not exist until the // connect has already succeeded. A booting guest answers ARP long before // it answers SYNs, so without an explicit ceiling here each probe hangs // for the platform default — around 75 s — and `waitForSSH` gets a // handful of attempts inside its budget instead of one every couple of // seconds. Bounded by `timeout` so a caller asking for less than the // default gets what it asked for. let connectTimeout = min(timeout, Self.defaultConnectTimeout) let bootstrap = ClientBootstrap(group: group) .connectTimeout(.nanoseconds(Self.nanoseconds(connectTimeout))) .channelOption(ChannelOptions.socketOption(.tcp_nodelay), value: 1) .channelInitializer { channel in channel.eventLoop.makeCompletedFuture { let configuration = SSHClientConfiguration( userAuthDelegate: PasswordOnlyAuthDelegate( username: username, password: password, outcome: auth ), serverAuthDelegate: AcceptAnyHostKeyDelegate() ) try channel.pipeline.syncOperations.addHandler( NIOSSHHandler( role: .client(configuration), allocator: channel.allocator, inboundChildChannelInitializer: nil ) ) } } let channel: Channel do { channel = try await bootstrap.connect(host: host, port: port).get() } catch { // No TCP connection at all: sshd is not listening yet (or the guest // is unreachable). Recoverable — this is what waitForSSH retries on. throw SSHTransportError.connectFailed(host: host, port: port, underlying: error) } let loop = channel.eventLoop let resultPromise = loop.makePromise(of: SSHCommandResult.self) let stdinBuffer = stdin.map { ByteBuffer(bytes: $0) } let description = command let timeoutTask = loop.scheduleTask(in: .nanoseconds(Self.nanoseconds(timeout))) { resultPromise.fail(CoreError.timeout("ssh command on \(host): \(description)")) channel.close(promise: nil) } // Completing an already-completed NIO promise is a no-op, so these // racing completions are safe: whichever fires first wins. channel.closeFuture.whenComplete { _ in if auth.wasRejected { resultPromise.fail( SSHTransportError.authenticationFailed(host: host, username: username) ) } else { resultPromise.fail( CoreError.sshFailed("ssh connection to \(host):\(port) closed before the command finished") ) } } channel.pipeline.handler(type: NIOSSHHandler.self).flatMap { sshHandler -> EventLoopFuture in let childPromise = loop.makePromise(of: Channel.self) sshHandler.createChannel(childPromise, channelType: .session) { child, channelType in guard channelType == .session else { return child.eventLoop.makeFailedFuture( CoreError.sshFailed("unexpected SSH channel type \(channelType)") ) } return child.eventLoop.makeCompletedFuture { try child.pipeline.syncOperations.addHandler( ExecChannelHandler( command: description, stdin: stdinBuffer, promise: resultPromise ) ) } // Without this the guest's EOF would close the channel before the // exit-status request arrives. .flatMap { child.setOption(ChannelOptions.allowRemoteHalfClosure, value: true) } } return childPromise.futureResult }.whenFailure { error in resultPromise.fail(error) channel.close(promise: nil) } do { let result = try await resultPromise.futureResult.get() timeoutTask.cancel() channel.close(promise: nil) return result } catch { timeoutTask.cancel() channel.close(promise: nil) throw error } } /// Wraps a path (or any argument) so `/bin/sh` sees it literally. static func shellQuote(_ value: String) -> String { "'" + value.replacingOccurrences(of: "'", with: "'\\''") + "'" } /// Ceiling on the TCP connect alone. /// /// Long enough that a loaded host's slow-but-working connect is not cut /// short, short enough that a silently dropped SYN costs one poll interval /// rather than the platform's ~75 s. static let defaultConnectTimeout = Duration.seconds(10) static func nanoseconds(_ duration: Duration) -> Int64 { let components = duration.components let seconds = components.seconds.multipliedReportingOverflow(by: 1_000_000_000) guard !seconds.overflow else { return .max } let sum = seconds.partialValue.addingReportingOverflow( Int64(components.attoseconds / 1_000_000_000) ) return sum.overflow ? .max : sum.partialValue } } // MARK: - Transport failures /// Connection-level failures, kept distinct from ``CoreError`` so that /// ``waitForSSH(host:port:username:password:timeout:pollInterval:reportInterval:onAttemptFailure:)`` can tell /// "sshd is not up yet" (retry) from "the password is wrong" (give up now). enum SSHTransportError: Error { /// No TCP connection could be established. case connectFailed(host: String, port: Int, underlying: Error) /// The server rejected our credentials. case authenticationFailed(host: String, username: String) var asCoreError: CoreError { switch self { case .connectFailed(let host, let port, let underlying): return .sshFailed( "cannot connect to \(host):\(port): \(underlying)" + Self.localNetworkHint(for: underlying) ) case .authenticationFailed(let host, let username): return .sshFailed("authentication failed for \(username)@\(host)") } } /// Extra guidance for the one connect failure that is usually not a network /// problem at all. /// /// macOS 15 and newer filter local-network traffic per app, and a blocked /// flow is not reported as "denied": the filter answers `EHOSTUNREACH` /// (errno 65, "No route to host"), which is exactly what a guest that is /// genuinely off the network looks like. Guests here sit on a host-private /// NAT link that is reachable whenever the VM is up, so on this code path /// that errno is more often the privacy filter than a routing failure — /// worth naming rather than leaving the operator to guess. /// /// It matters most right after a rebuild. Per /// [TN3179](https://developer.apple.com/documentation/technotes/tn3179-understanding-local-network-privacy) /// the grant "uses your main executable UUID", and the linker mints a fresh /// `LC_UUID` on essentially every build — so `make install` can present a /// program macOS has never seen, whose permission is undetermined again, /// even though the previous binary worked minutes earlier. static func localNetworkHint(for underlying: any Error) -> String { let text = "\(underlying)".lowercased() guard text.contains("errno: 65") || text.contains("no route to host") || text.contains("host is unreachable") else { return "" } return """ (on macOS 15+ this is also what Local Network privacy returns when \ it blocks an app — and the grant is keyed on the executable's UUID, \ so every rebuild withdraws it. Pre-authorize the guest subnet \ instead: sudo defaults write com.apple.network.local-network \ AllowedEthernetLocalNetworkAddresses -array "192.168.64.0/18" — same \ for AllowedWiFiLocalNetworkAddresses — then reboot. \ See docs/troubleshooting.md) """ } } /// Shared, thread-safe record of whether the server rejected our password. /// /// The auth delegate runs on the event loop; the value is read from the async /// caller, hence the lock. final class SSHAuthOutcome: @unchecked Sendable { private let lock = NSLock() private var rejected = false var wasRejected: Bool { lock.lock() defer { lock.unlock() } return rejected } func markRejected() { lock.lock() defer { lock.unlock() } rejected = true } } // MARK: - Delegates /// Accepts every host key. /// /// The peer is a VM this process booted seconds ago, on a host-private NAT link /// no other machine shares, from an image that is destroyed after one job. Its /// host key is freshly generated per clone, so there is nothing to pin: /// trust-on-first-use would accept whatever the first connection presented — /// exactly what this does — while a pinned key would reject every legitimate /// guest. See docs/DESIGN.md §6. final class AcceptAnyHostKeyDelegate: NIOSSHClientServerAuthenticationDelegate { func validateHostKey(hostKey: NIOSSHPublicKey, validationCompletePromise: EventLoopPromise) { validationCompletePromise.succeed(()) } } /// Offers the configured password once, then reports rejection. /// /// NIOSSH asks again after a failed attempt; a second ask means the server /// refused the first, which is a provisioning bug rather than a transient /// condition, so it is recorded for the caller to fail fast on. final class PasswordOnlyAuthDelegate: NIOSSHClientUserAuthenticationDelegate { private let username: String private let password: String private let outcome: SSHAuthOutcome private var offered = false init(username: String, password: String, outcome: SSHAuthOutcome) { self.username = username self.password = password self.outcome = outcome } func nextAuthenticationType( availableMethods: NIOSSHAvailableUserAuthenticationMethods, nextChallengePromise: EventLoopPromise ) { guard !offered, availableMethods.contains(.password) else { // Either the server refused our password, or it never offered // password auth at all. Both mean this guest will not let us in. outcome.markRejected() nextChallengePromise.succeed(nil) return } offered = true nextChallengePromise.succeed( NIOSSHUserAuthenticationOffer( username: username, serviceName: "", offer: .password(.init(password: password)) ) ) } } // MARK: - Exec channel /// Drives one exec channel: sends the request, streams stdin, splits stdout from /// stderr, and captures the exit status. final class ExecChannelHandler: ChannelInboundHandler { typealias InboundIn = SSHChannelData typealias OutboundOut = SSHChannelData /// SSH channel data is framed into packets; keep writes comfortably under /// the 128 KiB default maximum packet size. private static let chunkSize = 32 * 1024 private let command: String private var stdin: ByteBuffer? private var promise: EventLoopPromise? private var stdout = ByteBufferAllocator().buffer(capacity: 0) private var stderr = ByteBufferAllocator().buffer(capacity: 0) private var exitCode: Int32? init(command: String, stdin: ByteBuffer?, promise: EventLoopPromise) { self.command = command self.stdin = stdin self.promise = promise } func channelActive(context: ChannelHandlerContext) { let request = SSHChannelRequestEvent.ExecRequest(command: command, wantReply: true) let sent = context.eventLoop.makePromise(of: Void.self) // Capture only Sendable values: the handler and its context must not // escape onto another thread. let resultPromise = promise let channel = context.channel sent.futureResult.whenFailure { error in resultPromise?.fail(error) channel.close(promise: nil) } context.triggerUserOutboundEvent(request, promise: sent) context.fireChannelActive() } func userInboundEventTriggered(context: ChannelHandlerContext, event: Any) { switch event { case is ChannelSuccessEvent: sendStandardInput(context: context) case is ChannelFailureEvent: fail(context: context, error: CoreError.sshFailed("guest refused to exec: \(command)")) case let status as SSHChannelRequestEvent.ExitStatus: exitCode = Int32(truncatingIfNeeded: status.exitStatus) case let signal as SSHChannelRequestEvent.ExitSignal: // A signalled process has no exit status; report it the way a shell // would, and keep the signal name in stderr so it is not lost. exitCode = 128 var note = ByteBuffer(string: "\nterminated by SIG\(signal.signalName): \(signal.errorMessage)\n") stderr.writeBuffer(¬e) default: context.fireUserInboundEventTriggered(event) } } func channelRead(context: ChannelHandlerContext, data: NIOAny) { let channelData = unwrapInboundIn(data) guard case .byteBuffer(var bytes) = channelData.data else { return } switch channelData.type { case .channel: stdout.writeBuffer(&bytes) case .stdErr: stderr.writeBuffer(&bytes) default: break // An extended data type we did not ask for. } } func channelInactive(context: ChannelHandlerContext) { complete() context.fireChannelInactive() } func handlerRemoved(context: ChannelHandlerContext) { complete() } func errorCaught(context: ChannelHandlerContext, error: Error) { fail(context: context, error: error) } private func sendStandardInput(context: ChannelHandlerContext) { if var payload = stdin { stdin = nil while payload.readableBytes > 0 { let slice = payload.readSlice(length: min(Self.chunkSize, payload.readableBytes))! context.write( wrapOutboundOut(SSHChannelData(type: .channel, data: .byteBuffer(slice))), promise: nil ) } context.flush() } // EOF either way: `cat > file` needs it to finish, and a command that // would otherwise block reading stdin gets an immediate end of input. context.close(mode: .output, promise: nil) } private func complete() { guard let promise else { return } self.promise = nil if let exitCode { promise.succeed( SSHCommandResult( exitCode: exitCode, stdout: String(buffer: stdout), stderr: String(buffer: stderr) ) ) } else { promise.fail( CoreError.sshFailed("guest closed the channel without an exit status: \(command)") ) } } private func fail(context: ChannelHandlerContext, error: Error) { if let promise { self.promise = nil promise.fail(error) } context.close(promise: nil) } } /// One failed probe, handed to ``waitForSSH(host:port:username:password:timeout:pollInterval:reportInterval:onAttemptFailure:)``'s /// reporting callback. /// /// Carries a rendered `error` rather than the `Error` itself so the whole value /// is `Sendable` and can cross into a logger on another isolation domain. public struct SSHWaitAttempt: Sendable { /// 1-based probe count. public let attempt: Int /// Time since the wait began. public let elapsed: Duration /// The failure, rendered through `asCoreError` where applicable so the /// Local Network privacy hint survives. public let error: String public init(attempt: Int, elapsed: Duration, error: String) { self.attempt = attempt self.elapsed = elapsed self.error = error } } /// Blocks until a guest accepts an authenticated SSH session, or the deadline /// passes. /// /// Called after a DHCP lease appears but before any provisioning: a fresh guest /// answers on port 22 only once `launchd` has started `sshd`, which lags the /// lease by tens of seconds — and by minutes on a host running several guests /// at once. /// /// - Important: A caller whose own supervisor also enforces a deadline must pass /// a `timeout` strictly smaller than the supervisor's *remaining* budget. /// Otherwise the supervisor always fires first, this function is cancelled /// mid-`Task.sleep`, and the `CoreError.timeout` below — the only place the /// last error is ever rendered — is never thrown. That is why failures are /// also reported as they happen via `onAttemptFailure` rather than solely at /// the end. /// /// - Parameters: /// - host: Guest IP. /// - port: SSH port. Defaults to `22`. /// - username: Guest account. /// - password: Guest password. /// - timeout: Overall ceiling. /// - pollInterval: Delay between attempts. Defaults to 2 s. /// - reportInterval: Floor on the gap between `onAttemptFailure` calls. /// Defaults to 30 s. The first failure is always reported. /// - onAttemptFailure: Called for the first failure and then no more often /// than `reportInterval`, so a boot that is merely slow is visible while it /// is happening instead of only in the post-mortem. /// - Throws: ``CoreError/timeout(_:)`` if the guest never answers. public func waitForSSH( host: String, port: Int = 22, username: String, password: String, timeout: Duration, pollInterval: Duration = .seconds(2), reportInterval: Duration = .seconds(30), onAttemptFailure: (@Sendable (SSHWaitAttempt) -> Void)? = nil ) async throws { let executor = SSHExecutor(host: host, port: port, username: username, password: password) let started = ContinuousClock.now var lastError: Error? var attempt = 0 var lastReport: ContinuousClock.Instant? while true { attempt += 1 do { // A real authenticated session running a trivial command, not a bare // TCP probe: sshd binds the port before it is ready to authenticate, // so a connect that succeeds proves very little. _ = try await executor.execute("true", stdin: nil, timeout: .seconds(20)) return } catch let error as SSHTransportError { if case .authenticationFailed = error { // Wrong credentials will not become right by waiting: the guest // was provisioned with a different account or password, which is // a build failure, not a boot delay. throw error.asCoreError } lastError = error } catch { // Timeouts and mid-handshake closures are what a guest that is still // starting `sshd` looks like. Keep waiting. lastError = error } if let onAttemptFailure, let lastError { let now = ContinuousClock.now // Every probe of a guest that is still booting fails, so reporting // each one would bury the log. First one, then a heartbeat. if lastReport.map({ now - $0 >= reportInterval }) ?? true { lastReport = now onAttemptFailure( SSHWaitAttempt( attempt: attempt, elapsed: now - started, error: renderSSHWaitError(lastError) ) ) } } guard ContinuousClock.now - started < timeout else { break } try await Task.sleep(for: pollInterval) guard ContinuousClock.now - started < timeout else { break } } let detail = lastError.map { "; last error: \(renderSSHWaitError($0))" } ?? "" throw CoreError.timeout("ssh on \(host):\(port) after \(attempt) attempts\(detail)") } /// Renders a probe failure for humans. /// /// Goes through `asCoreError` rather than interpolating raw: a connect failure /// is where the Local Network privacy hint lives, and these strings are the only /// place most operators will ever see why a boot stalled. private func renderSSHWaitError(_ error: Error) -> String { (error as? SSHTransportError).map { "\($0.asCoreError)" } ?? "\(error)" }