nvrsion: Add: Fix auto-scroll instability in iOS session chat so the transcript no longer jitters on scroll, and begin MACOS_VM_NATIVE_AGENT_Markdown in ./docs/MACOS_VM_NATIVE_AGENT.md.

Nucleic-Promote: 1
Co-authored-by: Nucleic <[email protected]>
This commit is contained in:
2026-07-06 04:01:31 -07:00
co-authored by nucleic
parent a7dde9691d
commit 843db6151b
30 changed files with 2098 additions and 76 deletions
@@ -1071,6 +1071,7 @@ public actor ClaudeCodeBackend: AgentBackend {
defer { Task { await manager.finished(name: name) } }
let (image, summary) = try await manager.performComputerAction(
name: name, action: call.action, x: call.x, y: call.y, text: call.text,
ref: call.ref, value: call.value,
scrollDirection: call.scrollDirection, scrollAmount: call.scrollAmount,
durationMs: call.durationMs)
return .ok(text: summary, imageBase64: image)
@@ -166,14 +166,22 @@ public actor MCPApprovalServer {
/// exposure is the opt-in boundary.
public struct MacVMComputerCall: Sendable {
/// The action verb: `screenshot`, `left_click`, `right_click`, `double_click`, `mouse_move`,
/// `left_click_drag`, `type`, `key`, `scroll`, `cursor_position`, `launch_app`, `wait`.
/// `left_click_drag`, `type`, `key`, `scroll`, `cursor_position`, `launch_app`, `wait`; plus
/// the AX-semantic actions when the guest runs the native agent
/// (docs/MACOS_VM_NATIVE_AGENT.md): `ax_dump`, `ax_element_at`, `ax_press`, `ax_set_value`,
/// `ax_focus`.
public let action: String
/// Target point (screen pixels; the screenshot is 1:1 so the agent targets what it sees).
public let x: Int?
public let y: Int?
/// Payload for `type` (literal text), `key` (a chord like `cmd+s`, `return`, `tab`), or
/// `launch_app` (an application name).
/// Payload for `type` (literal text), `key` (a chord like `cmd+s`, `return`, `tab`),
/// `launch_app` (an application name), or `ax_press` (an optional AX action name to perform
/// instead of the default `AXPress`, e.g. `AXShowMenu`).
public let text: String?
/// For the `ax_*` element actions: the element handle from the last `ax_dump`/`ax_element_at`.
public let ref: String?
/// For `ax_set_value`: the value to write into the element.
public let value: String?
/// For `scroll`: `up`/`down`/`left`/`right` and a click/step amount.
public let scrollDirection: String?
public let scrollAmount: Int?
@@ -182,12 +190,15 @@ public actor MCPApprovalServer {
public init(
action: String, x: Int? = nil, y: Int? = nil, text: String? = nil,
ref: String? = nil, value: String? = nil,
scrollDirection: String? = nil, scrollAmount: Int? = nil, durationMs: Int? = nil
) {
self.action = action
self.x = x
self.y = y
self.text = text
self.ref = ref
self.value = value
self.scrollDirection = scrollDirection
self.scrollAmount = scrollAmount
self.durationMs = durationMs
@@ -965,17 +976,29 @@ public actor MCPApprovalServer {
+ "Each call performs ONE `action` and returns a fresh screenshot of the "
+ "VM screen, so work in a loop: screenshot → decide → act → screenshot. "
+ "The screenshot is the full screen at its native pixels, so target "
+ "`x`,`y` exactly as you see them. Actions: `screenshot` (just look); "
+ "`left_click`/`right_click`/`double_click` at `x`,`y`; `mouse_move` to "
+ "`x`,`y`; `left_click_drag` to `x`,`y` (from the current cursor); "
+ "`type` the literal `text`; `key` a chord in `text` (e.g. \"cmd+s\", "
+ "\"return\", \"tab\", \"cmd+shift+4\"); `scroll` with "
+ "`x`,`y` exactly as you see them. Pixel actions: `screenshot` (just "
+ "look); `left_click`/`right_click`/`double_click` at `x`,`y`; "
+ "`mouse_move` to `x`,`y`; `left_click_drag` to `x`,`y` (from the "
+ "current cursor); `type` the literal `text`; `key` a chord in `text` "
+ "(e.g. \"cmd+s\", \"return\", \"tab\", \"cmd+shift+4\"); `scroll` with "
+ "`scroll_direction`+`scroll_amount`; `launch_app` the app named in "
+ "`text` (e.g. \"Safari\", \"Xcode\"); `cursor_position`; `wait` "
+ "`duration_ms` (let a window open). This runs in the VM sandbox, so "
+ "actions are NOT individually approval-gated. Prefer `mac_vm_exec` for "
+ "headless build/test commands; use this when you need to look at or "
+ "operate the screen."),
+ "`duration_ms` (let a window open). SEMANTIC actions (preferred when "
+ "available — they act on controls by identity, not coordinates, and "
+ "keep working even when screenshots come back blank): `ax_dump` "
+ "returns the frontmost app's accessibility tree (every control's "
+ "role/title/value/frame/actions, each with a `ref`); `ax_press` a "
+ "`ref` (optionally a specific AX action name in `text`); "
+ "`ax_set_value` writes `value` into a `ref`'d field; `ax_focus` a "
+ "`ref`; `ax_element_at` resolves the element under `x`,`y`. Convention: "
+ "ax_dump → act on a ref → ax_dump again (refs go stale after the UI "
+ "changes). The ax_* actions need the VM's native agent; if the reply "
+ "says it's unavailable, use screenshots + pixel actions. If screenshots "
+ "come back blank (a macOS 26 guest), rely on `ax_dump` — it is "
+ "framebuffer-independent. This runs in the VM sandbox, so actions are "
+ "NOT individually approval-gated. Prefer `mac_vm_exec` for headless "
+ "build/test commands; use this when you need to look at or operate the "
+ "screen."),
"inputSchema": .object([
"type": .string("object"),
"properties": .object([
@@ -984,15 +1007,27 @@ public actor MCPApprovalServer {
"description": .string(
"One of: screenshot, left_click, right_click, double_click, "
+ "mouse_move, left_click_drag, type, key, scroll, "
+ "cursor_position, launch_app, wait."),
+ "cursor_position, launch_app, wait, ax_dump, "
+ "ax_element_at, ax_press, ax_set_value, ax_focus."),
]),
"x": .object(["type": .string("integer")]),
"y": .object(["type": .string("integer")]),
"text": .object([
"type": .string("string"),
"description": .string(
"Text to type, a key/chord for `key`, or an app name for "
+ "`launch_app`."),
"Text to type, a key/chord for `key`, an app name for "
+ "`launch_app`, or an AX action name for `ax_press` "
+ "(default AXPress)."),
]),
"ref": .object([
"type": .string("string"),
"description": .string(
"Element handle from the last ax_dump/ax_element_at (for "
+ "ax_press/ax_set_value/ax_focus)."),
]),
"value": .object([
"type": .string("string"),
"description": .string("The value to write, for ax_set_value."),
]),
"scroll_direction": .object(["type": .string("string")]),
"scroll_amount": .object(["type": .string("integer")]),
@@ -1065,6 +1100,8 @@ public actor MCPApprovalServer {
x: arguments?["x"]?.intValue,
y: arguments?["y"]?.intValue,
text: arguments?["text"]?.stringValue,
ref: arguments?["ref"]?.stringValue,
value: arguments?["value"]?.stringValue,
scrollDirection: arguments?["scroll_direction"]?.stringValue,
scrollAmount: arguments?["scroll_amount"]?.intValue,
durationMs: arguments?["duration_ms"]?.intValue)
@@ -0,0 +1,267 @@
import Foundation
import NucleicProtocol
// Unconditional: `VZVirtioSocketConnection` (unlike the arm64-only `VZMac*` guest APIs) exists on
// both architectures, and the client actor must be a real type wherever the engine compiles.
import Virtualization
/// Wire constants + framing for the host in-guest-agent protocol
/// (docs/MACOS_VM_NATIVE_AGENT.md §3§4): NDJSON over vsock one JSON request object per line
/// (carrying an `op`), one JSON reply per line (carrying `ok` + op fields, or `ok:false, error`).
///
/// The guest half lives in `guest/NucleicVMAgent` (its own package the agent runs on macOS 14+
/// guests while this package's floor is 26), which mirrors these constants in `VMAgentCore.AgentWire`;
/// keep the two in lockstep.
enum MacVMAgentWire {
/// The Nucleic-reserved vsock port the guest agent listens on.
static let port: UInt32 = 2035
/// Protocol version this client speaks; the agent reports its own in the `ping` reply.
static let version = 1
/// Default per-request deadline. Generous enough for a full-display SCK screenshot or a deep
/// AX walk; a hung agent surfaces as `.timeout` instead of wedging the tool call.
static let requestTimeout: TimeInterval = 20
/// Encode one request line (`{"op": , }\n`). Pure, unit-tested.
static func requestLine(op: String, fields: [String: JSONValue] = [:]) -> Data {
var object = fields
object["op"] = .string(op)
var line = (try? JSONValue.object(object).encodedData()) ?? Data("{}".utf8)
line.append(0x0A)
return line
}
/// Decode one reply line into its JSON object. Pure, unit-tested.
static func parseReply(_ line: Data) throws -> JSONValue {
guard let reply = try? JSONValue(parsing: line), reply.objectValue != nil else {
throw MacVMAgentError.protocolError(
"malformed agent reply: \(String(decoding: line.prefix(200), as: UTF8.self))")
}
return reply
}
}
/// Errors from the native-agent channel. Deliberately separate from ``MacVMError``: these are
/// routing signals the computer-use path catches them and falls back to SSH not user-facing
/// VM-lifecycle failures.
enum MacVMAgentError: Error, CustomStringConvertible {
/// The VM was built without a vsock device (pre-agent configuration still booted).
case noSocketDevice
/// The vsock connection died or the fd errored; the client is unusable.
case ioFailed(String)
/// No reply within the deadline (agent hung, or nothing listening behind the connect).
case timeout
/// The reply wasn't the protocol's shape.
case protocolError(String)
/// The agent executed the op and reported failure (`ok:false`) transport is healthy.
case agentError(String)
var description: String {
switch self {
case .noSocketDevice: return "the VM has no vsock device"
case .ioFailed(let why): return "agent connection failed: \(why)"
case .timeout: return "the in-guest agent did not reply in time"
case .protocolError(let why): return why
case .agentError(let message): return message
}
}
/// Whether the underlying connection should be discarded (vs. an op-level `agentError`, after
/// which the channel remains perfectly usable).
var isTransportFailure: Bool {
switch self {
case .agentError: return false
default: return true
}
}
}
/// One live NDJSON request/reply channel to the in-guest `NucleicVMAgent`, over the vsock
/// connection ``MacVMInstance/connectAgent(port:)`` opened. The actor serializes requests (the
/// protocol is strictly one-reply-per-request), owns the read buffer, and keeps the
/// `VZVirtioSocketConnection` alive while I/O is in flight (its fd closes with it).
///
/// I/O is nonblocking `read`/`write` on the connection's fd with `Task.sleep` backoff no thread
/// is ever parked on a guest that stops answering; the deadline turns it into `.timeout`.
actor MacVMAgentClient {
private let connection: UncheckedSendableBox<VZVirtioSocketConnection>
private let fd: Int32
private var buffer = Data()
private var closed = false
init(connection: UncheckedSendableBox<VZVirtioSocketConnection>) {
self.connection = connection
self.fd = connection.value.fileDescriptor
// Nonblocking so the actor yields (rather than blocks a cooperative thread) while waiting.
let flags = fcntl(fd, F_GETFL, 0)
_ = fcntl(fd, F_SETFL, flags | O_NONBLOCK)
}
/// Send one op and await its reply object. Throws ``MacVMAgentError/agentError(_:)`` when the
/// agent answers `ok:false`, a transport error when the channel itself broke.
func request(
op: String, fields: [String: JSONValue] = [:],
timeout: TimeInterval = MacVMAgentWire.requestTimeout
) async throws -> JSONValue {
guard !closed, fd >= 0 else { throw MacVMAgentError.ioFailed("connection closed") }
let deadline = Date().addingTimeInterval(timeout)
try await writeAll(MacVMAgentWire.requestLine(op: op, fields: fields), deadline: deadline)
let line = try await readLine(deadline: deadline)
let reply = try MacVMAgentWire.parseReply(line)
guard reply["ok"]?.boolValue == true else {
throw MacVMAgentError.agentError(
reply["error"]?.stringValue ?? "the agent reported an unspecified failure")
}
return reply
}
/// Close the vsock connection. Idempotent; subsequent requests throw.
func close() {
guard !closed else { return }
closed = true
connection.value.close()
}
// MARK: - Nonblocking fd I/O
private func writeAll(_ data: Data, deadline: Date) async throws {
var remaining = data
while !remaining.isEmpty {
let written = remaining.withUnsafeBytes { raw in
write(fd, raw.baseAddress, raw.count)
}
if written > 0 {
remaining = remaining.dropFirst(written)
} else if written < 0, errno == EAGAIN || errno == EINTR {
guard Date() < deadline else { throw MacVMAgentError.timeout }
try? await Task.sleep(nanoseconds: 20_000_000)
} else {
closed = true
throw MacVMAgentError.ioFailed("write errno \(errno)")
}
}
}
private func readLine(deadline: Date) async throws -> Data {
var scratch = [UInt8](repeating: 0, count: 256 * 1024)
while true {
if let newline = buffer.firstIndex(of: 0x0A) {
let line = buffer.subdata(in: buffer.startIndex..<newline)
buffer.removeSubrange(buffer.startIndex...newline)
return line
}
// A screenshot reply is a few MB of base64 cap far above that, but cap.
if buffer.count > 32 * 1024 * 1024 {
closed = true
throw MacVMAgentError.protocolError("agent reply exceeded 32 MB without a newline")
}
let count = read(fd, &scratch, scratch.count)
if count > 0 {
buffer.append(contentsOf: scratch[0..<count])
} else if count == 0 {
closed = true
throw MacVMAgentError.ioFailed("the agent closed the connection")
} else if errno == EAGAIN || errno == EINTR {
guard Date() < deadline else { throw MacVMAgentError.timeout }
try? await Task.sleep(nanoseconds: 25_000_000)
} else {
closed = true
throw MacVMAgentError.ioFailed("read errno \(errno)")
}
}
}
}
#if arch(arm64)
extension MacVMInstance {
/// Open a vsock connection to `port` in this VM's guest. Runs the connect on the VM's serial
/// queue (all VZ calls must); resolves once the guest's listener accepts. If nothing in the
/// guest listens on the port (no agent baked in / not yet launched), the framework fails the
/// connect the engine's probe turns that into "agent unavailable", never an error the user sees.
func connectAgent(port: UInt32) async throws -> UncheckedSendableBox<VZVirtioSocketConnection> {
try await withCheckedThrowingContinuation { cont in
queue.async { [self] in
guard let device = vm.socketDevices.first as? VZVirtioSocketDevice else {
cont.resume(throwing: MacVMAgentError.noSocketDevice)
return
}
device.connect(toPort: port) { result in
switch result {
case .success(let connection):
cont.resume(returning: UncheckedSendableBox(value: connection))
case .failure(let error):
cont.resume(throwing: error)
}
}
}
}
}
}
#endif
extension MacVMEngine {
/// The native in-guest agent channel for a live VM, or `nil` when the VM has none (older base
/// image, agent not running, or vsock connect unsupported) the caller then uses the SSH path.
///
/// First call per VM probes: vsock connect + `ping`, both under a short deadline. The result is
/// cached on the ``LiveVM`` (see ``MacVMAgentState``); a concurrent duplicate probe is harmless
/// (one extra short-lived connection the agent accepts several).
func agentClient(name: String) async -> MacVMAgentClient? {
#if arch(arm64)
guard let entry = live[name] else { return nil }
switch entry.agent {
case .available(let client): return client
case .unavailable: return nil
case .unprobed: break
}
do {
let instance = entry.instance
let box = try await withAgentTimeout(seconds: 5) {
try await instance.connectAgent(port: MacVMAgentWire.port)
}
let client = MacVMAgentClient(connection: box)
let pong = try await client.request(op: "ping", timeout: 5)
// Readiness is informational (each op re-checks its own TCC grant and reports
// precisely); version is the only hard gate.
guard let version = pong["version"]?.intValue, version == MacVMAgentWire.version else {
await client.close()
throw MacVMAgentError.protocolError(
"agent protocol v\(pong["version"]?.intValue ?? 0) ≠ host v\(MacVMAgentWire.version)")
}
live[name]?.agent = .available(client)
return client
} catch {
live[name]?.agent = .unavailable
return nil
}
#else
return nil
#endif
}
/// Discard a VM's agent channel after a transport failure so the next action re-probes once
/// (the guest LaunchAgent's `KeepAlive` may have already restarted the agent).
func discardAgentClient(name: String, _ client: MacVMAgentClient) async {
#if arch(arm64)
await client.close()
if live[name] != nil { live[name]?.agent = .unprobed }
#endif
}
}
/// Race `body` against a wall-clock deadline. The vsock `connect` has no timeout of its own and
/// Apple documents the no-listener case only as "does nothing" this guarantees the agent probe
/// resolves either way.
func withAgentTimeout<T: Sendable>(
seconds: TimeInterval, _ body: @escaping @Sendable () async throws -> T
) async throws -> T {
try await withThrowingTaskGroup(of: T.self) { group in
group.addTask { try await body() }
group.addTask {
try await Task.sleep(nanoseconds: UInt64(seconds * 1_000_000_000))
throw MacVMAgentError.timeout
}
// One of the two always completes; cancel the loser.
let winner = try await group.next()!
group.cancelAll()
return winner
}
}
@@ -156,6 +156,12 @@ extension MacVMEngine {
net.attachment = VZNATNetworkDeviceAttachment()
config.networkDevices = [net]
// vsock channel for the native in-guest agent (docs/MACOS_VM_NATIVE_AGENT.md §3): the host
// reaches the agent via VZVirtioSocketDevice.connect(toPort:), no NAT/DHCP/SSH involved.
// Exactly one socket device per VM (the framework's limit); harmless on a base image that
// doesn't run the agent the engine's probe just falls back to SSH.
config.socketDevices = [VZVirtioSocketDeviceConfiguration()]
config.graphicsDevices = [Self.makeGraphics()]
config.keyboards = [VZMacKeyboardConfiguration()]
config.pointingDevices = [VZMacTrackpadConfiguration()]
@@ -19,10 +19,49 @@ extension MacVMEngine {
static let openPath = "/usr/bin/open"
/// Perform one computer-use action in the named guest and return the post-action screenshot
/// (base64 PNG, ready for the MCP image block) plus a short text summary. `screenshot` just
/// (base64 JPEG, ready for the MCP image block) plus a short text summary. `screenshot` just
/// captures; `cursor_position` returns the pointer location as text; `wait` pauses then captures.
/// Invalid/missing coordinates yield a `denied`-style text with no image (the caller maps it).
///
/// Routing (docs/MACOS_VM_NATIVE_AGENT.md §10): the **native in-guest agent** is preferred when
/// its vsock `ping` answers semantic AX actions, ScreenCaptureKit capture, CGEvent input
/// and this SSH + screencapture/cliclick path is the transparent fallback, so a base image
/// without the agent behaves exactly as before. The `ax_*` actions are agent-only (the
/// accessibility tree has no SSH equivalent); `ref`/`value` are their arguments.
public func performComputerAction(
name: String, action: String, x: Int?, y: Int?, text: String?,
ref: String? = nil, value: String? = nil,
scrollDirection: String?, scrollAmount: Int?, durationMs: Int?
) async throws -> (imageBase64: String?, summary: String) {
if Self.axActions.contains(action) {
guard let client = await agentClient(name: name) else {
return (
nil,
"The '\(action)' action needs the native in-guest agent, which this VM doesn't "
+ "have (base image without NucleicVMAgent, or the agent isn't "
+ "running/authorized). Use screenshot + pixel actions instead."
)
}
return await performAXComputerAction(
name: name, client: client, action: action, x: x, y: y, text: text,
ref: ref, value: value)
}
if let client = await agentClient(name: name),
let native = await performAgentComputerAction(
name: name, client: client, action: action, x: x, y: y, text: text,
scrollDirection: scrollDirection, scrollAmount: scrollAmount,
durationMs: durationMs)
{
return native
}
return try await performSSHComputerAction(
name: name, action: action, x: x, y: y, text: text,
scrollDirection: scrollDirection, scrollAmount: scrollAmount, durationMs: durationMs)
}
/// Today's SSH + `launchctl asuser` + screencapture/cliclick implementation the fallback path
/// (and the only one on a pre-agent base image).
func performSSHComputerAction(
name: String, action: String, x: Int?, y: Int?, text: String?,
scrollDirection: String?, scrollAmount: Int?, durationMs: Int?
) async throws -> (imageBase64: String?, summary: String) {
@@ -98,7 +137,7 @@ extension MacVMEngine {
+ "macOS 15 base image for computer-use; see docs/MACOS_VM.md).")
}
private func screenSummary(for action: String) -> String {
func screenSummary(for action: String) -> String {
action == "screenshot" ? "Screenshot of the macOS VM screen." : "Did '\(action)'; here's the screen now."
}
@@ -0,0 +1,227 @@
import Foundation
import NucleicProtocol
/// Computer-use over the **native in-guest agent** (docs/MACOS_VM_NATIVE_AGENT.md): the preferred
/// path when the guest runs `NucleicVMAgent` ScreenCaptureKit capture, CGEvent input (with a REAL
/// scroll wheel), and the AX-semantic ops that have no SSH/cliclick equivalent at all. The SSH path
/// (`MacVMEngine+Computer`) remains the transparent fallback, so a base image without the agent
/// loses nothing.
extension MacVMEngine {
/// The AX-semantic tool actions (§10). Agent-only: the accessibility tree is not reachable over
/// the SSH/cliclick path, so these never fall back.
static let axActions: Set<String> = [
"ax_dump", "ax_element_at", "ax_press", "ax_set_value", "ax_focus",
]
/// Try one pixel/input action natively. Returns the reply when the action ran (never falls
/// back past this point re-running a click over SSH would double-click), or `nil` when the
/// action did NOT run (op failed before side effects, or the args don't validate) and the SSH
/// path should take over.
func performAgentComputerAction(
name: String, client: MacVMAgentClient, action: String, x: Int?, y: Int?, text: String?,
scrollDirection: String?, scrollAmount: Int?, durationMs: Int?
) async -> (imageBase64: String?, summary: String)? {
do {
switch action {
case "screenshot":
break // nothing to do but capture below
case "cursor_position":
let reply = try await client.request(op: "cursor_position")
guard let cx = reply["x"]?.intValue, let cy = reply["y"]?.intValue else {
return nil
}
return (nil, "Cursor position: \(cx),\(cy)")
case "wait":
let ms = max(0, min(durationMs ?? 1000, 30_000))
try? await Task.sleep(nanoseconds: UInt64(ms) * 1_000_000)
case "left_click", "right_click", "double_click":
guard let x, let y else { return nil }
var fields: [String: JSONValue] = [
"x": .number(Double(x)), "y": .number(Double(y)),
]
if action == "right_click" { fields["button"] = .string("right") }
if action == "double_click" { fields["count"] = .number(2) }
_ = try await client.request(op: "click", fields: fields)
case "mouse_move":
guard let x, let y else { return nil }
_ = try await client.request(
op: "move", fields: ["x": .number(Double(x)), "y": .number(Double(y))])
case "left_click_drag":
// Drag from the CURRENT cursor to the target (the agent defaults `from` to the
// live pointer, so no cursor-position round-trip is needed).
guard let x, let y else { return nil }
_ = try await client.request(
op: "drag", fields: ["toX": .number(Double(x)), "toY": .number(Double(y))])
case "type":
guard let text, !text.isEmpty else { return nil }
_ = try await client.request(op: "type", fields: ["text": .string(text)])
case "key":
guard let chord = text, !chord.isEmpty else { return nil }
_ = try await client.request(op: "key", fields: ["chord": .string(chord)])
case "scroll":
let amount = max(1, min(scrollAmount ?? 3, 30))
let (dx, dy) = Self.scrollDelta(
direction: scrollDirection ?? "down", amount: amount)
_ = try await client.request(
op: "scroll",
fields: ["dx": .number(Double(dx)), "dy": .number(Double(dy))])
case "launch_app":
guard let app = text, !app.isEmpty else { return nil }
_ = try await client.request(op: "launch_app", fields: ["name": .string(app)])
default:
return nil // not a native action the SSH path owns the "unknown action" reply
}
} catch {
await noteAgentFailure(name: name, client: client, error: error)
return nil // the op failed before any side effect safe to re-run over SSH
}
// The action ran natively; capture the post-action frame. Capture-only fallbacks from here
// (the action itself must never re-run).
if let shot = await agentScreenshot(name: name, client: client) {
return (shot, screenSummary(for: action))
}
if let shot = try? await captureScreenshotBase64(name: name) {
return (shot, screenSummary(for: action))
}
return (
nil,
screenSummary(for: action) + " No screenshot came back — the guest may lack Screen "
+ "Recording permission, or (on a macOS 26 guest) the VZ framebuffer bug blanks all "
+ "capture. Use `ax_dump` to inspect the UI semantically instead."
)
}
/// Direction/amount (the tool's scroll vocabulary) wheel-line deltas (the agent's §4 `scroll`
/// op; dy > 0 scrolls content up, dx > 0 scrolls left CGEvent wheel conventions).
static func scrollDelta(direction: String, amount: Int) -> (dx: Int, dy: Int) {
switch direction.lowercased() {
case "up": return (0, amount)
case "left": return (amount, 0)
case "right": return (-amount, 0)
default: return (0, -amount) // down
}
}
// MARK: - AX-semantic actions (§6, §10)
/// Perform one AX action. Unlike the pixel path this never falls back to SSH (there is no SSH
/// equivalent), so failures come back as actionable text for the model.
func performAXComputerAction(
name: String, client: MacVMAgentClient, action: String, x: Int?, y: Int?, text: String?,
ref: String?, value: String?
) async -> (imageBase64: String?, summary: String) {
do {
switch action {
case "ax_dump":
let reply = try await client.request(op: "ax_dump")
let app = reply["app"]?.stringValue ?? "frontmost app"
let tree = reply["tree"] ?? .object([:])
var header = "Accessibility tree of \(app) (roles/titles/values/frames; act on an "
+ "element by its `ref` with ax_press / ax_set_value / ax_focus — refs stay "
+ "valid until the next ax_dump):"
if reply["truncated"]?.boolValue == true {
header += " [tree truncated — deep/rich UI]"
}
return (nil, header + "\n" + tree.canonicalString())
case "ax_element_at":
guard let x, let y else {
return (nil, "ax_element_at requires x and y coordinates.")
}
let reply = try await client.request(
op: "ax_element_at",
fields: ["x": .number(Double(x)), "y": .number(Double(y))])
let node = reply["node"] ?? .object([:])
return (nil, "Element at (\(x), \(y)): " + node.canonicalString())
case "ax_press":
guard let ref, !ref.isEmpty else {
return (nil, "ax_press requires an element `ref` (from ax_dump).")
}
var fields: [String: JSONValue] = ["ref": .string(ref)]
// `text` optionally names a non-default AX action (AXShowMenu, AXConfirm, ).
if let axAction = text, !axAction.isEmpty { fields["action"] = .string(axAction) }
let reply = try await client.request(op: "ax_action", fields: fields)
var summary = "Performed \(text?.isEmpty == false ? text! : "AXPress") on \(ref)."
if let note = reply["note"]?.stringValue { summary += " Note: \(note)." }
return await withPostActionScreenshot(
name: name, client: client, summary: summary)
case "ax_set_value":
guard let ref, !ref.isEmpty else {
return (nil, "ax_set_value requires an element `ref` (from ax_dump).")
}
guard let value else {
return (nil, "ax_set_value requires `value` (the text to set).")
}
_ = try await client.request(
op: "ax_set_value",
fields: ["ref": .string(ref), "value": .string(value)])
return await withPostActionScreenshot(
name: name, client: client, summary: "Set the value of \(ref).")
case "ax_focus":
guard let ref, !ref.isEmpty else {
return (nil, "ax_focus requires an element `ref` (from ax_dump).")
}
_ = try await client.request(op: "ax_focus", fields: ["ref": .string(ref)])
return await withPostActionScreenshot(
name: name, client: client, summary: "Focused \(ref) — you can now `type` into it.")
default:
return (nil, "Unknown AX action '\(action)'.")
}
} catch {
await noteAgentFailure(name: name, client: client, error: error)
let reason = (error as? MacVMAgentError)?.description ?? String(describing: error)
return (
nil,
"\(action) failed: \(reason) If the element may have changed, re-run ax_dump and "
+ "retry with a fresh ref; otherwise fall back to screenshot + pixel actions."
)
}
}
/// The mutating AX actions return a screenshot when capture works (helpful on healthy guests)
/// and a re-dump hint always (the ONLY observation channel on a blank-framebuffer guest).
private func withPostActionScreenshot(
name: String, client: MacVMAgentClient, summary: String
) async -> (imageBase64: String?, summary: String) {
let hint = " Re-run ax_dump to observe the result semantically (refs may now be stale)."
if let shot = await agentScreenshot(name: name, client: client) {
return (shot, summary + hint)
}
return (nil, summary + hint)
}
/// One native screenshot, or `nil` (after noting a transport failure). Never throws capture
/// is always best-effort around an action that already ran.
private func agentScreenshot(name: String, client: MacVMAgentClient) async -> String? {
do {
let reply = try await client.request(op: "screenshot")
return reply["image"]?.stringValue
} catch {
await noteAgentFailure(name: name, client: client, error: error)
return nil
}
}
/// Bookkeeping for a failed agent request: a transport failure discards the channel (the next
/// action re-probes once); an op-level `agentError` keeps the healthy channel.
private func noteAgentFailure(name: String, client: MacVMAgentClient, error: Error) async {
if let agentError = error as? MacVMAgentError, agentError.isTransportFailure {
await discardAgentClient(name: name, client)
}
}
}
+23 -1
View File
@@ -24,6 +24,9 @@ public actor MacVMEngine {
struct LiveVM {
#if arch(arm64)
let instance: MacVMInstance
/// The native in-guest agent channel for this VM (docs/MACOS_VM_NATIVE_AGENT.md), probed
/// lazily by ``MacVMEngine/agentClient(name:)`` and cached here for the VM's lifetime.
var agent: MacVMAgentState = .unprobed
#endif
/// The clone bundle this VM booted from (its writable disk + aux storage live here).
let bundle: MacVMBundle
@@ -51,6 +54,19 @@ public actor MacVMEngine {
/// Last spec seen per name, so `restart` (which only has a name) can recreate from disk.
var lastSpec: [String: MacVMSpec] = [:]
#if arch(arm64)
/// Probe state of a live VM's native in-guest agent (the vsock channel). `unavailable` is
/// sticky for the VM's lifetime a base image without the agent will never grow one mid-boot,
/// so re-probing every action would just pay the connect timeout repeatedly. A transport
/// failure on a previously `available` channel resets to `unprobed` instead (the LaunchAgent's
/// `KeepAlive` restarts a crashed agent, so one reconnect attempt is worth it).
enum MacVMAgentState {
case unprobed
case unavailable
case available(MacVMAgentClient)
}
#endif
/// Held while at least one VM is live, to keep the host app from being App-Napped / auto-terminated
/// out from under a running guest (mirrors ``ContainerEngine``'s assertion; the runtime is
/// daemonless so the app process bounds every VM's life). `nil` when nothing is live.
@@ -227,9 +243,14 @@ public actor MacVMEngine {
/// crashed/powered-off guest doesn't keep consuming a concurrency slot or hold the app-activity
/// assertion. Idempotent: a no-op if the engine already tore it down.
private func handleGuestStopped(_ name: String) {
guard live[name] != nil else { return }
guard let entry = live[name] else { return }
live[name] = nil
syncBackgroundActivity()
#if arch(arm64)
if case .available(let client) = entry.agent {
Task { await client.close() }
}
#endif
}
/// Run `argv` inside the named guest over SSH, returning a ``ProcessHandle`` whose stdio is the
@@ -298,6 +319,7 @@ public actor MacVMEngine {
public func stop(name: String) async {
guard let entry = live[name] else { return }
#if arch(arm64)
if case .available(let client) = entry.agent { await client.close() }
await entry.instance.stop()
#endif
live[name] = nil
+6 -3
View File
@@ -76,14 +76,17 @@ public actor MacVMManager {
try await engine.run(name: name, command: command, workdir: workdir, env: env)
}
/// Perform one computer-use action (screenshot / click / type / ) in the named VM and return the
/// post-action screenshot (base64 PNG) + a summary (the `mac_vm_computer` tool path). No lifecycle effect.
/// Perform one computer-use action (screenshot / click / type / ax_dump / ax_press / ) in the
/// named VM and return the post-action screenshot (base64 JPEG) + a summary (the
/// `mac_vm_computer` tool path). `ref`/`value` feed the AX-semantic actions (native agent).
/// No lifecycle effect.
public func performComputerAction(
name: String, action: String, x: Int?, y: Int?, text: String?,
ref: String? = nil, value: String? = nil,
scrollDirection: String?, scrollAmount: Int?, durationMs: Int?
) async throws -> (imageBase64: String?, summary: String) {
try await engine.performComputerAction(
name: name, action: action, x: x, y: y, text: text,
name: name, action: action, x: x, y: y, text: text, ref: ref, value: value,
scrollDirection: scrollDirection, scrollAmount: scrollAmount, durationMs: durationMs)
}
+7 -1
View File
@@ -257,7 +257,13 @@ public actor SessionController {
iteratively: screenshot → decide → act → screenshot. Reach for it to visually debug a \
macOS/iOS app you built (launch it and look at / click through its window), drive the Xcode \
or Simulator UI, or check how something renders — things a headless command can't show you. \
Coordinates are the screen's real pixels (target exactly what you see). These actions run in \
Coordinates are the screen's real pixels (target exactly what you see). When the VM's \
native agent is available, PREFER the semantic actions: `ax_dump` shows the frontmost \
app's accessibility tree (every control's role/title/value/frame with a `ref`), then act \
by identity with `ax_press`/`ax_set_value`/`ax_focus` — far more reliable than guessing \
pixel coordinates, and it keeps working even when screenshots come back blank (a known \
macOS 26 guest bug). Fall back to screenshot + pixel clicks when a control has no AX \
action or the reply says the native agent is unavailable. These actions run in \
the VM sandbox and are NOT individually approved, so you can move quickly. Prefer \
`mac_vm_exec` for headless build/test; use `mac_vm_computer` when you need to look at or \
operate the screen.
+68
View File
@@ -1,4 +1,5 @@
import Foundation
import NucleicProtocol
import Testing
@testable import NucleicCore
@@ -281,6 +282,73 @@ import Testing
#expect(MacVMEngine.launchAppCommand("Xcode").hasSuffix("/usr/bin/open -a 'Xcode'"))
}
// MARK: - Native in-guest agent (docs/MACOS_VM_NATIVE_AGENT.md)
@Test func agentRequestLineIsOneJSONObjectPerLine() throws {
let line = MacVMAgentWire.requestLine(
op: "click", fields: ["x": .number(10), "y": .number(20)])
#expect(line.last == 0x0A) // newline-terminated (NDJSON)
#expect(!line.dropLast().contains(0x0A)) // and exactly ONE line
let parsed = try JSONValue(parsing: line.dropLast())
#expect(parsed["op"]?.stringValue == "click")
#expect(parsed["x"]?.intValue == 10)
#expect(parsed["y"]?.intValue == 20)
}
@Test func agentRequestLineOpWinsOverFieldCollision() throws {
// `op` is the dispatcher's field; a caller-supplied collision must not hijack it.
let line = MacVMAgentWire.requestLine(op: "ping", fields: ["op": .string("evil")])
let parsed = try JSONValue(parsing: line.dropLast())
#expect(parsed["op"]?.stringValue == "ping")
}
@Test func agentReplyParsingAcceptsObjectsRejectsGarbage() throws {
let reply = try MacVMAgentWire.parseReply(Data(#"{"ok":true,"version":1}"#.utf8))
#expect(reply["ok"]?.boolValue == true)
#expect(reply["version"]?.intValue == 1)
#expect(throws: (any Error).self) {
try MacVMAgentWire.parseReply(Data("not json".utf8))
}
#expect(throws: (any Error).self) {
try MacVMAgentWire.parseReply(Data(#"[1,2,3]"#.utf8)) // an array is not a reply
}
}
@Test func agentErrorSeparatesTransportFromOpFailures() {
// Op-level failures keep the channel; transport failures discard it (re-probe next action).
#expect(!MacVMAgentError.agentError("no such ref").isTransportFailure)
#expect(MacVMAgentError.ioFailed("EOF").isTransportFailure)
#expect(MacVMAgentError.timeout.isTransportFailure)
#expect(MacVMAgentError.protocolError("bad line").isTransportFailure)
#expect(MacVMAgentError.noSocketDevice.isTransportFailure)
}
@Test func scrollDeltaMapsDirectionToWheelLines() {
// CGEvent wheel conventions: wheel1 > 0 scrolls content up, wheel2 > 0 scrolls left.
#expect(MacVMEngine.scrollDelta(direction: "up", amount: 3) == (0, 3))
#expect(MacVMEngine.scrollDelta(direction: "down", amount: 3) == (0, -3))
#expect(MacVMEngine.scrollDelta(direction: "left", amount: 2) == (2, 0))
#expect(MacVMEngine.scrollDelta(direction: "right", amount: 2) == (-2, 0))
#expect(MacVMEngine.scrollDelta(direction: "diagonal??", amount: 1) == (0, -1)) // down
}
@Test func axActionsAreAgentOnlyVocabulary() {
#expect(MacVMEngine.axActions == [
"ax_dump", "ax_element_at", "ax_press", "ax_set_value", "ax_focus",
])
// The pixel vocabulary must stay disjoint routing switches on membership.
for pixel in ["screenshot", "left_click", "type", "key", "scroll", "launch_app", "wait"] {
#expect(!MacVMEngine.axActions.contains(pixel))
}
}
@Test func computerUseGuidanceTeachesAXFirst() {
let on = SessionController.sandboxBuildGuidance(
allowHostExec: true, allowMacVMExec: true, allowMacVMComputer: true)
#expect(on.contains("ax_dump"))
#expect(on.contains("ax_press"))
}
@Test func computerUseSettingGatedOnService() async throws {
let suite = UserDefaults(suiteName: "macvm-cu-\(UUID().uuidString)")!
try await ContainerServiceSettings.withDefaults(suite) {
+43 -28
View File
@@ -6,11 +6,17 @@ capture, **CGEvent** for raw input — reached from the host over **vsock**. It
`mac_vm_computer` transport (SSH → `launchctl asuser``screencapture`/`cliclick` → base64), which is
a stack of shims over those same frameworks.
**Status:** design / spec. Not implemented. The vsock transport and the AXUIElement surface below are
verified against Apple's live documentation (see §11 sources); the one empirical claim to validate on a
real guest (guest-side vsock `listen`, and AX/CGEvent behaviour under the Tahoe framebuffer bug) is
called out in §9 and §10. Companion to [MACOS_VM](MACOS_VM.md) (which this augments) and
[VSOCK_CONTROL_PLANE](VSOCK_CONTROL_PLANE.md) (Nucleic's existing vsock work for the Linux container).
**Status:** implemented through §13 phases 14 (code + provisioning), **pending real-guest
validation** (§13 phase 5 / §14). What exists: the guest agent (`guest/NucleicVMAgent` — its own
SwiftPM package, macOS 14 floor — vsock listener, all §4 ops), the host side (`MacVMAgentClient` +
the `VZVirtioSocketDevice` config + the ping-probe/fallback routing in `MacVMEngine+ComputerAgent`),
the extended `mac_vm_computer` tool surface (`ax_*` actions + `ref`/`value`), the app-bundle build
(`scripts/build-vm-agent.sh`), and provisioning Phase 7d (`scripts/provision-macos-guest.sh`). The
empirical claims to validate on a real guest (guest-side vsock `listen`, and AX/CGEvent behaviour
under the Tahoe framebuffer bug) are called out in §9 and §10 — until then the SSH path remains the
proven default and the agent is strictly additive. Companion to [MACOS_VM](MACOS_VM.md) (which this
augments) and [VSOCK_CONTROL_PLANE](VSOCK_CONTROL_PLANE.md) (Nucleic's existing vsock work for the
Linux container).
---
@@ -347,36 +353,45 @@ present.
---
## 12. Fit with the existing code (when built)
## 12. Fit with the existing code (as built)
- **New target `NucleicVMAgent`** — a macOS executable built into a `.app` bundle (its own SwiftPM
executable target or a small Xcode target invoked from the base-image build). Uses `ApplicationServices`,
`ScreenCaptureKit`, `CoreGraphics`, `AppKit`.
- **`MacVMEngine`** — add `VZVirtioSocketDeviceConfiguration` in `buildConfiguration`; a `MacVMAgentClient`
(vsock connect on the VM queue, NDJSON request/reply); a `ping`-based capability check cached per live VM.
- **`MacVMEngine+Computer`** — route `performComputerAction` through the agent when present, SSH/cliclick
otherwise; add the `ax_*` ops (no SSH fallback for those).
- **`MCPApprovalServer`** — extend the `mac_vm_computer` schema with the AX actions + `ref`/`value` fields;
return the AX tree as text content (and screenshots as image content, as now).
- **`scripts/provision-macos-guest.sh`** — install the agent app + LaunchAgent + its TCC grants.
- **`MacVMSettings`** — a toggle is optional; the agent is auto-preferred when baked in, so no new
user-facing switch is strictly required (a "prefer native agent" override could aid debugging).
- **`guest/NucleicVMAgent`** — the agent, as its own SwiftPM **package** (not a target of the main
one): the main package's platform floor is macOS 26, but the agent must run on a macOS 14/15
(Sequoia) guest, and SwiftPM has no per-target floors. Targets: `CVsock` (C shim — Swift's Darwin
overlay doesn't surface `sockaddr_vm`), `VMAgentCore` (pure wire/keymap logic, unit-tested), and
the `NucleicVMAgent` executable (`ApplicationServices`, `ScreenCaptureKit`, `CoreGraphics`,
`AppKit`). `scripts/build-vm-agent.sh` wraps it into the signed `.app` (+ the LaunchAgent plist).
- **`MacVMEngine`** — `buildConfiguration` attaches one `VZVirtioSocketDeviceConfiguration`;
`MacVMAgentClient.swift` holds the NDJSON client (an actor over the vsock fd), the wire constants
(`MacVMAgentWire`, port 2035 — mirrored in `VMAgentCore.AgentWire`), and the `ping`-probe
capability cache per live VM (`LiveVM.agent`: unprobed → available/unavailable; a transport
failure re-probes once, an absent agent stays absent for the VM's lifetime).
- **`MacVMEngine+ComputerAgent`** — routes `performComputerAction` through the agent when present
(action ran natively ⇒ never re-run over SSH; capture-only fallbacks after), SSH/cliclick
otherwise; implements the `ax_*` ops (no SSH fallback for those).
- **`MCPApprovalServer`** — the `mac_vm_computer` schema carries the AX actions + `ref`/`value`
fields; the AX tree returns as text content (and screenshots as image content, as before).
- **`scripts/provision-macos-guest.sh`** — Phase 7d installs the agent app + LaunchAgent + its three
TCC grants (bundle-id-keyed rows), skipping gracefully when the app isn't staged or SIP is on.
- **`MacVMSettings`** — no new toggle: the agent is auto-preferred when present (the ping probe is
the switch), and the SSH path engages transparently otherwise.
---
## 13. Rollout plan (phased, each independently landable)
1. **Transport core**`NucleicVMAgent` skeleton with the vsock listener + `ping`/`screenshot`; host
`VZVirtioSocketDevice` config + client; prove the round-trip on a real guest (validates §3.2). SSH
path stays the default until this is proven.
2. **Input + capture over the agent**`click`/`move`/`type`/`key`/`scroll` (real scroll) + SCK
`screenshot`; flip `mac_vm_computer` to prefer the agent, SSH fallback intact.
3. **Accessibility**`ax_dump`/`ax_element_at`/`ax_action`/`ax_set_value` + the tool-schema actions +
1. **Transport core**`NucleicVMAgent` skeleton with the vsock listener + `ping`/`screenshot`; host
`VZVirtioSocketDevice` config + client. *Code landed; the real-guest round-trip that proves §3.2 is
still owed (see 5).* SSH path stays the default until this is proven.
2. **Input + capture over the agent**`click`/`move`/`type`/`key`/`scroll` (real scroll) + SCK
`screenshot`; `mac_vm_computer` prefers the agent, SSH fallback intact.
3. **Accessibility**`ax_dump`/`ax_element_at`/`ax_action`/`ax_set_value` + the tool-schema actions +
model guidance. This is where the semantic + framebuffer-independent wins land.
4. **Provisioning + base image** bake the agent, LaunchAgent, and TCC grants into
`provision-macos-guest.sh`; document.
5. **Validate on Tahoe** — confirm AX observe+control works with blank screenshots (§9); if so, Tahoe
becomes viable for computer-use *via AX*, relaxing the Sequoia-only recommendation for the control path.
4. **Provisioning + base image** — the agent, LaunchAgent, and TCC grants in
`provision-macos-guest.sh` Phase 7d (agent app staged from `scripts/build-vm-agent.sh` output).
5. **Validate on a real guest** — smoke-test the vsock round-trip (§3.2), then confirm on Tahoe that
AX observe+control works with blank screenshots (§9); if so, Tahoe becomes viable for computer-use
*via AX*, relaxing the Sequoia-only recommendation for the control path.
---
+2
View File
@@ -0,0 +1,2 @@
.build/
dist/
+33
View File
@@ -0,0 +1,33 @@
// swift-tools-version: 6.2
import PackageDescription
// The native in-guest automation agent for Nucleic's macOS VMs (docs/MACOS_VM_NATIVE_AGENT.md).
//
// A SEPARATE package from the main Nucleic one on purpose: the main package's platform floor is
// macOS 26 (the host app needs the macOS-26 containerization framework), but this binary runs INSIDE
// the guest and the recommended computer-use guest is macOS 15 (Sequoia), because of the Tahoe
// framebuffer bug (docs/MACOS_VM.md §12.6). SwiftPM has no per-target platform floors, so the agent
// gets its own package with a macOS 14 floor (`SCScreenshotManager` is macOS 14+).
//
// Built on the HOST (scripts/build-vm-agent.sh wraps it into a signed NucleicVMAgent.app) and baked
// into the golden base image by scripts/provision-macos-guest.sh; never shipped with the host app.
let package = Package(
name: "NucleicVMAgent",
platforms: [.macOS(.v14)],
targets: [
// C shim for the AF_VSOCK listener: `struct sockaddr_vm` (<sys/vsock.h>) isn't surfaced to
// Swift by the Darwin overlay, so the socket/bind/listen dance lives in C.
.target(name: "CVsock"),
// Pure, framework-light logic (wire constants, NDJSON line splitting, key-chord parsing)
// unit-testable on any Mac without TCC grants or a GUI session.
.target(name: "VMAgentCore"),
// The agent itself: vsock accept loop + the op handlers (AXUIElement / ScreenCaptureKit /
// CGEvent). Swift 5 language mode: the handlers are thread-confined by construction (one
// blocking thread per connection), which strict concurrency can't see past the CF/AX types.
.executableTarget(
name: "NucleicVMAgent",
dependencies: ["CVsock", "VMAgentCore"],
swiftSettings: [.swiftLanguageMode(.v5)]),
.testTarget(name: "VMAgentCoreTests", dependencies: ["VMAgentCore"]),
]
)
+32
View File
@@ -0,0 +1,32 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<!-- Bundle metadata for NucleicVMAgent.app (docs/MACOS_VM_NATIVE_AGENT.md §11). A real signed
.app bundle is mandatory-in-practice: TCC mis-handles bare executables, and on macOS 26
ScreenCaptureKit returns -3801 ("declined") for a non-bundled binary even with a TCC row. -->
<dict>
<key>CFBundleIdentifier</key>
<string>xyz.blakeslee.nucleic.vmagent</string>
<key>CFBundleName</key>
<string>NucleicVMAgent</string>
<key>CFBundleDisplayName</key>
<string>Nucleic VM Agent</string>
<key>CFBundleExecutable</key>
<string>NucleicVMAgent</string>
<key>CFBundlePackageType</key>
<string>APPL</string>
<key>CFBundleShortVersionString</key>
<string>1.0</string>
<key>CFBundleVersion</key>
<string>1</string>
<key>LSMinimumSystemVersion</key>
<string>14.0</string>
<!-- Background agent: no Dock icon, no menu bar. -->
<key>LSUIElement</key>
<true/>
<key>NSScreenCaptureUsageDescription</key>
<string>Nucleic's in-guest agent captures this virtual machine's screen so the driving agent can see it.</string>
<key>NSAccessibilityUsageDescription</key>
<string>Nucleic's in-guest agent reads and drives app UIs in this virtual machine via accessibility.</string>
</dict>
</plist>
@@ -0,0 +1,29 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<!-- LaunchAgent for NucleicVMAgent (docs/MACOS_VM_NATIVE_AGENT.md §11). Installed by
scripts/provision-macos-guest.sh at /Library/LaunchAgents/ so it runs in the auto-login
`agent` account's Aqua session — AX, ScreenCaptureKit and CGEvent all require an active
WindowServer session, which is why this is a LaunchAgent, not a LaunchDaemon. -->
<dict>
<key>Label</key>
<string>xyz.blakeslee.nucleic.vmagent</string>
<key>ProgramArguments</key>
<array>
<string>/Applications/NucleicVMAgent.app/Contents/MacOS/NucleicVMAgent</string>
</array>
<key>RunAtLoad</key>
<true/>
<key>KeepAlive</key>
<true/>
<!-- Aqua only: never launch in the pre-login / background domains where the frameworks fail. -->
<key>LimitLoadToSessionType</key>
<string>Aqua</string>
<key>AssociatedBundleIdentifiers</key>
<string>xyz.blakeslee.nucleic.vmagent</string>
<key>StandardOutPath</key>
<string>/tmp/nucleic-vmagent.log</string>
<key>StandardErrorPath</key>
<string>/tmp/nucleic-vmagent.log</string>
</dict>
</plist>
+35
View File
@@ -0,0 +1,35 @@
# NucleicVMAgent
The **native in-guest automation agent** for Nucleic's macOS VMs — design and rationale in
[docs/MACOS_VM_NATIVE_AGENT.md](../../docs/MACOS_VM_NATIVE_AGENT.md). It runs inside the guest as a
LaunchAgent in the auto-login Aqua session, listens on **vsock port 2035**, and executes NDJSON ops
with Apple's frameworks directly:
- **AXUIElement** — semantic observe + control (`ax_dump`, `ax_action`, `ax_set_value`, `ax_focus`,
`ax_element_at`). Framebuffer-independent: keeps working on a macOS 26 guest whose screenshots are
blank (the Tahoe VZ compositing bug).
- **ScreenCaptureKit** — in-process `screenshot`.
- **CGEvent** — raw `click`/`move`/`drag`/`type`/`key`/`scroll` (a real wheel event).
A **separate SwiftPM package** from the main Nucleic one: the host app's floor is macOS 26, but this
binary must run on macOS 14/15 guests (the recommended computer-use base is Sequoia), and SwiftPM has
no per-target platform floors.
## Build & install
```sh
# On the host (needs Swift; sign stably or TCC grants break on rebuild):
NUCLEIC_VMAGENT_SIGN_IDENTITY="Developer ID Application: …" ../../scripts/build-vm-agent.sh
# → dist/NucleicVMAgent.app + dist/xyz.blakeslee.nucleic.vmagent.plist
# Stage dist/* next to scripts/provision-macos-guest.sh inside the guest and run the provisioner —
# Phase 7d installs the app, the LaunchAgent, and the three TCC grants (Accessibility, PostEvent,
# ScreenCapture; needs SIP off, same as the rest of Phase 7).
```
The host side (`MacVMAgentClient` in `Sources/NucleicCore/MacVM/`) probes the agent with a vsock
`ping` per VM boot and transparently falls back to the SSH + cliclick path when it's absent — the
agent is strictly additive. Wire constants are mirrored between `VMAgentCore.AgentWire` (here) and
`MacVMAgentWire` (host); keep them in lockstep.
Agent log inside the guest: `/tmp/nucleic-vmagent.log`.
@@ -0,0 +1,31 @@
#include "include/cvsock.h"
#include <errno.h>
#include <string.h>
#include <sys/socket.h>
#include <sys/vsock.h>
#include <unistd.h>
int nucleic_vsock_listen(uint32_t port, int backlog) {
int fd = socket(AF_VSOCK, SOCK_STREAM, 0);
if (fd < 0) return -errno;
struct sockaddr_vm addr;
memset(&addr, 0, sizeof(addr));
addr.svm_len = sizeof(addr); // macOS-specific BSD length field (Linux has none)
addr.svm_family = AF_VSOCK;
addr.svm_cid = VMADDR_CID_ANY;
addr.svm_port = port;
if (bind(fd, (const struct sockaddr *)&addr, sizeof(addr)) != 0) {
int e = errno;
close(fd);
return -e;
}
if (listen(fd, backlog) != 0) {
int e = errno;
close(fd);
return -e;
}
return fd;
}
@@ -0,0 +1,12 @@
#ifndef NUCLEIC_CVSOCK_H
#define NUCLEIC_CVSOCK_H
#include <stdint.h>
// Create + bind + listen an AF_VSOCK stream socket on `port`, listening on any CID (inside a macOS
// guest the guest's CID is assigned by the hypervisor; the host connects via VZVirtioSocketDevice).
// Returns the listening fd (>= 0), or -errno on failure. Lives in C because Swift's Darwin overlay
// does not surface `struct sockaddr_vm` from <sys/vsock.h>.
int nucleic_vsock_listen(uint32_t port, int backlog);
#endif
@@ -0,0 +1,152 @@
import CVsock
import Darwin
import Foundation
import VMAgentCore
enum AgentError: Error, CustomStringConvertible {
case posix(String, Int32)
var description: String {
switch self {
case let .posix(what, errno): return "\(what) failed: \(String(cString: strerror(errno)))"
}
}
}
/// Timestamped line to stderr the LaunchAgent routes it to the agent's log file.
func agentLog(_ message: String) {
FileHandle.standardError.write(Data("[NucleicVMAgent] \(message)\n".utf8))
}
/// Owns the AF_VSOCK listening socket and the blocking accept loop. One `ConnectionHandler` thread
/// per accepted connection; in practice there is exactly one client (the host's `MacVMAgentClient`),
/// but a reconnect after a host restart must not require an agent restart.
final class AgentListener {
private let fd: Int32
init(port: UInt32) throws {
let result = nucleic_vsock_listen(port, 4)
guard result >= 0 else { throw AgentError.posix("vsock listen(:\(port))", -result) }
fd = result
}
func run() {
while true {
let conn = accept(fd, nil, nil)
if conn < 0 {
if errno == EINTR { continue }
agentLog("accept failed (errno \(errno)); retrying")
Thread.sleep(forTimeInterval: 1)
continue
}
agentLog("host connected (fd \(conn))")
let handler = ConnectionHandler(fd: conn)
Thread.detachNewThread { handler.run() }
}
}
}
/// One host connection: a blocking read loop that frames NDJSON requests, dispatches each op, and
/// writes the one-line JSON reply. Everything for this connection (including the AX element
/// registry) is confined to this thread requests on a connection are strictly serial.
final class ConnectionHandler {
private let fd: Int32
private var lines = LineSplitBuffer()
/// AX element handles (`ref` live `AXUIElement`) issued by the last dump on this connection.
let axRegistry = AXRegistry()
init(fd: Int32) {
self.fd = fd
}
func run() {
defer {
close(fd)
agentLog("host disconnected (fd \(fd))")
}
var scratch = [UInt8](repeating: 0, count: 64 * 1024)
while true {
let n = read(fd, &scratch, scratch.count)
if n == 0 { return } // EOF host closed
if n < 0 {
if errno == EINTR { continue }
return
}
lines.append(Data(bytes: scratch, count: n))
while let line = lines.nextLine() {
guard !line.isEmpty else { continue }
var reply = handle(requestLine: line)
reply.append(0x0A)
guard writeAll(reply) else { return }
}
// Runaway-line guard: a client that never sends a newline can't balloon memory.
if lines.pendingBytes > 4 * 1024 * 1024 { return }
}
}
private func writeAll(_ data: Data) -> Bool {
var remaining = data
while !remaining.isEmpty {
let written = remaining.withUnsafeBytes { raw in
write(fd, raw.baseAddress, raw.count)
}
if written < 0 {
if errno == EINTR { continue }
return false
}
remaining = remaining.dropFirst(written)
}
return true
}
// MARK: - Dispatch
private func handle(requestLine: Data) -> Data {
let reply = dispatch(requestLine: requestLine)
if let data = try? JSONSerialization.data(withJSONObject: reply) { return data }
return Data(#"{"ok":false,"error":"reply serialization failed"}"#.utf8)
}
private func dispatch(requestLine: Data) -> [String: Any] {
guard
let parsed = try? JSONSerialization.jsonObject(with: requestLine),
let request = parsed as? [String: Any],
let op = request["op"] as? String
else {
return AgentReply.err("malformed request (want one JSON object per line with an \"op\")")
}
// Each op is individually guarded so a framework-level surprise degrades to an error reply,
// never a dead agent.
switch op {
case "ping": return opPing()
case "screenshot": return opScreenshot(request)
case "ax_dump": return opAXDump(request)
case "ax_element_at": return opAXElementAt(request)
case "ax_action": return opAXAction(request)
case "ax_set_value": return opAXSetValue(request)
case "ax_focus": return opAXFocus(request)
case "click": return opClick(request)
case "move": return opMove(request)
case "drag": return opDrag(request)
case "type": return opType(request)
case "key": return opKey(request)
case "scroll": return opScroll(request)
case "cursor_position": return opCursorPosition()
case "launch_app": return opLaunchApp(request)
default:
return AgentReply.err("unknown op '\(op)' (agent protocol v\(AgentWire.version))")
}
}
}
/// Reply-shape helpers (§4): `{ok:true, }` or `{ok:false, error}`.
enum AgentReply {
static func ok(_ fields: [String: Any] = [:]) -> [String: Any] {
var reply = fields
reply["ok"] = true
return reply
}
static func err(_ message: String) -> [String: Any] {
["ok": false, "error": message]
}
}
@@ -0,0 +1,277 @@
import AppKit
import ApplicationServices
import Foundation
/// Per-connection registry of the element handles (`ref` live `AXUIElementRef`) issued by the
/// last `ax_dump`/`ax_element_at` (§6.2). `AXUIElementRef`s go stale as the UI mutates, so refs are
/// valid until the next dump replaces them; a stale ref surfaces as "re-dump and retry".
final class AXRegistry {
private var refs: [String: AXUIElement] = [:]
private var counter = 0
func clear() { refs.removeAll() }
func register(_ element: AXUIElement) -> String {
counter += 1
let ref = "e\(counter)"
refs[ref] = element
return ref
}
subscript(ref: String) -> AXUIElement? { refs[ref] }
}
/// Accessibility observe + control (§6): the semantic, framebuffer-independent surface. All calls
/// are synchronous cross-process IPC into the target app, bounded by the messaging timeout so a
/// hung app can't wedge the walker.
extension ConnectionHandler {
private var axTrusted: Bool { AXIsProcessTrusted() }
private static let axDenied =
"Accessibility (kTCCServiceAccessibility) is not granted to the agent — re-run provisioning"
/// `ax_dump` walk the frontmost (or `pid`-addressed) app's element tree into a JSON node tree,
/// registering a fresh `ref` per element. This is the model's semantic "look" primitive.
func opAXDump(_ request: [String: Any]) -> [String: Any] {
guard axTrusted else { return AgentReply.err(Self.axDenied) }
let pid: pid_t
if let requested = request["pid"] as? Int {
pid = pid_t(requested)
} else if let front = NSWorkspace.shared.frontmostApplication {
pid = front.processIdentifier
} else {
return AgentReply.err("no frontmost application to dump")
}
let maxDepth = min(max((request["maxDepth"] as? Int) ?? 12, 1), 25)
// New dump new handle generation; every previously issued ref is now invalid by contract.
axRegistry.clear()
let app = AXUIElementCreateApplication(pid)
// A tight timeout so ONE hung app can't stall the whole walk for the default ~6 s per call.
AXUIElementSetMessagingTimeout(app, 1.5)
// Total node budget: a rich app's full tree is thousands of IPC round-trips (§6.1); the cap
// bounds both the walk time and the payload the model has to read.
var budget = 800
let tree = buildNode(app, depth: 0, maxDepth: maxDepth, budget: &budget)
var fields: [String: Any] = ["tree": tree]
fields["app"] = NSRunningApplication(processIdentifier: pid)?.localizedName ?? "pid \(pid)"
if budget <= 0 { fields["truncated"] = true }
return AgentReply.ok(fields)
}
/// `ax_element_at` resolve the element under a screen point (hit-test via the system-wide
/// element) into a single node (no children). Registers a ref alongside the last dump's.
func opAXElementAt(_ request: [String: Any]) -> [String: Any] {
guard axTrusted else { return AgentReply.err(Self.axDenied) }
guard let x = request["x"] as? Int, let y = request["y"] as? Int else {
return AgentReply.err("ax_element_at requires x and y")
}
let systemWide = AXUIElementCreateSystemWide()
AXUIElementSetMessagingTimeout(systemWide, 1.5)
var element: AXUIElement?
let result = AXUIElementCopyElementAtPosition(systemWide, Float(x), Float(y), &element)
guard result == .success, let element else {
return AgentReply.err("no accessible element at (\(x), \(y)) (\(describe(result)))")
}
var budget = 1
return AgentReply.ok(["node": buildNode(element, depth: 0, maxDepth: 0, budget: &budget)])
}
/// `ax_action` perform a named action (`AXPress`, `AXShowMenu`, ) on a ref'd element: the
/// control's own callback runs in the app process, no rendering or synthetic click involved.
func opAXAction(_ request: [String: Any]) -> [String: Any] {
guard axTrusted else { return AgentReply.err(Self.axDenied) }
guard let element = resolveRef(request) else { return staleRefReply(request) }
let action = (request["action"] as? String).flatMap { $0.isEmpty ? nil : $0 } ?? "AXPress"
let result = AXUIElementPerformAction(element, action as CFString)
switch result {
case .success:
return AgentReply.ok()
case .cannotComplete:
// Apps often run modal processing inside the action callback and never return within
// the AX timeout the action may well have fired (§6.3). Tell the host the truth.
return AgentReply.ok([
"note": "the app did not confirm within the AX timeout — the action possibly "
+ "succeeded; re-dump to verify"
])
case .invalidUIElement:
return staleRefReply(request)
default:
return AgentReply.err("\(action) failed (\(describe(result)))")
}
}
/// `ax_set_value` write a field's `AXValue` directly (the reliable way to fill text fields).
/// Non-settable / non-AppKit fields are reported as such so the host can fall back to
/// `ax_focus` + CGEvent typing.
func opAXSetValue(_ request: [String: Any]) -> [String: Any] {
guard axTrusted else { return AgentReply.err(Self.axDenied) }
guard let element = resolveRef(request) else { return staleRefReply(request) }
guard let value = request["value"] as? String else {
return AgentReply.err("ax_set_value requires a string \"value\"")
}
var settable = DarwinBoolean(false)
AXUIElementIsAttributeSettable(element, kAXValueAttribute as CFString, &settable)
guard settable.boolValue else {
return AgentReply.err(
"this element's AXValue is not settable — focus it (ax_focus) and type instead")
}
let result = AXUIElementSetAttributeValue(
element, kAXValueAttribute as CFString, value as CFString)
guard result == .success else {
if result == .invalidUIElement { return staleRefReply(request) }
return AgentReply.err("setting AXValue failed (\(describe(result)))")
}
return AgentReply.ok()
}
/// `ax_focus` give a ref'd element keyboard focus (precedes CGEvent typing into it).
func opAXFocus(_ request: [String: Any]) -> [String: Any] {
guard axTrusted else { return AgentReply.err(Self.axDenied) }
guard let element = resolveRef(request) else { return staleRefReply(request) }
let result = AXUIElementSetAttributeValue(
element, kAXFocusedAttribute as CFString, kCFBooleanTrue)
guard result == .success else {
if result == .invalidUIElement { return staleRefReply(request) }
return AgentReply.err("focusing failed (\(describe(result)))")
}
return AgentReply.ok()
}
// MARK: - Tree walking
/// Serialize one element into the §4 node shape, recursing into children while depth and the
/// total node budget allow. Empty/absent attributes are omitted to keep the payload the model
/// reads small.
private func buildNode(
_ element: AXUIElement, depth: Int, maxDepth: Int, budget: inout Int
) -> [String: Any] {
budget -= 1
var node: [String: Any] = ["ref": axRegistry.register(element)]
if let role = copyString(element, kAXRoleAttribute) { node["role"] = role }
if let subrole = copyString(element, kAXSubroleAttribute) { node["subrole"] = subrole }
if let title = copyString(element, kAXTitleAttribute), !title.isEmpty {
node["title"] = title
}
if let value = copyValueDescription(element) { node["value"] = value }
if copyBool(element, kAXEnabledAttribute) == false { node["enabled"] = false }
if copyBool(element, kAXFocusedAttribute) == true { node["focused"] = true }
if let frame = copyFrame(element) { node["frame"] = frame }
var actionNames: CFArray?
if AXUIElementCopyActionNames(element, &actionNames) == .success,
let actions = actionNames as? [String], !actions.isEmpty
{
node["actions"] = actions
}
if depth < maxDepth, budget > 0 {
var childrenRef: CFTypeRef?
if AXUIElementCopyAttributeValue(
element, kAXChildrenAttribute as CFString, &childrenRef) == .success,
let children = childrenRef as? [AXUIElement], !children.isEmpty
{
var serialized: [[String: Any]] = []
// Per-node child cap: a 10k-row table must not eat the entire budget at one level.
for child in children.prefix(48) {
guard budget > 0 else { break }
serialized.append(
buildNode(child, depth: depth + 1, maxDepth: maxDepth, budget: &budget))
}
if !serialized.isEmpty { node["children"] = serialized }
if children.count > serialized.count {
node["childrenOmitted"] = children.count - serialized.count
}
}
}
return node
}
private func copyString(_ element: AXUIElement, _ attribute: String) -> String? {
var value: CFTypeRef?
guard
AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success
else { return nil }
return value as? String
}
private func copyBool(_ element: AXUIElement, _ attribute: String) -> Bool? {
var value: CFTypeRef?
guard
AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success
else { return nil }
return (value as? NSNumber)?.boolValue
}
/// A human/model-readable rendering of `AXValue` (string, number, bool), truncated so one giant
/// text view can't dominate the tree payload.
private func copyValueDescription(_ element: AXUIElement) -> String? {
var value: CFTypeRef?
guard
AXUIElementCopyAttributeValue(element, kAXValueAttribute as CFString, &value) == .success,
let value
else { return nil }
let text: String
if let string = value as? String {
text = string
} else if let number = value as? NSNumber {
text = number.stringValue
} else if let url = value as? URL {
text = url.absoluteString
} else {
return nil // AXValueRef boxes (ranges etc.) aren't useful as text
}
if text.isEmpty { return nil }
return text.count > 250 ? String(text.prefix(250)) + "" : text
}
/// `kAXPosition`/`kAXSize` come back as `AXValueRef` boxes unwrap into the §4 frame shape
/// (top-left-origin screen coordinates, matching screenshots and CGEvent clicks).
private func copyFrame(_ element: AXUIElement) -> [String: Any]? {
var positionRef: CFTypeRef?
var sizeRef: CFTypeRef?
var point = CGPoint.zero
var size = CGSize.zero
guard
AXUIElementCopyAttributeValue(
element, kAXPositionAttribute as CFString, &positionRef) == .success,
AXUIElementCopyAttributeValue(
element, kAXSizeAttribute as CFString, &sizeRef) == .success,
let positionRef, let sizeRef,
AXValueGetValue(positionRef as! AXValue, .cgPoint, &point),
AXValueGetValue(sizeRef as! AXValue, .cgSize, &size)
else { return nil }
return [
"x": Int(point.x.rounded()), "y": Int(point.y.rounded()),
"w": Int(size.width.rounded()), "h": Int(size.height.rounded()),
]
}
// MARK: - Ref plumbing
private func resolveRef(_ request: [String: Any]) -> AXUIElement? {
guard let ref = request["ref"] as? String else { return nil }
return axRegistry[ref]
}
private func staleRefReply(_ request: [String: Any]) -> [String: Any] {
let ref = (request["ref"] as? String) ?? "?"
return AgentReply.err(
"element ref \"\(ref)\" is unknown or stale — refs are only valid until the next "
+ "ax_dump; re-dump and retry")
}
private func describe(_ error: AXError) -> String {
switch error {
case .success: return "success"
case .apiDisabled: return "kAXErrorAPIDisabled — accessibility is disabled for this app"
case .cannotComplete: return "kAXErrorCannotComplete — the app did not respond in time"
case .invalidUIElement: return "kAXErrorInvalidUIElement — stale element"
case .attributeUnsupported: return "attribute unsupported"
case .actionUnsupported: return "action unsupported"
case .noValue: return "no value"
case .notImplemented: return "the app does not implement this AX request"
default: return "AXError \(error.rawValue)"
}
}
}
@@ -0,0 +1,163 @@
import CoreGraphics
import Foundation
import VMAgentCore
/// Low-level input synthesis CGEvent posted to the HID event tap (§8). The raw fallback for
/// targets with no AX action (custom-drawn UI); AX actions (§6) are preferred where they exist.
/// All of it needs the `kTCCServicePostEvent` grant and an Aqua session.
extension ConnectionHandler {
private var canPostEvents: Bool { CGPreflightPostEventAccess() }
private func intField(_ request: [String: Any], _ key: String) -> Int? {
(request[key] as? Int) ?? (request[key] as? Double).map(Int.init)
}
/// `click` move the pointer to (x, y), then press/release. `count` 2 raises the click-state
/// sequence AppKit needs to see a real double-click.
func opClick(_ request: [String: Any]) -> [String: Any] {
guard canPostEvents else { return AgentReply.err(Self.postEventDenied) }
guard let x = intField(request, "x"), let y = intField(request, "y") else {
return AgentReply.err("click requires x and y")
}
let position = CGPoint(x: x, y: y)
let count = max(1, min(intField(request, "count") ?? 1, 3))
let (down, up, button): (CGEventType, CGEventType, CGMouseButton) = {
switch request["button"] as? String {
case "right": return (.rightMouseDown, .rightMouseUp, .right)
case "middle": return (.otherMouseDown, .otherMouseUp, .center)
default: return (.leftMouseDown, .leftMouseUp, .left)
}
}()
post(mouse: .mouseMoved, at: position, button: .left)
for clickState in 1...count {
guard
let downEvent = CGEvent(
mouseEventSource: nil, mouseType: down, mouseCursorPosition: position,
mouseButton: button),
let upEvent = CGEvent(
mouseEventSource: nil, mouseType: up, mouseCursorPosition: position,
mouseButton: button)
else { return AgentReply.err("could not create the mouse event") }
downEvent.setIntegerValueField(.mouseEventClickState, value: Int64(clickState))
upEvent.setIntegerValueField(.mouseEventClickState, value: Int64(clickState))
downEvent.post(tap: .cghidEventTap)
usleep(40_000)
upEvent.post(tap: .cghidEventTap)
if clickState < count { usleep(90_000) } // within the double-click interval
}
return AgentReply.ok()
}
/// `move` reposition the pointer without clicking.
func opMove(_ request: [String: Any]) -> [String: Any] {
guard canPostEvents else { return AgentReply.err(Self.postEventDenied) }
guard let x = intField(request, "x"), let y = intField(request, "y") else {
return AgentReply.err("move requires x and y")
}
post(mouse: .mouseMoved, at: CGPoint(x: x, y: y), button: .left)
return AgentReply.ok()
}
/// `drag` press at (fromX, fromY) (default: the current pointer), glide in interpolated
/// dragged events (apps ignore a teleporting drag), release at (toX, toY).
func opDrag(_ request: [String: Any]) -> [String: Any] {
guard canPostEvents else { return AgentReply.err(Self.postEventDenied) }
guard let toX = intField(request, "toX"), let toY = intField(request, "toY") else {
return AgentReply.err("drag requires toX and toY")
}
let current = CGEvent(source: nil)?.location ?? .zero
let from = CGPoint(
x: intField(request, "fromX").map(Double.init) ?? current.x,
y: intField(request, "fromY").map(Double.init) ?? current.y)
let to = CGPoint(x: toX, y: toY)
post(mouse: .mouseMoved, at: from, button: .left)
usleep(60_000)
post(mouse: .leftMouseDown, at: from, button: .left)
let steps = 12
for step in 1...steps {
let t = Double(step) / Double(steps)
let waypoint = CGPoint(
x: from.x + (to.x - from.x) * t, y: from.y + (to.y - from.y) * t)
usleep(15_000)
post(mouse: .leftMouseDragged, at: waypoint, button: .left)
}
usleep(60_000)
post(mouse: .leftMouseUp, at: to, button: .left)
return AgentReply.ok()
}
/// `type` inject literal text as unicode keyboard events (no key-code mapping, so any text
/// works regardless of layout). Chunked: `keyboardSetUnicodeString` reliably carries only short
/// runs per event.
func opType(_ request: [String: Any]) -> [String: Any] {
guard canPostEvents else { return AgentReply.err(Self.postEventDenied) }
guard let text = request["text"] as? String, !text.isEmpty else {
return AgentReply.err("type requires non-empty \"text\"")
}
let units = Array(text.utf16)
var index = 0
while index < units.count {
let chunk = Array(units[index..<min(index + 16, units.count)])
guard
let down = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: true),
let up = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: false)
else { return AgentReply.err("could not create the keyboard event") }
down.keyboardSetUnicodeString(stringLength: chunk.count, unicodeString: chunk)
up.keyboardSetUnicodeString(stringLength: chunk.count, unicodeString: chunk)
down.post(tap: .cghidEventTap)
up.post(tap: .cghidEventTap)
usleep(12_000)
index += 16
}
return AgentReply.ok()
}
/// `key` a chord like `cmd+s` / `return` / `cmd+shift+4`, as a real key-code press with the
/// modifier flags held (VMAgentCore.KeyMap owns the parse).
func opKey(_ request: [String: Any]) -> [String: Any] {
guard canPostEvents else { return AgentReply.err(Self.postEventDenied) }
guard let chord = request["chord"] as? String, !chord.isEmpty else {
return AgentReply.err("key requires a \"chord\" (e.g. \"cmd+s\")")
}
guard let stroke = KeyMap.stroke(forChord: chord) else {
return AgentReply.err("unrecognized key in chord \"\(chord)\"")
}
guard
let down = CGEvent(keyboardEventSource: nil, virtualKey: stroke.keyCode, keyDown: true),
let up = CGEvent(keyboardEventSource: nil, virtualKey: stroke.keyCode, keyDown: false)
else { return AgentReply.err("could not create the keyboard event") }
down.flags = stroke.flags
up.flags = stroke.flags
down.post(tap: .cghidEventTap)
usleep(30_000)
up.post(tap: .cghidEventTap)
return AgentReply.ok()
}
/// `scroll` a REAL wheel event (dy > 0 scrolls content up, dx > 0 scrolls left, in lines),
/// unlike the SSH path's arrow-key approximation.
func opScroll(_ request: [String: Any]) -> [String: Any] {
guard canPostEvents else { return AgentReply.err(Self.postEventDenied) }
let dy = max(-40, min(intField(request, "dy") ?? 0, 40))
let dx = max(-40, min(intField(request, "dx") ?? 0, 40))
guard dx != 0 || dy != 0 else { return AgentReply.err("scroll requires dx and/or dy") }
guard
let event = CGEvent(
scrollWheelEvent2Source: nil, units: .line, wheelCount: 2,
wheel1: Int32(dy), wheel2: Int32(dx), wheel3: 0)
else { return AgentReply.err("could not create the scroll event") }
event.post(tap: .cghidEventTap)
return AgentReply.ok()
}
private func post(mouse type: CGEventType, at position: CGPoint, button: CGMouseButton) {
CGEvent(
mouseEventSource: nil, mouseType: type, mouseCursorPosition: position,
mouseButton: button)?
.post(tap: .cghidEventTap)
}
private static let postEventDenied =
"input synthesis (kTCCServicePostEvent) is not granted to the agent — re-run provisioning"
}
@@ -0,0 +1,83 @@
import AppKit
import ApplicationServices
import CoreGraphics
import Foundation
import VMAgentCore
extension ConnectionHandler {
/// `ping` capability negotiation: protocol version, displays, and TCC readiness, so the host
/// can distinguish "agent present" from "agent present but not authorized" (§5).
func opPing() -> [String: Any] {
var displayIDs = [CGDirectDisplayID](repeating: 0, count: 8)
var displayCount: UInt32 = 0
CGGetActiveDisplayList(UInt32(displayIDs.count), &displayIDs, &displayCount)
let displays: [[String: Any]] = (0..<Int(displayCount)).map { i in
let id = displayIDs[i]
return [
"id": Int(id),
"width": CGDisplayPixelsWide(id),
"height": CGDisplayPixelsHigh(id),
"main": CGDisplayIsMain(id) != 0,
]
}
return AgentReply.ok([
"version": AgentWire.version,
"displays": displays,
"readiness": [
"accessibility": AXIsProcessTrusted(),
"screenRecording": CGPreflightScreenCaptureAccess(),
"postEvent": CGPreflightPostEventAccess(),
],
])
}
/// `cursor_position` the pointer's current screen location (top-left-origin pixels).
func opCursorPosition() -> [String: Any] {
guard let event = CGEvent(source: nil) else {
return AgentReply.err("could not read the cursor position (no event source)")
}
let p = event.location
return AgentReply.ok(["x": Int(p.x.rounded()), "y": Int(p.y.rounded())])
}
/// `launch_app` open an application by name, `NSWorkspace` when the bundle is findable in the
/// standard app directories, else `/usr/bin/open -a` (which searches LaunchServices).
func opLaunchApp(_ request: [String: Any]) -> [String: Any] {
guard let name = request["name"] as? String, !name.isEmpty else {
return AgentReply.err("launch_app requires a non-empty \"name\"")
}
let leaf = name.hasSuffix(".app") ? name : name + ".app"
let candidates = [
"/Applications/\(leaf)",
"/System/Applications/\(leaf)",
"/System/Applications/Utilities/\(leaf)",
]
if let path = candidates.first(where: { FileManager.default.fileExists(atPath: $0) }) {
let semaphore = DispatchSemaphore(value: 0)
var failure: String?
NSWorkspace.shared.openApplication(
at: URL(fileURLWithPath: path), configuration: NSWorkspace.OpenConfiguration()
) { _, error in
failure = error.map { String(describing: $0) }
semaphore.signal()
}
_ = semaphore.wait(timeout: .now() + 15)
if let failure { return AgentReply.err("could not launch \(name): \(failure)") }
return AgentReply.ok()
}
// Not in the standard directories let LaunchServices resolve it.
let open = Process()
open.executableURL = URL(fileURLWithPath: "/usr/bin/open")
open.arguments = ["-a", name]
do {
try open.run()
} catch {
return AgentReply.err("could not launch \(name): \(error)")
}
open.waitUntilExit()
guard open.terminationStatus == 0 else {
return AgentReply.err("no application named \"\(name)\" (open -a exited \(open.terminationStatus))")
}
return AgentReply.ok()
}
}
@@ -0,0 +1,75 @@
import AppKit
import CoreGraphics
import Foundation
import ScreenCaptureKit
/// Pixel capture ScreenCaptureKit in-process (§7): no `screencapture` CLI, no `launchctl asuser`,
/// no base64-over-SSH. Needs the Screen Recording grant + an Aqua session. On a Tahoe
/// framebuffer-bug guest this (correctly) returns blank app windows which is exactly why AX (§6)
/// is the primary "look" there.
extension ConnectionHandler {
/// `screenshot` capture one display (default: the first/main) and return base64 JPEG plus its
/// pixel geometry. JPEG keeps a screenshotact loop's payload ~10× smaller than PNG.
func opScreenshot(_ request: [String: Any]) -> [String: Any] {
guard CGPreflightScreenCaptureAccess() else {
return AgentReply.err(
"Screen Recording (kTCCServiceScreenCapture) is not granted to the agent — re-run "
+ "provisioning")
}
let displayIndex = (request["display"] as? Int) ?? 0
// The op handlers run on a plain blocking connection thread; SCK's surface is async-only,
// so bridge with a semaphore (the thread has nothing else to do but wait).
let semaphore = DispatchSemaphore(value: 0)
let box = ResultBox<[String: Any]>()
Task {
box.value = await Self.captureDisplay(index: displayIndex)
semaphore.signal()
}
guard semaphore.wait(timeout: .now() + 15) == .success, let reply = box.value else {
return AgentReply.err("screenshot timed out")
}
return reply
}
private static func captureDisplay(index: Int) async -> [String: Any] {
do {
let content = try await SCShareableContent.current
let displays = content.displays
guard !displays.isEmpty else { return AgentReply.err("no shareable displays") }
let display = displays.indices.contains(index) ? displays[index] : displays[0]
let filter = SCContentFilter(display: display, excludingWindows: [])
let configuration = SCStreamConfiguration()
// Capture at the filter's native pixel size (`contentRect` is points; the scale is the
// pointpixel factor Float in the SDK, hence the CGFloat conversion).
let scale = CGFloat(filter.pointPixelScale)
configuration.width = Int(filter.contentRect.width * scale)
configuration.height = Int(filter.contentRect.height * scale)
configuration.showsCursor = true
let image = try await SCScreenshotManager.captureImage(
contentFilter: filter, configuration: configuration)
let bitmap = NSBitmapImageRep(cgImage: image)
guard
let jpeg = bitmap.representation(
using: .jpeg, properties: [.compressionFactor: 0.7])
else { return AgentReply.err("JPEG encoding failed") }
return AgentReply.ok([
"image": jpeg.base64EncodedString(),
"format": "jpeg",
"width": image.width,
"height": image.height,
"scale": Double(scale),
])
} catch {
return AgentReply.err("ScreenCaptureKit capture failed: \(error)")
}
}
}
/// Hands one value from a bridging Task back to the blocking connection thread; the semaphore
/// orders the accesses (signal happens-after the write, wait happens-before the read).
final class ResultBox<T> {
var value: T?
}
@@ -0,0 +1,40 @@
import AppKit
import ApplicationServices
import CoreGraphics
import Foundation
import VMAgentCore
// NucleicVMAgent the native in-guest automation agent (docs/MACOS_VM_NATIVE_AGENT.md).
//
// Runs as a per-user LaunchAgent in the auto-login `agent` account's Aqua session (AX, SCK and
// CGEvent all need an active WindowServer session), listens on a vsock port, and executes the §4
// wire ops with Apple's frameworks directly: AXUIElement (observe + control), ScreenCaptureKit
// (capture), CGEvent (raw input). It is a dumb, well-scoped actuator the host keeps ownership of
// VM lifecycle, gating, and the approval boundary.
// A peer that vanishes mid-write must not kill the process writes surface EPIPE instead.
signal(SIGPIPE, SIG_IGN)
let port = ProcessInfo.processInfo.environment["NUCLEIC_AGENT_PORT"]
.flatMap(UInt32.init) ?? AgentWire.port
agentLog("NucleicVMAgent protocol v\(AgentWire.version) starting on vsock port \(port)")
// Self-check the TCC grants once at startup so the LaunchAgent log shows an actionable state; the
// same readiness rides in every `ping` reply so the HOST can surface "present but not authorized".
agentLog(
"readiness: accessibility=\(AXIsProcessTrusted()) "
+ "screenRecording=\(CGPreflightScreenCaptureAccess()) "
+ "postEvent=\(CGPreflightPostEventAccess())")
let listener: AgentListener
do {
listener = try AgentListener(port: port)
} catch {
agentLog("FATAL: \(error) — is this running inside a macOS guest VM? (AF_VSOCK fails ENODEV on a host)")
exit(1)
}
Thread.detachNewThread { listener.run() }
// Keep the main run loop alive: AppKit-adjacent calls (NSWorkspace, event posting) expect a main
// thread with a live dispatch queue in a GUI session.
dispatchMain()
@@ -0,0 +1,42 @@
import Foundation
/// Wire constants for the host agent protocol (docs/MACOS_VM_NATIVE_AGENT.md §3§4).
///
/// The transport is NDJSON over vsock: one JSON request object per line, one JSON reply per line.
/// Requests carry an `op`; replies carry `ok` (plus op-specific fields, or `error` when `ok:false`).
///
/// The port constant is mirrored on the host side in `MacVMAgentWire`
/// (Sources/NucleicCore/MacVM/MacVMAgentClient.swift) the two packages are deliberately separate
/// (different platform floors), so keep the values in lockstep.
public enum AgentWire {
/// The Nucleic-reserved vsock port the agent listens on and the host connects to.
public static let port: UInt32 = 2035
/// Protocol version reported in the `ping` reply, bumped on incompatible wire changes.
public static let version = 1
}
/// Accumulates raw bytes and yields complete newline-terminated lines the NDJSON framing both
/// sides use (one request/reply object per line). Pure and unit-tested; the connection loop feeds
/// it whatever `read(2)` returns.
public struct LineSplitBuffer {
private var data = Data()
public init() {}
public mutating func append(_ chunk: Data) {
data.append(chunk)
}
/// The next complete line (without its trailing `\n`), or `nil` when no full line is buffered.
public mutating func nextLine() -> Data? {
guard let nl = data.firstIndex(of: 0x0A) else { return nil }
let line = data.subdata(in: data.startIndex..<nl)
data.removeSubrange(data.startIndex...nl)
// Tolerate CRLF framing from a debugging client.
if line.last == 0x0D { return line.dropLast() }
return line
}
/// Bytes currently buffered without a terminating newline the runaway-line guard reads this.
public var pendingBytes: Int { data.count }
}
@@ -0,0 +1,70 @@
import CoreGraphics
import Foundation
/// One synthesizable keystroke: a hardware virtual key code plus the modifier flags held around it.
public struct KeyStroke: Equatable, Sendable {
public let keyCode: CGKeyCode
public let flags: CGEventFlags
public init(keyCode: CGKeyCode, flags: CGEventFlags) {
self.keyCode = keyCode
self.flags = flags
}
}
/// Translates the tool-surface chord syntax (`"cmd+s"`, `"return"`, `"cmd+shift+4"`) into a CGEvent
/// keystroke. Pure and unit-tested the CGEvent posting lives in the agent's input ops.
///
/// Key codes are the ANSI-US hardware codes from Carbon's `Events.h` (`kVK_ANSI_A` ); the VZ guest
/// keyboard is ANSI, so the table is exact for the layouts a guest ships with.
public enum KeyMap {
/// Parse a chord like `cmd+shift+s` into a keystroke, or `nil` when the (final) key is unknown.
/// Unknown modifier tokens are ignored (matching the host-side cliclick path's leniency).
public static func stroke(forChord chord: String) -> KeyStroke? {
let parts = chord.lowercased().split(separator: "+").map(String.init)
guard let last = parts.last, let code = keyCodes[normalizeKeyName(last)] else { return nil }
var flags = CGEventFlags()
for token in parts.dropLast() {
if let f = modifierFlags[token] { flags.insert(f) }
}
return KeyStroke(keyCode: code, flags: flags)
}
/// Alias common key-name spellings to the table's canonical names.
static func normalizeKeyName(_ key: String) -> String {
switch key {
case "enter": return "return"
case "escape": return "esc"
case "backspace": return "delete"
case "arrowup": return "up"
case "arrowdown": return "down"
case "arrowleft": return "left"
case "arrowright": return "right"
case "spacebar": return "space"
default: return key
}
}
static let modifierFlags: [String: CGEventFlags] = [
"cmd": .maskCommand, "command": .maskCommand, "meta": .maskCommand, "super": .maskCommand,
"ctrl": .maskControl, "control": .maskControl,
"alt": .maskAlternate, "opt": .maskAlternate, "option": .maskAlternate,
"shift": .maskShift,
"fn": .maskSecondaryFn,
]
/// ANSI-US virtual key codes (Carbon `kVK_*`).
static let keyCodes: [String: CGKeyCode] = [
"a": 0x00, "s": 0x01, "d": 0x02, "f": 0x03, "h": 0x04, "g": 0x05, "z": 0x06, "x": 0x07,
"c": 0x08, "v": 0x09, "b": 0x0B, "q": 0x0C, "w": 0x0D, "e": 0x0E, "r": 0x0F, "y": 0x10,
"t": 0x11, "1": 0x12, "2": 0x13, "3": 0x14, "4": 0x15, "6": 0x16, "5": 0x17, "=": 0x18,
"9": 0x19, "7": 0x1A, "-": 0x1B, "8": 0x1C, "0": 0x1D, "]": 0x1E, "o": 0x1F, "u": 0x20,
"[": 0x21, "i": 0x22, "p": 0x23, "l": 0x25, "j": 0x26, "'": 0x27, "k": 0x28, ";": 0x29,
"\\": 0x2A, ",": 0x2B, "/": 0x2C, "n": 0x2D, "m": 0x2E, ".": 0x2F, "`": 0x32,
"return": 0x24, "tab": 0x30, "space": 0x31, "delete": 0x33, "esc": 0x35,
"home": 0x73, "end": 0x77, "pageup": 0x74, "pagedown": 0x79, "forwarddelete": 0x75,
"left": 0x7B, "right": 0x7C, "down": 0x7D, "up": 0x7E,
"f1": 0x7A, "f2": 0x78, "f3": 0x63, "f4": 0x76, "f5": 0x60, "f6": 0x61, "f7": 0x62,
"f8": 0x64, "f9": 0x65, "f10": 0x6D, "f11": 0x67, "f12": 0x6F, "f13": 0x69, "f14": 0x6B,
"f15": 0x71,
]
}
@@ -0,0 +1,69 @@
import CoreGraphics
import Foundation
import Testing
@testable import VMAgentCore
// MARK: - NDJSON line framing
@Test func lineSplitterYieldsCompleteLines() {
var buffer = LineSplitBuffer()
buffer.append(Data("{\"op\":\"pi".utf8))
#expect(buffer.nextLine() == nil)
buffer.append(Data("ng\"}\n{\"op\":".utf8))
#expect(buffer.nextLine() == Data("{\"op\":\"ping\"}".utf8))
#expect(buffer.nextLine() == nil)
buffer.append(Data("\"click\"}\n".utf8))
#expect(buffer.nextLine() == Data("{\"op\":\"click\"}".utf8))
#expect(buffer.nextLine() == nil)
#expect(buffer.pendingBytes == 0)
}
@Test func lineSplitterHandlesCRLFAndEmptyLines() {
var buffer = LineSplitBuffer()
buffer.append(Data("a\r\n\nb\n".utf8))
#expect(buffer.nextLine() == Data("a".utf8))
#expect(buffer.nextLine() == Data())
#expect(buffer.nextLine() == Data("b".utf8))
#expect(buffer.nextLine() == nil)
}
@Test func lineSplitterTracksPendingBytes() {
var buffer = LineSplitBuffer()
buffer.append(Data(repeating: UInt8(ascii: "x"), count: 100))
#expect(buffer.pendingBytes == 100)
#expect(buffer.nextLine() == nil)
}
// MARK: - Key chords
@Test func chordParsesModifiersAndKey() {
let stroke = KeyMap.stroke(forChord: "cmd+shift+s")
#expect(stroke == KeyStroke(keyCode: 0x01, flags: [.maskCommand, .maskShift]))
}
@Test func chordParsesBareSpecialKeys() {
#expect(KeyMap.stroke(forChord: "return") == KeyStroke(keyCode: 0x24, flags: []))
#expect(KeyMap.stroke(forChord: "enter") == KeyStroke(keyCode: 0x24, flags: []))
#expect(KeyMap.stroke(forChord: "escape") == KeyStroke(keyCode: 0x35, flags: []))
#expect(KeyMap.stroke(forChord: "backspace") == KeyStroke(keyCode: 0x33, flags: []))
#expect(KeyMap.stroke(forChord: "tab") == KeyStroke(keyCode: 0x30, flags: []))
}
@Test func chordParsesArrowAliasesAndFKeys() {
#expect(KeyMap.stroke(forChord: "arrowdown") == KeyMap.stroke(forChord: "down"))
#expect(KeyMap.stroke(forChord: "f5") == KeyStroke(keyCode: 0x60, flags: []))
#expect(KeyMap.stroke(forChord: "cmd+shift+4") == KeyStroke(
keyCode: 0x15, flags: [.maskCommand, .maskShift]))
}
@Test func chordIsCaseInsensitiveAndToleratesUnknownModifiers() {
#expect(KeyMap.stroke(forChord: "CMD+S") == KeyStroke(keyCode: 0x01, flags: [.maskCommand]))
// Unknown modifier tokens are dropped (host-side leniency parity), the key still lands.
#expect(KeyMap.stroke(forChord: "hyper+s") == KeyStroke(keyCode: 0x01, flags: []))
}
@Test func chordRejectsUnknownKey() {
#expect(KeyMap.stroke(forChord: "cmd+definitely-not-a-key") == nil)
#expect(KeyMap.stroke(forChord: "") == nil)
}
@@ -18,13 +18,18 @@ struct SessionDetailView: View {
// Bumped on send to jump the transcript to the bottom, even if the user had scrolled up to
// read history sending is a deliberate "show me what happens next" (mirrors the Mac).
@State private var scrollToBottomRequest = 0
// Owned here (not inside `TranscriptList`) so the jump-to-bottom chevron can ride in the chat
// bar just above the composer. `TranscriptList` drives it from the scroll geometry.
@State private var isScrolledToBottom = true
private var summary: WireSessionSummary? {
store.sessions.first { $0.sessionID == sessionID }
}
var body: some View {
TranscriptList(events: store.openEvents, scrollToBottomRequest: scrollToBottomRequest)
TranscriptList(events: store.openEvents,
scrollToBottomRequest: scrollToBottomRequest,
isScrolledToBottom: $isScrolledToBottom)
// The chat bar floats over the scrolling content on Liquid Glass instead of sitting
// in a boxed strip below it, so the transcript runs the full height of the screen.
.safeAreaInset(edge: .bottom) { actionArea }
@@ -236,6 +241,15 @@ struct SessionDetailView: View {
/// pending. Content scrolls beneath it; nothing renders when there's nothing to act on.
private var actionArea: some View {
VStack(spacing: 8) {
// The jump-to-bottom chevron rides here, immediately above the chat bar not as a
// transcript overlay, which aligned to the scroll view's full-height bounds and so sat
// behind this floating bar at the screen's bottom edge. Shown only while scrolled up; a
// tap bumps the same scroll request the send button uses, so following resumes once the
// transcript reaches the bottom. Mirrors the Mac's `JumpToBottomButton`.
if !isScrolledToBottom {
JumpToBottomButton { scrollToBottomRequest += 1 }
.transition(.move(edge: .bottom).combined(with: .opacity))
}
// Interacting with a chat while the owning Mac is unreachable surfaces this first, so a
// disabled composer reads as "offline / read-only history" rather than broken.
if !store.connectivity.isLive { disconnectedBanner }
@@ -243,6 +257,7 @@ struct SessionDetailView: View {
}
.padding(.horizontal, 12)
.padding(.bottom, 8)
.animation(.easeInOut(duration: 0.15), value: isScrolledToBottom)
}
/// The offline notice over the transcript: the chat's history is cached and readable, but the
@@ -401,8 +416,9 @@ struct TranscriptList: View {
/// True while the user is parked at (within a hair of) the bottom, so live output keeps
/// following; once they scroll up it flips false, following stops, and the jump-to-bottom
/// chevron appears. Starts true a freshly opened chat is anchored at the bottom.
@State private var isScrolledToBottom = true
/// chevron appears. Starts true a freshly opened chat is anchored at the bottom. Owned by
/// the parent so the chevron can live in the chat bar above the composer.
@Binding var isScrolledToBottom: Bool
/// True for a short window right after the transcript appears, while it runs its first
/// layout passes and its events fill in. During it we accept only "at bottom" scroll
@@ -426,10 +442,22 @@ struct TranscriptList: View {
var body: some View {
ScrollViewReader { proxy in
ScrollView {
LazyVStack(alignment: .leading, spacing: 14) {
// Eager VStack (not Lazy): the whole transcript is in memory, and eager layout
// hands the ScrollView its true, *fixed* total content height up front. A
// LazyVStack instead realizes rows as they scroll into view, so its reported
// content height shifts mid-scroll as rows resolve and that shifting height fed
// straight back through the geometry reader below (flip at-bottom toggle the
// `.defaultScrollAnchor` re-pin new height ), which is what made the whole
// transcript jitter up and down under a drag. A stable height breaks the loop.
// Matches the Mac transcript, for the same reason.
VStack(alignment: .leading, spacing: 14) {
ForEach(items) { item in
TranscriptRow(item: item).id(item.id)
}
// Fixed, zero-content scroll target for "jump to bottom". Scrolling to the last
// row's id instead would chase a moving target the final row's identity
// changes as its streaming deltas coalesce nudging the viewport mid-stream.
Color.clear.frame(height: 1).id(Self.bottomAnchorID)
}
.padding()
// Cap the transcript to a readable measure on a wide iPad so lines don't run
@@ -483,35 +511,27 @@ struct TranscriptList: View {
// so every streamed delta keeps the view pinned to the bottom but only while the
// user is already parked there. Scrolling up to read history is never yanked down.
.onChange(of: events.count) {
if isScrolledToBottom { scrollToLast(proxy, animated: !transcriptSettling) }
}
// An explicit jump the chevron or sending a message always wins.
.onChange(of: scrollToBottomRequest) { scrollToLast(proxy) }
// Floating chevron above the chat bar, shown only while scrolled up; tap to jump back
// to the latest output. Mirrors the Mac's `JumpToBottomButton`.
.overlay(alignment: .bottom) {
ZStack {
if !isScrolledToBottom {
JumpToBottomButton { scrollToLast(proxy) }
.padding(.bottom, 12)
.transition(.move(edge: .bottom).combined(with: .opacity))
}
}
.animation(.easeInOut(duration: 0.15), value: isScrolledToBottom)
if isScrolledToBottom { scrollToEnd(proxy, animated: !transcriptSettling) }
}
// An explicit jump the chevron or sending a message always wins. The chevron
// itself lives in the parent's chat bar (above the composer), not as an overlay here,
// so it sits over the composer instead of behind the floating bar; a tap bumps this
// same request, and following resumes once the geometry reader sees the bottom.
.onChange(of: scrollToBottomRequest) { scrollToEnd(proxy) }
}
}
/// Scroll the last message to the bottom edge; `animated: false` during the open-settle
/// Scroll the fixed bottom anchor into view; `animated: false` during the open-settle
/// window so a chat that's still laying out snaps to the tail instead of easing into place.
private func scrollToLast(_ proxy: ScrollViewProxy, animated: Bool = true) {
guard let last = items.last else { return }
private func scrollToEnd(_ proxy: ScrollViewProxy, animated: Bool = true) {
if animated {
withAnimation { proxy.scrollTo(last.id, anchor: .bottom) }
withAnimation { proxy.scrollTo(Self.bottomAnchorID, anchor: .bottom) }
} else {
proxy.scrollTo(last.id, anchor: .bottom)
proxy.scrollTo(Self.bottomAnchorID, anchor: .bottom)
}
}
private static let bottomAnchorID = "nucleic.transcript-bottom"
}
/// The floating "jump to the latest" chevron shown above the chat bar while the user has
+53
View File
@@ -0,0 +1,53 @@
#!/usr/bin/env bash
#
# Build guest/NucleicVMAgent into a signed NucleicVMAgent.app bundle, ready to be baked into the
# macOS-VM golden base image (docs/MACOS_VM_NATIVE_AGENT.md §11).
#
# ./scripts/build-vm-agent.sh [output-dir] # default output: guest/NucleicVMAgent/dist
#
# Signing: TCC keys its grants to the app's DESIGNATED REQUIREMENT, and an ad-hoc signature changes
# identity every build — so a pre-inserted TCC row would stop matching after a rebuild. Sign with a
# stable identity via $NUCLEIC_VMAGENT_SIGN_IDENTITY (a Developer ID, or a self-signed code-signing
# cert in the login keychain). Falls back to ad-hoc WITH A WARNING so a first local round-trip still
# works (you must re-provision TCC after every ad-hoc rebuild).
#
# Runs on the HOST (needs the Swift toolchain); the produced .app runs inside the GUEST. Stage the
# app next to scripts/provision-macos-guest.sh (or in the shared workspace) when provisioning.
set -euo pipefail
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
PKG_DIR="$REPO_ROOT/guest/NucleicVMAgent"
OUT_DIR="${1:-$PKG_DIR/dist}"
APP="$OUT_DIR/NucleicVMAgent.app"
IDENTITY="${NUCLEIC_VMAGENT_SIGN_IDENTITY:-}"
echo "▸ Building NucleicVMAgent (release) …"
swift build --package-path "$PKG_DIR" -c release
BIN="$(swift build --package-path "$PKG_DIR" -c release --show-bin-path)/NucleicVMAgent"
[ -x "$BIN" ] || { echo "✗ build produced no binary at $BIN" >&2; exit 1; }
echo "▸ Assembling $APP"
rm -rf "$APP"
mkdir -p "$APP/Contents/MacOS"
cp "$PKG_DIR/Packaging/Info.plist" "$APP/Contents/Info.plist"
cp "$BIN" "$APP/Contents/MacOS/NucleicVMAgent"
# The LaunchAgent plist rides along so the provisioner can install it from one staged directory.
cp "$PKG_DIR/Packaging/xyz.blakeslee.nucleic.vmagent.plist" "$OUT_DIR/"
echo "▸ Signing …"
if [ -n "$IDENTITY" ]; then
codesign --force --options runtime --sign "$IDENTITY" "$APP"
echo " ✓ signed with '$IDENTITY' (hardened runtime)."
else
codesign --force --sign - "$APP"
echo " ⚠ AD-HOC signed (set NUCLEIC_VMAGENT_SIGN_IDENTITY for a stable identity)." >&2
echo " An ad-hoc identity changes EVERY build, so the guest's pre-granted TCC rows stop" >&2
echo " matching after a rebuild — re-run provisioning Phase 7d after installing this build." >&2
fi
# Never let a quarantine xattr ride into the guest (the app is never downloaded, but be safe).
xattr -dr com.apple.quarantine "$APP" 2>/dev/null || true
codesign --verify --deep "$APP"
echo "$APP"
echo " Stage it for provisioning: copy $OUT_DIR/* next to scripts/provision-macos-guest.sh in the guest."
+115 -2
View File
@@ -22,8 +22,10 @@
# sourced by those shells; /etc/zshenv always is.
# 6. Warms the iOS simulator.
# 7. COMPUTER USE (GUI automation): installs cliclick at /usr/local/bin, enables auto-login for the
# agent (so a headless boot has an Aqua session), and pre-grants Screen Recording + Accessibility
# in the SYSTEM TCC.db so screencapture/cliclick work unattended (needs SIP off — see Phase 7).
# agent (so a headless boot has an Aqua session), pre-grants Screen Recording + Accessibility
# in the SYSTEM TCC.db so screencapture/cliclick work unattended (needs SIP off — see Phase 7),
# and installs the NATIVE IN-GUEST AGENT (NucleicVMAgent.app + its LaunchAgent + TCC grants)
# when the app is staged alongside this script — see docs/MACOS_VM_NATIVE_AGENT.md.
# 8. Shuts the guest DOWN cleanly so the base can be cloned.
#
# Run it as the `agent` user (it will `sudo` for the system-level bits). It's idempotent — safe to
@@ -369,6 +371,115 @@ else
2>/dev/null | sed 's/^/ /' || true
fi
# ── 7d: the NATIVE IN-GUEST AGENT (NucleicVMAgent.app + LaunchAgent + TCC) ────────────────────────
# The preferred computer-use path (docs/MACOS_VM_NATIVE_AGENT.md): a small signed .app, run as a
# per-user LaunchAgent in the auto-login Aqua session, that the host reaches over VSOCK (no SSH) and
# that drives the guest with Apple's frameworks directly — AXUIElement (semantic observe/control,
# and the ONLY observation channel on a macOS 26 guest whose framebuffer captures come back blank),
# ScreenCaptureKit (capture), CGEvent (raw input). Entirely OPTIONAL: when the app isn't staged, the
# base still works via the SSH + cliclick path above (Nucleic probes and falls back automatically).
#
# Build it on the HOST with scripts/build-vm-agent.sh and stage dist/* (the .app + the LaunchAgent
# plist) next to this script, in the agent home, or in the shared workspace.
echo " ▸ [7d] Installing the native in-guest agent (NucleicVMAgent.app) …"
AGENT_APP_DEST="/Applications/NucleicVMAgent.app"
AGENT_LA_PLIST="/Library/LaunchAgents/xyz.blakeslee.nucleic.vmagent.plist"
AGENT_BUNDLE_ID="xyz.blakeslee.nucleic.vmagent"
AGENT_APP_SRC=""
for cand in \
"$(cd "$(dirname "$0")" && pwd)/NucleicVMAgent.app" \
"$AGENT_HOME/NucleicVMAgent.app" \
"$SHARED_WORKSPACE/NucleicVMAgent.app"; do
[ -d "$cand" ] && { AGENT_APP_SRC="$cand"; break; }
done
if [ -z "$AGENT_APP_SRC" ]; then
echo " ⚠ NucleicVMAgent.app not staged — SKIPPING the native agent. Computer use will fall back"
echo " to the SSH + cliclick path (no ax_* semantic actions, and NO computer-use at all on a"
echo " macOS 26 guest whose screenshots are blank). To add it: on the host run"
echo " scripts/build-vm-agent.sh, stage its dist/* next to this script, and re-run."
else
# Install the app (a fresh copy every run — the bundle is tiny) and strip any quarantine xattr.
sudo rm -rf "$AGENT_APP_DEST"
sudo cp -R "$AGENT_APP_SRC" "$AGENT_APP_DEST"
sudo xattr -dr com.apple.quarantine "$AGENT_APP_DEST" 2>/dev/null || true
echo " ✓ installed $AGENT_APP_DEST (from $AGENT_APP_SRC)."
# LaunchAgent: Aqua-session-only, RunAtLoad + KeepAlive (staged copy preferred, generated else).
AGENT_LA_SRC="$(dirname "$AGENT_APP_SRC")/xyz.blakeslee.nucleic.vmagent.plist"
if [ -f "$AGENT_LA_SRC" ]; then
sudo cp "$AGENT_LA_SRC" "$AGENT_LA_PLIST"
else
sudo tee "$AGENT_LA_PLIST" >/dev/null <<'PLIST'
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key>
<string>xyz.blakeslee.nucleic.vmagent</string>
<key>ProgramArguments</key>
<array>
<string>/Applications/NucleicVMAgent.app/Contents/MacOS/NucleicVMAgent</string>
</array>
<key>RunAtLoad</key>
<true/>
<key>KeepAlive</key>
<true/>
<key>LimitLoadToSessionType</key>
<string>Aqua</string>
<key>AssociatedBundleIdentifiers</key>
<string>xyz.blakeslee.nucleic.vmagent</string>
<key>StandardOutPath</key>
<string>/tmp/nucleic-vmagent.log</string>
<key>StandardErrorPath</key>
<string>/tmp/nucleic-vmagent.log</string>
</dict>
</plist>
PLIST
fi
sudo chown root:wheel "$AGENT_LA_PLIST"
sudo chmod 644 "$AGENT_LA_PLIST"
echo " ✓ LaunchAgent at $AGENT_LA_PLIST (Aqua session, RunAtLoad, KeepAlive)."
# TCC: the agent needs THREE grants (docs/MACOS_VM_NATIVE_AGENT.md §11), each a separate service
# row, keyed by BUNDLE ID (client_type=0) + the app's designated-requirement csreq blob:
# • kTCCServiceAccessibility — AXUIElement observe/control (AXIsProcessTrusted)
# • kTCCServicePostEvent — CGEvent input synthesis (a DISTINCT row from Accessibility)
# • kTCCServiceScreenCapture — ScreenCaptureKit
# Same SIP-off reality as 7c: with SIP on, the system TCC.db is not writable — skip gracefully.
if [ "$SIP_OFF" -ne 1 ]; then
echo " ⚠ SIP is ENABLED — the agent's TCC grants are SKIPPED (see 7c for the one-time"
echo " recoveryOS 'csrutil disable' step, then re-run this provisioner)." >&2
else
TCC_DB="/Library/Application Support/com.apple.TCC/TCC.db"
AGENT_HEXREQ="NULL"
AGENT_REQ=$(codesign -d -r- "$AGENT_APP_DEST" 2>&1 | sed -n 's/^designated => //p')
if [ -n "$AGENT_REQ" ] && echo "$AGENT_REQ" | csreq -r- -b /tmp/csreq-vmagent.bin 2>/dev/null; then
AGENT_HEXREQ="X'$(xxd -p /tmp/csreq-vmagent.bin | tr -d '\n')'"
else
echo " ⚠ no usable designated requirement (ad-hoc signed?) — writing csreq NULL rows."
echo " NOTE an ad-hoc identity changes every rebuild; sign with a stable identity" >&2
echo " (scripts/build-vm-agent.sh + NUCLEIC_VMAGENT_SIGN_IDENTITY) for durable grants." >&2
fi
for svc in kTCCServiceAccessibility kTCCServicePostEvent kTCCServiceScreenCapture; do
echo " • $svc ← $AGENT_BUNDLE_ID"
sudo sqlite3 "$TCC_DB" \
"INSERT OR REPLACE INTO access
(service,client,client_type,auth_value,auth_reason,auth_version,csreq,policy_id,indirect_object_identifier_type,indirect_object_identifier,indirect_object_code_identity,flags,last_modified)
VALUES('$svc','$AGENT_BUNDLE_ID',0,2,4,1,$AGENT_HEXREQ,NULL,0,'UNUSED',NULL,0,strftime('%s','now'));"
done
rm -f /tmp/csreq-vmagent.bin 2>/dev/null || true
# tccd must reload the db, and the agent must RESTART to pick the grants up (Screen Recording
# in particular is only read at process start).
sudo killall tccd 2>/dev/null || true
AGENT_UID="$(id -u "$AGENT_USER")"
sudo launchctl bootout "gui/$AGENT_UID/$AGENT_BUNDLE_ID" 2>/dev/null || true
sudo launchctl bootstrap "gui/$AGENT_UID" "$AGENT_LA_PLIST" 2>/dev/null || \
echo " (LaunchAgent will start on next login/boot — no live Aqua session to bootstrap into.)"
echo " ✓ agent TCC grants written; agent (re)started. Log: /tmp/nucleic-vmagent.log"
fi
fi
echo ""
echo " ── Computer use needs ALL THREE of the above (cliclick + auto-login + TCC) ─────────────────"
echo " If screenshots come back BLACK or clicks NO-OP after this base is live, check, in the guest:"
@@ -376,6 +487,8 @@ echo " • SIP is disabled: csrutil status"
echo " • TCC rows exist & allowed: sudo sqlite3 \"/Library/Application Support/com.apple.TCC/TCC.db\" \\"
echo " \"SELECT service,client,auth_value FROM access;\""
echo " • agent is auto-logged-in: an Aqua/desktop session is active (not the login window)"
echo " • native agent alive (7d): cat /tmp/nucleic-vmagent.log — should show 'starting on"
echo " vsock port 2035' and readiness true×3 (if the app was staged)"
echo " ──────────────────────────────────────────────────────────────────────────────────────────────"
echo ""