nvrsion: Add: Fix auto-scroll instability in iOS session chat so the transcript no longer jitters on scroll, and begin MACOS_VM_NATIVE_AGENT_Markdown in ./docs/MACOS_VM_NATIVE_AGENT.md.
Nucleic-Promote: 1 Co-authored-by: Nucleic <[email protected]>
This commit is contained in:
@@ -1071,6 +1071,7 @@ public actor ClaudeCodeBackend: AgentBackend {
|
||||
defer { Task { await manager.finished(name: name) } }
|
||||
let (image, summary) = try await manager.performComputerAction(
|
||||
name: name, action: call.action, x: call.x, y: call.y, text: call.text,
|
||||
ref: call.ref, value: call.value,
|
||||
scrollDirection: call.scrollDirection, scrollAmount: call.scrollAmount,
|
||||
durationMs: call.durationMs)
|
||||
return .ok(text: summary, imageBase64: image)
|
||||
|
||||
@@ -166,14 +166,22 @@ public actor MCPApprovalServer {
|
||||
/// exposure is the opt-in boundary.
|
||||
public struct MacVMComputerCall: Sendable {
|
||||
/// The action verb: `screenshot`, `left_click`, `right_click`, `double_click`, `mouse_move`,
|
||||
/// `left_click_drag`, `type`, `key`, `scroll`, `cursor_position`, `launch_app`, `wait`.
|
||||
/// `left_click_drag`, `type`, `key`, `scroll`, `cursor_position`, `launch_app`, `wait`; plus
|
||||
/// the AX-semantic actions when the guest runs the native agent
|
||||
/// (docs/MACOS_VM_NATIVE_AGENT.md): `ax_dump`, `ax_element_at`, `ax_press`, `ax_set_value`,
|
||||
/// `ax_focus`.
|
||||
public let action: String
|
||||
/// Target point (screen pixels; the screenshot is 1:1 so the agent targets what it sees).
|
||||
public let x: Int?
|
||||
public let y: Int?
|
||||
/// Payload for `type` (literal text), `key` (a chord like `cmd+s`, `return`, `tab`), or
|
||||
/// `launch_app` (an application name).
|
||||
/// Payload for `type` (literal text), `key` (a chord like `cmd+s`, `return`, `tab`),
|
||||
/// `launch_app` (an application name), or `ax_press` (an optional AX action name to perform
|
||||
/// instead of the default `AXPress`, e.g. `AXShowMenu`).
|
||||
public let text: String?
|
||||
/// For the `ax_*` element actions: the element handle from the last `ax_dump`/`ax_element_at`.
|
||||
public let ref: String?
|
||||
/// For `ax_set_value`: the value to write into the element.
|
||||
public let value: String?
|
||||
/// For `scroll`: `up`/`down`/`left`/`right` and a click/step amount.
|
||||
public let scrollDirection: String?
|
||||
public let scrollAmount: Int?
|
||||
@@ -182,12 +190,15 @@ public actor MCPApprovalServer {
|
||||
|
||||
public init(
|
||||
action: String, x: Int? = nil, y: Int? = nil, text: String? = nil,
|
||||
ref: String? = nil, value: String? = nil,
|
||||
scrollDirection: String? = nil, scrollAmount: Int? = nil, durationMs: Int? = nil
|
||||
) {
|
||||
self.action = action
|
||||
self.x = x
|
||||
self.y = y
|
||||
self.text = text
|
||||
self.ref = ref
|
||||
self.value = value
|
||||
self.scrollDirection = scrollDirection
|
||||
self.scrollAmount = scrollAmount
|
||||
self.durationMs = durationMs
|
||||
@@ -965,17 +976,29 @@ public actor MCPApprovalServer {
|
||||
+ "Each call performs ONE `action` and returns a fresh screenshot of the "
|
||||
+ "VM screen, so work in a loop: screenshot → decide → act → screenshot. "
|
||||
+ "The screenshot is the full screen at its native pixels, so target "
|
||||
+ "`x`,`y` exactly as you see them. Actions: `screenshot` (just look); "
|
||||
+ "`left_click`/`right_click`/`double_click` at `x`,`y`; `mouse_move` to "
|
||||
+ "`x`,`y`; `left_click_drag` to `x`,`y` (from the current cursor); "
|
||||
+ "`type` the literal `text`; `key` a chord in `text` (e.g. \"cmd+s\", "
|
||||
+ "\"return\", \"tab\", \"cmd+shift+4\"); `scroll` with "
|
||||
+ "`x`,`y` exactly as you see them. Pixel actions: `screenshot` (just "
|
||||
+ "look); `left_click`/`right_click`/`double_click` at `x`,`y`; "
|
||||
+ "`mouse_move` to `x`,`y`; `left_click_drag` to `x`,`y` (from the "
|
||||
+ "current cursor); `type` the literal `text`; `key` a chord in `text` "
|
||||
+ "(e.g. \"cmd+s\", \"return\", \"tab\", \"cmd+shift+4\"); `scroll` with "
|
||||
+ "`scroll_direction`+`scroll_amount`; `launch_app` the app named in "
|
||||
+ "`text` (e.g. \"Safari\", \"Xcode\"); `cursor_position`; `wait` "
|
||||
+ "`duration_ms` (let a window open). This runs in the VM sandbox, so "
|
||||
+ "actions are NOT individually approval-gated. Prefer `mac_vm_exec` for "
|
||||
+ "headless build/test commands; use this when you need to look at or "
|
||||
+ "operate the screen."),
|
||||
+ "`duration_ms` (let a window open). SEMANTIC actions (preferred when "
|
||||
+ "available — they act on controls by identity, not coordinates, and "
|
||||
+ "keep working even when screenshots come back blank): `ax_dump` "
|
||||
+ "returns the frontmost app's accessibility tree (every control's "
|
||||
+ "role/title/value/frame/actions, each with a `ref`); `ax_press` a "
|
||||
+ "`ref` (optionally a specific AX action name in `text`); "
|
||||
+ "`ax_set_value` writes `value` into a `ref`'d field; `ax_focus` a "
|
||||
+ "`ref`; `ax_element_at` resolves the element under `x`,`y`. Convention: "
|
||||
+ "ax_dump → act on a ref → ax_dump again (refs go stale after the UI "
|
||||
+ "changes). The ax_* actions need the VM's native agent; if the reply "
|
||||
+ "says it's unavailable, use screenshots + pixel actions. If screenshots "
|
||||
+ "come back blank (a macOS 26 guest), rely on `ax_dump` — it is "
|
||||
+ "framebuffer-independent. This runs in the VM sandbox, so actions are "
|
||||
+ "NOT individually approval-gated. Prefer `mac_vm_exec` for headless "
|
||||
+ "build/test commands; use this when you need to look at or operate the "
|
||||
+ "screen."),
|
||||
"inputSchema": .object([
|
||||
"type": .string("object"),
|
||||
"properties": .object([
|
||||
@@ -984,15 +1007,27 @@ public actor MCPApprovalServer {
|
||||
"description": .string(
|
||||
"One of: screenshot, left_click, right_click, double_click, "
|
||||
+ "mouse_move, left_click_drag, type, key, scroll, "
|
||||
+ "cursor_position, launch_app, wait."),
|
||||
+ "cursor_position, launch_app, wait, ax_dump, "
|
||||
+ "ax_element_at, ax_press, ax_set_value, ax_focus."),
|
||||
]),
|
||||
"x": .object(["type": .string("integer")]),
|
||||
"y": .object(["type": .string("integer")]),
|
||||
"text": .object([
|
||||
"type": .string("string"),
|
||||
"description": .string(
|
||||
"Text to type, a key/chord for `key`, or an app name for "
|
||||
+ "`launch_app`."),
|
||||
"Text to type, a key/chord for `key`, an app name for "
|
||||
+ "`launch_app`, or an AX action name for `ax_press` "
|
||||
+ "(default AXPress)."),
|
||||
]),
|
||||
"ref": .object([
|
||||
"type": .string("string"),
|
||||
"description": .string(
|
||||
"Element handle from the last ax_dump/ax_element_at (for "
|
||||
+ "ax_press/ax_set_value/ax_focus)."),
|
||||
]),
|
||||
"value": .object([
|
||||
"type": .string("string"),
|
||||
"description": .string("The value to write, for ax_set_value."),
|
||||
]),
|
||||
"scroll_direction": .object(["type": .string("string")]),
|
||||
"scroll_amount": .object(["type": .string("integer")]),
|
||||
@@ -1065,6 +1100,8 @@ public actor MCPApprovalServer {
|
||||
x: arguments?["x"]?.intValue,
|
||||
y: arguments?["y"]?.intValue,
|
||||
text: arguments?["text"]?.stringValue,
|
||||
ref: arguments?["ref"]?.stringValue,
|
||||
value: arguments?["value"]?.stringValue,
|
||||
scrollDirection: arguments?["scroll_direction"]?.stringValue,
|
||||
scrollAmount: arguments?["scroll_amount"]?.intValue,
|
||||
durationMs: arguments?["duration_ms"]?.intValue)
|
||||
|
||||
@@ -0,0 +1,267 @@
|
||||
import Foundation
|
||||
import NucleicProtocol
|
||||
// Unconditional: `VZVirtioSocketConnection` (unlike the arm64-only `VZMac*` guest APIs) exists on
|
||||
// both architectures, and the client actor must be a real type wherever the engine compiles.
|
||||
import Virtualization
|
||||
|
||||
/// Wire constants + framing for the host ⇄ in-guest-agent protocol
|
||||
/// (docs/MACOS_VM_NATIVE_AGENT.md §3–§4): NDJSON over vsock — one JSON request object per line
|
||||
/// (carrying an `op`), one JSON reply per line (carrying `ok` + op fields, or `ok:false, error`).
|
||||
///
|
||||
/// The guest half lives in `guest/NucleicVMAgent` (its own package — the agent runs on macOS 14+
|
||||
/// guests while this package's floor is 26), which mirrors these constants in `VMAgentCore.AgentWire`;
|
||||
/// keep the two in lockstep.
|
||||
enum MacVMAgentWire {
|
||||
/// The Nucleic-reserved vsock port the guest agent listens on.
|
||||
static let port: UInt32 = 2035
|
||||
/// Protocol version this client speaks; the agent reports its own in the `ping` reply.
|
||||
static let version = 1
|
||||
/// Default per-request deadline. Generous enough for a full-display SCK screenshot or a deep
|
||||
/// AX walk; a hung agent surfaces as `.timeout` instead of wedging the tool call.
|
||||
static let requestTimeout: TimeInterval = 20
|
||||
|
||||
/// Encode one request line (`{"op": …, …}\n`). Pure, unit-tested.
|
||||
static func requestLine(op: String, fields: [String: JSONValue] = [:]) -> Data {
|
||||
var object = fields
|
||||
object["op"] = .string(op)
|
||||
var line = (try? JSONValue.object(object).encodedData()) ?? Data("{}".utf8)
|
||||
line.append(0x0A)
|
||||
return line
|
||||
}
|
||||
|
||||
/// Decode one reply line into its JSON object. Pure, unit-tested.
|
||||
static func parseReply(_ line: Data) throws -> JSONValue {
|
||||
guard let reply = try? JSONValue(parsing: line), reply.objectValue != nil else {
|
||||
throw MacVMAgentError.protocolError(
|
||||
"malformed agent reply: \(String(decoding: line.prefix(200), as: UTF8.self))")
|
||||
}
|
||||
return reply
|
||||
}
|
||||
}
|
||||
|
||||
/// Errors from the native-agent channel. Deliberately separate from ``MacVMError``: these are
|
||||
/// routing signals — the computer-use path catches them and falls back to SSH — not user-facing
|
||||
/// VM-lifecycle failures.
|
||||
enum MacVMAgentError: Error, CustomStringConvertible {
|
||||
/// The VM was built without a vsock device (pre-agent configuration still booted).
|
||||
case noSocketDevice
|
||||
/// The vsock connection died or the fd errored; the client is unusable.
|
||||
case ioFailed(String)
|
||||
/// No reply within the deadline (agent hung, or nothing listening behind the connect).
|
||||
case timeout
|
||||
/// The reply wasn't the protocol's shape.
|
||||
case protocolError(String)
|
||||
/// The agent executed the op and reported failure (`ok:false`) — transport is healthy.
|
||||
case agentError(String)
|
||||
|
||||
var description: String {
|
||||
switch self {
|
||||
case .noSocketDevice: return "the VM has no vsock device"
|
||||
case .ioFailed(let why): return "agent connection failed: \(why)"
|
||||
case .timeout: return "the in-guest agent did not reply in time"
|
||||
case .protocolError(let why): return why
|
||||
case .agentError(let message): return message
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the underlying connection should be discarded (vs. an op-level `agentError`, after
|
||||
/// which the channel remains perfectly usable).
|
||||
var isTransportFailure: Bool {
|
||||
switch self {
|
||||
case .agentError: return false
|
||||
default: return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One live NDJSON request/reply channel to the in-guest `NucleicVMAgent`, over the vsock
|
||||
/// connection ``MacVMInstance/connectAgent(port:)`` opened. The actor serializes requests (the
|
||||
/// protocol is strictly one-reply-per-request), owns the read buffer, and keeps the
|
||||
/// `VZVirtioSocketConnection` alive while I/O is in flight (its fd closes with it).
|
||||
///
|
||||
/// I/O is nonblocking `read`/`write` on the connection's fd with `Task.sleep` backoff — no thread
|
||||
/// is ever parked on a guest that stops answering; the deadline turns it into `.timeout`.
|
||||
actor MacVMAgentClient {
|
||||
private let connection: UncheckedSendableBox<VZVirtioSocketConnection>
|
||||
private let fd: Int32
|
||||
private var buffer = Data()
|
||||
private var closed = false
|
||||
|
||||
init(connection: UncheckedSendableBox<VZVirtioSocketConnection>) {
|
||||
self.connection = connection
|
||||
self.fd = connection.value.fileDescriptor
|
||||
// Nonblocking so the actor yields (rather than blocks a cooperative thread) while waiting.
|
||||
let flags = fcntl(fd, F_GETFL, 0)
|
||||
_ = fcntl(fd, F_SETFL, flags | O_NONBLOCK)
|
||||
}
|
||||
|
||||
/// Send one op and await its reply object. Throws ``MacVMAgentError/agentError(_:)`` when the
|
||||
/// agent answers `ok:false`, a transport error when the channel itself broke.
|
||||
func request(
|
||||
op: String, fields: [String: JSONValue] = [:],
|
||||
timeout: TimeInterval = MacVMAgentWire.requestTimeout
|
||||
) async throws -> JSONValue {
|
||||
guard !closed, fd >= 0 else { throw MacVMAgentError.ioFailed("connection closed") }
|
||||
let deadline = Date().addingTimeInterval(timeout)
|
||||
try await writeAll(MacVMAgentWire.requestLine(op: op, fields: fields), deadline: deadline)
|
||||
let line = try await readLine(deadline: deadline)
|
||||
let reply = try MacVMAgentWire.parseReply(line)
|
||||
guard reply["ok"]?.boolValue == true else {
|
||||
throw MacVMAgentError.agentError(
|
||||
reply["error"]?.stringValue ?? "the agent reported an unspecified failure")
|
||||
}
|
||||
return reply
|
||||
}
|
||||
|
||||
/// Close the vsock connection. Idempotent; subsequent requests throw.
|
||||
func close() {
|
||||
guard !closed else { return }
|
||||
closed = true
|
||||
connection.value.close()
|
||||
}
|
||||
|
||||
// MARK: - Nonblocking fd I/O
|
||||
|
||||
private func writeAll(_ data: Data, deadline: Date) async throws {
|
||||
var remaining = data
|
||||
while !remaining.isEmpty {
|
||||
let written = remaining.withUnsafeBytes { raw in
|
||||
write(fd, raw.baseAddress, raw.count)
|
||||
}
|
||||
if written > 0 {
|
||||
remaining = remaining.dropFirst(written)
|
||||
} else if written < 0, errno == EAGAIN || errno == EINTR {
|
||||
guard Date() < deadline else { throw MacVMAgentError.timeout }
|
||||
try? await Task.sleep(nanoseconds: 20_000_000)
|
||||
} else {
|
||||
closed = true
|
||||
throw MacVMAgentError.ioFailed("write errno \(errno)")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func readLine(deadline: Date) async throws -> Data {
|
||||
var scratch = [UInt8](repeating: 0, count: 256 * 1024)
|
||||
while true {
|
||||
if let newline = buffer.firstIndex(of: 0x0A) {
|
||||
let line = buffer.subdata(in: buffer.startIndex..<newline)
|
||||
buffer.removeSubrange(buffer.startIndex...newline)
|
||||
return line
|
||||
}
|
||||
// A screenshot reply is a few MB of base64 — cap far above that, but cap.
|
||||
if buffer.count > 32 * 1024 * 1024 {
|
||||
closed = true
|
||||
throw MacVMAgentError.protocolError("agent reply exceeded 32 MB without a newline")
|
||||
}
|
||||
let count = read(fd, &scratch, scratch.count)
|
||||
if count > 0 {
|
||||
buffer.append(contentsOf: scratch[0..<count])
|
||||
} else if count == 0 {
|
||||
closed = true
|
||||
throw MacVMAgentError.ioFailed("the agent closed the connection")
|
||||
} else if errno == EAGAIN || errno == EINTR {
|
||||
guard Date() < deadline else { throw MacVMAgentError.timeout }
|
||||
try? await Task.sleep(nanoseconds: 25_000_000)
|
||||
} else {
|
||||
closed = true
|
||||
throw MacVMAgentError.ioFailed("read errno \(errno)")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#if arch(arm64)
|
||||
extension MacVMInstance {
|
||||
/// Open a vsock connection to `port` in this VM's guest. Runs the connect on the VM's serial
|
||||
/// queue (all VZ calls must); resolves once the guest's listener accepts. If nothing in the
|
||||
/// guest listens on the port (no agent baked in / not yet launched), the framework fails the
|
||||
/// connect — the engine's probe turns that into "agent unavailable", never an error the user sees.
|
||||
func connectAgent(port: UInt32) async throws -> UncheckedSendableBox<VZVirtioSocketConnection> {
|
||||
try await withCheckedThrowingContinuation { cont in
|
||||
queue.async { [self] in
|
||||
guard let device = vm.socketDevices.first as? VZVirtioSocketDevice else {
|
||||
cont.resume(throwing: MacVMAgentError.noSocketDevice)
|
||||
return
|
||||
}
|
||||
device.connect(toPort: port) { result in
|
||||
switch result {
|
||||
case .success(let connection):
|
||||
cont.resume(returning: UncheckedSendableBox(value: connection))
|
||||
case .failure(let error):
|
||||
cont.resume(throwing: error)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
extension MacVMEngine {
|
||||
/// The native in-guest agent channel for a live VM, or `nil` when the VM has none (older base
|
||||
/// image, agent not running, or vsock connect unsupported) — the caller then uses the SSH path.
|
||||
///
|
||||
/// First call per VM probes: vsock connect + `ping`, both under a short deadline. The result is
|
||||
/// cached on the ``LiveVM`` (see ``MacVMAgentState``); a concurrent duplicate probe is harmless
|
||||
/// (one extra short-lived connection — the agent accepts several).
|
||||
func agentClient(name: String) async -> MacVMAgentClient? {
|
||||
#if arch(arm64)
|
||||
guard let entry = live[name] else { return nil }
|
||||
switch entry.agent {
|
||||
case .available(let client): return client
|
||||
case .unavailable: return nil
|
||||
case .unprobed: break
|
||||
}
|
||||
do {
|
||||
let instance = entry.instance
|
||||
let box = try await withAgentTimeout(seconds: 5) {
|
||||
try await instance.connectAgent(port: MacVMAgentWire.port)
|
||||
}
|
||||
let client = MacVMAgentClient(connection: box)
|
||||
let pong = try await client.request(op: "ping", timeout: 5)
|
||||
// Readiness is informational (each op re-checks its own TCC grant and reports
|
||||
// precisely); version is the only hard gate.
|
||||
guard let version = pong["version"]?.intValue, version == MacVMAgentWire.version else {
|
||||
await client.close()
|
||||
throw MacVMAgentError.protocolError(
|
||||
"agent protocol v\(pong["version"]?.intValue ?? 0) ≠ host v\(MacVMAgentWire.version)")
|
||||
}
|
||||
live[name]?.agent = .available(client)
|
||||
return client
|
||||
} catch {
|
||||
live[name]?.agent = .unavailable
|
||||
return nil
|
||||
}
|
||||
#else
|
||||
return nil
|
||||
#endif
|
||||
}
|
||||
|
||||
/// Discard a VM's agent channel after a transport failure so the next action re-probes once
|
||||
/// (the guest LaunchAgent's `KeepAlive` may have already restarted the agent).
|
||||
func discardAgentClient(name: String, _ client: MacVMAgentClient) async {
|
||||
#if arch(arm64)
|
||||
await client.close()
|
||||
if live[name] != nil { live[name]?.agent = .unprobed }
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
/// Race `body` against a wall-clock deadline. The vsock `connect` has no timeout of its own and
|
||||
/// Apple documents the no-listener case only as "does nothing" — this guarantees the agent probe
|
||||
/// resolves either way.
|
||||
func withAgentTimeout<T: Sendable>(
|
||||
seconds: TimeInterval, _ body: @escaping @Sendable () async throws -> T
|
||||
) async throws -> T {
|
||||
try await withThrowingTaskGroup(of: T.self) { group in
|
||||
group.addTask { try await body() }
|
||||
group.addTask {
|
||||
try await Task.sleep(nanoseconds: UInt64(seconds * 1_000_000_000))
|
||||
throw MacVMAgentError.timeout
|
||||
}
|
||||
// One of the two always completes; cancel the loser.
|
||||
let winner = try await group.next()!
|
||||
group.cancelAll()
|
||||
return winner
|
||||
}
|
||||
}
|
||||
@@ -156,6 +156,12 @@ extension MacVMEngine {
|
||||
net.attachment = VZNATNetworkDeviceAttachment()
|
||||
config.networkDevices = [net]
|
||||
|
||||
// vsock channel for the native in-guest agent (docs/MACOS_VM_NATIVE_AGENT.md §3): the host
|
||||
// reaches the agent via VZVirtioSocketDevice.connect(toPort:), no NAT/DHCP/SSH involved.
|
||||
// Exactly one socket device per VM (the framework's limit); harmless on a base image that
|
||||
// doesn't run the agent — the engine's probe just falls back to SSH.
|
||||
config.socketDevices = [VZVirtioSocketDeviceConfiguration()]
|
||||
|
||||
config.graphicsDevices = [Self.makeGraphics()]
|
||||
config.keyboards = [VZMacKeyboardConfiguration()]
|
||||
config.pointingDevices = [VZMacTrackpadConfiguration()]
|
||||
|
||||
@@ -19,10 +19,49 @@ extension MacVMEngine {
|
||||
static let openPath = "/usr/bin/open"
|
||||
|
||||
/// Perform one computer-use action in the named guest and return the post-action screenshot
|
||||
/// (base64 PNG, ready for the MCP image block) plus a short text summary. `screenshot` just
|
||||
/// (base64 JPEG, ready for the MCP image block) plus a short text summary. `screenshot` just
|
||||
/// captures; `cursor_position` returns the pointer location as text; `wait` pauses then captures.
|
||||
/// Invalid/missing coordinates yield a `denied`-style text with no image (the caller maps it).
|
||||
///
|
||||
/// Routing (docs/MACOS_VM_NATIVE_AGENT.md §10): the **native in-guest agent** is preferred when
|
||||
/// its vsock `ping` answers — semantic AX actions, ScreenCaptureKit capture, CGEvent input —
|
||||
/// and this SSH + screencapture/cliclick path is the transparent fallback, so a base image
|
||||
/// without the agent behaves exactly as before. The `ax_*` actions are agent-only (the
|
||||
/// accessibility tree has no SSH equivalent); `ref`/`value` are their arguments.
|
||||
public func performComputerAction(
|
||||
name: String, action: String, x: Int?, y: Int?, text: String?,
|
||||
ref: String? = nil, value: String? = nil,
|
||||
scrollDirection: String?, scrollAmount: Int?, durationMs: Int?
|
||||
) async throws -> (imageBase64: String?, summary: String) {
|
||||
if Self.axActions.contains(action) {
|
||||
guard let client = await agentClient(name: name) else {
|
||||
return (
|
||||
nil,
|
||||
"The '\(action)' action needs the native in-guest agent, which this VM doesn't "
|
||||
+ "have (base image without NucleicVMAgent, or the agent isn't "
|
||||
+ "running/authorized). Use screenshot + pixel actions instead."
|
||||
)
|
||||
}
|
||||
return await performAXComputerAction(
|
||||
name: name, client: client, action: action, x: x, y: y, text: text,
|
||||
ref: ref, value: value)
|
||||
}
|
||||
if let client = await agentClient(name: name),
|
||||
let native = await performAgentComputerAction(
|
||||
name: name, client: client, action: action, x: x, y: y, text: text,
|
||||
scrollDirection: scrollDirection, scrollAmount: scrollAmount,
|
||||
durationMs: durationMs)
|
||||
{
|
||||
return native
|
||||
}
|
||||
return try await performSSHComputerAction(
|
||||
name: name, action: action, x: x, y: y, text: text,
|
||||
scrollDirection: scrollDirection, scrollAmount: scrollAmount, durationMs: durationMs)
|
||||
}
|
||||
|
||||
/// Today's SSH + `launchctl asuser` + screencapture/cliclick implementation — the fallback path
|
||||
/// (and the only one on a pre-agent base image).
|
||||
func performSSHComputerAction(
|
||||
name: String, action: String, x: Int?, y: Int?, text: String?,
|
||||
scrollDirection: String?, scrollAmount: Int?, durationMs: Int?
|
||||
) async throws -> (imageBase64: String?, summary: String) {
|
||||
@@ -98,7 +137,7 @@ extension MacVMEngine {
|
||||
+ "macOS 15 base image for computer-use; see docs/MACOS_VM.md).")
|
||||
}
|
||||
|
||||
private func screenSummary(for action: String) -> String {
|
||||
func screenSummary(for action: String) -> String {
|
||||
action == "screenshot" ? "Screenshot of the macOS VM screen." : "Did '\(action)'; here's the screen now."
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,227 @@
|
||||
import Foundation
|
||||
import NucleicProtocol
|
||||
|
||||
/// Computer-use over the **native in-guest agent** (docs/MACOS_VM_NATIVE_AGENT.md): the preferred
|
||||
/// path when the guest runs `NucleicVMAgent` — ScreenCaptureKit capture, CGEvent input (with a REAL
|
||||
/// scroll wheel), and the AX-semantic ops that have no SSH/cliclick equivalent at all. The SSH path
|
||||
/// (`MacVMEngine+Computer`) remains the transparent fallback, so a base image without the agent
|
||||
/// loses nothing.
|
||||
extension MacVMEngine {
|
||||
/// The AX-semantic tool actions (§10). Agent-only: the accessibility tree is not reachable over
|
||||
/// the SSH/cliclick path, so these never fall back.
|
||||
static let axActions: Set<String> = [
|
||||
"ax_dump", "ax_element_at", "ax_press", "ax_set_value", "ax_focus",
|
||||
]
|
||||
|
||||
/// Try one pixel/input action natively. Returns the reply when the action ran (never falls
|
||||
/// back past this point — re-running a click over SSH would double-click), or `nil` when the
|
||||
/// action did NOT run (op failed before side effects, or the args don't validate) and the SSH
|
||||
/// path should take over.
|
||||
func performAgentComputerAction(
|
||||
name: String, client: MacVMAgentClient, action: String, x: Int?, y: Int?, text: String?,
|
||||
scrollDirection: String?, scrollAmount: Int?, durationMs: Int?
|
||||
) async -> (imageBase64: String?, summary: String)? {
|
||||
do {
|
||||
switch action {
|
||||
case "screenshot":
|
||||
break // nothing to do but capture below
|
||||
|
||||
case "cursor_position":
|
||||
let reply = try await client.request(op: "cursor_position")
|
||||
guard let cx = reply["x"]?.intValue, let cy = reply["y"]?.intValue else {
|
||||
return nil
|
||||
}
|
||||
return (nil, "Cursor position: \(cx),\(cy)")
|
||||
|
||||
case "wait":
|
||||
let ms = max(0, min(durationMs ?? 1000, 30_000))
|
||||
try? await Task.sleep(nanoseconds: UInt64(ms) * 1_000_000)
|
||||
|
||||
case "left_click", "right_click", "double_click":
|
||||
guard let x, let y else { return nil }
|
||||
var fields: [String: JSONValue] = [
|
||||
"x": .number(Double(x)), "y": .number(Double(y)),
|
||||
]
|
||||
if action == "right_click" { fields["button"] = .string("right") }
|
||||
if action == "double_click" { fields["count"] = .number(2) }
|
||||
_ = try await client.request(op: "click", fields: fields)
|
||||
|
||||
case "mouse_move":
|
||||
guard let x, let y else { return nil }
|
||||
_ = try await client.request(
|
||||
op: "move", fields: ["x": .number(Double(x)), "y": .number(Double(y))])
|
||||
|
||||
case "left_click_drag":
|
||||
// Drag from the CURRENT cursor to the target (the agent defaults `from` to the
|
||||
// live pointer, so no cursor-position round-trip is needed).
|
||||
guard let x, let y else { return nil }
|
||||
_ = try await client.request(
|
||||
op: "drag", fields: ["toX": .number(Double(x)), "toY": .number(Double(y))])
|
||||
|
||||
case "type":
|
||||
guard let text, !text.isEmpty else { return nil }
|
||||
_ = try await client.request(op: "type", fields: ["text": .string(text)])
|
||||
|
||||
case "key":
|
||||
guard let chord = text, !chord.isEmpty else { return nil }
|
||||
_ = try await client.request(op: "key", fields: ["chord": .string(chord)])
|
||||
|
||||
case "scroll":
|
||||
let amount = max(1, min(scrollAmount ?? 3, 30))
|
||||
let (dx, dy) = Self.scrollDelta(
|
||||
direction: scrollDirection ?? "down", amount: amount)
|
||||
_ = try await client.request(
|
||||
op: "scroll",
|
||||
fields: ["dx": .number(Double(dx)), "dy": .number(Double(dy))])
|
||||
|
||||
case "launch_app":
|
||||
guard let app = text, !app.isEmpty else { return nil }
|
||||
_ = try await client.request(op: "launch_app", fields: ["name": .string(app)])
|
||||
|
||||
default:
|
||||
return nil // not a native action — the SSH path owns the "unknown action" reply
|
||||
}
|
||||
} catch {
|
||||
await noteAgentFailure(name: name, client: client, error: error)
|
||||
return nil // the op failed before any side effect — safe to re-run over SSH
|
||||
}
|
||||
|
||||
// The action ran natively; capture the post-action frame. Capture-only fallbacks from here
|
||||
// (the action itself must never re-run).
|
||||
if let shot = await agentScreenshot(name: name, client: client) {
|
||||
return (shot, screenSummary(for: action))
|
||||
}
|
||||
if let shot = try? await captureScreenshotBase64(name: name) {
|
||||
return (shot, screenSummary(for: action))
|
||||
}
|
||||
return (
|
||||
nil,
|
||||
screenSummary(for: action) + " No screenshot came back — the guest may lack Screen "
|
||||
+ "Recording permission, or (on a macOS 26 guest) the VZ framebuffer bug blanks all "
|
||||
+ "capture. Use `ax_dump` to inspect the UI semantically instead."
|
||||
)
|
||||
}
|
||||
|
||||
/// Direction/amount (the tool's scroll vocabulary) → wheel-line deltas (the agent's §4 `scroll`
|
||||
/// op; dy > 0 scrolls content up, dx > 0 scrolls left — CGEvent wheel conventions).
|
||||
static func scrollDelta(direction: String, amount: Int) -> (dx: Int, dy: Int) {
|
||||
switch direction.lowercased() {
|
||||
case "up": return (0, amount)
|
||||
case "left": return (amount, 0)
|
||||
case "right": return (-amount, 0)
|
||||
default: return (0, -amount) // down
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - AX-semantic actions (§6, §10)
|
||||
|
||||
/// Perform one AX action. Unlike the pixel path this never falls back to SSH (there is no SSH
|
||||
/// equivalent), so failures come back as actionable text for the model.
|
||||
func performAXComputerAction(
|
||||
name: String, client: MacVMAgentClient, action: String, x: Int?, y: Int?, text: String?,
|
||||
ref: String?, value: String?
|
||||
) async -> (imageBase64: String?, summary: String) {
|
||||
do {
|
||||
switch action {
|
||||
case "ax_dump":
|
||||
let reply = try await client.request(op: "ax_dump")
|
||||
let app = reply["app"]?.stringValue ?? "frontmost app"
|
||||
let tree = reply["tree"] ?? .object([:])
|
||||
var header = "Accessibility tree of \(app) (roles/titles/values/frames; act on an "
|
||||
+ "element by its `ref` with ax_press / ax_set_value / ax_focus — refs stay "
|
||||
+ "valid until the next ax_dump):"
|
||||
if reply["truncated"]?.boolValue == true {
|
||||
header += " [tree truncated — deep/rich UI]"
|
||||
}
|
||||
return (nil, header + "\n" + tree.canonicalString())
|
||||
|
||||
case "ax_element_at":
|
||||
guard let x, let y else {
|
||||
return (nil, "ax_element_at requires x and y coordinates.")
|
||||
}
|
||||
let reply = try await client.request(
|
||||
op: "ax_element_at",
|
||||
fields: ["x": .number(Double(x)), "y": .number(Double(y))])
|
||||
let node = reply["node"] ?? .object([:])
|
||||
return (nil, "Element at (\(x), \(y)): " + node.canonicalString())
|
||||
|
||||
case "ax_press":
|
||||
guard let ref, !ref.isEmpty else {
|
||||
return (nil, "ax_press requires an element `ref` (from ax_dump).")
|
||||
}
|
||||
var fields: [String: JSONValue] = ["ref": .string(ref)]
|
||||
// `text` optionally names a non-default AX action (AXShowMenu, AXConfirm, …).
|
||||
if let axAction = text, !axAction.isEmpty { fields["action"] = .string(axAction) }
|
||||
let reply = try await client.request(op: "ax_action", fields: fields)
|
||||
var summary = "Performed \(text?.isEmpty == false ? text! : "AXPress") on \(ref)."
|
||||
if let note = reply["note"]?.stringValue { summary += " Note: \(note)." }
|
||||
return await withPostActionScreenshot(
|
||||
name: name, client: client, summary: summary)
|
||||
|
||||
case "ax_set_value":
|
||||
guard let ref, !ref.isEmpty else {
|
||||
return (nil, "ax_set_value requires an element `ref` (from ax_dump).")
|
||||
}
|
||||
guard let value else {
|
||||
return (nil, "ax_set_value requires `value` (the text to set).")
|
||||
}
|
||||
_ = try await client.request(
|
||||
op: "ax_set_value",
|
||||
fields: ["ref": .string(ref), "value": .string(value)])
|
||||
return await withPostActionScreenshot(
|
||||
name: name, client: client, summary: "Set the value of \(ref).")
|
||||
|
||||
case "ax_focus":
|
||||
guard let ref, !ref.isEmpty else {
|
||||
return (nil, "ax_focus requires an element `ref` (from ax_dump).")
|
||||
}
|
||||
_ = try await client.request(op: "ax_focus", fields: ["ref": .string(ref)])
|
||||
return await withPostActionScreenshot(
|
||||
name: name, client: client, summary: "Focused \(ref) — you can now `type` into it.")
|
||||
|
||||
default:
|
||||
return (nil, "Unknown AX action '\(action)'.")
|
||||
}
|
||||
} catch {
|
||||
await noteAgentFailure(name: name, client: client, error: error)
|
||||
let reason = (error as? MacVMAgentError)?.description ?? String(describing: error)
|
||||
return (
|
||||
nil,
|
||||
"\(action) failed: \(reason) If the element may have changed, re-run ax_dump and "
|
||||
+ "retry with a fresh ref; otherwise fall back to screenshot + pixel actions."
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// The mutating AX actions return a screenshot when capture works (helpful on healthy guests)
|
||||
/// and a re-dump hint always (the ONLY observation channel on a blank-framebuffer guest).
|
||||
private func withPostActionScreenshot(
|
||||
name: String, client: MacVMAgentClient, summary: String
|
||||
) async -> (imageBase64: String?, summary: String) {
|
||||
let hint = " Re-run ax_dump to observe the result semantically (refs may now be stale)."
|
||||
if let shot = await agentScreenshot(name: name, client: client) {
|
||||
return (shot, summary + hint)
|
||||
}
|
||||
return (nil, summary + hint)
|
||||
}
|
||||
|
||||
/// One native screenshot, or `nil` (after noting a transport failure). Never throws — capture
|
||||
/// is always best-effort around an action that already ran.
|
||||
private func agentScreenshot(name: String, client: MacVMAgentClient) async -> String? {
|
||||
do {
|
||||
let reply = try await client.request(op: "screenshot")
|
||||
return reply["image"]?.stringValue
|
||||
} catch {
|
||||
await noteAgentFailure(name: name, client: client, error: error)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
/// Bookkeeping for a failed agent request: a transport failure discards the channel (the next
|
||||
/// action re-probes once); an op-level `agentError` keeps the healthy channel.
|
||||
private func noteAgentFailure(name: String, client: MacVMAgentClient, error: Error) async {
|
||||
if let agentError = error as? MacVMAgentError, agentError.isTransportFailure {
|
||||
await discardAgentClient(name: name, client)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -24,6 +24,9 @@ public actor MacVMEngine {
|
||||
struct LiveVM {
|
||||
#if arch(arm64)
|
||||
let instance: MacVMInstance
|
||||
/// The native in-guest agent channel for this VM (docs/MACOS_VM_NATIVE_AGENT.md), probed
|
||||
/// lazily by ``MacVMEngine/agentClient(name:)`` and cached here for the VM's lifetime.
|
||||
var agent: MacVMAgentState = .unprobed
|
||||
#endif
|
||||
/// The clone bundle this VM booted from (its writable disk + aux storage live here).
|
||||
let bundle: MacVMBundle
|
||||
@@ -51,6 +54,19 @@ public actor MacVMEngine {
|
||||
/// Last spec seen per name, so `restart` (which only has a name) can recreate from disk.
|
||||
var lastSpec: [String: MacVMSpec] = [:]
|
||||
|
||||
#if arch(arm64)
|
||||
/// Probe state of a live VM's native in-guest agent (the vsock channel). `unavailable` is
|
||||
/// sticky for the VM's lifetime — a base image without the agent will never grow one mid-boot,
|
||||
/// so re-probing every action would just pay the connect timeout repeatedly. A transport
|
||||
/// failure on a previously `available` channel resets to `unprobed` instead (the LaunchAgent's
|
||||
/// `KeepAlive` restarts a crashed agent, so one reconnect attempt is worth it).
|
||||
enum MacVMAgentState {
|
||||
case unprobed
|
||||
case unavailable
|
||||
case available(MacVMAgentClient)
|
||||
}
|
||||
#endif
|
||||
|
||||
/// Held while at least one VM is live, to keep the host app from being App-Napped / auto-terminated
|
||||
/// out from under a running guest (mirrors ``ContainerEngine``'s assertion; the runtime is
|
||||
/// daemonless so the app process bounds every VM's life). `nil` when nothing is live.
|
||||
@@ -227,9 +243,14 @@ public actor MacVMEngine {
|
||||
/// crashed/powered-off guest doesn't keep consuming a concurrency slot or hold the app-activity
|
||||
/// assertion. Idempotent: a no-op if the engine already tore it down.
|
||||
private func handleGuestStopped(_ name: String) {
|
||||
guard live[name] != nil else { return }
|
||||
guard let entry = live[name] else { return }
|
||||
live[name] = nil
|
||||
syncBackgroundActivity()
|
||||
#if arch(arm64)
|
||||
if case .available(let client) = entry.agent {
|
||||
Task { await client.close() }
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
/// Run `argv` inside the named guest over SSH, returning a ``ProcessHandle`` whose stdio is the
|
||||
@@ -298,6 +319,7 @@ public actor MacVMEngine {
|
||||
public func stop(name: String) async {
|
||||
guard let entry = live[name] else { return }
|
||||
#if arch(arm64)
|
||||
if case .available(let client) = entry.agent { await client.close() }
|
||||
await entry.instance.stop()
|
||||
#endif
|
||||
live[name] = nil
|
||||
|
||||
@@ -76,14 +76,17 @@ public actor MacVMManager {
|
||||
try await engine.run(name: name, command: command, workdir: workdir, env: env)
|
||||
}
|
||||
|
||||
/// Perform one computer-use action (screenshot / click / type / …) in the named VM and return the
|
||||
/// post-action screenshot (base64 PNG) + a summary (the `mac_vm_computer` tool path). No lifecycle effect.
|
||||
/// Perform one computer-use action (screenshot / click / type / ax_dump / ax_press / …) in the
|
||||
/// named VM and return the post-action screenshot (base64 JPEG) + a summary (the
|
||||
/// `mac_vm_computer` tool path). `ref`/`value` feed the AX-semantic actions (native agent).
|
||||
/// No lifecycle effect.
|
||||
public func performComputerAction(
|
||||
name: String, action: String, x: Int?, y: Int?, text: String?,
|
||||
ref: String? = nil, value: String? = nil,
|
||||
scrollDirection: String?, scrollAmount: Int?, durationMs: Int?
|
||||
) async throws -> (imageBase64: String?, summary: String) {
|
||||
try await engine.performComputerAction(
|
||||
name: name, action: action, x: x, y: y, text: text,
|
||||
name: name, action: action, x: x, y: y, text: text, ref: ref, value: value,
|
||||
scrollDirection: scrollDirection, scrollAmount: scrollAmount, durationMs: durationMs)
|
||||
}
|
||||
|
||||
|
||||
@@ -257,7 +257,13 @@ public actor SessionController {
|
||||
iteratively: screenshot → decide → act → screenshot. Reach for it to visually debug a \
|
||||
macOS/iOS app you built (launch it and look at / click through its window), drive the Xcode \
|
||||
or Simulator UI, or check how something renders — things a headless command can't show you. \
|
||||
Coordinates are the screen's real pixels (target exactly what you see). These actions run in \
|
||||
Coordinates are the screen's real pixels (target exactly what you see). When the VM's \
|
||||
native agent is available, PREFER the semantic actions: `ax_dump` shows the frontmost \
|
||||
app's accessibility tree (every control's role/title/value/frame with a `ref`), then act \
|
||||
by identity with `ax_press`/`ax_set_value`/`ax_focus` — far more reliable than guessing \
|
||||
pixel coordinates, and it keeps working even when screenshots come back blank (a known \
|
||||
macOS 26 guest bug). Fall back to screenshot + pixel clicks when a control has no AX \
|
||||
action or the reply says the native agent is unavailable. These actions run in \
|
||||
the VM sandbox and are NOT individually approved, so you can move quickly. Prefer \
|
||||
`mac_vm_exec` for headless build/test; use `mac_vm_computer` when you need to look at or \
|
||||
operate the screen.
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import Foundation
|
||||
import NucleicProtocol
|
||||
import Testing
|
||||
|
||||
@testable import NucleicCore
|
||||
@@ -281,6 +282,73 @@ import Testing
|
||||
#expect(MacVMEngine.launchAppCommand("Xcode").hasSuffix("/usr/bin/open -a 'Xcode'"))
|
||||
}
|
||||
|
||||
// MARK: - Native in-guest agent (docs/MACOS_VM_NATIVE_AGENT.md)
|
||||
|
||||
@Test func agentRequestLineIsOneJSONObjectPerLine() throws {
|
||||
let line = MacVMAgentWire.requestLine(
|
||||
op: "click", fields: ["x": .number(10), "y": .number(20)])
|
||||
#expect(line.last == 0x0A) // newline-terminated (NDJSON)
|
||||
#expect(!line.dropLast().contains(0x0A)) // and exactly ONE line
|
||||
let parsed = try JSONValue(parsing: line.dropLast())
|
||||
#expect(parsed["op"]?.stringValue == "click")
|
||||
#expect(parsed["x"]?.intValue == 10)
|
||||
#expect(parsed["y"]?.intValue == 20)
|
||||
}
|
||||
|
||||
@Test func agentRequestLineOpWinsOverFieldCollision() throws {
|
||||
// `op` is the dispatcher's field; a caller-supplied collision must not hijack it.
|
||||
let line = MacVMAgentWire.requestLine(op: "ping", fields: ["op": .string("evil")])
|
||||
let parsed = try JSONValue(parsing: line.dropLast())
|
||||
#expect(parsed["op"]?.stringValue == "ping")
|
||||
}
|
||||
|
||||
@Test func agentReplyParsingAcceptsObjectsRejectsGarbage() throws {
|
||||
let reply = try MacVMAgentWire.parseReply(Data(#"{"ok":true,"version":1}"#.utf8))
|
||||
#expect(reply["ok"]?.boolValue == true)
|
||||
#expect(reply["version"]?.intValue == 1)
|
||||
#expect(throws: (any Error).self) {
|
||||
try MacVMAgentWire.parseReply(Data("not json".utf8))
|
||||
}
|
||||
#expect(throws: (any Error).self) {
|
||||
try MacVMAgentWire.parseReply(Data(#"[1,2,3]"#.utf8)) // an array is not a reply
|
||||
}
|
||||
}
|
||||
|
||||
@Test func agentErrorSeparatesTransportFromOpFailures() {
|
||||
// Op-level failures keep the channel; transport failures discard it (re-probe next action).
|
||||
#expect(!MacVMAgentError.agentError("no such ref").isTransportFailure)
|
||||
#expect(MacVMAgentError.ioFailed("EOF").isTransportFailure)
|
||||
#expect(MacVMAgentError.timeout.isTransportFailure)
|
||||
#expect(MacVMAgentError.protocolError("bad line").isTransportFailure)
|
||||
#expect(MacVMAgentError.noSocketDevice.isTransportFailure)
|
||||
}
|
||||
|
||||
@Test func scrollDeltaMapsDirectionToWheelLines() {
|
||||
// CGEvent wheel conventions: wheel1 > 0 scrolls content up, wheel2 > 0 scrolls left.
|
||||
#expect(MacVMEngine.scrollDelta(direction: "up", amount: 3) == (0, 3))
|
||||
#expect(MacVMEngine.scrollDelta(direction: "down", amount: 3) == (0, -3))
|
||||
#expect(MacVMEngine.scrollDelta(direction: "left", amount: 2) == (2, 0))
|
||||
#expect(MacVMEngine.scrollDelta(direction: "right", amount: 2) == (-2, 0))
|
||||
#expect(MacVMEngine.scrollDelta(direction: "diagonal??", amount: 1) == (0, -1)) // → down
|
||||
}
|
||||
|
||||
@Test func axActionsAreAgentOnlyVocabulary() {
|
||||
#expect(MacVMEngine.axActions == [
|
||||
"ax_dump", "ax_element_at", "ax_press", "ax_set_value", "ax_focus",
|
||||
])
|
||||
// The pixel vocabulary must stay disjoint — routing switches on membership.
|
||||
for pixel in ["screenshot", "left_click", "type", "key", "scroll", "launch_app", "wait"] {
|
||||
#expect(!MacVMEngine.axActions.contains(pixel))
|
||||
}
|
||||
}
|
||||
|
||||
@Test func computerUseGuidanceTeachesAXFirst() {
|
||||
let on = SessionController.sandboxBuildGuidance(
|
||||
allowHostExec: true, allowMacVMExec: true, allowMacVMComputer: true)
|
||||
#expect(on.contains("ax_dump"))
|
||||
#expect(on.contains("ax_press"))
|
||||
}
|
||||
|
||||
@Test func computerUseSettingGatedOnService() async throws {
|
||||
let suite = UserDefaults(suiteName: "macvm-cu-\(UUID().uuidString)")!
|
||||
try await ContainerServiceSettings.withDefaults(suite) {
|
||||
|
||||
@@ -6,11 +6,17 @@ capture, **CGEvent** for raw input — reached from the host over **vsock**. It
|
||||
`mac_vm_computer` transport (SSH → `launchctl asuser` → `screencapture`/`cliclick` → base64), which is
|
||||
a stack of shims over those same frameworks.
|
||||
|
||||
**Status:** design / spec. Not implemented. The vsock transport and the AXUIElement surface below are
|
||||
verified against Apple's live documentation (see §11 sources); the one empirical claim to validate on a
|
||||
real guest (guest-side vsock `listen`, and AX/CGEvent behaviour under the Tahoe framebuffer bug) is
|
||||
called out in §9 and §10. Companion to [MACOS_VM](MACOS_VM.md) (which this augments) and
|
||||
[VSOCK_CONTROL_PLANE](VSOCK_CONTROL_PLANE.md) (Nucleic's existing vsock work for the Linux container).
|
||||
**Status:** implemented through §13 phases 1–4 (code + provisioning), **pending real-guest
|
||||
validation** (§13 phase 5 / §14). What exists: the guest agent (`guest/NucleicVMAgent` — its own
|
||||
SwiftPM package, macOS 14 floor — vsock listener, all §4 ops), the host side (`MacVMAgentClient` +
|
||||
the `VZVirtioSocketDevice` config + the ping-probe/fallback routing in `MacVMEngine+ComputerAgent`),
|
||||
the extended `mac_vm_computer` tool surface (`ax_*` actions + `ref`/`value`), the app-bundle build
|
||||
(`scripts/build-vm-agent.sh`), and provisioning Phase 7d (`scripts/provision-macos-guest.sh`). The
|
||||
empirical claims to validate on a real guest (guest-side vsock `listen`, and AX/CGEvent behaviour
|
||||
under the Tahoe framebuffer bug) are called out in §9 and §10 — until then the SSH path remains the
|
||||
proven default and the agent is strictly additive. Companion to [MACOS_VM](MACOS_VM.md) (which this
|
||||
augments) and [VSOCK_CONTROL_PLANE](VSOCK_CONTROL_PLANE.md) (Nucleic's existing vsock work for the
|
||||
Linux container).
|
||||
|
||||
---
|
||||
|
||||
@@ -347,36 +353,45 @@ present.
|
||||
|
||||
---
|
||||
|
||||
## 12. Fit with the existing code (when built)
|
||||
## 12. Fit with the existing code (as built)
|
||||
|
||||
- **New target `NucleicVMAgent`** — a macOS executable built into a `.app` bundle (its own SwiftPM
|
||||
executable target or a small Xcode target invoked from the base-image build). Uses `ApplicationServices`,
|
||||
`ScreenCaptureKit`, `CoreGraphics`, `AppKit`.
|
||||
- **`MacVMEngine`** — add `VZVirtioSocketDeviceConfiguration` in `buildConfiguration`; a `MacVMAgentClient`
|
||||
(vsock connect on the VM queue, NDJSON request/reply); a `ping`-based capability check cached per live VM.
|
||||
- **`MacVMEngine+Computer`** — route `performComputerAction` through the agent when present, SSH/cliclick
|
||||
otherwise; add the `ax_*` ops (no SSH fallback for those).
|
||||
- **`MCPApprovalServer`** — extend the `mac_vm_computer` schema with the AX actions + `ref`/`value` fields;
|
||||
return the AX tree as text content (and screenshots as image content, as now).
|
||||
- **`scripts/provision-macos-guest.sh`** — install the agent app + LaunchAgent + its TCC grants.
|
||||
- **`MacVMSettings`** — a toggle is optional; the agent is auto-preferred when baked in, so no new
|
||||
user-facing switch is strictly required (a "prefer native agent" override could aid debugging).
|
||||
- **`guest/NucleicVMAgent`** — the agent, as its own SwiftPM **package** (not a target of the main
|
||||
one): the main package's platform floor is macOS 26, but the agent must run on a macOS 14/15
|
||||
(Sequoia) guest, and SwiftPM has no per-target floors. Targets: `CVsock` (C shim — Swift's Darwin
|
||||
overlay doesn't surface `sockaddr_vm`), `VMAgentCore` (pure wire/keymap logic, unit-tested), and
|
||||
the `NucleicVMAgent` executable (`ApplicationServices`, `ScreenCaptureKit`, `CoreGraphics`,
|
||||
`AppKit`). `scripts/build-vm-agent.sh` wraps it into the signed `.app` (+ the LaunchAgent plist).
|
||||
- **`MacVMEngine`** — `buildConfiguration` attaches one `VZVirtioSocketDeviceConfiguration`;
|
||||
`MacVMAgentClient.swift` holds the NDJSON client (an actor over the vsock fd), the wire constants
|
||||
(`MacVMAgentWire`, port 2035 — mirrored in `VMAgentCore.AgentWire`), and the `ping`-probe
|
||||
capability cache per live VM (`LiveVM.agent`: unprobed → available/unavailable; a transport
|
||||
failure re-probes once, an absent agent stays absent for the VM's lifetime).
|
||||
- **`MacVMEngine+ComputerAgent`** — routes `performComputerAction` through the agent when present
|
||||
(action ran natively ⇒ never re-run over SSH; capture-only fallbacks after), SSH/cliclick
|
||||
otherwise; implements the `ax_*` ops (no SSH fallback for those).
|
||||
- **`MCPApprovalServer`** — the `mac_vm_computer` schema carries the AX actions + `ref`/`value`
|
||||
fields; the AX tree returns as text content (and screenshots as image content, as before).
|
||||
- **`scripts/provision-macos-guest.sh`** — Phase 7d installs the agent app + LaunchAgent + its three
|
||||
TCC grants (bundle-id-keyed rows), skipping gracefully when the app isn't staged or SIP is on.
|
||||
- **`MacVMSettings`** — no new toggle: the agent is auto-preferred when present (the ping probe is
|
||||
the switch), and the SSH path engages transparently otherwise.
|
||||
|
||||
---
|
||||
|
||||
## 13. Rollout plan (phased, each independently landable)
|
||||
|
||||
1. **Transport core** — `NucleicVMAgent` skeleton with the vsock listener + `ping`/`screenshot`; host
|
||||
`VZVirtioSocketDevice` config + client; prove the round-trip on a real guest (validates §3.2). SSH
|
||||
path stays the default until this is proven.
|
||||
2. **Input + capture over the agent** — `click`/`move`/`type`/`key`/`scroll` (real scroll) + SCK
|
||||
`screenshot`; flip `mac_vm_computer` to prefer the agent, SSH fallback intact.
|
||||
3. **Accessibility** — `ax_dump`/`ax_element_at`/`ax_action`/`ax_set_value` + the tool-schema actions +
|
||||
1. ✅ **Transport core** — `NucleicVMAgent` skeleton with the vsock listener + `ping`/`screenshot`; host
|
||||
`VZVirtioSocketDevice` config + client. *Code landed; the real-guest round-trip that proves §3.2 is
|
||||
still owed (see 5).* SSH path stays the default until this is proven.
|
||||
2. ✅ **Input + capture over the agent** — `click`/`move`/`type`/`key`/`scroll` (real scroll) + SCK
|
||||
`screenshot`; `mac_vm_computer` prefers the agent, SSH fallback intact.
|
||||
3. ✅ **Accessibility** — `ax_dump`/`ax_element_at`/`ax_action`/`ax_set_value` + the tool-schema actions +
|
||||
model guidance. This is where the semantic + framebuffer-independent wins land.
|
||||
4. **Provisioning + base image** — bake the agent, LaunchAgent, and TCC grants into
|
||||
`provision-macos-guest.sh`; document.
|
||||
5. **Validate on Tahoe** — confirm AX observe+control works with blank screenshots (§9); if so, Tahoe
|
||||
becomes viable for computer-use *via AX*, relaxing the Sequoia-only recommendation for the control path.
|
||||
4. ✅ **Provisioning + base image** — the agent, LaunchAgent, and TCC grants in
|
||||
`provision-macos-guest.sh` Phase 7d (agent app staged from `scripts/build-vm-agent.sh` output).
|
||||
5. ⬜ **Validate on a real guest** — smoke-test the vsock round-trip (§3.2), then confirm on Tahoe that
|
||||
AX observe+control works with blank screenshots (§9); if so, Tahoe becomes viable for computer-use
|
||||
*via AX*, relaxing the Sequoia-only recommendation for the control path.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
.build/
|
||||
dist/
|
||||
@@ -0,0 +1,33 @@
|
||||
// swift-tools-version: 6.2
|
||||
import PackageDescription
|
||||
|
||||
// The native in-guest automation agent for Nucleic's macOS VMs (docs/MACOS_VM_NATIVE_AGENT.md).
|
||||
//
|
||||
// A SEPARATE package from the main Nucleic one on purpose: the main package's platform floor is
|
||||
// macOS 26 (the host app needs the macOS-26 containerization framework), but this binary runs INSIDE
|
||||
// the guest — and the recommended computer-use guest is macOS 15 (Sequoia), because of the Tahoe
|
||||
// framebuffer bug (docs/MACOS_VM.md §12.6). SwiftPM has no per-target platform floors, so the agent
|
||||
// gets its own package with a macOS 14 floor (`SCScreenshotManager` is macOS 14+).
|
||||
//
|
||||
// Built on the HOST (scripts/build-vm-agent.sh wraps it into a signed NucleicVMAgent.app) and baked
|
||||
// into the golden base image by scripts/provision-macos-guest.sh; never shipped with the host app.
|
||||
let package = Package(
|
||||
name: "NucleicVMAgent",
|
||||
platforms: [.macOS(.v14)],
|
||||
targets: [
|
||||
// C shim for the AF_VSOCK listener: `struct sockaddr_vm` (<sys/vsock.h>) isn't surfaced to
|
||||
// Swift by the Darwin overlay, so the socket/bind/listen dance lives in C.
|
||||
.target(name: "CVsock"),
|
||||
// Pure, framework-light logic (wire constants, NDJSON line splitting, key-chord parsing) —
|
||||
// unit-testable on any Mac without TCC grants or a GUI session.
|
||||
.target(name: "VMAgentCore"),
|
||||
// The agent itself: vsock accept loop + the op handlers (AXUIElement / ScreenCaptureKit /
|
||||
// CGEvent). Swift 5 language mode: the handlers are thread-confined by construction (one
|
||||
// blocking thread per connection), which strict concurrency can't see past the CF/AX types.
|
||||
.executableTarget(
|
||||
name: "NucleicVMAgent",
|
||||
dependencies: ["CVsock", "VMAgentCore"],
|
||||
swiftSettings: [.swiftLanguageMode(.v5)]),
|
||||
.testTarget(name: "VMAgentCoreTests", dependencies: ["VMAgentCore"]),
|
||||
]
|
||||
)
|
||||
@@ -0,0 +1,32 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<!-- Bundle metadata for NucleicVMAgent.app (docs/MACOS_VM_NATIVE_AGENT.md §11). A real signed
|
||||
.app bundle is mandatory-in-practice: TCC mis-handles bare executables, and on macOS 26
|
||||
ScreenCaptureKit returns -3801 ("declined") for a non-bundled binary even with a TCC row. -->
|
||||
<dict>
|
||||
<key>CFBundleIdentifier</key>
|
||||
<string>xyz.blakeslee.nucleic.vmagent</string>
|
||||
<key>CFBundleName</key>
|
||||
<string>NucleicVMAgent</string>
|
||||
<key>CFBundleDisplayName</key>
|
||||
<string>Nucleic VM Agent</string>
|
||||
<key>CFBundleExecutable</key>
|
||||
<string>NucleicVMAgent</string>
|
||||
<key>CFBundlePackageType</key>
|
||||
<string>APPL</string>
|
||||
<key>CFBundleShortVersionString</key>
|
||||
<string>1.0</string>
|
||||
<key>CFBundleVersion</key>
|
||||
<string>1</string>
|
||||
<key>LSMinimumSystemVersion</key>
|
||||
<string>14.0</string>
|
||||
<!-- Background agent: no Dock icon, no menu bar. -->
|
||||
<key>LSUIElement</key>
|
||||
<true/>
|
||||
<key>NSScreenCaptureUsageDescription</key>
|
||||
<string>Nucleic's in-guest agent captures this virtual machine's screen so the driving agent can see it.</string>
|
||||
<key>NSAccessibilityUsageDescription</key>
|
||||
<string>Nucleic's in-guest agent reads and drives app UIs in this virtual machine via accessibility.</string>
|
||||
</dict>
|
||||
</plist>
|
||||
@@ -0,0 +1,29 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<!-- LaunchAgent for NucleicVMAgent (docs/MACOS_VM_NATIVE_AGENT.md §11). Installed by
|
||||
scripts/provision-macos-guest.sh at /Library/LaunchAgents/ so it runs in the auto-login
|
||||
`agent` account's Aqua session — AX, ScreenCaptureKit and CGEvent all require an active
|
||||
WindowServer session, which is why this is a LaunchAgent, not a LaunchDaemon. -->
|
||||
<dict>
|
||||
<key>Label</key>
|
||||
<string>xyz.blakeslee.nucleic.vmagent</string>
|
||||
<key>ProgramArguments</key>
|
||||
<array>
|
||||
<string>/Applications/NucleicVMAgent.app/Contents/MacOS/NucleicVMAgent</string>
|
||||
</array>
|
||||
<key>RunAtLoad</key>
|
||||
<true/>
|
||||
<key>KeepAlive</key>
|
||||
<true/>
|
||||
<!-- Aqua only: never launch in the pre-login / background domains where the frameworks fail. -->
|
||||
<key>LimitLoadToSessionType</key>
|
||||
<string>Aqua</string>
|
||||
<key>AssociatedBundleIdentifiers</key>
|
||||
<string>xyz.blakeslee.nucleic.vmagent</string>
|
||||
<key>StandardOutPath</key>
|
||||
<string>/tmp/nucleic-vmagent.log</string>
|
||||
<key>StandardErrorPath</key>
|
||||
<string>/tmp/nucleic-vmagent.log</string>
|
||||
</dict>
|
||||
</plist>
|
||||
@@ -0,0 +1,35 @@
|
||||
# NucleicVMAgent
|
||||
|
||||
The **native in-guest automation agent** for Nucleic's macOS VMs — design and rationale in
|
||||
[docs/MACOS_VM_NATIVE_AGENT.md](../../docs/MACOS_VM_NATIVE_AGENT.md). It runs inside the guest as a
|
||||
LaunchAgent in the auto-login Aqua session, listens on **vsock port 2035**, and executes NDJSON ops
|
||||
with Apple's frameworks directly:
|
||||
|
||||
- **AXUIElement** — semantic observe + control (`ax_dump`, `ax_action`, `ax_set_value`, `ax_focus`,
|
||||
`ax_element_at`). Framebuffer-independent: keeps working on a macOS 26 guest whose screenshots are
|
||||
blank (the Tahoe VZ compositing bug).
|
||||
- **ScreenCaptureKit** — in-process `screenshot`.
|
||||
- **CGEvent** — raw `click`/`move`/`drag`/`type`/`key`/`scroll` (a real wheel event).
|
||||
|
||||
A **separate SwiftPM package** from the main Nucleic one: the host app's floor is macOS 26, but this
|
||||
binary must run on macOS 14/15 guests (the recommended computer-use base is Sequoia), and SwiftPM has
|
||||
no per-target platform floors.
|
||||
|
||||
## Build & install
|
||||
|
||||
```sh
|
||||
# On the host (needs Swift; sign stably or TCC grants break on rebuild):
|
||||
NUCLEIC_VMAGENT_SIGN_IDENTITY="Developer ID Application: …" ../../scripts/build-vm-agent.sh
|
||||
# → dist/NucleicVMAgent.app + dist/xyz.blakeslee.nucleic.vmagent.plist
|
||||
|
||||
# Stage dist/* next to scripts/provision-macos-guest.sh inside the guest and run the provisioner —
|
||||
# Phase 7d installs the app, the LaunchAgent, and the three TCC grants (Accessibility, PostEvent,
|
||||
# ScreenCapture; needs SIP off, same as the rest of Phase 7).
|
||||
```
|
||||
|
||||
The host side (`MacVMAgentClient` in `Sources/NucleicCore/MacVM/`) probes the agent with a vsock
|
||||
`ping` per VM boot and transparently falls back to the SSH + cliclick path when it's absent — the
|
||||
agent is strictly additive. Wire constants are mirrored between `VMAgentCore.AgentWire` (here) and
|
||||
`MacVMAgentWire` (host); keep them in lockstep.
|
||||
|
||||
Agent log inside the guest: `/tmp/nucleic-vmagent.log`.
|
||||
@@ -0,0 +1,31 @@
|
||||
#include "include/cvsock.h"
|
||||
|
||||
#include <errno.h>
|
||||
#include <string.h>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/vsock.h>
|
||||
#include <unistd.h>
|
||||
|
||||
int nucleic_vsock_listen(uint32_t port, int backlog) {
|
||||
int fd = socket(AF_VSOCK, SOCK_STREAM, 0);
|
||||
if (fd < 0) return -errno;
|
||||
|
||||
struct sockaddr_vm addr;
|
||||
memset(&addr, 0, sizeof(addr));
|
||||
addr.svm_len = sizeof(addr); // macOS-specific BSD length field (Linux has none)
|
||||
addr.svm_family = AF_VSOCK;
|
||||
addr.svm_cid = VMADDR_CID_ANY;
|
||||
addr.svm_port = port;
|
||||
|
||||
if (bind(fd, (const struct sockaddr *)&addr, sizeof(addr)) != 0) {
|
||||
int e = errno;
|
||||
close(fd);
|
||||
return -e;
|
||||
}
|
||||
if (listen(fd, backlog) != 0) {
|
||||
int e = errno;
|
||||
close(fd);
|
||||
return -e;
|
||||
}
|
||||
return fd;
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
#ifndef NUCLEIC_CVSOCK_H
|
||||
#define NUCLEIC_CVSOCK_H
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
// Create + bind + listen an AF_VSOCK stream socket on `port`, listening on any CID (inside a macOS
|
||||
// guest the guest's CID is assigned by the hypervisor; the host connects via VZVirtioSocketDevice).
|
||||
// Returns the listening fd (>= 0), or -errno on failure. Lives in C because Swift's Darwin overlay
|
||||
// does not surface `struct sockaddr_vm` from <sys/vsock.h>.
|
||||
int nucleic_vsock_listen(uint32_t port, int backlog);
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,152 @@
|
||||
import CVsock
|
||||
import Darwin
|
||||
import Foundation
|
||||
import VMAgentCore
|
||||
|
||||
enum AgentError: Error, CustomStringConvertible {
|
||||
case posix(String, Int32)
|
||||
var description: String {
|
||||
switch self {
|
||||
case let .posix(what, errno): return "\(what) failed: \(String(cString: strerror(errno)))"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Timestamped line to stderr — the LaunchAgent routes it to the agent's log file.
|
||||
func agentLog(_ message: String) {
|
||||
FileHandle.standardError.write(Data("[NucleicVMAgent] \(message)\n".utf8))
|
||||
}
|
||||
|
||||
/// Owns the AF_VSOCK listening socket and the blocking accept loop. One `ConnectionHandler` thread
|
||||
/// per accepted connection; in practice there is exactly one client (the host's `MacVMAgentClient`),
|
||||
/// but a reconnect after a host restart must not require an agent restart.
|
||||
final class AgentListener {
|
||||
private let fd: Int32
|
||||
|
||||
init(port: UInt32) throws {
|
||||
let result = nucleic_vsock_listen(port, 4)
|
||||
guard result >= 0 else { throw AgentError.posix("vsock listen(:\(port))", -result) }
|
||||
fd = result
|
||||
}
|
||||
|
||||
func run() {
|
||||
while true {
|
||||
let conn = accept(fd, nil, nil)
|
||||
if conn < 0 {
|
||||
if errno == EINTR { continue }
|
||||
agentLog("accept failed (errno \(errno)); retrying")
|
||||
Thread.sleep(forTimeInterval: 1)
|
||||
continue
|
||||
}
|
||||
agentLog("host connected (fd \(conn))")
|
||||
let handler = ConnectionHandler(fd: conn)
|
||||
Thread.detachNewThread { handler.run() }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One host connection: a blocking read loop that frames NDJSON requests, dispatches each op, and
|
||||
/// writes the one-line JSON reply. Everything for this connection (including the AX element
|
||||
/// registry) is confined to this thread — requests on a connection are strictly serial.
|
||||
final class ConnectionHandler {
|
||||
private let fd: Int32
|
||||
private var lines = LineSplitBuffer()
|
||||
/// AX element handles (`ref` → live `AXUIElement`) issued by the last dump on this connection.
|
||||
let axRegistry = AXRegistry()
|
||||
|
||||
init(fd: Int32) {
|
||||
self.fd = fd
|
||||
}
|
||||
|
||||
func run() {
|
||||
defer {
|
||||
close(fd)
|
||||
agentLog("host disconnected (fd \(fd))")
|
||||
}
|
||||
var scratch = [UInt8](repeating: 0, count: 64 * 1024)
|
||||
while true {
|
||||
let n = read(fd, &scratch, scratch.count)
|
||||
if n == 0 { return } // EOF — host closed
|
||||
if n < 0 {
|
||||
if errno == EINTR { continue }
|
||||
return
|
||||
}
|
||||
lines.append(Data(bytes: scratch, count: n))
|
||||
while let line = lines.nextLine() {
|
||||
guard !line.isEmpty else { continue }
|
||||
var reply = handle(requestLine: line)
|
||||
reply.append(0x0A)
|
||||
guard writeAll(reply) else { return }
|
||||
}
|
||||
// Runaway-line guard: a client that never sends a newline can't balloon memory.
|
||||
if lines.pendingBytes > 4 * 1024 * 1024 { return }
|
||||
}
|
||||
}
|
||||
|
||||
private func writeAll(_ data: Data) -> Bool {
|
||||
var remaining = data
|
||||
while !remaining.isEmpty {
|
||||
let written = remaining.withUnsafeBytes { raw in
|
||||
write(fd, raw.baseAddress, raw.count)
|
||||
}
|
||||
if written < 0 {
|
||||
if errno == EINTR { continue }
|
||||
return false
|
||||
}
|
||||
remaining = remaining.dropFirst(written)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// MARK: - Dispatch
|
||||
|
||||
private func handle(requestLine: Data) -> Data {
|
||||
let reply = dispatch(requestLine: requestLine)
|
||||
if let data = try? JSONSerialization.data(withJSONObject: reply) { return data }
|
||||
return Data(#"{"ok":false,"error":"reply serialization failed"}"#.utf8)
|
||||
}
|
||||
|
||||
private func dispatch(requestLine: Data) -> [String: Any] {
|
||||
guard
|
||||
let parsed = try? JSONSerialization.jsonObject(with: requestLine),
|
||||
let request = parsed as? [String: Any],
|
||||
let op = request["op"] as? String
|
||||
else {
|
||||
return AgentReply.err("malformed request (want one JSON object per line with an \"op\")")
|
||||
}
|
||||
// Each op is individually guarded so a framework-level surprise degrades to an error reply,
|
||||
// never a dead agent.
|
||||
switch op {
|
||||
case "ping": return opPing()
|
||||
case "screenshot": return opScreenshot(request)
|
||||
case "ax_dump": return opAXDump(request)
|
||||
case "ax_element_at": return opAXElementAt(request)
|
||||
case "ax_action": return opAXAction(request)
|
||||
case "ax_set_value": return opAXSetValue(request)
|
||||
case "ax_focus": return opAXFocus(request)
|
||||
case "click": return opClick(request)
|
||||
case "move": return opMove(request)
|
||||
case "drag": return opDrag(request)
|
||||
case "type": return opType(request)
|
||||
case "key": return opKey(request)
|
||||
case "scroll": return opScroll(request)
|
||||
case "cursor_position": return opCursorPosition()
|
||||
case "launch_app": return opLaunchApp(request)
|
||||
default:
|
||||
return AgentReply.err("unknown op '\(op)' (agent protocol v\(AgentWire.version))")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Reply-shape helpers (§4): `{ok:true, …}` or `{ok:false, error}`.
|
||||
enum AgentReply {
|
||||
static func ok(_ fields: [String: Any] = [:]) -> [String: Any] {
|
||||
var reply = fields
|
||||
reply["ok"] = true
|
||||
return reply
|
||||
}
|
||||
|
||||
static func err(_ message: String) -> [String: Any] {
|
||||
["ok": false, "error": message]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,277 @@
|
||||
import AppKit
|
||||
import ApplicationServices
|
||||
import Foundation
|
||||
|
||||
/// Per-connection registry of the element handles (`ref` → live `AXUIElementRef`) issued by the
|
||||
/// last `ax_dump`/`ax_element_at` (§6.2). `AXUIElementRef`s go stale as the UI mutates, so refs are
|
||||
/// valid until the next dump replaces them; a stale ref surfaces as "re-dump and retry".
|
||||
final class AXRegistry {
|
||||
private var refs: [String: AXUIElement] = [:]
|
||||
private var counter = 0
|
||||
|
||||
func clear() { refs.removeAll() }
|
||||
|
||||
func register(_ element: AXUIElement) -> String {
|
||||
counter += 1
|
||||
let ref = "e\(counter)"
|
||||
refs[ref] = element
|
||||
return ref
|
||||
}
|
||||
|
||||
subscript(ref: String) -> AXUIElement? { refs[ref] }
|
||||
}
|
||||
|
||||
/// Accessibility observe + control (§6): the semantic, framebuffer-independent surface. All calls
|
||||
/// are synchronous cross-process IPC into the target app, bounded by the messaging timeout so a
|
||||
/// hung app can't wedge the walker.
|
||||
extension ConnectionHandler {
|
||||
private var axTrusted: Bool { AXIsProcessTrusted() }
|
||||
private static let axDenied =
|
||||
"Accessibility (kTCCServiceAccessibility) is not granted to the agent — re-run provisioning"
|
||||
|
||||
/// `ax_dump` — walk the frontmost (or `pid`-addressed) app's element tree into a JSON node tree,
|
||||
/// registering a fresh `ref` per element. This is the model's semantic "look" primitive.
|
||||
func opAXDump(_ request: [String: Any]) -> [String: Any] {
|
||||
guard axTrusted else { return AgentReply.err(Self.axDenied) }
|
||||
let pid: pid_t
|
||||
if let requested = request["pid"] as? Int {
|
||||
pid = pid_t(requested)
|
||||
} else if let front = NSWorkspace.shared.frontmostApplication {
|
||||
pid = front.processIdentifier
|
||||
} else {
|
||||
return AgentReply.err("no frontmost application to dump")
|
||||
}
|
||||
let maxDepth = min(max((request["maxDepth"] as? Int) ?? 12, 1), 25)
|
||||
|
||||
// New dump ⇒ new handle generation; every previously issued ref is now invalid by contract.
|
||||
axRegistry.clear()
|
||||
let app = AXUIElementCreateApplication(pid)
|
||||
// A tight timeout so ONE hung app can't stall the whole walk for the default ~6 s per call.
|
||||
AXUIElementSetMessagingTimeout(app, 1.5)
|
||||
|
||||
// Total node budget: a rich app's full tree is thousands of IPC round-trips (§6.1); the cap
|
||||
// bounds both the walk time and the payload the model has to read.
|
||||
var budget = 800
|
||||
let tree = buildNode(app, depth: 0, maxDepth: maxDepth, budget: &budget)
|
||||
var fields: [String: Any] = ["tree": tree]
|
||||
fields["app"] = NSRunningApplication(processIdentifier: pid)?.localizedName ?? "pid \(pid)"
|
||||
if budget <= 0 { fields["truncated"] = true }
|
||||
return AgentReply.ok(fields)
|
||||
}
|
||||
|
||||
/// `ax_element_at` — resolve the element under a screen point (hit-test via the system-wide
|
||||
/// element) into a single node (no children). Registers a ref alongside the last dump's.
|
||||
func opAXElementAt(_ request: [String: Any]) -> [String: Any] {
|
||||
guard axTrusted else { return AgentReply.err(Self.axDenied) }
|
||||
guard let x = request["x"] as? Int, let y = request["y"] as? Int else {
|
||||
return AgentReply.err("ax_element_at requires x and y")
|
||||
}
|
||||
let systemWide = AXUIElementCreateSystemWide()
|
||||
AXUIElementSetMessagingTimeout(systemWide, 1.5)
|
||||
var element: AXUIElement?
|
||||
let result = AXUIElementCopyElementAtPosition(systemWide, Float(x), Float(y), &element)
|
||||
guard result == .success, let element else {
|
||||
return AgentReply.err("no accessible element at (\(x), \(y)) (\(describe(result)))")
|
||||
}
|
||||
var budget = 1
|
||||
return AgentReply.ok(["node": buildNode(element, depth: 0, maxDepth: 0, budget: &budget)])
|
||||
}
|
||||
|
||||
/// `ax_action` — perform a named action (`AXPress`, `AXShowMenu`, …) on a ref'd element: the
|
||||
/// control's own callback runs in the app process, no rendering or synthetic click involved.
|
||||
func opAXAction(_ request: [String: Any]) -> [String: Any] {
|
||||
guard axTrusted else { return AgentReply.err(Self.axDenied) }
|
||||
guard let element = resolveRef(request) else { return staleRefReply(request) }
|
||||
let action = (request["action"] as? String).flatMap { $0.isEmpty ? nil : $0 } ?? "AXPress"
|
||||
let result = AXUIElementPerformAction(element, action as CFString)
|
||||
switch result {
|
||||
case .success:
|
||||
return AgentReply.ok()
|
||||
case .cannotComplete:
|
||||
// Apps often run modal processing inside the action callback and never return within
|
||||
// the AX timeout — the action may well have fired (§6.3). Tell the host the truth.
|
||||
return AgentReply.ok([
|
||||
"note": "the app did not confirm within the AX timeout — the action possibly "
|
||||
+ "succeeded; re-dump to verify"
|
||||
])
|
||||
case .invalidUIElement:
|
||||
return staleRefReply(request)
|
||||
default:
|
||||
return AgentReply.err("\(action) failed (\(describe(result)))")
|
||||
}
|
||||
}
|
||||
|
||||
/// `ax_set_value` — write a field's `AXValue` directly (the reliable way to fill text fields).
|
||||
/// Non-settable / non-AppKit fields are reported as such so the host can fall back to
|
||||
/// `ax_focus` + CGEvent typing.
|
||||
func opAXSetValue(_ request: [String: Any]) -> [String: Any] {
|
||||
guard axTrusted else { return AgentReply.err(Self.axDenied) }
|
||||
guard let element = resolveRef(request) else { return staleRefReply(request) }
|
||||
guard let value = request["value"] as? String else {
|
||||
return AgentReply.err("ax_set_value requires a string \"value\"")
|
||||
}
|
||||
var settable = DarwinBoolean(false)
|
||||
AXUIElementIsAttributeSettable(element, kAXValueAttribute as CFString, &settable)
|
||||
guard settable.boolValue else {
|
||||
return AgentReply.err(
|
||||
"this element's AXValue is not settable — focus it (ax_focus) and type instead")
|
||||
}
|
||||
let result = AXUIElementSetAttributeValue(
|
||||
element, kAXValueAttribute as CFString, value as CFString)
|
||||
guard result == .success else {
|
||||
if result == .invalidUIElement { return staleRefReply(request) }
|
||||
return AgentReply.err("setting AXValue failed (\(describe(result)))")
|
||||
}
|
||||
return AgentReply.ok()
|
||||
}
|
||||
|
||||
/// `ax_focus` — give a ref'd element keyboard focus (precedes CGEvent typing into it).
|
||||
func opAXFocus(_ request: [String: Any]) -> [String: Any] {
|
||||
guard axTrusted else { return AgentReply.err(Self.axDenied) }
|
||||
guard let element = resolveRef(request) else { return staleRefReply(request) }
|
||||
let result = AXUIElementSetAttributeValue(
|
||||
element, kAXFocusedAttribute as CFString, kCFBooleanTrue)
|
||||
guard result == .success else {
|
||||
if result == .invalidUIElement { return staleRefReply(request) }
|
||||
return AgentReply.err("focusing failed (\(describe(result)))")
|
||||
}
|
||||
return AgentReply.ok()
|
||||
}
|
||||
|
||||
// MARK: - Tree walking
|
||||
|
||||
/// Serialize one element into the §4 node shape, recursing into children while depth and the
|
||||
/// total node budget allow. Empty/absent attributes are omitted to keep the payload the model
|
||||
/// reads small.
|
||||
private func buildNode(
|
||||
_ element: AXUIElement, depth: Int, maxDepth: Int, budget: inout Int
|
||||
) -> [String: Any] {
|
||||
budget -= 1
|
||||
var node: [String: Any] = ["ref": axRegistry.register(element)]
|
||||
if let role = copyString(element, kAXRoleAttribute) { node["role"] = role }
|
||||
if let subrole = copyString(element, kAXSubroleAttribute) { node["subrole"] = subrole }
|
||||
if let title = copyString(element, kAXTitleAttribute), !title.isEmpty {
|
||||
node["title"] = title
|
||||
}
|
||||
if let value = copyValueDescription(element) { node["value"] = value }
|
||||
if copyBool(element, kAXEnabledAttribute) == false { node["enabled"] = false }
|
||||
if copyBool(element, kAXFocusedAttribute) == true { node["focused"] = true }
|
||||
if let frame = copyFrame(element) { node["frame"] = frame }
|
||||
|
||||
var actionNames: CFArray?
|
||||
if AXUIElementCopyActionNames(element, &actionNames) == .success,
|
||||
let actions = actionNames as? [String], !actions.isEmpty
|
||||
{
|
||||
node["actions"] = actions
|
||||
}
|
||||
|
||||
if depth < maxDepth, budget > 0 {
|
||||
var childrenRef: CFTypeRef?
|
||||
if AXUIElementCopyAttributeValue(
|
||||
element, kAXChildrenAttribute as CFString, &childrenRef) == .success,
|
||||
let children = childrenRef as? [AXUIElement], !children.isEmpty
|
||||
{
|
||||
var serialized: [[String: Any]] = []
|
||||
// Per-node child cap: a 10k-row table must not eat the entire budget at one level.
|
||||
for child in children.prefix(48) {
|
||||
guard budget > 0 else { break }
|
||||
serialized.append(
|
||||
buildNode(child, depth: depth + 1, maxDepth: maxDepth, budget: &budget))
|
||||
}
|
||||
if !serialized.isEmpty { node["children"] = serialized }
|
||||
if children.count > serialized.count {
|
||||
node["childrenOmitted"] = children.count - serialized.count
|
||||
}
|
||||
}
|
||||
}
|
||||
return node
|
||||
}
|
||||
|
||||
private func copyString(_ element: AXUIElement, _ attribute: String) -> String? {
|
||||
var value: CFTypeRef?
|
||||
guard
|
||||
AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success
|
||||
else { return nil }
|
||||
return value as? String
|
||||
}
|
||||
|
||||
private func copyBool(_ element: AXUIElement, _ attribute: String) -> Bool? {
|
||||
var value: CFTypeRef?
|
||||
guard
|
||||
AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success
|
||||
else { return nil }
|
||||
return (value as? NSNumber)?.boolValue
|
||||
}
|
||||
|
||||
/// A human/model-readable rendering of `AXValue` (string, number, bool), truncated so one giant
|
||||
/// text view can't dominate the tree payload.
|
||||
private func copyValueDescription(_ element: AXUIElement) -> String? {
|
||||
var value: CFTypeRef?
|
||||
guard
|
||||
AXUIElementCopyAttributeValue(element, kAXValueAttribute as CFString, &value) == .success,
|
||||
let value
|
||||
else { return nil }
|
||||
let text: String
|
||||
if let string = value as? String {
|
||||
text = string
|
||||
} else if let number = value as? NSNumber {
|
||||
text = number.stringValue
|
||||
} else if let url = value as? URL {
|
||||
text = url.absoluteString
|
||||
} else {
|
||||
return nil // AXValueRef boxes (ranges etc.) aren't useful as text
|
||||
}
|
||||
if text.isEmpty { return nil }
|
||||
return text.count > 250 ? String(text.prefix(250)) + "…" : text
|
||||
}
|
||||
|
||||
/// `kAXPosition`/`kAXSize` come back as `AXValueRef` boxes — unwrap into the §4 frame shape
|
||||
/// (top-left-origin screen coordinates, matching screenshots and CGEvent clicks).
|
||||
private func copyFrame(_ element: AXUIElement) -> [String: Any]? {
|
||||
var positionRef: CFTypeRef?
|
||||
var sizeRef: CFTypeRef?
|
||||
var point = CGPoint.zero
|
||||
var size = CGSize.zero
|
||||
guard
|
||||
AXUIElementCopyAttributeValue(
|
||||
element, kAXPositionAttribute as CFString, &positionRef) == .success,
|
||||
AXUIElementCopyAttributeValue(
|
||||
element, kAXSizeAttribute as CFString, &sizeRef) == .success,
|
||||
let positionRef, let sizeRef,
|
||||
AXValueGetValue(positionRef as! AXValue, .cgPoint, &point),
|
||||
AXValueGetValue(sizeRef as! AXValue, .cgSize, &size)
|
||||
else { return nil }
|
||||
return [
|
||||
"x": Int(point.x.rounded()), "y": Int(point.y.rounded()),
|
||||
"w": Int(size.width.rounded()), "h": Int(size.height.rounded()),
|
||||
]
|
||||
}
|
||||
|
||||
// MARK: - Ref plumbing
|
||||
|
||||
private func resolveRef(_ request: [String: Any]) -> AXUIElement? {
|
||||
guard let ref = request["ref"] as? String else { return nil }
|
||||
return axRegistry[ref]
|
||||
}
|
||||
|
||||
private func staleRefReply(_ request: [String: Any]) -> [String: Any] {
|
||||
let ref = (request["ref"] as? String) ?? "?"
|
||||
return AgentReply.err(
|
||||
"element ref \"\(ref)\" is unknown or stale — refs are only valid until the next "
|
||||
+ "ax_dump; re-dump and retry")
|
||||
}
|
||||
|
||||
private func describe(_ error: AXError) -> String {
|
||||
switch error {
|
||||
case .success: return "success"
|
||||
case .apiDisabled: return "kAXErrorAPIDisabled — accessibility is disabled for this app"
|
||||
case .cannotComplete: return "kAXErrorCannotComplete — the app did not respond in time"
|
||||
case .invalidUIElement: return "kAXErrorInvalidUIElement — stale element"
|
||||
case .attributeUnsupported: return "attribute unsupported"
|
||||
case .actionUnsupported: return "action unsupported"
|
||||
case .noValue: return "no value"
|
||||
case .notImplemented: return "the app does not implement this AX request"
|
||||
default: return "AXError \(error.rawValue)"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
import CoreGraphics
|
||||
import Foundation
|
||||
import VMAgentCore
|
||||
|
||||
/// Low-level input synthesis — CGEvent posted to the HID event tap (§8). The raw fallback for
|
||||
/// targets with no AX action (custom-drawn UI); AX actions (§6) are preferred where they exist.
|
||||
/// All of it needs the `kTCCServicePostEvent` grant and an Aqua session.
|
||||
extension ConnectionHandler {
|
||||
private var canPostEvents: Bool { CGPreflightPostEventAccess() }
|
||||
|
||||
private func intField(_ request: [String: Any], _ key: String) -> Int? {
|
||||
(request[key] as? Int) ?? (request[key] as? Double).map(Int.init)
|
||||
}
|
||||
|
||||
/// `click` — move the pointer to (x, y), then press/release. `count` 2 raises the click-state
|
||||
/// sequence AppKit needs to see a real double-click.
|
||||
func opClick(_ request: [String: Any]) -> [String: Any] {
|
||||
guard canPostEvents else { return AgentReply.err(Self.postEventDenied) }
|
||||
guard let x = intField(request, "x"), let y = intField(request, "y") else {
|
||||
return AgentReply.err("click requires x and y")
|
||||
}
|
||||
let position = CGPoint(x: x, y: y)
|
||||
let count = max(1, min(intField(request, "count") ?? 1, 3))
|
||||
let (down, up, button): (CGEventType, CGEventType, CGMouseButton) = {
|
||||
switch request["button"] as? String {
|
||||
case "right": return (.rightMouseDown, .rightMouseUp, .right)
|
||||
case "middle": return (.otherMouseDown, .otherMouseUp, .center)
|
||||
default: return (.leftMouseDown, .leftMouseUp, .left)
|
||||
}
|
||||
}()
|
||||
post(mouse: .mouseMoved, at: position, button: .left)
|
||||
for clickState in 1...count {
|
||||
guard
|
||||
let downEvent = CGEvent(
|
||||
mouseEventSource: nil, mouseType: down, mouseCursorPosition: position,
|
||||
mouseButton: button),
|
||||
let upEvent = CGEvent(
|
||||
mouseEventSource: nil, mouseType: up, mouseCursorPosition: position,
|
||||
mouseButton: button)
|
||||
else { return AgentReply.err("could not create the mouse event") }
|
||||
downEvent.setIntegerValueField(.mouseEventClickState, value: Int64(clickState))
|
||||
upEvent.setIntegerValueField(.mouseEventClickState, value: Int64(clickState))
|
||||
downEvent.post(tap: .cghidEventTap)
|
||||
usleep(40_000)
|
||||
upEvent.post(tap: .cghidEventTap)
|
||||
if clickState < count { usleep(90_000) } // within the double-click interval
|
||||
}
|
||||
return AgentReply.ok()
|
||||
}
|
||||
|
||||
/// `move` — reposition the pointer without clicking.
|
||||
func opMove(_ request: [String: Any]) -> [String: Any] {
|
||||
guard canPostEvents else { return AgentReply.err(Self.postEventDenied) }
|
||||
guard let x = intField(request, "x"), let y = intField(request, "y") else {
|
||||
return AgentReply.err("move requires x and y")
|
||||
}
|
||||
post(mouse: .mouseMoved, at: CGPoint(x: x, y: y), button: .left)
|
||||
return AgentReply.ok()
|
||||
}
|
||||
|
||||
/// `drag` — press at (fromX, fromY) (default: the current pointer), glide in interpolated
|
||||
/// dragged events (apps ignore a teleporting drag), release at (toX, toY).
|
||||
func opDrag(_ request: [String: Any]) -> [String: Any] {
|
||||
guard canPostEvents else { return AgentReply.err(Self.postEventDenied) }
|
||||
guard let toX = intField(request, "toX"), let toY = intField(request, "toY") else {
|
||||
return AgentReply.err("drag requires toX and toY")
|
||||
}
|
||||
let current = CGEvent(source: nil)?.location ?? .zero
|
||||
let from = CGPoint(
|
||||
x: intField(request, "fromX").map(Double.init) ?? current.x,
|
||||
y: intField(request, "fromY").map(Double.init) ?? current.y)
|
||||
let to = CGPoint(x: toX, y: toY)
|
||||
|
||||
post(mouse: .mouseMoved, at: from, button: .left)
|
||||
usleep(60_000)
|
||||
post(mouse: .leftMouseDown, at: from, button: .left)
|
||||
let steps = 12
|
||||
for step in 1...steps {
|
||||
let t = Double(step) / Double(steps)
|
||||
let waypoint = CGPoint(
|
||||
x: from.x + (to.x - from.x) * t, y: from.y + (to.y - from.y) * t)
|
||||
usleep(15_000)
|
||||
post(mouse: .leftMouseDragged, at: waypoint, button: .left)
|
||||
}
|
||||
usleep(60_000)
|
||||
post(mouse: .leftMouseUp, at: to, button: .left)
|
||||
return AgentReply.ok()
|
||||
}
|
||||
|
||||
/// `type` — inject literal text as unicode keyboard events (no key-code mapping, so any text
|
||||
/// works regardless of layout). Chunked: `keyboardSetUnicodeString` reliably carries only short
|
||||
/// runs per event.
|
||||
func opType(_ request: [String: Any]) -> [String: Any] {
|
||||
guard canPostEvents else { return AgentReply.err(Self.postEventDenied) }
|
||||
guard let text = request["text"] as? String, !text.isEmpty else {
|
||||
return AgentReply.err("type requires non-empty \"text\"")
|
||||
}
|
||||
let units = Array(text.utf16)
|
||||
var index = 0
|
||||
while index < units.count {
|
||||
let chunk = Array(units[index..<min(index + 16, units.count)])
|
||||
guard
|
||||
let down = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: true),
|
||||
let up = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: false)
|
||||
else { return AgentReply.err("could not create the keyboard event") }
|
||||
down.keyboardSetUnicodeString(stringLength: chunk.count, unicodeString: chunk)
|
||||
up.keyboardSetUnicodeString(stringLength: chunk.count, unicodeString: chunk)
|
||||
down.post(tap: .cghidEventTap)
|
||||
up.post(tap: .cghidEventTap)
|
||||
usleep(12_000)
|
||||
index += 16
|
||||
}
|
||||
return AgentReply.ok()
|
||||
}
|
||||
|
||||
/// `key` — a chord like `cmd+s` / `return` / `cmd+shift+4`, as a real key-code press with the
|
||||
/// modifier flags held (VMAgentCore.KeyMap owns the parse).
|
||||
func opKey(_ request: [String: Any]) -> [String: Any] {
|
||||
guard canPostEvents else { return AgentReply.err(Self.postEventDenied) }
|
||||
guard let chord = request["chord"] as? String, !chord.isEmpty else {
|
||||
return AgentReply.err("key requires a \"chord\" (e.g. \"cmd+s\")")
|
||||
}
|
||||
guard let stroke = KeyMap.stroke(forChord: chord) else {
|
||||
return AgentReply.err("unrecognized key in chord \"\(chord)\"")
|
||||
}
|
||||
guard
|
||||
let down = CGEvent(keyboardEventSource: nil, virtualKey: stroke.keyCode, keyDown: true),
|
||||
let up = CGEvent(keyboardEventSource: nil, virtualKey: stroke.keyCode, keyDown: false)
|
||||
else { return AgentReply.err("could not create the keyboard event") }
|
||||
down.flags = stroke.flags
|
||||
up.flags = stroke.flags
|
||||
down.post(tap: .cghidEventTap)
|
||||
usleep(30_000)
|
||||
up.post(tap: .cghidEventTap)
|
||||
return AgentReply.ok()
|
||||
}
|
||||
|
||||
/// `scroll` — a REAL wheel event (dy > 0 scrolls content up, dx > 0 scrolls left, in lines),
|
||||
/// unlike the SSH path's arrow-key approximation.
|
||||
func opScroll(_ request: [String: Any]) -> [String: Any] {
|
||||
guard canPostEvents else { return AgentReply.err(Self.postEventDenied) }
|
||||
let dy = max(-40, min(intField(request, "dy") ?? 0, 40))
|
||||
let dx = max(-40, min(intField(request, "dx") ?? 0, 40))
|
||||
guard dx != 0 || dy != 0 else { return AgentReply.err("scroll requires dx and/or dy") }
|
||||
guard
|
||||
let event = CGEvent(
|
||||
scrollWheelEvent2Source: nil, units: .line, wheelCount: 2,
|
||||
wheel1: Int32(dy), wheel2: Int32(dx), wheel3: 0)
|
||||
else { return AgentReply.err("could not create the scroll event") }
|
||||
event.post(tap: .cghidEventTap)
|
||||
return AgentReply.ok()
|
||||
}
|
||||
|
||||
private func post(mouse type: CGEventType, at position: CGPoint, button: CGMouseButton) {
|
||||
CGEvent(
|
||||
mouseEventSource: nil, mouseType: type, mouseCursorPosition: position,
|
||||
mouseButton: button)?
|
||||
.post(tap: .cghidEventTap)
|
||||
}
|
||||
|
||||
private static let postEventDenied =
|
||||
"input synthesis (kTCCServicePostEvent) is not granted to the agent — re-run provisioning"
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
import AppKit
|
||||
import ApplicationServices
|
||||
import CoreGraphics
|
||||
import Foundation
|
||||
import VMAgentCore
|
||||
|
||||
extension ConnectionHandler {
|
||||
/// `ping` — capability negotiation: protocol version, displays, and TCC readiness, so the host
|
||||
/// can distinguish "agent present" from "agent present but not authorized" (§5).
|
||||
func opPing() -> [String: Any] {
|
||||
var displayIDs = [CGDirectDisplayID](repeating: 0, count: 8)
|
||||
var displayCount: UInt32 = 0
|
||||
CGGetActiveDisplayList(UInt32(displayIDs.count), &displayIDs, &displayCount)
|
||||
let displays: [[String: Any]] = (0..<Int(displayCount)).map { i in
|
||||
let id = displayIDs[i]
|
||||
return [
|
||||
"id": Int(id),
|
||||
"width": CGDisplayPixelsWide(id),
|
||||
"height": CGDisplayPixelsHigh(id),
|
||||
"main": CGDisplayIsMain(id) != 0,
|
||||
]
|
||||
}
|
||||
return AgentReply.ok([
|
||||
"version": AgentWire.version,
|
||||
"displays": displays,
|
||||
"readiness": [
|
||||
"accessibility": AXIsProcessTrusted(),
|
||||
"screenRecording": CGPreflightScreenCaptureAccess(),
|
||||
"postEvent": CGPreflightPostEventAccess(),
|
||||
],
|
||||
])
|
||||
}
|
||||
|
||||
/// `cursor_position` — the pointer's current screen location (top-left-origin pixels).
|
||||
func opCursorPosition() -> [String: Any] {
|
||||
guard let event = CGEvent(source: nil) else {
|
||||
return AgentReply.err("could not read the cursor position (no event source)")
|
||||
}
|
||||
let p = event.location
|
||||
return AgentReply.ok(["x": Int(p.x.rounded()), "y": Int(p.y.rounded())])
|
||||
}
|
||||
|
||||
/// `launch_app` — open an application by name, `NSWorkspace` when the bundle is findable in the
|
||||
/// standard app directories, else `/usr/bin/open -a` (which searches LaunchServices).
|
||||
func opLaunchApp(_ request: [String: Any]) -> [String: Any] {
|
||||
guard let name = request["name"] as? String, !name.isEmpty else {
|
||||
return AgentReply.err("launch_app requires a non-empty \"name\"")
|
||||
}
|
||||
let leaf = name.hasSuffix(".app") ? name : name + ".app"
|
||||
let candidates = [
|
||||
"/Applications/\(leaf)",
|
||||
"/System/Applications/\(leaf)",
|
||||
"/System/Applications/Utilities/\(leaf)",
|
||||
]
|
||||
if let path = candidates.first(where: { FileManager.default.fileExists(atPath: $0) }) {
|
||||
let semaphore = DispatchSemaphore(value: 0)
|
||||
var failure: String?
|
||||
NSWorkspace.shared.openApplication(
|
||||
at: URL(fileURLWithPath: path), configuration: NSWorkspace.OpenConfiguration()
|
||||
) { _, error in
|
||||
failure = error.map { String(describing: $0) }
|
||||
semaphore.signal()
|
||||
}
|
||||
_ = semaphore.wait(timeout: .now() + 15)
|
||||
if let failure { return AgentReply.err("could not launch \(name): \(failure)") }
|
||||
return AgentReply.ok()
|
||||
}
|
||||
// Not in the standard directories — let LaunchServices resolve it.
|
||||
let open = Process()
|
||||
open.executableURL = URL(fileURLWithPath: "/usr/bin/open")
|
||||
open.arguments = ["-a", name]
|
||||
do {
|
||||
try open.run()
|
||||
} catch {
|
||||
return AgentReply.err("could not launch \(name): \(error)")
|
||||
}
|
||||
open.waitUntilExit()
|
||||
guard open.terminationStatus == 0 else {
|
||||
return AgentReply.err("no application named \"\(name)\" (open -a exited \(open.terminationStatus))")
|
||||
}
|
||||
return AgentReply.ok()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
import AppKit
|
||||
import CoreGraphics
|
||||
import Foundation
|
||||
import ScreenCaptureKit
|
||||
|
||||
/// Pixel capture — ScreenCaptureKit in-process (§7): no `screencapture` CLI, no `launchctl asuser`,
|
||||
/// no base64-over-SSH. Needs the Screen Recording grant + an Aqua session. On a Tahoe
|
||||
/// framebuffer-bug guest this (correctly) returns blank app windows — which is exactly why AX (§6)
|
||||
/// is the primary "look" there.
|
||||
extension ConnectionHandler {
|
||||
/// `screenshot` — capture one display (default: the first/main) and return base64 JPEG plus its
|
||||
/// pixel geometry. JPEG keeps a screenshot→act loop's payload ~10× smaller than PNG.
|
||||
func opScreenshot(_ request: [String: Any]) -> [String: Any] {
|
||||
guard CGPreflightScreenCaptureAccess() else {
|
||||
return AgentReply.err(
|
||||
"Screen Recording (kTCCServiceScreenCapture) is not granted to the agent — re-run "
|
||||
+ "provisioning")
|
||||
}
|
||||
let displayIndex = (request["display"] as? Int) ?? 0
|
||||
|
||||
// The op handlers run on a plain blocking connection thread; SCK's surface is async-only,
|
||||
// so bridge with a semaphore (the thread has nothing else to do but wait).
|
||||
let semaphore = DispatchSemaphore(value: 0)
|
||||
let box = ResultBox<[String: Any]>()
|
||||
Task {
|
||||
box.value = await Self.captureDisplay(index: displayIndex)
|
||||
semaphore.signal()
|
||||
}
|
||||
guard semaphore.wait(timeout: .now() + 15) == .success, let reply = box.value else {
|
||||
return AgentReply.err("screenshot timed out")
|
||||
}
|
||||
return reply
|
||||
}
|
||||
|
||||
private static func captureDisplay(index: Int) async -> [String: Any] {
|
||||
do {
|
||||
let content = try await SCShareableContent.current
|
||||
let displays = content.displays
|
||||
guard !displays.isEmpty else { return AgentReply.err("no shareable displays") }
|
||||
let display = displays.indices.contains(index) ? displays[index] : displays[0]
|
||||
|
||||
let filter = SCContentFilter(display: display, excludingWindows: [])
|
||||
let configuration = SCStreamConfiguration()
|
||||
// Capture at the filter's native pixel size (`contentRect` is points; the scale is the
|
||||
// point→pixel factor — Float in the SDK, hence the CGFloat conversion).
|
||||
let scale = CGFloat(filter.pointPixelScale)
|
||||
configuration.width = Int(filter.contentRect.width * scale)
|
||||
configuration.height = Int(filter.contentRect.height * scale)
|
||||
configuration.showsCursor = true
|
||||
|
||||
let image = try await SCScreenshotManager.captureImage(
|
||||
contentFilter: filter, configuration: configuration)
|
||||
let bitmap = NSBitmapImageRep(cgImage: image)
|
||||
guard
|
||||
let jpeg = bitmap.representation(
|
||||
using: .jpeg, properties: [.compressionFactor: 0.7])
|
||||
else { return AgentReply.err("JPEG encoding failed") }
|
||||
return AgentReply.ok([
|
||||
"image": jpeg.base64EncodedString(),
|
||||
"format": "jpeg",
|
||||
"width": image.width,
|
||||
"height": image.height,
|
||||
"scale": Double(scale),
|
||||
])
|
||||
} catch {
|
||||
return AgentReply.err("ScreenCaptureKit capture failed: \(error)")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Hands one value from a bridging Task back to the blocking connection thread; the semaphore
|
||||
/// orders the accesses (signal happens-after the write, wait happens-before the read).
|
||||
final class ResultBox<T> {
|
||||
var value: T?
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import AppKit
|
||||
import ApplicationServices
|
||||
import CoreGraphics
|
||||
import Foundation
|
||||
import VMAgentCore
|
||||
|
||||
// NucleicVMAgent — the native in-guest automation agent (docs/MACOS_VM_NATIVE_AGENT.md).
|
||||
//
|
||||
// Runs as a per-user LaunchAgent in the auto-login `agent` account's Aqua session (AX, SCK and
|
||||
// CGEvent all need an active WindowServer session), listens on a vsock port, and executes the §4
|
||||
// wire ops with Apple's frameworks directly: AXUIElement (observe + control), ScreenCaptureKit
|
||||
// (capture), CGEvent (raw input). It is a dumb, well-scoped actuator — the host keeps ownership of
|
||||
// VM lifecycle, gating, and the approval boundary.
|
||||
|
||||
// A peer that vanishes mid-write must not kill the process — writes surface EPIPE instead.
|
||||
signal(SIGPIPE, SIG_IGN)
|
||||
|
||||
let port = ProcessInfo.processInfo.environment["NUCLEIC_AGENT_PORT"]
|
||||
.flatMap(UInt32.init) ?? AgentWire.port
|
||||
|
||||
agentLog("NucleicVMAgent protocol v\(AgentWire.version) starting on vsock port \(port)")
|
||||
// Self-check the TCC grants once at startup so the LaunchAgent log shows an actionable state; the
|
||||
// same readiness rides in every `ping` reply so the HOST can surface "present but not authorized".
|
||||
agentLog(
|
||||
"readiness: accessibility=\(AXIsProcessTrusted()) "
|
||||
+ "screenRecording=\(CGPreflightScreenCaptureAccess()) "
|
||||
+ "postEvent=\(CGPreflightPostEventAccess())")
|
||||
|
||||
let listener: AgentListener
|
||||
do {
|
||||
listener = try AgentListener(port: port)
|
||||
} catch {
|
||||
agentLog("FATAL: \(error) — is this running inside a macOS guest VM? (AF_VSOCK fails ENODEV on a host)")
|
||||
exit(1)
|
||||
}
|
||||
|
||||
Thread.detachNewThread { listener.run() }
|
||||
// Keep the main run loop alive: AppKit-adjacent calls (NSWorkspace, event posting) expect a main
|
||||
// thread with a live dispatch queue in a GUI session.
|
||||
dispatchMain()
|
||||
@@ -0,0 +1,42 @@
|
||||
import Foundation
|
||||
|
||||
/// Wire constants for the host ⇄ agent protocol (docs/MACOS_VM_NATIVE_AGENT.md §3–§4).
|
||||
///
|
||||
/// The transport is NDJSON over vsock: one JSON request object per line, one JSON reply per line.
|
||||
/// Requests carry an `op`; replies carry `ok` (plus op-specific fields, or `error` when `ok:false`).
|
||||
///
|
||||
/// The port constant is mirrored on the host side in `MacVMAgentWire`
|
||||
/// (Sources/NucleicCore/MacVM/MacVMAgentClient.swift) — the two packages are deliberately separate
|
||||
/// (different platform floors), so keep the values in lockstep.
|
||||
public enum AgentWire {
|
||||
/// The Nucleic-reserved vsock port the agent listens on and the host connects to.
|
||||
public static let port: UInt32 = 2035
|
||||
/// Protocol version reported in the `ping` reply, bumped on incompatible wire changes.
|
||||
public static let version = 1
|
||||
}
|
||||
|
||||
/// Accumulates raw bytes and yields complete newline-terminated lines — the NDJSON framing both
|
||||
/// sides use (one request/reply object per line). Pure and unit-tested; the connection loop feeds
|
||||
/// it whatever `read(2)` returns.
|
||||
public struct LineSplitBuffer {
|
||||
private var data = Data()
|
||||
|
||||
public init() {}
|
||||
|
||||
public mutating func append(_ chunk: Data) {
|
||||
data.append(chunk)
|
||||
}
|
||||
|
||||
/// The next complete line (without its trailing `\n`), or `nil` when no full line is buffered.
|
||||
public mutating func nextLine() -> Data? {
|
||||
guard let nl = data.firstIndex(of: 0x0A) else { return nil }
|
||||
let line = data.subdata(in: data.startIndex..<nl)
|
||||
data.removeSubrange(data.startIndex...nl)
|
||||
// Tolerate CRLF framing from a debugging client.
|
||||
if line.last == 0x0D { return line.dropLast() }
|
||||
return line
|
||||
}
|
||||
|
||||
/// Bytes currently buffered without a terminating newline — the runaway-line guard reads this.
|
||||
public var pendingBytes: Int { data.count }
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
import CoreGraphics
|
||||
import Foundation
|
||||
|
||||
/// One synthesizable keystroke: a hardware virtual key code plus the modifier flags held around it.
|
||||
public struct KeyStroke: Equatable, Sendable {
|
||||
public let keyCode: CGKeyCode
|
||||
public let flags: CGEventFlags
|
||||
public init(keyCode: CGKeyCode, flags: CGEventFlags) {
|
||||
self.keyCode = keyCode
|
||||
self.flags = flags
|
||||
}
|
||||
}
|
||||
|
||||
/// Translates the tool-surface chord syntax (`"cmd+s"`, `"return"`, `"cmd+shift+4"`) into a CGEvent
|
||||
/// keystroke. Pure and unit-tested — the CGEvent posting lives in the agent's input ops.
|
||||
///
|
||||
/// Key codes are the ANSI-US hardware codes from Carbon's `Events.h` (`kVK_ANSI_A` …); the VZ guest
|
||||
/// keyboard is ANSI, so the table is exact for the layouts a guest ships with.
|
||||
public enum KeyMap {
|
||||
/// Parse a chord like `cmd+shift+s` into a keystroke, or `nil` when the (final) key is unknown.
|
||||
/// Unknown modifier tokens are ignored (matching the host-side cliclick path's leniency).
|
||||
public static func stroke(forChord chord: String) -> KeyStroke? {
|
||||
let parts = chord.lowercased().split(separator: "+").map(String.init)
|
||||
guard let last = parts.last, let code = keyCodes[normalizeKeyName(last)] else { return nil }
|
||||
var flags = CGEventFlags()
|
||||
for token in parts.dropLast() {
|
||||
if let f = modifierFlags[token] { flags.insert(f) }
|
||||
}
|
||||
return KeyStroke(keyCode: code, flags: flags)
|
||||
}
|
||||
|
||||
/// Alias common key-name spellings to the table's canonical names.
|
||||
static func normalizeKeyName(_ key: String) -> String {
|
||||
switch key {
|
||||
case "enter": return "return"
|
||||
case "escape": return "esc"
|
||||
case "backspace": return "delete"
|
||||
case "arrowup": return "up"
|
||||
case "arrowdown": return "down"
|
||||
case "arrowleft": return "left"
|
||||
case "arrowright": return "right"
|
||||
case "spacebar": return "space"
|
||||
default: return key
|
||||
}
|
||||
}
|
||||
|
||||
static let modifierFlags: [String: CGEventFlags] = [
|
||||
"cmd": .maskCommand, "command": .maskCommand, "meta": .maskCommand, "super": .maskCommand,
|
||||
"ctrl": .maskControl, "control": .maskControl,
|
||||
"alt": .maskAlternate, "opt": .maskAlternate, "option": .maskAlternate,
|
||||
"shift": .maskShift,
|
||||
"fn": .maskSecondaryFn,
|
||||
]
|
||||
|
||||
/// ANSI-US virtual key codes (Carbon `kVK_*`).
|
||||
static let keyCodes: [String: CGKeyCode] = [
|
||||
"a": 0x00, "s": 0x01, "d": 0x02, "f": 0x03, "h": 0x04, "g": 0x05, "z": 0x06, "x": 0x07,
|
||||
"c": 0x08, "v": 0x09, "b": 0x0B, "q": 0x0C, "w": 0x0D, "e": 0x0E, "r": 0x0F, "y": 0x10,
|
||||
"t": 0x11, "1": 0x12, "2": 0x13, "3": 0x14, "4": 0x15, "6": 0x16, "5": 0x17, "=": 0x18,
|
||||
"9": 0x19, "7": 0x1A, "-": 0x1B, "8": 0x1C, "0": 0x1D, "]": 0x1E, "o": 0x1F, "u": 0x20,
|
||||
"[": 0x21, "i": 0x22, "p": 0x23, "l": 0x25, "j": 0x26, "'": 0x27, "k": 0x28, ";": 0x29,
|
||||
"\\": 0x2A, ",": 0x2B, "/": 0x2C, "n": 0x2D, "m": 0x2E, ".": 0x2F, "`": 0x32,
|
||||
"return": 0x24, "tab": 0x30, "space": 0x31, "delete": 0x33, "esc": 0x35,
|
||||
"home": 0x73, "end": 0x77, "pageup": 0x74, "pagedown": 0x79, "forwarddelete": 0x75,
|
||||
"left": 0x7B, "right": 0x7C, "down": 0x7D, "up": 0x7E,
|
||||
"f1": 0x7A, "f2": 0x78, "f3": 0x63, "f4": 0x76, "f5": 0x60, "f6": 0x61, "f7": 0x62,
|
||||
"f8": 0x64, "f9": 0x65, "f10": 0x6D, "f11": 0x67, "f12": 0x6F, "f13": 0x69, "f14": 0x6B,
|
||||
"f15": 0x71,
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
import CoreGraphics
|
||||
import Foundation
|
||||
import Testing
|
||||
|
||||
@testable import VMAgentCore
|
||||
|
||||
// MARK: - NDJSON line framing
|
||||
|
||||
@Test func lineSplitterYieldsCompleteLines() {
|
||||
var buffer = LineSplitBuffer()
|
||||
buffer.append(Data("{\"op\":\"pi".utf8))
|
||||
#expect(buffer.nextLine() == nil)
|
||||
buffer.append(Data("ng\"}\n{\"op\":".utf8))
|
||||
#expect(buffer.nextLine() == Data("{\"op\":\"ping\"}".utf8))
|
||||
#expect(buffer.nextLine() == nil)
|
||||
buffer.append(Data("\"click\"}\n".utf8))
|
||||
#expect(buffer.nextLine() == Data("{\"op\":\"click\"}".utf8))
|
||||
#expect(buffer.nextLine() == nil)
|
||||
#expect(buffer.pendingBytes == 0)
|
||||
}
|
||||
|
||||
@Test func lineSplitterHandlesCRLFAndEmptyLines() {
|
||||
var buffer = LineSplitBuffer()
|
||||
buffer.append(Data("a\r\n\nb\n".utf8))
|
||||
#expect(buffer.nextLine() == Data("a".utf8))
|
||||
#expect(buffer.nextLine() == Data())
|
||||
#expect(buffer.nextLine() == Data("b".utf8))
|
||||
#expect(buffer.nextLine() == nil)
|
||||
}
|
||||
|
||||
@Test func lineSplitterTracksPendingBytes() {
|
||||
var buffer = LineSplitBuffer()
|
||||
buffer.append(Data(repeating: UInt8(ascii: "x"), count: 100))
|
||||
#expect(buffer.pendingBytes == 100)
|
||||
#expect(buffer.nextLine() == nil)
|
||||
}
|
||||
|
||||
// MARK: - Key chords
|
||||
|
||||
@Test func chordParsesModifiersAndKey() {
|
||||
let stroke = KeyMap.stroke(forChord: "cmd+shift+s")
|
||||
#expect(stroke == KeyStroke(keyCode: 0x01, flags: [.maskCommand, .maskShift]))
|
||||
}
|
||||
|
||||
@Test func chordParsesBareSpecialKeys() {
|
||||
#expect(KeyMap.stroke(forChord: "return") == KeyStroke(keyCode: 0x24, flags: []))
|
||||
#expect(KeyMap.stroke(forChord: "enter") == KeyStroke(keyCode: 0x24, flags: []))
|
||||
#expect(KeyMap.stroke(forChord: "escape") == KeyStroke(keyCode: 0x35, flags: []))
|
||||
#expect(KeyMap.stroke(forChord: "backspace") == KeyStroke(keyCode: 0x33, flags: []))
|
||||
#expect(KeyMap.stroke(forChord: "tab") == KeyStroke(keyCode: 0x30, flags: []))
|
||||
}
|
||||
|
||||
@Test func chordParsesArrowAliasesAndFKeys() {
|
||||
#expect(KeyMap.stroke(forChord: "arrowdown") == KeyMap.stroke(forChord: "down"))
|
||||
#expect(KeyMap.stroke(forChord: "f5") == KeyStroke(keyCode: 0x60, flags: []))
|
||||
#expect(KeyMap.stroke(forChord: "cmd+shift+4") == KeyStroke(
|
||||
keyCode: 0x15, flags: [.maskCommand, .maskShift]))
|
||||
}
|
||||
|
||||
@Test func chordIsCaseInsensitiveAndToleratesUnknownModifiers() {
|
||||
#expect(KeyMap.stroke(forChord: "CMD+S") == KeyStroke(keyCode: 0x01, flags: [.maskCommand]))
|
||||
// Unknown modifier tokens are dropped (host-side leniency parity), the key still lands.
|
||||
#expect(KeyMap.stroke(forChord: "hyper+s") == KeyStroke(keyCode: 0x01, flags: []))
|
||||
}
|
||||
|
||||
@Test func chordRejectsUnknownKey() {
|
||||
#expect(KeyMap.stroke(forChord: "cmd+definitely-not-a-key") == nil)
|
||||
#expect(KeyMap.stroke(forChord: "") == nil)
|
||||
}
|
||||
@@ -18,13 +18,18 @@ struct SessionDetailView: View {
|
||||
// Bumped on send to jump the transcript to the bottom, even if the user had scrolled up to
|
||||
// read history — sending is a deliberate "show me what happens next" (mirrors the Mac).
|
||||
@State private var scrollToBottomRequest = 0
|
||||
// Owned here (not inside `TranscriptList`) so the jump-to-bottom chevron can ride in the chat
|
||||
// bar just above the composer. `TranscriptList` drives it from the scroll geometry.
|
||||
@State private var isScrolledToBottom = true
|
||||
|
||||
private var summary: WireSessionSummary? {
|
||||
store.sessions.first { $0.sessionID == sessionID }
|
||||
}
|
||||
|
||||
var body: some View {
|
||||
TranscriptList(events: store.openEvents, scrollToBottomRequest: scrollToBottomRequest)
|
||||
TranscriptList(events: store.openEvents,
|
||||
scrollToBottomRequest: scrollToBottomRequest,
|
||||
isScrolledToBottom: $isScrolledToBottom)
|
||||
// The chat bar floats over the scrolling content on Liquid Glass instead of sitting
|
||||
// in a boxed strip below it, so the transcript runs the full height of the screen.
|
||||
.safeAreaInset(edge: .bottom) { actionArea }
|
||||
@@ -236,6 +241,15 @@ struct SessionDetailView: View {
|
||||
/// pending. Content scrolls beneath it; nothing renders when there's nothing to act on.
|
||||
private var actionArea: some View {
|
||||
VStack(spacing: 8) {
|
||||
// The jump-to-bottom chevron rides here, immediately above the chat bar — not as a
|
||||
// transcript overlay, which aligned to the scroll view's full-height bounds and so sat
|
||||
// behind this floating bar at the screen's bottom edge. Shown only while scrolled up; a
|
||||
// tap bumps the same scroll request the send button uses, so following resumes once the
|
||||
// transcript reaches the bottom. Mirrors the Mac's `JumpToBottomButton`.
|
||||
if !isScrolledToBottom {
|
||||
JumpToBottomButton { scrollToBottomRequest += 1 }
|
||||
.transition(.move(edge: .bottom).combined(with: .opacity))
|
||||
}
|
||||
// Interacting with a chat while the owning Mac is unreachable surfaces this first, so a
|
||||
// disabled composer reads as "offline / read-only history" rather than broken.
|
||||
if !store.connectivity.isLive { disconnectedBanner }
|
||||
@@ -243,6 +257,7 @@ struct SessionDetailView: View {
|
||||
}
|
||||
.padding(.horizontal, 12)
|
||||
.padding(.bottom, 8)
|
||||
.animation(.easeInOut(duration: 0.15), value: isScrolledToBottom)
|
||||
}
|
||||
|
||||
/// The offline notice over the transcript: the chat's history is cached and readable, but the
|
||||
@@ -401,8 +416,9 @@ struct TranscriptList: View {
|
||||
|
||||
/// True while the user is parked at (within a hair of) the bottom, so live output keeps
|
||||
/// following; once they scroll up it flips false, following stops, and the jump-to-bottom
|
||||
/// chevron appears. Starts true — a freshly opened chat is anchored at the bottom.
|
||||
@State private var isScrolledToBottom = true
|
||||
/// chevron appears. Starts true — a freshly opened chat is anchored at the bottom. Owned by
|
||||
/// the parent so the chevron can live in the chat bar above the composer.
|
||||
@Binding var isScrolledToBottom: Bool
|
||||
|
||||
/// True for a short window right after the transcript appears, while it runs its first
|
||||
/// layout passes and its events fill in. During it we accept only "at bottom" scroll
|
||||
@@ -426,10 +442,22 @@ struct TranscriptList: View {
|
||||
var body: some View {
|
||||
ScrollViewReader { proxy in
|
||||
ScrollView {
|
||||
LazyVStack(alignment: .leading, spacing: 14) {
|
||||
// Eager VStack (not Lazy): the whole transcript is in memory, and eager layout
|
||||
// hands the ScrollView its true, *fixed* total content height up front. A
|
||||
// LazyVStack instead realizes rows as they scroll into view, so its reported
|
||||
// content height shifts mid-scroll as rows resolve — and that shifting height fed
|
||||
// straight back through the geometry reader below (flip at-bottom → toggle the
|
||||
// `.defaultScrollAnchor` → re-pin → new height → …), which is what made the whole
|
||||
// transcript jitter up and down under a drag. A stable height breaks the loop.
|
||||
// Matches the Mac transcript, for the same reason.
|
||||
VStack(alignment: .leading, spacing: 14) {
|
||||
ForEach(items) { item in
|
||||
TranscriptRow(item: item).id(item.id)
|
||||
}
|
||||
// Fixed, zero-content scroll target for "jump to bottom". Scrolling to the last
|
||||
// row's id instead would chase a moving target — the final row's identity
|
||||
// changes as its streaming deltas coalesce — nudging the viewport mid-stream.
|
||||
Color.clear.frame(height: 1).id(Self.bottomAnchorID)
|
||||
}
|
||||
.padding()
|
||||
// Cap the transcript to a readable measure on a wide iPad so lines don't run
|
||||
@@ -483,35 +511,27 @@ struct TranscriptList: View {
|
||||
// so every streamed delta keeps the view pinned to the bottom — but only while the
|
||||
// user is already parked there. Scrolling up to read history is never yanked down.
|
||||
.onChange(of: events.count) {
|
||||
if isScrolledToBottom { scrollToLast(proxy, animated: !transcriptSettling) }
|
||||
}
|
||||
// An explicit jump — the chevron or sending a message — always wins.
|
||||
.onChange(of: scrollToBottomRequest) { scrollToLast(proxy) }
|
||||
// Floating chevron above the chat bar, shown only while scrolled up; tap to jump back
|
||||
// to the latest output. Mirrors the Mac's `JumpToBottomButton`.
|
||||
.overlay(alignment: .bottom) {
|
||||
ZStack {
|
||||
if !isScrolledToBottom {
|
||||
JumpToBottomButton { scrollToLast(proxy) }
|
||||
.padding(.bottom, 12)
|
||||
.transition(.move(edge: .bottom).combined(with: .opacity))
|
||||
}
|
||||
}
|
||||
.animation(.easeInOut(duration: 0.15), value: isScrolledToBottom)
|
||||
if isScrolledToBottom { scrollToEnd(proxy, animated: !transcriptSettling) }
|
||||
}
|
||||
// An explicit jump — the chevron or sending a message — always wins. The chevron
|
||||
// itself lives in the parent's chat bar (above the composer), not as an overlay here,
|
||||
// so it sits over the composer instead of behind the floating bar; a tap bumps this
|
||||
// same request, and following resumes once the geometry reader sees the bottom.
|
||||
.onChange(of: scrollToBottomRequest) { scrollToEnd(proxy) }
|
||||
}
|
||||
}
|
||||
|
||||
/// Scroll the last message to the bottom edge; `animated: false` during the open-settle
|
||||
/// Scroll the fixed bottom anchor into view; `animated: false` during the open-settle
|
||||
/// window so a chat that's still laying out snaps to the tail instead of easing into place.
|
||||
private func scrollToLast(_ proxy: ScrollViewProxy, animated: Bool = true) {
|
||||
guard let last = items.last else { return }
|
||||
private func scrollToEnd(_ proxy: ScrollViewProxy, animated: Bool = true) {
|
||||
if animated {
|
||||
withAnimation { proxy.scrollTo(last.id, anchor: .bottom) }
|
||||
withAnimation { proxy.scrollTo(Self.bottomAnchorID, anchor: .bottom) }
|
||||
} else {
|
||||
proxy.scrollTo(last.id, anchor: .bottom)
|
||||
proxy.scrollTo(Self.bottomAnchorID, anchor: .bottom)
|
||||
}
|
||||
}
|
||||
|
||||
private static let bottomAnchorID = "nucleic.transcript-bottom"
|
||||
}
|
||||
|
||||
/// The floating "jump to the latest" chevron shown above the chat bar while the user has
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Build guest/NucleicVMAgent into a signed NucleicVMAgent.app bundle, ready to be baked into the
|
||||
# macOS-VM golden base image (docs/MACOS_VM_NATIVE_AGENT.md §11).
|
||||
#
|
||||
# ./scripts/build-vm-agent.sh [output-dir] # default output: guest/NucleicVMAgent/dist
|
||||
#
|
||||
# Signing: TCC keys its grants to the app's DESIGNATED REQUIREMENT, and an ad-hoc signature changes
|
||||
# identity every build — so a pre-inserted TCC row would stop matching after a rebuild. Sign with a
|
||||
# stable identity via $NUCLEIC_VMAGENT_SIGN_IDENTITY (a Developer ID, or a self-signed code-signing
|
||||
# cert in the login keychain). Falls back to ad-hoc WITH A WARNING so a first local round-trip still
|
||||
# works (you must re-provision TCC after every ad-hoc rebuild).
|
||||
#
|
||||
# Runs on the HOST (needs the Swift toolchain); the produced .app runs inside the GUEST. Stage the
|
||||
# app next to scripts/provision-macos-guest.sh (or in the shared workspace) when provisioning.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
PKG_DIR="$REPO_ROOT/guest/NucleicVMAgent"
|
||||
OUT_DIR="${1:-$PKG_DIR/dist}"
|
||||
APP="$OUT_DIR/NucleicVMAgent.app"
|
||||
IDENTITY="${NUCLEIC_VMAGENT_SIGN_IDENTITY:-}"
|
||||
|
||||
echo "▸ Building NucleicVMAgent (release) …"
|
||||
swift build --package-path "$PKG_DIR" -c release
|
||||
BIN="$(swift build --package-path "$PKG_DIR" -c release --show-bin-path)/NucleicVMAgent"
|
||||
[ -x "$BIN" ] || { echo "✗ build produced no binary at $BIN" >&2; exit 1; }
|
||||
|
||||
echo "▸ Assembling $APP …"
|
||||
rm -rf "$APP"
|
||||
mkdir -p "$APP/Contents/MacOS"
|
||||
cp "$PKG_DIR/Packaging/Info.plist" "$APP/Contents/Info.plist"
|
||||
cp "$BIN" "$APP/Contents/MacOS/NucleicVMAgent"
|
||||
# The LaunchAgent plist rides along so the provisioner can install it from one staged directory.
|
||||
cp "$PKG_DIR/Packaging/xyz.blakeslee.nucleic.vmagent.plist" "$OUT_DIR/"
|
||||
|
||||
echo "▸ Signing …"
|
||||
if [ -n "$IDENTITY" ]; then
|
||||
codesign --force --options runtime --sign "$IDENTITY" "$APP"
|
||||
echo " ✓ signed with '$IDENTITY' (hardened runtime)."
|
||||
else
|
||||
codesign --force --sign - "$APP"
|
||||
echo " ⚠ AD-HOC signed (set NUCLEIC_VMAGENT_SIGN_IDENTITY for a stable identity)." >&2
|
||||
echo " An ad-hoc identity changes EVERY build, so the guest's pre-granted TCC rows stop" >&2
|
||||
echo " matching after a rebuild — re-run provisioning Phase 7d after installing this build." >&2
|
||||
fi
|
||||
# Never let a quarantine xattr ride into the guest (the app is never downloaded, but be safe).
|
||||
xattr -dr com.apple.quarantine "$APP" 2>/dev/null || true
|
||||
|
||||
codesign --verify --deep "$APP"
|
||||
echo "✓ $APP"
|
||||
echo " Stage it for provisioning: copy $OUT_DIR/* next to scripts/provision-macos-guest.sh in the guest."
|
||||
@@ -22,8 +22,10 @@
|
||||
# sourced by those shells; /etc/zshenv always is.
|
||||
# 6. Warms the iOS simulator.
|
||||
# 7. COMPUTER USE (GUI automation): installs cliclick at /usr/local/bin, enables auto-login for the
|
||||
# agent (so a headless boot has an Aqua session), and pre-grants Screen Recording + Accessibility
|
||||
# in the SYSTEM TCC.db so screencapture/cliclick work unattended (needs SIP off — see Phase 7).
|
||||
# agent (so a headless boot has an Aqua session), pre-grants Screen Recording + Accessibility
|
||||
# in the SYSTEM TCC.db so screencapture/cliclick work unattended (needs SIP off — see Phase 7),
|
||||
# and installs the NATIVE IN-GUEST AGENT (NucleicVMAgent.app + its LaunchAgent + TCC grants)
|
||||
# when the app is staged alongside this script — see docs/MACOS_VM_NATIVE_AGENT.md.
|
||||
# 8. Shuts the guest DOWN cleanly so the base can be cloned.
|
||||
#
|
||||
# Run it as the `agent` user (it will `sudo` for the system-level bits). It's idempotent — safe to
|
||||
@@ -369,6 +371,115 @@ else
|
||||
2>/dev/null | sed 's/^/ /' || true
|
||||
fi
|
||||
|
||||
# ── 7d: the NATIVE IN-GUEST AGENT (NucleicVMAgent.app + LaunchAgent + TCC) ────────────────────────
|
||||
# The preferred computer-use path (docs/MACOS_VM_NATIVE_AGENT.md): a small signed .app, run as a
|
||||
# per-user LaunchAgent in the auto-login Aqua session, that the host reaches over VSOCK (no SSH) and
|
||||
# that drives the guest with Apple's frameworks directly — AXUIElement (semantic observe/control,
|
||||
# and the ONLY observation channel on a macOS 26 guest whose framebuffer captures come back blank),
|
||||
# ScreenCaptureKit (capture), CGEvent (raw input). Entirely OPTIONAL: when the app isn't staged, the
|
||||
# base still works via the SSH + cliclick path above (Nucleic probes and falls back automatically).
|
||||
#
|
||||
# Build it on the HOST with scripts/build-vm-agent.sh and stage dist/* (the .app + the LaunchAgent
|
||||
# plist) next to this script, in the agent home, or in the shared workspace.
|
||||
echo " ▸ [7d] Installing the native in-guest agent (NucleicVMAgent.app) …"
|
||||
AGENT_APP_DEST="/Applications/NucleicVMAgent.app"
|
||||
AGENT_LA_PLIST="/Library/LaunchAgents/xyz.blakeslee.nucleic.vmagent.plist"
|
||||
AGENT_BUNDLE_ID="xyz.blakeslee.nucleic.vmagent"
|
||||
AGENT_APP_SRC=""
|
||||
for cand in \
|
||||
"$(cd "$(dirname "$0")" && pwd)/NucleicVMAgent.app" \
|
||||
"$AGENT_HOME/NucleicVMAgent.app" \
|
||||
"$SHARED_WORKSPACE/NucleicVMAgent.app"; do
|
||||
[ -d "$cand" ] && { AGENT_APP_SRC="$cand"; break; }
|
||||
done
|
||||
|
||||
if [ -z "$AGENT_APP_SRC" ]; then
|
||||
echo " ⚠ NucleicVMAgent.app not staged — SKIPPING the native agent. Computer use will fall back"
|
||||
echo " to the SSH + cliclick path (no ax_* semantic actions, and NO computer-use at all on a"
|
||||
echo " macOS 26 guest whose screenshots are blank). To add it: on the host run"
|
||||
echo " scripts/build-vm-agent.sh, stage its dist/* next to this script, and re-run."
|
||||
else
|
||||
# Install the app (a fresh copy every run — the bundle is tiny) and strip any quarantine xattr.
|
||||
sudo rm -rf "$AGENT_APP_DEST"
|
||||
sudo cp -R "$AGENT_APP_SRC" "$AGENT_APP_DEST"
|
||||
sudo xattr -dr com.apple.quarantine "$AGENT_APP_DEST" 2>/dev/null || true
|
||||
echo " ✓ installed $AGENT_APP_DEST (from $AGENT_APP_SRC)."
|
||||
|
||||
# LaunchAgent: Aqua-session-only, RunAtLoad + KeepAlive (staged copy preferred, generated else).
|
||||
AGENT_LA_SRC="$(dirname "$AGENT_APP_SRC")/xyz.blakeslee.nucleic.vmagent.plist"
|
||||
if [ -f "$AGENT_LA_SRC" ]; then
|
||||
sudo cp "$AGENT_LA_SRC" "$AGENT_LA_PLIST"
|
||||
else
|
||||
sudo tee "$AGENT_LA_PLIST" >/dev/null <<'PLIST'
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>Label</key>
|
||||
<string>xyz.blakeslee.nucleic.vmagent</string>
|
||||
<key>ProgramArguments</key>
|
||||
<array>
|
||||
<string>/Applications/NucleicVMAgent.app/Contents/MacOS/NucleicVMAgent</string>
|
||||
</array>
|
||||
<key>RunAtLoad</key>
|
||||
<true/>
|
||||
<key>KeepAlive</key>
|
||||
<true/>
|
||||
<key>LimitLoadToSessionType</key>
|
||||
<string>Aqua</string>
|
||||
<key>AssociatedBundleIdentifiers</key>
|
||||
<string>xyz.blakeslee.nucleic.vmagent</string>
|
||||
<key>StandardOutPath</key>
|
||||
<string>/tmp/nucleic-vmagent.log</string>
|
||||
<key>StandardErrorPath</key>
|
||||
<string>/tmp/nucleic-vmagent.log</string>
|
||||
</dict>
|
||||
</plist>
|
||||
PLIST
|
||||
fi
|
||||
sudo chown root:wheel "$AGENT_LA_PLIST"
|
||||
sudo chmod 644 "$AGENT_LA_PLIST"
|
||||
echo " ✓ LaunchAgent at $AGENT_LA_PLIST (Aqua session, RunAtLoad, KeepAlive)."
|
||||
|
||||
# TCC: the agent needs THREE grants (docs/MACOS_VM_NATIVE_AGENT.md §11), each a separate service
|
||||
# row, keyed by BUNDLE ID (client_type=0) + the app's designated-requirement csreq blob:
|
||||
# • kTCCServiceAccessibility — AXUIElement observe/control (AXIsProcessTrusted)
|
||||
# • kTCCServicePostEvent — CGEvent input synthesis (a DISTINCT row from Accessibility)
|
||||
# • kTCCServiceScreenCapture — ScreenCaptureKit
|
||||
# Same SIP-off reality as 7c: with SIP on, the system TCC.db is not writable — skip gracefully.
|
||||
if [ "$SIP_OFF" -ne 1 ]; then
|
||||
echo " ⚠ SIP is ENABLED — the agent's TCC grants are SKIPPED (see 7c for the one-time"
|
||||
echo " recoveryOS 'csrutil disable' step, then re-run this provisioner)." >&2
|
||||
else
|
||||
TCC_DB="/Library/Application Support/com.apple.TCC/TCC.db"
|
||||
AGENT_HEXREQ="NULL"
|
||||
AGENT_REQ=$(codesign -d -r- "$AGENT_APP_DEST" 2>&1 | sed -n 's/^designated => //p')
|
||||
if [ -n "$AGENT_REQ" ] && echo "$AGENT_REQ" | csreq -r- -b /tmp/csreq-vmagent.bin 2>/dev/null; then
|
||||
AGENT_HEXREQ="X'$(xxd -p /tmp/csreq-vmagent.bin | tr -d '\n')'"
|
||||
else
|
||||
echo " ⚠ no usable designated requirement (ad-hoc signed?) — writing csreq NULL rows."
|
||||
echo " NOTE an ad-hoc identity changes every rebuild; sign with a stable identity" >&2
|
||||
echo " (scripts/build-vm-agent.sh + NUCLEIC_VMAGENT_SIGN_IDENTITY) for durable grants." >&2
|
||||
fi
|
||||
for svc in kTCCServiceAccessibility kTCCServicePostEvent kTCCServiceScreenCapture; do
|
||||
echo " • $svc ← $AGENT_BUNDLE_ID"
|
||||
sudo sqlite3 "$TCC_DB" \
|
||||
"INSERT OR REPLACE INTO access
|
||||
(service,client,client_type,auth_value,auth_reason,auth_version,csreq,policy_id,indirect_object_identifier_type,indirect_object_identifier,indirect_object_code_identity,flags,last_modified)
|
||||
VALUES('$svc','$AGENT_BUNDLE_ID',0,2,4,1,$AGENT_HEXREQ,NULL,0,'UNUSED',NULL,0,strftime('%s','now'));"
|
||||
done
|
||||
rm -f /tmp/csreq-vmagent.bin 2>/dev/null || true
|
||||
# tccd must reload the db, and the agent must RESTART to pick the grants up (Screen Recording
|
||||
# in particular is only read at process start).
|
||||
sudo killall tccd 2>/dev/null || true
|
||||
AGENT_UID="$(id -u "$AGENT_USER")"
|
||||
sudo launchctl bootout "gui/$AGENT_UID/$AGENT_BUNDLE_ID" 2>/dev/null || true
|
||||
sudo launchctl bootstrap "gui/$AGENT_UID" "$AGENT_LA_PLIST" 2>/dev/null || \
|
||||
echo " (LaunchAgent will start on next login/boot — no live Aqua session to bootstrap into.)"
|
||||
echo " ✓ agent TCC grants written; agent (re)started. Log: /tmp/nucleic-vmagent.log"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo " ── Computer use needs ALL THREE of the above (cliclick + auto-login + TCC) ─────────────────"
|
||||
echo " If screenshots come back BLACK or clicks NO-OP after this base is live, check, in the guest:"
|
||||
@@ -376,6 +487,8 @@ echo " • SIP is disabled: csrutil status"
|
||||
echo " • TCC rows exist & allowed: sudo sqlite3 \"/Library/Application Support/com.apple.TCC/TCC.db\" \\"
|
||||
echo " \"SELECT service,client,auth_value FROM access;\""
|
||||
echo " • agent is auto-logged-in: an Aqua/desktop session is active (not the login window)"
|
||||
echo " • native agent alive (7d): cat /tmp/nucleic-vmagent.log — should show 'starting on"
|
||||
echo " vsock port 2035' and readiness true×3 (if the app was staged)"
|
||||
echo " ──────────────────────────────────────────────────────────────────────────────────────────────"
|
||||
|
||||
echo ""
|
||||
|
||||
Reference in New Issue
Block a user