187 lines
9.2 KiB
Swift
187 lines
9.2 KiB
Swift
import Foundation
|
||
|
||
/// **Host-side computer-use surface** for a macOS guest (docs/MACOS_VM.md §12). A computer-use VM is
|
||
/// booted on the main queue with a `VZVirtualMachineView` bound to it; Nucleic then captures the
|
||
/// guest **framebuffer** and injects **keyboard/mouse HID** entirely host-side, through Apple's
|
||
/// Virtualization virtual display + virtual input devices. The guest grants **nothing** — no TCC, no
|
||
/// SIP — because these are ordinary virtual hardware, not in-guest automation.
|
||
///
|
||
/// The concrete implementation (`MacVMComputerSurface`) lives in the app layer (it needs AppKit /
|
||
/// `VZVirtualMachineView`); the engine drives it through this protocol, exactly like `MacVMManager`
|
||
/// is injected. All coordinates are the guest's **1920×1200** framebuffer space, 1:1 with the JPEG
|
||
/// screenshots `capture` returns (the engine normalizes captures to that size, matching the existing
|
||
/// computer-use coordinate contract).
|
||
public protocol MacVMSurfaceHost: AnyObject, Sendable {
|
||
/// Bind a fresh `VZVirtualMachineView` to the (main-queue) VM `name`. `virtualMachine` boxes the
|
||
/// non-`Sendable` `VZVirtualMachine` for the actor→main hop; the surface casts it back on the main
|
||
/// actor where it's safe to touch.
|
||
func attach(name: String, virtualMachine: UncheckedSendableBox<AnyObject>) async
|
||
/// Release the surface for `name` (VM stopped / removed). Idempotent.
|
||
func detach(name: String) async
|
||
/// Capture the current framebuffer as **JPEG bytes at 1920×1200**, or `nil` when unavailable.
|
||
func capture(name: String) async -> Data?
|
||
/// Inject one input action into `name`; no-op when `name` has no surface.
|
||
func send(name: String, _ input: MacVMSurfaceInput) async
|
||
/// The host-tracked pointer position (we synthesize the moves, so we know it), for `cursor_position`.
|
||
func cursorPosition(name: String) async -> Point?
|
||
}
|
||
|
||
/// A guest-space point (1920×1200 framebuffer coordinates).
|
||
public struct Point: Sendable, Equatable {
|
||
public let x: Int
|
||
public let y: Int
|
||
public init(x: Int, y: Int) {
|
||
self.x = x
|
||
self.y = y
|
||
}
|
||
}
|
||
|
||
/// One host-side input action to inject via the VM's `VZVirtualMachineView`. Coordinates are in the
|
||
/// guest's 1920×1200 framebuffer space (1:1 with `capture`'s screenshots).
|
||
public enum MacVMSurfaceInput: Sendable {
|
||
case move(x: Int, y: Int)
|
||
case click(x: Int, y: Int, button: Button, count: Int)
|
||
/// Press at `from`, move to `to`, release (a left-button drag).
|
||
case drag(fromX: Int, fromY: Int, toX: Int, toY: Int)
|
||
/// Scroll wheel by line deltas at `(x, y)` (positive `dy` scrolls up, negative down).
|
||
case scroll(x: Int, y: Int, dx: Int, dy: Int)
|
||
/// A chord like `cmd+s`, `return`, `cmd+shift+4`.
|
||
case key(chord: String)
|
||
/// Type a literal string, one synthesized keystroke per character (ASCII).
|
||
case text(String)
|
||
|
||
public enum Button: Sendable { case left, right }
|
||
}
|
||
|
||
/// One synthesizable keystroke: an ANSI-US hardware key code plus which modifiers are held.
|
||
/// Framework-light + `Sendable` so the app-layer surface (and unit tests) can map it to an `NSEvent`.
|
||
public struct MacVMKeyStroke: Sendable, Equatable {
|
||
public let keyCode: UInt16
|
||
public let command: Bool
|
||
public let shift: Bool
|
||
public let option: Bool
|
||
public let control: Bool
|
||
public let function: Bool
|
||
public init(
|
||
keyCode: UInt16, command: Bool = false, shift: Bool = false, option: Bool = false,
|
||
control: Bool = false, function: Bool = false
|
||
) {
|
||
self.keyCode = keyCode
|
||
self.command = command
|
||
self.shift = shift
|
||
self.option = option
|
||
self.control = control
|
||
self.function = function
|
||
}
|
||
}
|
||
|
||
/// Host-side chord/character → `MacVMKeyStroke` translation for synthesizing input into the guest's
|
||
/// virtual keyboard. ANSI-US hardware key codes (Carbon `kVK_*`); the VZ guest keyboard is ANSI, so
|
||
/// the table is exact for the layouts a guest ships with. Mirrors the guest agent's
|
||
/// `VMAgentCore.KeyMap` (a separate package we can't import) plus a character table for `type`.
|
||
public enum MacVMKeyMap {
|
||
/// Parse a chord like `cmd+shift+s` into a keystroke, or `nil` when the final key is unknown.
|
||
public static func stroke(forChord chord: String) -> MacVMKeyStroke? {
|
||
let parts = chord.lowercased().split(separator: "+").map(String.init)
|
||
guard let last = parts.last, let code = keyCodes[normalizeKeyName(last)] else { return nil }
|
||
var s = MacVMKeyStroke(keyCode: code)
|
||
for token in parts.dropLast() {
|
||
switch token {
|
||
case "cmd", "command", "meta", "super": s = s.with(command: true)
|
||
case "ctrl", "control": s = s.with(control: true)
|
||
case "alt", "opt", "option": s = s.with(option: true)
|
||
case "shift": s = s.with(shift: true)
|
||
case "fn": s = s.with(function: true)
|
||
default: break
|
||
}
|
||
}
|
||
return s
|
||
}
|
||
|
||
/// Keystroke to TYPE one character (adds shift for uppercase/shifted symbols). `nil` for a
|
||
/// character with no ANSI-US key (the caller skips it — `type` is best-effort ASCII).
|
||
public static func stroke(forCharacter ch: Character) -> MacVMKeyStroke? {
|
||
if let (code, shift) = characterKeys[ch] {
|
||
return MacVMKeyStroke(keyCode: code, shift: shift)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
static func normalizeKeyName(_ key: String) -> String {
|
||
switch key {
|
||
case "enter": return "return"
|
||
case "escape": return "esc"
|
||
case "backspace": return "delete"
|
||
case "arrowup": return "up"
|
||
case "arrowdown": return "down"
|
||
case "arrowleft": return "left"
|
||
case "arrowright": return "right"
|
||
case "spacebar": return "space"
|
||
default: return key
|
||
}
|
||
}
|
||
|
||
/// ANSI-US virtual key codes (Carbon `kVK_*`) for named keys + chords.
|
||
static let keyCodes: [String: UInt16] = [
|
||
"a": 0x00, "s": 0x01, "d": 0x02, "f": 0x03, "h": 0x04, "g": 0x05, "z": 0x06, "x": 0x07,
|
||
"c": 0x08, "v": 0x09, "b": 0x0B, "q": 0x0C, "w": 0x0D, "e": 0x0E, "r": 0x0F, "y": 0x10,
|
||
"t": 0x11, "1": 0x12, "2": 0x13, "3": 0x14, "4": 0x15, "6": 0x16, "5": 0x17, "=": 0x18,
|
||
"9": 0x19, "7": 0x1A, "-": 0x1B, "8": 0x1C, "0": 0x1D, "]": 0x1E, "o": 0x1F, "u": 0x20,
|
||
"[": 0x21, "i": 0x22, "p": 0x23, "l": 0x25, "j": 0x26, "'": 0x27, "k": 0x28, ";": 0x29,
|
||
"\\": 0x2A, ",": 0x2B, "/": 0x2C, "n": 0x2D, "m": 0x2E, ".": 0x2F, "`": 0x32,
|
||
"return": 0x24, "tab": 0x30, "space": 0x31, "delete": 0x33, "esc": 0x35,
|
||
"home": 0x73, "end": 0x77, "pageup": 0x74, "pagedown": 0x79, "forwarddelete": 0x75,
|
||
"left": 0x7B, "right": 0x7C, "down": 0x7D, "up": 0x7E,
|
||
"f1": 0x7A, "f2": 0x78, "f3": 0x63, "f4": 0x76, "f5": 0x60, "f6": 0x61, "f7": 0x62,
|
||
"f8": 0x64, "f9": 0x65, "f10": 0x6D, "f11": 0x67, "f12": 0x6F, "f13": 0x69, "f14": 0x6B,
|
||
"f15": 0x71,
|
||
]
|
||
|
||
/// Printable ASCII → (key code, whether shift is required), for `type`. Built from the unshifted
|
||
/// key table plus the shifted symbols and uppercase letters.
|
||
static let characterKeys: [Character: (UInt16, Bool)] = {
|
||
var t: [Character: (UInt16, Bool)] = [:]
|
||
// Unshifted keys: letters, digits, space, and the base symbols.
|
||
let unshifted: [Character: UInt16] = [
|
||
"a": 0x00, "b": 0x0B, "c": 0x08, "d": 0x02, "e": 0x0E, "f": 0x03, "g": 0x05, "h": 0x04,
|
||
"i": 0x22, "j": 0x26, "k": 0x28, "l": 0x25, "m": 0x2E, "n": 0x2D, "o": 0x1F, "p": 0x23,
|
||
"q": 0x0C, "r": 0x0F, "s": 0x01, "t": 0x11, "u": 0x20, "v": 0x09, "w": 0x0D, "x": 0x07,
|
||
"y": 0x10, "z": 0x06,
|
||
"0": 0x1D, "1": 0x12, "2": 0x13, "3": 0x14, "4": 0x15, "5": 0x17, "6": 0x16, "7": 0x1A,
|
||
"8": 0x1C, "9": 0x19,
|
||
" ": 0x31, "\t": 0x30, "\n": 0x24,
|
||
"-": 0x1B, "=": 0x18, "[": 0x21, "]": 0x1E, "\\": 0x2A, ";": 0x29, "'": 0x27,
|
||
",": 0x2B, ".": 0x2F, "/": 0x2C, "`": 0x32,
|
||
]
|
||
for (c, code) in unshifted { t[c] = (code, false) }
|
||
// Uppercase letters = the letter key + shift.
|
||
for scalar in UInt8(ascii: "a")...UInt8(ascii: "z") {
|
||
let lower = Character(UnicodeScalar(scalar))
|
||
let upper = Character(UnicodeScalar(scalar).properties.uppercaseMapping)
|
||
if let (code, _) = t[lower] { t[upper] = (code, true) }
|
||
}
|
||
// Shifted symbols (US ANSI): the key that produces them, held with shift.
|
||
let shifted: [Character: Character] = [
|
||
"!": "1", "@": "2", "#": "3", "$": "4", "%": "5", "^": "6", "&": "7", "*": "8",
|
||
"(": "9", ")": "0", "_": "-", "+": "=", "{": "[", "}": "]", "|": "\\", ":": ";",
|
||
"\"": "'", "<": ",", ">": ".", "?": "/", "~": "`",
|
||
]
|
||
for (sym, base) in shifted {
|
||
if let (code, _) = t[base] { t[sym] = (code, true) }
|
||
}
|
||
return t
|
||
}()
|
||
}
|
||
|
||
extension MacVMKeyStroke {
|
||
func with(
|
||
command: Bool? = nil, shift: Bool? = nil, option: Bool? = nil, control: Bool? = nil,
|
||
function: Bool? = nil
|
||
) -> MacVMKeyStroke {
|
||
MacVMKeyStroke(
|
||
keyCode: keyCode, command: command ?? self.command, shift: shift ?? self.shift,
|
||
option: option ?? self.option, control: control ?? self.control,
|
||
function: function ?? self.function)
|
||
}
|
||
}
|