abkslmandnucleic
9a70972a48
Nucleic-Session: 28559516-7571-4295-A21C-28B0B95D9427 Co-authored-by: Nucleic <[email protected]>
246 lines
11 KiB
Swift
246 lines
11 KiB
Swift
import AppKit
|
||
import CoreGraphics
|
||
import Foundation
|
||
import NucleicCore
|
||
import Virtualization
|
||
|
||
// Surface spike (NUCLEIC_MACVM_SPIKE_SURFACE=1) — exercise the REAL shipping computer-use pipeline
|
||
// end-to-end against a live VM: the `MacVMSurfaceHost` protocol contract, the real `MacVMKeyMap`
|
||
// chord/character tables, the full `MacVMSurfaceInput` vocabulary (move/click/drag/scroll/key/text),
|
||
// and the 1920×1200 **JPEG** capture the `mac_vm_computer` tool actually returns.
|
||
//
|
||
// The concrete `MacVMComputerSurface` lives in the app layer (AppKit) and can't be imported by this
|
||
// CLI, so `SpikeSurface` below mirrors it — same cacheDisplay capture + `NSEvent` synthesis, driven
|
||
// through the same public protocol + keymap. It boots a disposable CLONE (the golden base stays
|
||
// pristine) so this needs no provisioned account/SSH: host-side IO grants nothing in the guest.
|
||
//
|
||
// Writes /tmp/nucleic-spike-surface-{1,2}.jpg (before/after input) and logs frame stats + the
|
||
// host-tracked cursor position.
|
||
|
||
/// The app-layer `MacVMComputerSurface`, mirrored for the spike (verbatim capture/input logic, driven
|
||
/// through the public `MacVMSurfaceHost` protocol + `MacVMKeyMap`).
|
||
@MainActor
|
||
final class SpikeSurface: MacVMSurfaceHost {
|
||
static let fbWidth = 1920
|
||
static let fbHeight = 1200
|
||
|
||
private struct Entry {
|
||
let window: NSWindow
|
||
let view: VZVirtualMachineView
|
||
var cursor: CGPoint
|
||
}
|
||
private var entries: [String: Entry] = [:]
|
||
|
||
func attach(name: String, virtualMachine box: UncheckedSendableBox<AnyObject>) async {
|
||
guard let vm = box.value as? VZVirtualMachine else { return }
|
||
let frame = NSRect(x: 0, y: 0, width: Self.fbWidth, height: Self.fbHeight)
|
||
let view = VZVirtualMachineView(frame: frame)
|
||
view.virtualMachine = vm
|
||
view.capturesSystemKeys = true
|
||
let window = NSWindow(
|
||
contentRect: frame, styleMask: [.borderless], backing: .buffered, defer: false)
|
||
window.contentView = view
|
||
window.isReleasedWhenClosed = false
|
||
window.setFrameOrigin(NSPoint(x: -30_000, y: -30_000))
|
||
window.orderFrontRegardless()
|
||
window.makeFirstResponder(view)
|
||
entries[name] = Entry(
|
||
window: window, view: view,
|
||
cursor: CGPoint(x: Self.fbWidth / 2, y: Self.fbHeight / 2))
|
||
}
|
||
|
||
func detach(name: String) async {
|
||
guard let e = entries.removeValue(forKey: name) else { return }
|
||
e.view.virtualMachine = nil
|
||
e.window.orderOut(nil)
|
||
}
|
||
|
||
func cursorPosition(name: String) async -> Point? {
|
||
guard let e = entries[name] else { return nil }
|
||
return Point(x: Int(e.cursor.x), y: Int(e.cursor.y))
|
||
}
|
||
|
||
func capture(name: String) async -> Data? {
|
||
guard let e = entries[name] else { return nil }
|
||
let view = e.view
|
||
guard let rep = view.bitmapImageRepForCachingDisplay(in: view.bounds) else { return nil }
|
||
view.cacheDisplay(in: view.bounds, to: rep)
|
||
let target = NSSize(width: CGFloat(Self.fbWidth), height: CGFloat(Self.fbHeight))
|
||
guard let out = NSBitmapImageRep(
|
||
bitmapDataPlanes: nil, pixelsWide: Self.fbWidth, pixelsHigh: Self.fbHeight,
|
||
bitsPerSample: 8, samplesPerPixel: 4, hasAlpha: true, isPlanar: false,
|
||
colorSpaceName: .deviceRGB, bytesPerRow: 0, bitsPerPixel: 0)
|
||
else { return nil }
|
||
out.size = target
|
||
NSGraphicsContext.saveGraphicsState()
|
||
NSGraphicsContext.current = NSGraphicsContext(bitmapImageRep: out)
|
||
NSGraphicsContext.current?.imageInterpolation = .high
|
||
let img = NSImage(size: target)
|
||
img.addRepresentation(rep)
|
||
img.draw(in: NSRect(origin: .zero, size: target))
|
||
NSGraphicsContext.restoreGraphicsState()
|
||
return out.representation(using: .jpeg, properties: [.compressionFactor: 0.7])
|
||
}
|
||
|
||
func send(name: String, _ input: MacVMSurfaceInput) async {
|
||
guard var e = entries[name] else { return }
|
||
let view = e.view, window = e.window
|
||
switch input {
|
||
case let .move(x, y):
|
||
postMouse(.mouseMoved, at: guestPoint(x, y), window: window, view: view)
|
||
e.cursor = CGPoint(x: x, y: y)
|
||
case let .click(x, y, button, count):
|
||
let p = guestPoint(x, y)
|
||
postMouse(.mouseMoved, at: p, window: window, view: view)
|
||
let (down, up): (NSEvent.EventType, NSEvent.EventType) =
|
||
button == .right ? (.rightMouseDown, .rightMouseUp) : (.leftMouseDown, .leftMouseUp)
|
||
postMouse(down, at: p, window: window, view: view, clickCount: count)
|
||
postMouse(up, at: p, window: window, view: view, clickCount: count)
|
||
e.cursor = CGPoint(x: x, y: y)
|
||
case let .drag(fromX, fromY, toX, toY):
|
||
let from = guestPoint(fromX, fromY), to = guestPoint(toX, toY)
|
||
postMouse(.leftMouseDown, at: from, window: window, view: view)
|
||
postMouse(.leftMouseDragged, at: to, window: window, view: view)
|
||
postMouse(.leftMouseUp, at: to, window: window, view: view)
|
||
e.cursor = CGPoint(x: toX, y: toY)
|
||
case let .scroll(_, _, dx, dy):
|
||
postScroll(dx: dx, dy: dy, view: view)
|
||
case let .key(chord):
|
||
if let s = MacVMKeyMap.stroke(forChord: chord) { postStroke(s, window: window, view: view) }
|
||
case let .text(text):
|
||
for ch in text {
|
||
if let s = MacVMKeyMap.stroke(forCharacter: ch) {
|
||
postStroke(s, window: window, view: view, characters: String(ch))
|
||
}
|
||
}
|
||
}
|
||
entries[name] = e
|
||
}
|
||
|
||
private func guestPoint(_ x: Int, _ y: Int) -> NSPoint {
|
||
NSPoint(x: CGFloat(x), y: CGFloat(Self.fbHeight - y))
|
||
}
|
||
|
||
private func postMouse(
|
||
_ type: NSEvent.EventType, at p: NSPoint, window: NSWindow, view: NSView, clickCount: Int = 1
|
||
) {
|
||
let down = type == .leftMouseDown || type == .rightMouseDown
|
||
guard let e = NSEvent.mouseEvent(
|
||
with: type, location: p, modifierFlags: [],
|
||
timestamp: ProcessInfo.processInfo.systemUptime, windowNumber: window.windowNumber,
|
||
context: nil, eventNumber: 0, clickCount: clickCount, pressure: down ? 1 : 0)
|
||
else { return }
|
||
window.sendEvent(e)
|
||
}
|
||
|
||
private func postScroll(dx: Int, dy: Int, view: NSView) {
|
||
guard let cg = CGEvent(
|
||
scrollWheelEvent2Source: nil, units: .line, wheelCount: 2,
|
||
wheel1: Int32(dy), wheel2: Int32(dx), wheel3: 0),
|
||
let e = NSEvent(cgEvent: cg)
|
||
else { return }
|
||
view.scrollWheel(with: e)
|
||
}
|
||
|
||
private func postStroke(
|
||
_ s: MacVMKeyStroke, window: NSWindow, view: NSView, characters: String = ""
|
||
) {
|
||
var flags: NSEvent.ModifierFlags = []
|
||
if s.command { flags.insert(.command) }
|
||
if s.shift { flags.insert(.shift) }
|
||
if s.option { flags.insert(.option) }
|
||
if s.control { flags.insert(.control) }
|
||
if s.function { flags.insert(.function) }
|
||
for type in [NSEvent.EventType.keyDown, .keyUp] {
|
||
if let e = NSEvent.keyEvent(
|
||
with: type, location: .zero, modifierFlags: flags,
|
||
timestamp: ProcessInfo.processInfo.systemUptime, windowNumber: window.windowNumber,
|
||
context: nil, characters: characters, charactersIgnoringModifiers: characters,
|
||
isARepeat: false, keyCode: s.keyCode)
|
||
{
|
||
window.sendEvent(e)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Drives `SpikeSurface` through the public protocol against a freshly booted clone.
|
||
@MainActor
|
||
final class SurfaceSpikeDriver {
|
||
let surface = SpikeSurface()
|
||
let name = "surface-spike"
|
||
var cloneBundle: MacVMBundle!
|
||
var vm: VZVirtualMachine!
|
||
|
||
func run() {
|
||
Task { @MainActor in
|
||
do { try await drive() } catch { die("surface spike: \(error)") }
|
||
}
|
||
}
|
||
|
||
private func startVM() async throws {
|
||
try await withCheckedThrowingContinuation { (cont: CheckedContinuation<Void, Error>) in
|
||
vm.start { result in
|
||
switch result {
|
||
case .success: cont.resume()
|
||
case .failure(let e): cont.resume(throwing: e)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
private func stopVM() async {
|
||
guard vm.canStop else { return }
|
||
await withCheckedContinuation { (cont: CheckedContinuation<Void, Never>) in
|
||
vm.stop { _ in cont.resume() }
|
||
}
|
||
}
|
||
|
||
private func write(_ data: Data?, _ label: String) {
|
||
guard let data else { log(" [surface] capture \(label): nil"); return }
|
||
let path = "/tmp/nucleic-spike-surface-\(label).jpg"
|
||
try? data.write(to: URL(fileURLWithPath: path))
|
||
log(" [surface] capture \(label): \(data.count) JPEG bytes → \(path)")
|
||
}
|
||
|
||
private func drive() async throws {
|
||
cloneBundle = try cloneBaseForSpike()
|
||
log("[surface] cloned base → \(cloneBundle.root.lastPathComponent); building main-queue VM …")
|
||
let config = try MacVMEngine.makeConfiguration(
|
||
bundle: cloneBundle, cpus: 4, memoryGiB: 8,
|
||
mac: VZMACAddress.randomLocallyAdministered().string, mounts: [])
|
||
vm = VZVirtualMachine(configuration: config) // default init ⇒ main-queue VM
|
||
try await startVM()
|
||
|
||
// Bind the surface through the SAME protocol call the engine makes at boot.
|
||
await surface.attach(name: name, virtualMachine: UncheckedSendableBox(value: vm as AnyObject))
|
||
log("[surface] booted + attached via MacVMSurfaceHost; waiting ~45s for a visible screen …")
|
||
try await Task.sleep(for: .seconds(45))
|
||
|
||
write(await surface.capture(name: name), "1")
|
||
|
||
// Exercise the FULL input vocabulary through the protocol, using the real MacVMKeyMap.
|
||
log("[surface] driving MacVMSurfaceInput: move, text('hello'), key('return'), click, scroll …")
|
||
await surface.send(name: name, .move(x: 960, y: 700))
|
||
await surface.send(name: name, .text("hello"))
|
||
await surface.send(name: name, .key(chord: "return"))
|
||
await surface.send(name: name, .click(x: 960, y: 700, button: .left, count: 1))
|
||
await surface.send(name: name, .scroll(x: 960, y: 700, dx: 0, dy: -3))
|
||
let cursor = await surface.cursorPosition(name: name)
|
||
log("[surface] cursorPosition (host-tracked): \(cursor.map { "(\($0.x), \($0.y))" } ?? "nil")")
|
||
|
||
try await Task.sleep(for: .seconds(4))
|
||
write(await surface.capture(name: name), "2")
|
||
|
||
await surface.detach(name: name)
|
||
await stopVM()
|
||
try? FileManager.default.removeItem(at: cloneBundle.root)
|
||
log("")
|
||
log("─── SURFACE-SPIKE RESULTS ───")
|
||
log(" Drove the real MacVMSurfaceHost protocol + MacVMKeyMap + JPEG capture on a live VM.")
|
||
log(" JPEGs: /tmp/nucleic-spike-surface-1.jpg (initial), -2.jpg (after input vocabulary).")
|
||
log(" Non-blank #1 ⇒ host JPEG capture works; a change in #2 ⇒ the input vocabulary lands.")
|
||
exit(0)
|
||
}
|
||
}
|