Provisioned VM Image Preparation
Nucleic-Session: 28559516-7571-4295-A21C-28B0B95D9427 Co-authored-by: Nucleic <[email protected]>
This commit is contained in:
@@ -0,0 +1,245 @@
|
||||
import AppKit
|
||||
import CoreGraphics
|
||||
import Foundation
|
||||
import NucleicCore
|
||||
import Virtualization
|
||||
|
||||
// Surface spike (NUCLEIC_MACVM_SPIKE_SURFACE=1) — exercise the REAL shipping computer-use pipeline
|
||||
// end-to-end against a live VM: the `MacVMSurfaceHost` protocol contract, the real `MacVMKeyMap`
|
||||
// chord/character tables, the full `MacVMSurfaceInput` vocabulary (move/click/drag/scroll/key/text),
|
||||
// and the 1920×1200 **JPEG** capture the `mac_vm_computer` tool actually returns.
|
||||
//
|
||||
// The concrete `MacVMComputerSurface` lives in the app layer (AppKit) and can't be imported by this
|
||||
// CLI, so `SpikeSurface` below mirrors it — same cacheDisplay capture + `NSEvent` synthesis, driven
|
||||
// through the same public protocol + keymap. It boots a disposable CLONE (the golden base stays
|
||||
// pristine) so this needs no provisioned account/SSH: host-side IO grants nothing in the guest.
|
||||
//
|
||||
// Writes /tmp/nucleic-spike-surface-{1,2}.jpg (before/after input) and logs frame stats + the
|
||||
// host-tracked cursor position.
|
||||
|
||||
/// The app-layer `MacVMComputerSurface`, mirrored for the spike (verbatim capture/input logic, driven
|
||||
/// through the public `MacVMSurfaceHost` protocol + `MacVMKeyMap`).
|
||||
@MainActor
|
||||
final class SpikeSurface: MacVMSurfaceHost {
|
||||
static let fbWidth = 1920
|
||||
static let fbHeight = 1200
|
||||
|
||||
private struct Entry {
|
||||
let window: NSWindow
|
||||
let view: VZVirtualMachineView
|
||||
var cursor: CGPoint
|
||||
}
|
||||
private var entries: [String: Entry] = [:]
|
||||
|
||||
func attach(name: String, virtualMachine box: UncheckedSendableBox<AnyObject>) async {
|
||||
guard let vm = box.value as? VZVirtualMachine else { return }
|
||||
let frame = NSRect(x: 0, y: 0, width: Self.fbWidth, height: Self.fbHeight)
|
||||
let view = VZVirtualMachineView(frame: frame)
|
||||
view.virtualMachine = vm
|
||||
view.capturesSystemKeys = true
|
||||
let window = NSWindow(
|
||||
contentRect: frame, styleMask: [.borderless], backing: .buffered, defer: false)
|
||||
window.contentView = view
|
||||
window.isReleasedWhenClosed = false
|
||||
window.setFrameOrigin(NSPoint(x: -30_000, y: -30_000))
|
||||
window.orderFrontRegardless()
|
||||
window.makeFirstResponder(view)
|
||||
entries[name] = Entry(
|
||||
window: window, view: view,
|
||||
cursor: CGPoint(x: Self.fbWidth / 2, y: Self.fbHeight / 2))
|
||||
}
|
||||
|
||||
func detach(name: String) async {
|
||||
guard let e = entries.removeValue(forKey: name) else { return }
|
||||
e.view.virtualMachine = nil
|
||||
e.window.orderOut(nil)
|
||||
}
|
||||
|
||||
func cursorPosition(name: String) async -> Point? {
|
||||
guard let e = entries[name] else { return nil }
|
||||
return Point(x: Int(e.cursor.x), y: Int(e.cursor.y))
|
||||
}
|
||||
|
||||
func capture(name: String) async -> Data? {
|
||||
guard let e = entries[name] else { return nil }
|
||||
let view = e.view
|
||||
guard let rep = view.bitmapImageRepForCachingDisplay(in: view.bounds) else { return nil }
|
||||
view.cacheDisplay(in: view.bounds, to: rep)
|
||||
let target = NSSize(width: CGFloat(Self.fbWidth), height: CGFloat(Self.fbHeight))
|
||||
guard let out = NSBitmapImageRep(
|
||||
bitmapDataPlanes: nil, pixelsWide: Self.fbWidth, pixelsHigh: Self.fbHeight,
|
||||
bitsPerSample: 8, samplesPerPixel: 4, hasAlpha: true, isPlanar: false,
|
||||
colorSpaceName: .deviceRGB, bytesPerRow: 0, bitsPerPixel: 0)
|
||||
else { return nil }
|
||||
out.size = target
|
||||
NSGraphicsContext.saveGraphicsState()
|
||||
NSGraphicsContext.current = NSGraphicsContext(bitmapImageRep: out)
|
||||
NSGraphicsContext.current?.imageInterpolation = .high
|
||||
let img = NSImage(size: target)
|
||||
img.addRepresentation(rep)
|
||||
img.draw(in: NSRect(origin: .zero, size: target))
|
||||
NSGraphicsContext.restoreGraphicsState()
|
||||
return out.representation(using: .jpeg, properties: [.compressionFactor: 0.7])
|
||||
}
|
||||
|
||||
func send(name: String, _ input: MacVMSurfaceInput) async {
|
||||
guard var e = entries[name] else { return }
|
||||
let view = e.view, window = e.window
|
||||
switch input {
|
||||
case let .move(x, y):
|
||||
postMouse(.mouseMoved, at: guestPoint(x, y), window: window, view: view)
|
||||
e.cursor = CGPoint(x: x, y: y)
|
||||
case let .click(x, y, button, count):
|
||||
let p = guestPoint(x, y)
|
||||
postMouse(.mouseMoved, at: p, window: window, view: view)
|
||||
let (down, up): (NSEvent.EventType, NSEvent.EventType) =
|
||||
button == .right ? (.rightMouseDown, .rightMouseUp) : (.leftMouseDown, .leftMouseUp)
|
||||
postMouse(down, at: p, window: window, view: view, clickCount: count)
|
||||
postMouse(up, at: p, window: window, view: view, clickCount: count)
|
||||
e.cursor = CGPoint(x: x, y: y)
|
||||
case let .drag(fromX, fromY, toX, toY):
|
||||
let from = guestPoint(fromX, fromY), to = guestPoint(toX, toY)
|
||||
postMouse(.leftMouseDown, at: from, window: window, view: view)
|
||||
postMouse(.leftMouseDragged, at: to, window: window, view: view)
|
||||
postMouse(.leftMouseUp, at: to, window: window, view: view)
|
||||
e.cursor = CGPoint(x: toX, y: toY)
|
||||
case let .scroll(_, _, dx, dy):
|
||||
postScroll(dx: dx, dy: dy, view: view)
|
||||
case let .key(chord):
|
||||
if let s = MacVMKeyMap.stroke(forChord: chord) { postStroke(s, window: window, view: view) }
|
||||
case let .text(text):
|
||||
for ch in text {
|
||||
if let s = MacVMKeyMap.stroke(forCharacter: ch) {
|
||||
postStroke(s, window: window, view: view, characters: String(ch))
|
||||
}
|
||||
}
|
||||
}
|
||||
entries[name] = e
|
||||
}
|
||||
|
||||
private func guestPoint(_ x: Int, _ y: Int) -> NSPoint {
|
||||
NSPoint(x: CGFloat(x), y: CGFloat(Self.fbHeight - y))
|
||||
}
|
||||
|
||||
private func postMouse(
|
||||
_ type: NSEvent.EventType, at p: NSPoint, window: NSWindow, view: NSView, clickCount: Int = 1
|
||||
) {
|
||||
let down = type == .leftMouseDown || type == .rightMouseDown
|
||||
guard let e = NSEvent.mouseEvent(
|
||||
with: type, location: p, modifierFlags: [],
|
||||
timestamp: ProcessInfo.processInfo.systemUptime, windowNumber: window.windowNumber,
|
||||
context: nil, eventNumber: 0, clickCount: clickCount, pressure: down ? 1 : 0)
|
||||
else { return }
|
||||
window.sendEvent(e)
|
||||
}
|
||||
|
||||
private func postScroll(dx: Int, dy: Int, view: NSView) {
|
||||
guard let cg = CGEvent(
|
||||
scrollWheelEvent2Source: nil, units: .line, wheelCount: 2,
|
||||
wheel1: Int32(dy), wheel2: Int32(dx), wheel3: 0),
|
||||
let e = NSEvent(cgEvent: cg)
|
||||
else { return }
|
||||
view.scrollWheel(with: e)
|
||||
}
|
||||
|
||||
private func postStroke(
|
||||
_ s: MacVMKeyStroke, window: NSWindow, view: NSView, characters: String = ""
|
||||
) {
|
||||
var flags: NSEvent.ModifierFlags = []
|
||||
if s.command { flags.insert(.command) }
|
||||
if s.shift { flags.insert(.shift) }
|
||||
if s.option { flags.insert(.option) }
|
||||
if s.control { flags.insert(.control) }
|
||||
if s.function { flags.insert(.function) }
|
||||
for type in [NSEvent.EventType.keyDown, .keyUp] {
|
||||
if let e = NSEvent.keyEvent(
|
||||
with: type, location: .zero, modifierFlags: flags,
|
||||
timestamp: ProcessInfo.processInfo.systemUptime, windowNumber: window.windowNumber,
|
||||
context: nil, characters: characters, charactersIgnoringModifiers: characters,
|
||||
isARepeat: false, keyCode: s.keyCode)
|
||||
{
|
||||
window.sendEvent(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Drives `SpikeSurface` through the public protocol against a freshly booted clone.
|
||||
@MainActor
|
||||
final class SurfaceSpikeDriver {
|
||||
let surface = SpikeSurface()
|
||||
let name = "surface-spike"
|
||||
var cloneBundle: MacVMBundle!
|
||||
var vm: VZVirtualMachine!
|
||||
|
||||
func run() {
|
||||
Task { @MainActor in
|
||||
do { try await drive() } catch { die("surface spike: \(error)") }
|
||||
}
|
||||
}
|
||||
|
||||
private func startVM() async throws {
|
||||
try await withCheckedThrowingContinuation { (cont: CheckedContinuation<Void, Error>) in
|
||||
vm.start { result in
|
||||
switch result {
|
||||
case .success: cont.resume()
|
||||
case .failure(let e): cont.resume(throwing: e)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func stopVM() async {
|
||||
guard vm.canStop else { return }
|
||||
await withCheckedContinuation { (cont: CheckedContinuation<Void, Never>) in
|
||||
vm.stop { _ in cont.resume() }
|
||||
}
|
||||
}
|
||||
|
||||
private func write(_ data: Data?, _ label: String) {
|
||||
guard let data else { log(" [surface] capture \(label): nil"); return }
|
||||
let path = "/tmp/nucleic-spike-surface-\(label).jpg"
|
||||
try? data.write(to: URL(fileURLWithPath: path))
|
||||
log(" [surface] capture \(label): \(data.count) JPEG bytes → \(path)")
|
||||
}
|
||||
|
||||
private func drive() async throws {
|
||||
cloneBundle = try cloneBaseForSpike()
|
||||
log("[surface] cloned base → \(cloneBundle.root.lastPathComponent); building main-queue VM …")
|
||||
let config = try MacVMEngine.makeConfiguration(
|
||||
bundle: cloneBundle, cpus: 4, memoryGiB: 8,
|
||||
mac: VZMACAddress.randomLocallyAdministered().string, mounts: [])
|
||||
vm = VZVirtualMachine(configuration: config) // default init ⇒ main-queue VM
|
||||
try await startVM()
|
||||
|
||||
// Bind the surface through the SAME protocol call the engine makes at boot.
|
||||
await surface.attach(name: name, virtualMachine: UncheckedSendableBox(value: vm as AnyObject))
|
||||
log("[surface] booted + attached via MacVMSurfaceHost; waiting ~45s for a visible screen …")
|
||||
try await Task.sleep(for: .seconds(45))
|
||||
|
||||
write(await surface.capture(name: name), "1")
|
||||
|
||||
// Exercise the FULL input vocabulary through the protocol, using the real MacVMKeyMap.
|
||||
log("[surface] driving MacVMSurfaceInput: move, text('hello'), key('return'), click, scroll …")
|
||||
await surface.send(name: name, .move(x: 960, y: 700))
|
||||
await surface.send(name: name, .text("hello"))
|
||||
await surface.send(name: name, .key(chord: "return"))
|
||||
await surface.send(name: name, .click(x: 960, y: 700, button: .left, count: 1))
|
||||
await surface.send(name: name, .scroll(x: 960, y: 700, dx: 0, dy: -3))
|
||||
let cursor = await surface.cursorPosition(name: name)
|
||||
log("[surface] cursorPosition (host-tracked): \(cursor.map { "(\($0.x), \($0.y))" } ?? "nil")")
|
||||
|
||||
try await Task.sleep(for: .seconds(4))
|
||||
write(await surface.capture(name: name), "2")
|
||||
|
||||
await surface.detach(name: name)
|
||||
await stopVM()
|
||||
try? FileManager.default.removeItem(at: cloneBundle.root)
|
||||
log("")
|
||||
log("─── SURFACE-SPIKE RESULTS ───")
|
||||
log(" Drove the real MacVMSurfaceHost protocol + MacVMKeyMap + JPEG capture on a live VM.")
|
||||
log(" JPEGs: /tmp/nucleic-spike-surface-1.jpg (initial), -2.jpg (after input vocabulary).")
|
||||
log(" Non-blank #1 ⇒ host JPEG capture works; a change in #2 ⇒ the input vocabulary lands.")
|
||||
exit(0)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user