blakeslee-research/gitea-macos-vm-orchestrator
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge nucleic/mellow-dewy-falcon-rjhr into main

Browse Source
This commit is contained in:
abkslm
2026-08-07 03:12:18 -07:00
parent e7163de22f
commit adb7dedd80
5 changed files with 292 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Sources/RunnerHost/ImageBuilder.swift
+156 -16
View File
@@ -79,9 +79,29 @@ public struct ImageBuilder: Sendable {
// Checked against the filesystem rather than `store.image(named:)`: a
// half-built bundle from a previous failed run is exactly the thing this
// needs to catch, and it would not read back as a valid image.
// needs to look at, and it would not read back as a valid image.
let bundleURL = store.imagesDir.appendingPathComponent(name, isDirectory: true)
if FileManager.default.fileExists(atPath: bundleURL.path) {
let existing = VMBundle(rootURL: bundleURL)
let existingConfig = try? existing.loadConfig()
// Installed but never provisioned: resume rather than throw away an
// hour of installing. This is safe precisely because provisioning is
// what got skipped — the guest has never been booted, so its *true*
// first boot is still ahead of it and `VZMacGuestProvisioningOptions`
// will still be evaluated. (macOS only honours those options on the
// first boot after a restore; a guest that already booted once has
// consumed that chance, which is the ambiguous case handled by the
// SSH probe in `bootProvisionAndSeal`.)
if existing.isComplete(), let existingConfig, !existingConfig.provisioned {
progress?(
.note("image '\(name)' already installed — resuming first boot + provisioning"))
try await firstBootAndProvision(
bundle: existing, config: config, isResume: true, progress: progress)
progress?(.done)
return
}
throw CoreError.configInvalid(
"image '\(name)' already exists at \(bundleURL.path). "
+ "Delete it first (`image delete \(name)`), or build under a different --name."
@@ -370,19 +390,47 @@ public struct ImageBuilder: Sendable {
}
installer.install { result in
// `VZVirtualMachine` holds an exclusive lock on the bundle's
// auxiliary storage (nvram.bin) for its whole lifetime, and
// releases it in `dealloc`. The next thing the caller does is
// build a *second* VM over the same bundle for first boot, so
// if this one is still alive at that moment the new one fails
// validation with "Failed to lock auxiliary storage" — which
// is exactly what operators hit.
//
// Hence: drop every strong reference here, on the queue that
// owns these objects...
session.observation?.invalidate()
session.observation = nil
session.installer = nil
session.virtualMachine = nil
switch result {
case .success:
progress?(1.0)
continuation.resume()
case .failure(let error):
continuation.resume(throwing: VMInstance.mapVZError(error))
// ...and resume the caller only from a *later* block on that
// same serial queue. Returning from this handler is what lets
// the framework's own frame unwind and release its references,
// and a serial queue guarantees that has happened before the
// block below runs. Resuming inline instead would race the
// deallocation against the first-boot VM.
let boxedResult = UncheckedBox(result)
queue.async {
switch boxedResult.value {
case .success:
progress?(1.0)
continuation.resume()
case .failure(let error):
continuation.resume(throwing: VMInstance.mapVZError(error))
}
}
}
}
}
// One more hop to the back of the same queue: by the time an empty block
// gets to run, everything enqueued above it — including the release of
// the last reference to the VM — has finished.
await withCheckedContinuation { (continuation: CheckedContinuation<Void, Never>) in
queue.async { continuation.resume() }
}
}
/// Boots the freshly installed guest, gets it onto the network, and hands it
@@ -416,10 +464,14 @@ public struct ImageBuilder: Sendable {
/// - Parameters:
/// - bundle: The installed bundle.
/// - config: Guest credentials and timeouts.
/// - isResume: `true` when this is picking up a bundle that was installed
/// by an earlier run. Only affects the advice given if the guest never
/// answers on SSH — see ``bootProvisionAndSeal(bundle:config:startOptions:isFirstBoot:isResume:xcodeXIPPath:progress:)``.
/// - progress: Stage callback.
public func firstBootAndProvision(
bundle: VMBundle,
config: RunnerConfig,
isResume: Bool = false,
progress: (@Sendable (ImageBuildStage) -> Void)? = nil
) async throws {
guard #available(macOS 27.0, *) else {
@@ -460,6 +512,7 @@ public struct ImageBuilder: Sendable {
config: config,
startOptions: startOptions,
isFirstBoot: true,
isResume: isResume,
xcodeXIPPath: nil,
progress: progress
)
@@ -504,6 +557,7 @@ public struct ImageBuilder: Sendable {
config: config,
startOptions: nil,
isFirstBoot: false,
isResume: false,
xcodeXIPPath: xcodeXIPPath,
progress: progress
)
@@ -519,6 +573,7 @@ public struct ImageBuilder: Sendable {
config: RunnerConfig,
startOptions: VZMacOSVirtualMachineStartOptions?,
isFirstBoot: Bool,
isResume: Bool,
xcodeXIPPath: String?,
progress: (@Sendable (ImageBuildStage) -> Void)?
) async throws {
@@ -527,15 +582,11 @@ public struct ImageBuilder: Sendable {
let bootTimeout = Duration.seconds(max(60, config.scheduler.bootTimeoutSeconds))
progress?(.firstBoot)
let instance = try VMInstance(bundle: bundle, label: "image:\(bundle.name)", headless: true)
do {
try await instance.start(options: startOptions)
} catch {
throw CoreError.provisioningFailed(
"could not boot image '\(bundle.name)': \(error)"
)
}
let instance = try await Self.bootRetryingAuxStorageLock(
bundle: bundle,
startOptions: startOptions,
progress: progress
)
let address: String
do {
@@ -552,6 +603,37 @@ public struct ImageBuilder: Sendable {
} catch {
_ = await instance.requestStopThenForce()
if isFirstBoot {
// A resumed build has a second candidate cause, and it is
// unrecoverable rather than merely slow: macOS evaluates
// `VZMacGuestProvisioningOptions` only on the first boot after a
// restore. If the earlier run got far enough to boot the guest —
// which the bundle on disk cannot tell us — that chance is spent,
// and no amount of retrying will produce an account or sshd.
// Reaching SSH is the only way to distinguish the two, so this is
// said here, after the probe has failed, rather than refusing to
// resume in the first place.
if isResume {
throw CoreError.provisioningFailed(
"""
the resumed guest never became reachable over SSH within \
\(config.scheduler.bootTimeoutSeconds)s.
Either the guest is older than macOS 27 (see below), or an earlier run \
already consumed its first boot — macOS applies automated Setup Assistant \
provisioning only once, on the first boot after a restore, so a guest that \
has booted before can no longer be provisioned unattended.
There is no way to re-arm it: delete the image and build again with a \
macOS 27 or newer restore image.
gitea-macos-runner image delete \(bundle.name)
gitea-macos-runner image build --ipsw <path>
Underlying error: \(error)
"""
)
}
// The most likely cause by far, and the one with no diagnostic of
// its own: a pre-27 guest accepts the provisioning options and
// ignores them, so it sits at Setup Assistant with no account and
@@ -624,6 +706,64 @@ public struct ImageBuilder: Sendable {
/// Full name for the account Setup Assistant automation creates.
static let guestAccountFullName = "Gitea Runner"
/// Creates and starts the VM, tolerating a still-held auxiliary-storage lock.
///
/// `VZVirtualMachine` takes an exclusive lock on the bundle's `nvram.bin`
/// and gives it up only when the object deallocates. `install(bundle:…)`
/// now drains its queue before returning, so its installer VM is gone by
/// the time we get here — but "gone" is an ARC and Objective-C runtime
/// property, and a stray autorelease pool or a framework thread that has
/// not yet unwound can still be holding the last reference for a moment.
/// The failure that produces is not ambiguous and not persistent:
///
/// Invalid virtual machine configuration. Failed to lock auxiliary storage.
///
/// So it is retried, briefly and only for that message. Anything else fails
/// on the first attempt, because a genuinely invalid configuration does not
/// become valid by waiting.
static func bootRetryingAuxStorageLock(
bundle: VMBundle,
startOptions: VZMacOSVirtualMachineStartOptions?,
progress: (@Sendable (ImageBuildStage) -> Void)?,
timeout: Duration = .seconds(30),
pollInterval: Duration = .seconds(2)
) async throws -> VMInstance {
let started = ContinuousClock.now
var announced = false
while true {
do {
let instance = try VMInstance(
bundle: bundle, label: "image:\(bundle.name)", headless: true)
try await instance.start(options: startOptions)
return instance
} catch {
guard isAuxiliaryStorageLockFailure(error),
ContinuousClock.now - started < timeout
else {
throw CoreError.provisioningFailed(
"could not boot image '\(bundle.name)': \(error)"
)
}
if !announced {
announced = true
progress?(.note("waiting for installer to release the VM bundle…"))
}
try await Task.sleep(for: pollInterval)
}
}
}
/// Whether an error is the transient "someone else still has nvram.bin".
///
/// Matched on the message because the framework reports it as a generic
/// `VZError.invalidVirtualMachineConfiguration` with the detail only in the
/// description — there is no distinct code to switch on.
static func isAuxiliaryStorageLockFailure(_ error: any Error) -> Bool {
let text = "\(error)".lowercased()
return text.contains("auxiliary storage") && text.contains("lock")
}
/// Polls `/var/db/dhcpd_leases` until the guest's MAC appears.
///
/// - Parameters: