Merge nucleic/bold-grove-lynx-dpel into dev
This commit is contained in:
@@ -50,6 +50,35 @@ extension MacVMEngine {
|
||||
return out
|
||||
}
|
||||
|
||||
/// Where a FULL Xcode lives on the host, most-preferred first. Stable before beta: both build a
|
||||
/// macOS-27 guest fine, and the stable one is the less surprising thing to bake in.
|
||||
static let hostXcodeSearchPaths = ["/Applications/Xcode.app", "/Applications/Xcode-beta.app"]
|
||||
|
||||
/// The first path in `candidates` that is a real full Xcode — i.e. carries an `xcodebuild`, which
|
||||
/// the Command Line Tools do **not** ship at all. That binary is the whole point: it's the only
|
||||
/// supported installer for the Metal toolchain (`xcodebuild -downloadComponent MetalToolchain`),
|
||||
/// and the Metal compiler only resolves through `xcrun` when a full Xcode is the selected developer
|
||||
/// dir — so a CLT-only guest can neither install nor use `metal`.
|
||||
static func preferredHostXcode(
|
||||
_ candidates: [String] = hostXcodeSearchPaths,
|
||||
exists: (String) -> Bool = { FileManager.default.fileExists(atPath: $0) }
|
||||
) -> String? {
|
||||
candidates.first { exists("\($0)/Contents/Developer/usr/bin/xcodebuild") }
|
||||
}
|
||||
|
||||
/// Append the host's full Xcode to the `.app` bundles being baked into the base, so every session
|
||||
/// clone is born able to compile Metal (see ``preferredHostXcode(_:exists:)`` and the provisioner's
|
||||
/// Phase 7⅞). No host Xcode ⇒ unchanged, and an operator who already staged an `Xcode*.app` of
|
||||
/// their own wins — we never stage a second Xcode alongside theirs.
|
||||
static func addingHostXcode(_ staged: [String], hostXcode: String?) -> [String] {
|
||||
guard let hostXcode else { return staged }
|
||||
let alreadyStaged = staged.contains { path in
|
||||
let leaf = URL(fileURLWithPath: path).lastPathComponent
|
||||
return leaf.hasPrefix("Xcode") && leaf.hasSuffix(".app")
|
||||
}
|
||||
return alreadyStaged ? staged : staged + [hostXcode]
|
||||
}
|
||||
|
||||
/// Fold app host paths just injected into `bundle` at runtime (``installAppsIntoBase``) into its
|
||||
/// recorded ``MacVMBaseStatus/installedAppPaths``, so a later from-scratch rebuild that replaces
|
||||
/// this base carries them forward. Union preserves the existing order; new paths append.
|
||||
|
||||
@@ -780,10 +780,18 @@ extension MacVMEngine {
|
||||
// deduped by leaf name so a carried path and a settings path for the same app don't double-stage.
|
||||
let userApps = Self.dedupedByLeaf(
|
||||
Self.existingAppBundles(Self.orderedUnion(MacVMSettings.bundledAppPaths, carryAppPaths)))
|
||||
if !userApps.isEmpty {
|
||||
// Bake the host's FULL Xcode in too, unasked — it is the ONLY way a guest can have a `metal`
|
||||
// compiler. The Command Line Tools ship no `xcodebuild` (so `-downloadComponent MetalToolchain`,
|
||||
// the only supported Metal installer, is unreachable) and `xcrun` can't resolve `metal` unless a
|
||||
// full Xcode is the selected developer dir, so a CLT-only clone simply cannot build a shader.
|
||||
// Not recorded in `installedAppPaths`: this is re-derived from the host on every build rather
|
||||
// than an operator choice to carry forward, so a host that later loses Xcode honestly yields a
|
||||
// base without one. No host Xcode ⇒ nothing added, and the provisioner skips Phase 7⅞ loudly.
|
||||
let stagedApps = Self.addingHostXcode(userApps, hostXcode: Self.preferredHostXcode())
|
||||
if !stagedApps.isEmpty {
|
||||
let userAppsDir = stageDir.appendingPathComponent("user-apps", isDirectory: true)
|
||||
try? fm.createDirectory(at: userAppsDir, withIntermediateDirectories: true)
|
||||
for path in userApps {
|
||||
for path in stagedApps {
|
||||
let src = URL(fileURLWithPath: path)
|
||||
try? fm.copyItem(
|
||||
at: src, to: userAppsDir.appendingPathComponent(src.lastPathComponent))
|
||||
|
||||
@@ -63,7 +63,13 @@ extension MacVMEngine {
|
||||
// agent silently never launched on later boots. Fixed: bootout the agent first, copy with
|
||||
// `ditto` (bundle-aware), and verify with `codesign -v`. Every v9 base was built with the
|
||||
// broken copy, so force them all to reprovision onto the valid-bundle install.
|
||||
public static let macOSProvisioningRecipe = 10
|
||||
// v11: every base now bakes in the HOST's full Xcode (`addingHostXcode`, staged with the operator's
|
||||
// "Included apps"), so clones ship a working `metal`. A CLT-only guest cannot have one at all:
|
||||
// the Command Line Tools ship no `xcodebuild`, so Apple's only supported Metal installer
|
||||
// (`-downloadComponent MetalToolchain`) is unreachable, and `xcrun` won't resolve `metal`
|
||||
// unless a full Xcode is the selected developer dir. Provisioner Phase 7⅞ now also VERIFIES
|
||||
// the result by compiling a probe kernel. Every v10 base is Metal-less, so force them all.
|
||||
public static let macOSProvisioningRecipe = 11
|
||||
|
||||
/// The current Linux base **provisioning-recipe version**. Bump when the bundled Linux provisioning
|
||||
/// assets change in a way that must re-provision an already-built base — a new step in
|
||||
|
||||
@@ -57,16 +57,18 @@ extension MacVMPackage {
|
||||
/// code alone can't report this).
|
||||
static let okMarker = "__NUCLEIC_PKG_OK__"
|
||||
|
||||
/// Full **Xcode** + iOS simulators. **Placeholder for now** (`available: false`): a macOS 27 guest
|
||||
/// requires an Xcode *beta*, whose download is Apple-ID-gated and so can't be fetched unattended.
|
||||
/// Shown disabled until a supported, automatable path exists — the scaffolding is here so wiring it
|
||||
/// up later is a one-line flip plus an install script.
|
||||
/// Full **Xcode** + iOS simulators. Not a *package* — nothing to opt into and nothing to download:
|
||||
/// Xcode's own download is Apple-ID-gated, so instead the base build copies the HOST's
|
||||
/// `/Applications/Xcode[-beta].app` into every base automatically (`MacVMEngine.addingHostXcode`),
|
||||
/// which is also what gives the guest a `metal` compiler. Kept in the catalog, disabled, so the
|
||||
/// Settings list explains where Xcode comes from rather than leaving its absence a mystery.
|
||||
public static let xcode = MacVMPackage(
|
||||
id: "xcode",
|
||||
name: "Xcode",
|
||||
summary: "Full Xcode with the iOS simulators.",
|
||||
summary: "Full Xcode with the iOS simulators and the Metal toolchain.",
|
||||
available: false,
|
||||
unavailableNote: "Coming soon — macOS 27 guests currently require an Apple-ID-gated Xcode beta.",
|
||||
unavailableNote:
|
||||
"Installed automatically — the base image copies the Mac's own Xcode from /Applications.",
|
||||
installScript: nil)
|
||||
|
||||
/// Build the combined guest-side shell that installs every **installable** package whose id is in
|
||||
|
||||
@@ -427,6 +427,36 @@ import UniformTypeIdentifiers
|
||||
== ["/one/Foo.app", "/three/Bar.app"])
|
||||
}
|
||||
|
||||
@Test func preferredHostXcodePicksStableOverBetaAndNeedsXcodebuild() {
|
||||
// A full Xcode is identified by the `xcodebuild` the Command Line Tools don't ship — that
|
||||
// binary is the only supported Metal-toolchain installer, so a candidate without it is useless.
|
||||
let has: (String) -> Bool = { path in
|
||||
path.hasPrefix("/Applications/Xcode.app") || path.hasPrefix("/Applications/Xcode-beta.app")
|
||||
}
|
||||
#expect(MacVMEngine.preferredHostXcode(exists: has) == "/Applications/Xcode.app")
|
||||
// Only the beta installed → the beta; stable is preferred only when it's really there.
|
||||
#expect(
|
||||
MacVMEngine.preferredHostXcode(exists: { $0.hasPrefix("/Applications/Xcode-beta.app") })
|
||||
== "/Applications/Xcode-beta.app")
|
||||
// A bare bundle with no xcodebuild (a CLT-only host, or a stub) is not a full Xcode.
|
||||
#expect(MacVMEngine.preferredHostXcode(exists: { _ in false }) == nil)
|
||||
}
|
||||
|
||||
@Test func addingHostXcodeAppendsUnlessTheOperatorStagedTheirOwn() {
|
||||
// The host Xcode rides along with the operator's picks so every clone can compile Metal.
|
||||
#expect(
|
||||
MacVMEngine.addingHostXcode(["/Applications/Foo.app"], hostXcode: "/Applications/Xcode.app")
|
||||
== ["/Applications/Foo.app", "/Applications/Xcode.app"])
|
||||
// An operator-staged Xcode wins — never two Xcodes in one /Applications.
|
||||
#expect(
|
||||
MacVMEngine.addingHostXcode(
|
||||
["/Volumes/x/Xcode-beta.app"], hostXcode: "/Applications/Xcode.app")
|
||||
== ["/Volumes/x/Xcode-beta.app"])
|
||||
// No host Xcode ⇒ the staged set is untouched (the base builds, just without `metal`).
|
||||
#expect(MacVMEngine.addingHostXcode(["/Applications/Foo.app"], hostXcode: nil)
|
||||
== ["/Applications/Foo.app"])
|
||||
}
|
||||
|
||||
@Test func pendingBaseCarryPersistsSetAndClears() async throws {
|
||||
let suite = UserDefaults(suiteName: "macvm-carry-\(UUID().uuidString)")!
|
||||
await ContainerServiceSettings.withDefaults(suite) {
|
||||
@@ -1411,6 +1441,23 @@ import UniformTypeIdentifiers
|
||||
#expect(commonPackagesInstalled.upperBound < metalDownloaded.lowerBound)
|
||||
}
|
||||
|
||||
@Test func macOSGuestProvisionerVerifiesMetalByCompilingAProbe() throws {
|
||||
// "Downloaded" isn't "installed": `-downloadComponent` can exit 0 with an unusable toolchain,
|
||||
// so the provisioner must resolve `metal` through xcrun and actually compile something — after
|
||||
// the download, or it would be proving the wrong thing.
|
||||
let repoRoot = URL(fileURLWithPath: #filePath)
|
||||
.deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent()
|
||||
let script = try String(
|
||||
contentsOf: repoRoot.appendingPathComponent("scripts/provision-macos-guest.sh"),
|
||||
encoding: .utf8)
|
||||
|
||||
let metalDownloaded = try #require(
|
||||
script.range(of: "-downloadComponent MetalToolchain", options: .backwards))
|
||||
let probeCompiled = try #require(script.range(of: "probe.metal", options: .backwards))
|
||||
#expect(metalDownloaded.upperBound < probeCompiled.lowerBound)
|
||||
#expect(script.contains("--find metal"))
|
||||
}
|
||||
|
||||
@Test func randomPasswordIsHexAndLongEnough() {
|
||||
let a = MacVMEngine.randomPassword()
|
||||
let b = MacVMEngine.randomPassword()
|
||||
@@ -2552,7 +2599,7 @@ import UniformTypeIdentifiers
|
||||
provisioned: true, agent: false, grants: true), os: .macOS))
|
||||
#expect(MacVMEngine.baseIsUsable(status(provisioned: true, agent: false), os: .linux))
|
||||
#expect(!MacVMEngine.baseIsUsable(status(provisioned: false, agent: false), os: .linux))
|
||||
#expect(MacVMEngine.macOSProvisioningRecipe == 10)
|
||||
#expect(MacVMEngine.macOSProvisioningRecipe == 11)
|
||||
}
|
||||
|
||||
/// Clone-time admission duplicates the recipe-v8 publication invariant: neither an agentless
|
||||
|
||||
+30
-4
@@ -462,7 +462,8 @@ advertised (and pre-allowed) whenever the session can use the macOS VM at all, i
|
||||
|
||||
### 9.1 Baking in apps & common packages
|
||||
|
||||
Every base unconditionally gets the Homebrew dev toolchain (`provision-macos-guest.sh` §4) and **Chrome
|
||||
Every base unconditionally gets the Homebrew dev toolchain (`provision-macos-guest.sh` §4), the host's
|
||||
**full Xcode**, and **Chrome
|
||||
for Testing** (§7⅝) — the latter is what the `chrome` browser target resolves to for computer-use
|
||||
(`knownBrowsers`, §12). Chrome for Testing rather than stock Chrome because it is a *pinned* build with
|
||||
auto-update stripped out, so a disposable clone can't have Chrome updating (or nagging) under an agent
|
||||
@@ -476,9 +477,10 @@ Settings → "macOS virtual machines":
|
||||
stages each into the provisioning share's `user-apps/` and the provisioner copies them into the
|
||||
guest's `/Applications` (`provision-macos-guest.sh` §7½).
|
||||
- **Common packages** (`selectedPackages`) — curated, well-known *optional* tools the guest fetches and
|
||||
installs itself (`MacVMPackage.catalog`). Currently only **Xcode**, a grayed-out placeholder — a
|
||||
macOS 27 guest needs an Apple-ID-gated Xcode *beta* that can't be fetched unattended, so it's
|
||||
scaffolding for a future automatable path. `MacVMPackage.installShell(forIDs:)` is the single source
|
||||
installs itself (`MacVMPackage.catalog`). Currently only **Xcode**, and it's grayed out because
|
||||
Xcode is no longer optional: Apple gates its download behind an Apple ID, so the base build copies
|
||||
the *host's* `/Applications/Xcode[-beta].app` in instead (see "Xcode and the Metal compiler" below).
|
||||
`MacVMPackage.installShell(forIDs:)` is the single source
|
||||
of truth for the guest-side install shell: the engine writes it into the share as
|
||||
`install-packages.sh`, which the provisioner runs (`provision-macos-guest.sh` §7¾). With no
|
||||
installable entry in the catalog today that composer returns `nil`, so the step is skipped.
|
||||
@@ -499,6 +501,30 @@ old base survives the update automatically even if it's since been dropped from
|
||||
(e.g. its source bundle moved, so `bundledAppPaths` filtered it out). Best-effort: a carried app whose
|
||||
source `.app` no longer exists on the host is skipped, exactly as at first install.
|
||||
|
||||
### 9.2 Xcode and the Metal compiler
|
||||
|
||||
Every base bakes in the **host's own full Xcode**. `MacVMEngine.preferredHostXcode()` picks
|
||||
`/Applications/Xcode.app`, else `Xcode-beta.app` (identified by the `xcodebuild` inside it), and
|
||||
`addingHostXcode` appends it to the staged "Included apps" — unless the operator already staged an
|
||||
`Xcode*.app` of their own, who wins. It is deliberately **not** recorded in `installedAppPaths`: it's
|
||||
re-derived from the host on every build rather than an operator choice to carry forward.
|
||||
|
||||
This is what gives a guest a `metal` compiler, and there is no alternative:
|
||||
|
||||
- The Command Line Tools ship **no `xcodebuild` at all** (`/Library/Developer/CommandLineTools/usr/bin/`
|
||||
has none), so `xcodebuild -downloadComponent MetalToolchain` — Apple's only supported Metal installer
|
||||
since Xcode 16 — is unreachable in a CLT-only guest.
|
||||
- Even where the toolchain asset *is* installed, `DEVELOPER_DIR=…/CommandLineTools xcrun -f metal`
|
||||
answers *"unable to find utility metal"*. The compiler resolves only when a full Xcode is the
|
||||
selected developer dir, so removing Xcode after the download would take `metal` with it.
|
||||
|
||||
`provision-macos-guest.sh` §7⅞ then finishes the job once per *base* rather than once per clone:
|
||||
`xcode-select` onto it, `-license accept`, `-runFirstLaunch`, `-downloadComponent MetalToolchain`
|
||||
(falling back to `-downloadPlatform macOS`), and finally a **verification** — resolve `metal` through
|
||||
`xcrun` and compile a probe kernel, because `-downloadComponent` can exit 0 with an unusable toolchain.
|
||||
A host with no Xcode in `/Applications` still yields a working base; §7⅞ warns that it has no `metal`.
|
||||
Budget roughly **6 GB** of base disk for this (Xcode ≈ 3.6 GB, the Metal toolchain asset ≈ 2.5 GB).
|
||||
|
||||
---
|
||||
|
||||
## 10. Lifecycle
|
||||
|
||||
@@ -756,16 +756,23 @@ fi
|
||||
# ══════════════════════════════════════════════════════════════════════════════════════════════════
|
||||
# ── Phase 7⅞: FULL Xcode first-launch + Metal toolchain (bake into the base, not per-run) ───────────
|
||||
# ══════════════════════════════════════════════════════════════════════════════════════════════════
|
||||
# If the operator baked a FULL Xcode into the base (staged as an "Included app" in Phase 7½, or dropped
|
||||
# in by a common package), finish its one-time setup HERE so every disposable session clone is born
|
||||
# Every base gets a FULL Xcode: the host's own /Applications/Xcode[-beta].app is staged automatically
|
||||
# alongside the operator's "Included apps" (MacVMEngine+Provision.swift → `addingHostXcode`) and lands
|
||||
# in Phase 7½. This phase finishes its one-time setup so every disposable session clone is born
|
||||
# build-ready. Without this, the first agent turn that builds a full-Xcode / Metal target pays for the
|
||||
# first-launch component install AND the Metal toolchain download itself — on EVERY clone, since clones
|
||||
# are thrown away. That's the "Xcode[-beta] needs its first-launch component install, then retry the
|
||||
# Metal toolchain download" churn this phase eliminates.
|
||||
#
|
||||
# It is also the ONLY way this guest can have a `metal` compiler at all. The Command Line Tools ship no
|
||||
# `xcodebuild`, so `-downloadComponent MetalToolchain` — Apple's only supported Metal installer — is
|
||||
# unreachable without a full Xcode; and even with the toolchain asset installed, `xcrun` refuses to
|
||||
# resolve `metal` unless a full Xcode is the selected developer dir. Hence the verification at the end:
|
||||
# a base that can't compile a shader should say so HERE, not inside some agent's build an hour later.
|
||||
#
|
||||
# Runs AFTER Phase 7½/7¾ (that's when Xcode.app lands) and is fully guarded + idempotent: no full Xcode
|
||||
# in /Applications ⇒ skip silently (CLT-only bases are unaffected). `run_timeout` bounds each step so a
|
||||
# wedged download can't stall the base build; failures never abort provisioning.
|
||||
# in /Applications ⇒ warn and skip (the base stays usable, just Metal-less). `run_timeout` bounds each
|
||||
# step so a wedged download can't stall the base build; failures never abort provisioning.
|
||||
echo "▸ [7⅞/8] Finalizing full Xcode (first-launch components + Metal toolchain), if present …"
|
||||
# Locate a FULL Xcode by BUNDLE, not by a fixed name — it may be "Xcode.app" (stable) or "Xcode-beta.app"
|
||||
# (beta channel), and the operator could stage either depending on their OS version/channel. Accept any
|
||||
@@ -776,7 +783,12 @@ for cand in /Applications/Xcode.app /Applications/Xcode-beta.app /Applications/X
|
||||
done
|
||||
|
||||
if [ -z "$XCODE_APP" ]; then
|
||||
echo " (no full Xcode in /Applications — skipping; the CLT-only base is unaffected.)"
|
||||
# The host had no /Applications/Xcode[-beta].app to stage (or the copy failed). Everything else in
|
||||
# this base still works — but nothing in it can compile a `.metal` file, so say so plainly rather
|
||||
# than letting an agent discover it mid-build.
|
||||
echo " ⚠ no full Xcode in /Applications — this base has NO 'metal' compiler." >&2
|
||||
echo " (Install Xcode on the HOST, in /Applications, and rebuild the base image; the Command" >&2
|
||||
echo " Line Tools alone can neither install nor resolve the Metal toolchain.)" >&2
|
||||
else
|
||||
XCODE_DEVDIR="$XCODE_APP/Contents/Developer"
|
||||
echo " ▸ Using $XCODE_APP"
|
||||
@@ -812,6 +824,24 @@ else
|
||||
echo " ⚠ Metal toolchain download failed/timed out — the first agent build may re-attempt it." >&2
|
||||
fi
|
||||
fi
|
||||
# VERIFY, don't assume. `-downloadComponent` can exit 0 while `metal` still isn't usable, and the
|
||||
# failure would otherwise surface an hour later inside an agent's build. Resolve the tool the way a
|
||||
# build system does (`xcrun`) and actually compile a trivial kernel — that's the only proof the
|
||||
# toolchain is really installed and matched to this Xcode.
|
||||
METAL_BIN="$("$XCODE_DEVDIR/usr/bin/xcrun" --find metal 2>/dev/null || true)"
|
||||
if [ -z "$METAL_BIN" ]; then
|
||||
echo " ✗ 'metal' is STILL unresolvable via xcrun — this base cannot compile shaders." >&2
|
||||
else
|
||||
METAL_PROBE="$(mktemp -d)"
|
||||
printf '#include <metal_stdlib>\nkernel void nucleic_probe() {}\n' > "$METAL_PROBE/probe.metal"
|
||||
if run_timeout 300 "$XCODE_DEVDIR/usr/bin/xcrun" -sdk macosx metal \
|
||||
-c "$METAL_PROBE/probe.metal" -o "$METAL_PROBE/probe.air" 2>/dev/null; then
|
||||
echo " ✓ metal verified — $METAL_BIN compiled a probe kernel."
|
||||
else
|
||||
echo " ✗ 'metal' resolves ($METAL_BIN) but failed to compile a trivial kernel." >&2
|
||||
fi
|
||||
rm -rf "$METAL_PROBE"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
Reference in New Issue
Block a user