diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml index 30e0e38..f8aef7f 100644 --- a/.gitea/workflows/build.yml +++ b/.gitea/workflows/build.yml @@ -44,10 +44,13 @@ jobs: - name: unit tests run: swift test - # Release build, .app assembly, ad-hoc signature. `codesign --sign -` - # needs no signing identity and no keychain, so it works unattended in a - # throwaway guest — which is the same property that makes it the - # project's shipping signature. + # Release build, .app assembly, signature. `make sign` looks for a + # Developer ID identity and falls back to ad-hoc when there is none — which + # is always the case here, since this runs in a throwaway guest with no + # keychain and no certificate. That fallback is why this step works + # unattended, and it is deliberately not treated as a failure: the point of + # the CI signature is to prove the entitlement survives, not to produce a + # distributable artifact. Release builds are signed on a real host. - name: build and sign the app bundle run: make all diff --git a/Makefile b/Makefile index b6ec9aa..bd5377e 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,8 @@ # Virtualization.framework refuses to start a VM unless the calling process # carries the `com.apple.security.virtualization` entitlement, and entitlements # only survive on a signed bundle. So the shipping artifact is not a bare -# executable but a minimal `.app` bundle that we ad-hoc sign. See docs/DESIGN.md +# executable but a minimal `.app` bundle that we sign -- with a Developer ID +# certificate when one is in the keychain, ad-hoc otherwise. See docs/DESIGN.md # ("Verified Facts", item 10). SHELL := /bin/bash @@ -18,8 +19,9 @@ INFO_PLIST := Resources/Info.plist # The entitlements plist grants exactly one entitlement, # `com.apple.security.virtualization`. Virtualization.framework refuses to -# create a VM without it, and it is granted by ad-hoc signing -# (`codesign --sign -`) -- no Apple developer account required. +# create a VM without it. It is not a restricted entitlement: ad-hoc signing +# (`codesign --sign -`) grants it, and a Developer ID certificate grants it +# without a provisioning profile. # # Deliberately absent: com.apple.vm.networking, which would be needed for a # bridged network attachment. That one IS restricted and requires an approved @@ -43,6 +45,31 @@ APP_RESOURCES := Resources/provision.sh \ INSTALL_DIR := $(HOME)/Applications LINK_PATH := /usr/local/bin/$(BIN_NAME) +# Code signing identity. +# +# A real Developer ID certificate is what makes the bundle's code identity +# *stable across rebuilds*. Its designated requirement is anchored to the team +# ("... and certificate leaf[subject.OU] = L7UDTQ6F5W"), so macOS recognises +# every subsequent build as the same program. An ad-hoc signature has no such +# anchor, so the system falls back to the main executable's Mach-O UUID -- which +# the linker regenerates on essentially every link. Each `make install` then +# presents a program macOS has never seen, and per TN3179 that silently +# withdraws the app's Local Network grant. See docs/troubleshooting.md. +# +# TEAM_ID picks the certificate out of the keychain. When no matching +# "Developer ID Application" identity is present the build still succeeds -- +# ad-hoc, with a warning -- because CI runs `make all` inside a throwaway guest +# that has neither a keychain nor a certificate, and that path must keep +# working. Override with `make sign TEAM_ID=...`, or `TEAM_ID=` to force ad-hoc. +TEAM_ID ?= L7UDTQ6F5W + +# Hardened runtime plus a trusted timestamp: the pair notarization requires. +# Neither costs anything at runtime here, and having them means the bundle can +# be notarized later without re-signing. `--timestamp` contacts Apple's +# timestamp authority, so signing needs network access. Both are rejected by an +# ad-hoc signature, hence they are only passed on the Developer ID path. +SIGN_OPTS ?= --options runtime --timestamp + # Release by default; `make dev` overrides to debug. CONFIG ?= release BIN_PATH = $(BUILD_DIR)/$(CONFIG)/$(BIN_NAME) @@ -72,11 +99,31 @@ bundle: cp $(APP_RESOURCES) "$(RES_DIR)/" chmod +x "$(RES_DIR)/provision.sh" -## sign: ad-hoc sign the bundle with the virtualization entitlement +## sign: sign the bundle (Developer ID when available, else ad-hoc) with the virtualization entitlement sign: - codesign --sign - --entitlements "$(ENTITLEMENTS)" --force "$(APP_DIR)" + @identity=$$(security find-identity -v -p codesigning 2>/dev/null \ + | grep "Developer ID Application" | grep -F "($(TEAM_ID))" \ + | head -1 | awk '{print $$2}'); \ + if [ -n "$$identity" ]; then \ + echo "signing with Developer ID $$identity (team $(TEAM_ID))"; \ + codesign --sign "$$identity" $(SIGN_OPTS) \ + --entitlements "$(ENTITLEMENTS)" --force "$(APP_DIR)"; \ + else \ + echo "warning: no 'Developer ID Application' identity for team '$(TEAM_ID)' in the keychain."; \ + echo " Falling back to an ad-hoc signature. The bundle runs and the entitlement"; \ + echo " works, but its code identity changes on every rebuild, so a macOS Local"; \ + echo " Network grant will not survive the next 'make install'."; \ + echo " See docs/troubleshooting.md."; \ + codesign --sign - --entitlements "$(ENTITLEMENTS)" --force "$(APP_DIR)"; \ + fi @echo "--- entitlements ---" @codesign -d --entitlements - "$(APP_DIR)" 2>/dev/null || true + @echo "--- identity ---" + @# -dvv, not -dv: the Authority chain is only printed at the second -v. + @# The CodeDirectory line is where `flags=0x10000(runtime)` shows up, which + @# is the only proof the hardened runtime actually landed. + @codesign -dvv "$(APP_DIR)" 2>&1 \ + | grep -E "^(Identifier|TeamIdentifier|Authority|Timestamp|CodeDirectory)" || true ## dev: debug build + bundle + sign (fast iteration loop) dev: diff --git a/README.md b/README.md index 945d6be..c5da65a 100644 --- a/README.md +++ b/README.md @@ -51,17 +51,20 @@ were killed uncleanly, so the Gitea runner list does not accumulate dead entries - **Gitea 1.25 or newer** (1.26+ recommended). 1.25 added the admin jobs API with the `labels` field this daemon depends on. - A code-signed app bundle. The binary must carry the `com.apple.security.virtualization` - entitlement; ad-hoc signing (`codesign -s -`) is sufficient, so no paid Apple developer account - is required. + entitlement, which is not a restricted entitlement — ad-hoc signing (`codesign -s -`) grants it, + so the runner *works* with no Apple developer account. A **Developer ID Application** certificate + is nonetheless recommended: it anchors the bundle's code identity to your team, which is what + keeps a macOS Local Network grant alive across rebuilds. See + [Code signing](docs/setup.md#code-signing). ## Quickstart ```sh git clone && cd gitea-macos-runner -# Build, bundle (binary + Resources + Info.plist), ad-hoc sign with the -# virtualization entitlement, then copy to ~/Applications and symlink the CLI -# into /usr/local/bin. +# Build, bundle (binary + Resources + Info.plist), sign with the virtualization +# entitlement (Developer ID if a matching certificate is in the keychain, ad-hoc +# otherwise), then copy to ~/Applications and symlink the CLI into /usr/local/bin. make install # = make build bundle sign, then the install step # Write a starter config to ~/.config/gitea-macos-runner/config.json @@ -145,7 +148,7 @@ Every subcommand accepts the global options `--config PATH` (`-c`, default | `image delete NAME [--force]` | Delete a base image and its disk. `--force` (`-f`) skips the confirmation prompt. | | `vm boot [--image NAME] [--slot N] [--keep]` | Clone an image, boot it, print its IP, and wait for Ctrl-C. `--slot` picks which persistent per-slot MAC to use (default `0`); `--keep` leaves the clone on disk. | | `vm list` | List ephemeral VM clones on disk. | -| `service install [--executable PATH]` | Write and load `~/Library/LaunchAgents/xyz.blakeslee.gitea-macos-runner.plist`. | +| `service install [--executable PATH]` | Write and load `~/Library/LaunchAgents/xyz.blakeslee.gitea-macos-vm-orchestrator.plist`. Also evicts any agent left behind under a previous label. | | `service uninstall` | Unload the LaunchAgent and remove its plist. | | `service status` | Report LaunchAgent installation and run state. | | `doctor [--json] [--no-fail]` | Preflight checks. `--json` emits machine-readable results; `--no-fail` exits zero even when checks fail. | diff --git a/Resources/Info.plist b/Resources/Info.plist index 3be3d87..82d32fd 100644 --- a/Resources/Info.plist +++ b/Resources/Info.plist @@ -3,7 +3,7 @@ CFBundleIdentifier - xyz.blakeslee.gitea-macos-runner + xyz.blakeslee.gitea-macos-vm-orchestrator CFBundleName GiteaMacosRunner diff --git a/Sources/RunnerHost/Doctor.swift b/Sources/RunnerHost/Doctor.swift index fd844a6..36a306f 100644 --- a/Sources/RunnerHost/Doctor.swift +++ b/Sources/RunnerHost/Doctor.swift @@ -78,25 +78,29 @@ public enum Doctor { /// running binary, read with `codesign -d --entitlements - `. /// Running from `.build/` instead of the signed `.app` is the single most /// common setup mistake, and this is what catches it. - /// 5. **Free disk ≥ `storage.minFreeDiskGB`.** CoW clones grow as guests + /// 5. **Code identity is stable**, i.e. the bundle is signed with a real + /// team-anchored certificate rather than ad-hoc. Warns on ad-hoc, + /// because that is what makes Local Network grants evaporate on every + /// rebuild (check 12). + /// 6. **Free disk ≥ `storage.minFreeDiskGB`.** CoW clones grow as guests /// write. - /// 6. **`login.keychain` unlocked**, via `security show-keychain-info + /// 7. **`login.keychain` unlocked**, via `security show-keychain-info /// login.keychain`. macOS 15+ refuses to start a VM otherwise — the /// reason the daemon must be a LaunchAgent in a logged-in session. - /// 7. **Gitea reachable and the token has admin scope**, probed with + /// 8. **Gitea reachable and the token has admin scope**, probed with /// ``GiteaClient/listRunners()``. A non-admin token fails here rather /// than at the first poll. - /// 8. **Registration token resolvable** from file, inline value, or (if + /// 9. **Registration token resolvable** from file, inline value, or (if /// enabled) the API. - /// 9. **Runner download URL is live**, via a one-byte ranged `GET` — the - /// same verb the real download uses, because the presigned redirect - /// target is signed per method. Catches a version bump that no longer - /// has a darwin-arm64 asset. - /// 10. **Guest SSH**, against whichever slot currently holds a DHCP lease — + /// 10. **Runner download URL is live**, via a one-byte ranged `GET` — the + /// same verb the real download uses, because the presigned redirect + /// target is signed per method. Catches a version bump that no longer + /// has a darwin-arm64 asset. + /// 11. **Guest SSH**, against whichever slot currently holds a DHCP lease — /// the one check that exercises host → vmnet → guest `sshd` → password /// auth end to end. Informational when no guest is up, since `doctor` /// will not boot one. - /// 11. **Local Network privacy**. Passes when a subnet allowlist is set in + /// 12. **Local Network privacy**. Passes when a subnet allowlist is set in /// `com.apple.network.local-network`; otherwise informational. On /// macOS 15+ the first attempt to reach a guest over the NAT link can /// be blocked by the Local Network permission prompt, which a @@ -160,12 +164,13 @@ public enum Doctor { } /// The configuration-independent host checks: architecture, OS version, - /// framework support, entitlement. + /// framework support, entitlement, code identity. public static func hostChecks() -> [DoctorCheck] { [ checkHostCapability(), checkVirtualizationSupported(), checkVirtualizationEntitlement(), + checkCodeSignature(), ] } @@ -193,11 +198,7 @@ public enum Doctor { // The entitlement lives on the signature, so a bare binary copied out of // the bundle loses it. Report where we are as well as what we found. - let inAppBundle = executable.contains(".app/Contents/MacOS/") - let signedTarget = inAppBundle - ? String(executable.prefix(upTo: executable.range(of: ".app/Contents/MacOS/")!.upperBound) - .dropLast("/Contents/MacOS/".count)) - : executable + let (signedTarget, inAppBundle) = signableTarget(for: executable) let result = DoctorShell.run( "/usr/bin/codesign", @@ -232,6 +233,112 @@ public enum Doctor { ) } + /// Whether the bundle's code identity is stable across rebuilds. + /// + /// This is not a cosmetic "is it properly signed" check — it is the root + /// cause of the project's most confusing failure. A Developer ID signature + /// carries a designated requirement anchored to the team + /// (`certificate leaf[subject.OU] = "…"`), so macOS recognises every later + /// build as the same program and the app's Local Network grant persists. An + /// **ad-hoc** signature has no such anchor, so per + /// [TN3179](https://developer.apple.com/documentation/technotes/tn3179-understanding-local-network-privacy) + /// the system identifies the app by its main executable's Mach-O UUID — + /// which the linker regenerates on essentially every link. Each + /// `make install` therefore presents a program macOS has never seen, its + /// permission reverts to undetermined, and guest SSH starts failing with + /// `No route to host` minutes after a build that worked. + /// + /// Ad-hoc is a `warn`, not a `fail`: everything still runs, and it is the + /// only option on a host without a certificate (CI signs this way + /// deliberately). It just needs the subnet allowlist to compensate. + /// + /// - Parameter binaryPath: Defaults to the current executable. + /// - Returns: The check result. + public static func checkCodeSignature( + binaryPath: String = CommandLine.arguments.first ?? "" + ) -> DoctorCheck { + let name = "code identity" + + guard let executable = resolveExecutablePath(binaryPath) else { + return DoctorCheck( + name: name, + result: .warn, + detail: "could not locate the running executable to inspect", + remediation: "build and install the signed bundle: `make install`" + ) + } + + let (target, inAppBundle) = signableTarget(for: executable) + let result = DoctorShell.run("/usr/bin/codesign", ["-dv", target]) + + guard result.exitCode == 0 else { + return DoctorCheck( + name: name, + result: inAppBundle ? .fail : .warn, + detail: "\(target) carries no code signature", + remediation: "sign the bundle: `make sign` (or `make install`)" + ) + } + + let team = value(of: "TeamIdentifier", in: result.output) + let identifier = value(of: "Identifier", in: result.output) ?? "?" + let hardened = result.output.contains("flags=") && result.output.contains("runtime") + + guard let team, team != "not set" else { + return DoctorCheck( + name: name, + result: .warn, + detail: "\(identifier) is ad-hoc signed (no team identifier)", + remediation: """ + An ad-hoc signature has no stable designated requirement, so macOS falls back \ + to identifying this app by its Mach-O UUID — regenerated on every build. Any \ + Local Network grant is withdrawn by the next `make install`, and guests then \ + fail with "No route to host". Sign with a Developer ID certificate \ + (`make sign TEAM_ID=`), or set the subnet allowlist so no grant is \ + needed at all — see the "local network access" check. + """ + ) + } + + return DoctorCheck( + name: name, + result: .pass, + detail: "\(identifier), team \(team)" + + (hardened ? ", hardened runtime" : "") + ) + } + + /// Reads a `Key=value` line out of `codesign -dv` output. + /// + /// `codesign` writes this block to stderr, one `Key=value` per line, and + /// repeats some keys (`Authority`); the first match is the one that matters. + private static func value(of key: String, in output: String) -> String? { + for line in output.split(separator: "\n") { + let trimmed = line.trimmingCharacters(in: .whitespaces) + guard trimmed.hasPrefix("\(key)=") else { continue } + return String(trimmed.dropFirst(key.count + 1)) + } + return nil + } + + /// The artifact `codesign` should be pointed at: the enclosing `.app` when + /// the executable lives inside one, otherwise the executable itself. + /// + /// Signatures and entitlements are sealed on the bundle, so querying the + /// bare Mach-O inside it — or one copied out of it — answers the wrong + /// question. + /// + /// - Parameter executable: An absolute, symlink-resolved executable path. + /// - Returns: The path to query, and whether it is an `.app` bundle. + private static func signableTarget(for executable: String) -> (path: String, inAppBundle: Bool) { + guard let marker = executable.range(of: ".app/Contents/MacOS/") else { + return (executable, false) + } + let bundle = executable.prefix(upTo: marker.upperBound) + .dropLast("/Contents/MacOS/".count) + return (String(bundle), true) + } + /// Whether `login.keychain` is currently unlocked. public static func checkLoginKeychain() -> DoctorCheck { let name = "login.keychain unlocked" @@ -696,14 +803,15 @@ public enum Doctor { detail: "guests are reached over the host-private NAT link", remediation: """ on macOS 15+ the first connection to a guest can be blocked by the Local Network \ - privacy prompt, which a background LaunchAgent cannot answer. The app cannot be \ - pre-approved: it only appears under System Settings → Privacy & Security → Local \ - Network once it has actually attempted a guest connection. To trigger and answer \ - the prompt by hand, run `gitea-macos-runner vm boot --image default` once from a \ - Terminal in the GUI session. On an unattended CI host prefer the subnet \ - allowlist, which needs no prompt and survives rebuilds: sudo defaults write \ - com.apple.network.local-network AllowedEthernetLocalNetworkAddresses -array \ - "192.168.64.0/18" (then reboot). See docs/setup.md §2.6. + privacy prompt, and frequently there is nothing able to answer it. A LaunchAgent \ + has no UI to show it in; a run started from a shell is attributed to the \ + *responsible* process, so both the prompt and the System Settings → Privacy & \ + Security → Local Network row belong to Terminal rather than to this app — and \ + granting it to Terminal does not carry over to the LaunchAgent. Prefer the subnet \ + allowlist: it needs no prompt, covers every process, and survives rebuilds. \ + sudo defaults write com.apple.network.local-network \ + AllowedEthernetLocalNetworkAddresses -array "192.168.64.0/18" (same for \ + AllowedWiFiLocalNetworkAddresses), then reboot. See docs/setup.md §2.6. """ ) } diff --git a/Sources/RunnerHost/LaunchdService.swift b/Sources/RunnerHost/LaunchdService.swift index bfb8ed2..1857c98 100644 --- a/Sources/RunnerHost/LaunchdService.swift +++ b/Sources/RunnerHost/LaunchdService.swift @@ -47,10 +47,25 @@ public struct ServiceStatus: Sendable, Equatable { /// because this is the failure people hit first. public enum LaunchdService { /// The `launchd` label, matching `CFBundleIdentifier`. - public static let label = "xyz.blakeslee.gitea-macos-runner" + public static let label = "xyz.blakeslee.gitea-macos-vm-orchestrator" - /// `~/Library/LaunchAgents/xyz.blakeslee.gitea-macos-runner.plist`. + /// Labels this service used to install under. + /// + /// Renaming the label renames the plist, so an upgrade that only wrote the + /// new one would leave the old job bootstrapped and still running the old + /// binary — two daemons polling the same Gitea instance, racing to claim + /// the same queued jobs, with no hint in the logs that a second one exists. + /// ``install(executablePath:configPath:)`` and ``uninstall()`` therefore + /// evict these first. Append, never edit, when the label changes again. + public static let legacyLabels = ["xyz.blakeslee.gitea-macos-runner"] + + /// `~/Library/LaunchAgents/xyz.blakeslee.gitea-macos-vm-orchestrator.plist`. public static var agentPlistURL: URL { + agentPlistURL(for: label) + } + + /// The LaunchAgent plist path for an arbitrary label. + public static func agentPlistURL(for label: String) -> URL { URL(fileURLWithPath: RunnerConfig.expandTilde("~/Library/LaunchAgents/\(label).plist")) } @@ -106,6 +121,11 @@ public enum LaunchdService { withIntermediateDirectories: true ) + // Upgrading from a build that installed under an older label: evict it + // before bootstrapping this one, or both run at once. See + // ``legacyLabels``. + removeLegacyAgents() + // A reinstall over a loaded job is the common case (upgrade, config // change), so unload before rewriting rather than failing on "already // bootstrapped". @@ -140,13 +160,48 @@ public enum LaunchdService { } /// Unloads the job and removes the plist. Safe when not installed. + /// + /// Also evicts any ``legacyLabels`` job, so `service uninstall` leaves + /// nothing of this project loaded regardless of which version installed it. public static func uninstall() throws { + removeLegacyAgents() _ = try? uninstallJobOnly() if FileManager.default.fileExists(atPath: agentPlistURL.path) { try FileManager.default.removeItem(at: agentPlistURL) } } + /// Boots out and deletes any LaunchAgent installed under a ``legacyLabels`` + /// entry. + /// + /// Best effort by design: a legacy job that was never installed, is not + /// loaded, or whose plist is already gone is not an error, and failing to + /// evict one must not block installing the current job. + /// + /// - Returns: The legacy labels that were actually found and removed, for + /// callers that want to tell the operator a migration happened. + @discardableResult + public static func removeLegacyAgents() -> [String] { + var removed: [String] = [] + for legacy in legacyLabels { + let plist = agentPlistURL(for: legacy) + let bootout = LaunchdShell.run( + "/bin/launchctl", ["bootout", "\(domainTarget)/\(legacy)"]) + if bootout.exitCode != 0 { + _ = LaunchdShell.run("/bin/launchctl", ["unload", "-w", plist.path]) + } + + if FileManager.default.fileExists(atPath: plist.path) { + try? FileManager.default.removeItem(at: plist) + removed.append(legacy) + } else if bootout.exitCode == 0 { + // Loaded, but from a plist that is no longer on disk. + removed.append(legacy) + } + } + return removed + } + /// Unloads the job but leaves the plist on disk. private static func uninstallJobOnly() throws { let bootout = LaunchdShell.run("/bin/launchctl", ["bootout", serviceTarget]) diff --git a/Sources/gitea-macos-runner/CommandDaemon.swift b/Sources/gitea-macos-runner/CommandDaemon.swift index 40a0398..a9ae53f 100644 --- a/Sources/gitea-macos-runner/CommandDaemon.swift +++ b/Sources/gitea-macos-runner/CommandDaemon.swift @@ -133,7 +133,7 @@ enum VZAppRuntime { /// Signals land here rather than on `.main`. See ``run(onSignal:body:)``. private static let signalQueue = DispatchQueue( - label: "xyz.blakeslee.gitea-macos-runner.signals") + label: "xyz.blakeslee.gitea-macos-vm-orchestrator.signals") /// Starts the run loop and runs `body` alongside it. Never returns. /// diff --git a/Sources/gitea-macos-runner/CommandService.swift b/Sources/gitea-macos-runner/CommandService.swift index df2f037..453f599 100644 --- a/Sources/gitea-macos-runner/CommandService.swift +++ b/Sources/gitea-macos-runner/CommandService.swift @@ -21,7 +21,7 @@ struct ServiceCommand: AsyncParsableCommand { struct Install: AsyncParsableCommand { static let configuration = CommandConfiguration( commandName: "install", - abstract: "Write ~/Library/LaunchAgents/xyz.blakeslee.gitea-macos-runner.plist and load it.", + abstract: "Write ~/Library/LaunchAgents/\(LaunchdService.label).plist and load it.", discussion: """ Points the agent at the installed, signed .app bundle — not at a bare \ binary. The com.apple.security.virtualization entitlement only survives \ @@ -46,6 +46,13 @@ struct ServiceCommand: AsyncParsableCommand { CLI.note("warning: \(RunnerConfig.expandTilde(options.configPath)) is missing or invalid; the agent will fail to start until it is fixed") } + // Done before install (which also does it) purely so the operator + // is told: an agent silently vanishing from launchctl is alarming + // if you do not know a rename happened. + for legacy in LaunchdService.removeLegacyAgents() { + CLI.note("removed legacy agent \(legacy) (renamed to \(LaunchdService.label))") + } + try LaunchdService.install(executablePath: executablePath, configPath: configPath) print("installed \(LaunchdService.agentPlistURL.path)") @@ -68,7 +75,11 @@ struct ServiceCommand: AsyncParsableCommand { func run() async throws { let path = LaunchdService.agentPlistURL.path let existed = FileManager.default.fileExists(atPath: path) + let legacy = LaunchdService.removeLegacyAgents() try LaunchdService.uninstall() + for label in legacy { + print("removed legacy agent \(label)") + } print(existed ? "removed \(path)" : "not installed (\(path))") } } diff --git a/docs/DESIGN.md b/docs/DESIGN.md index dc24884..bad0185 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -21,8 +21,9 @@ Three constraints shape everything below: `VZError.virtualMachineLimitExceeded` from `start()`. Concurrency is therefore 2, permanently, and the config value is clamped rather than trusted. 2. **Virtualization needs a GUI session and a signed bundle.** The daemon runs as - a LaunchAgent in a logged-in user session, from inside an ad-hoc-signed `.app` - carrying `com.apple.security.virtualization`. + a LaunchAgent in a logged-in user session, from inside a signed `.app` carrying + `com.apple.security.virtualization` — Developer ID when a certificate is + available, ad-hoc otherwise (see "Verified facts", item 10). 3. **Gitea decides which job a runner claims, not us.** We supply capacity; the server matches. Trying to pin a specific job to a specific VM would mean reimplementing Gitea's matching rules, and would be wrong the moment they @@ -45,7 +46,7 @@ Three constraints shape everything below: ┌──────────────────────────────── Host (Apple Silicon Mac, macOS 26+) ─────────────────────────────┐ │ │ │ LaunchAgent (user session, auto-login, login.keychain unlocked) │ -│ └── GiteaMacosRunner.app (ad-hoc signed, com.apple.security.virtualization, LSUIElement) │ +│ └── GiteaMacosRunner.app (signed, com.apple.security.virtualization, LSUIElement) │ │ │ │ │ │ NSApplication(.prohibited).run() ── main thread, required by Virtualization │ │ │ │ @@ -382,7 +383,8 @@ disposable and isolated, not on the job being constrained inside it. and nothing else. * Networking is **NAT**, not bridged. Guests can reach the LAN and Gitea, but are not first-class hosts on it. Bridged networking would require the restricted - `com.apple.vm.networking` entitlement, which ad-hoc signing cannot grant — a + `com.apple.vm.networking` entitlement, which needs an Apple-approved + provisioning profile and which ad-hoc signing cannot grant at all — a constraint that happens to align with what we want anyway. * **SSH host keys are not verified.** The peer is a VM this process booted moments ago on a link no other machine shares; pinning would break on every @@ -527,9 +529,10 @@ timeout. The `--manual-setup` fallback is out of v1 scope (§4). **10. Headless Virtualization requires an `NSApplication` run loop with `.prohibited` activation policy, inside a signed `.app` bundle** carrying -`com.apple.security.virtualization`. Ad-hoc signing (`codesign -s -`) suffices. -Bridged networking would additionally need a restricted entitlement; NAT does -not. +`com.apple.security.virtualization`. That entitlement is unrestricted: ad-hoc +signing (`codesign -s -`) grants it, and a Developer ID certificate grants it +with no provisioning profile. Bridged networking would additionally need a +restricted entitlement; NAT does not. → *Consequence:* `CommandDaemon` starts `NSApplication` and runs the orchestrator in a detached `Task`. This applies to **every** command that starts a VM, not just the daemon: `vm boot`, `image build`, and `image provision` all go through @@ -550,6 +553,24 @@ downgrade that only surfaces as a failed VM start. And `bundle` must copy `.app` without them is a working binary with a broken `image build`, `service install`, and `config init`. +**10a. Which signature is used decides whether the app's code identity is stable +across rebuilds.** A Developer ID signature's designated requirement is anchored +to the team (`… and certificate leaf[subject.OU] = `), so every build +is the same program to macOS. An ad-hoc signature has no anchor, so identity +falls back to the main executable's Mach-O UUID, which the linker regenerates on +essentially every link. +→ *Consequence:* this is not cosmetic, because macOS Local Network privacy is +keyed on exactly that UUID (Fact 16). Under ad-hoc signing a grant is +withdrawn by the next `make install`; under Developer ID it persists. `make sign` +therefore selects a `Developer ID Application` identity matching `TEAM_ID` when +the keychain has one and falls back to ad-hoc with a warning when it does not — +the fallback is required because CI builds inside a throwaway guest with neither +keychain nor certificate. The Developer ID path also passes `--options runtime +--timestamp`, so the bundle is notarizable later without re-signing; notarization +itself is skipped, since it governs distribution to other Macs and this app is +built and installed in place. `Doctor.checkCodeSignature` reports which path was +taken and warns on ad-hoc. + **11. macOS 15+ requires an unlocked `login.keychain` to start a VM.** → *Consequence:* the service **must** be a LaunchAgent in the auto-logged-in user's session, never a LaunchDaemon (which has no session and no unlocked @@ -589,3 +610,26 @@ changing the MAC address or ECID.** prohibition collides directly with the per-slot MAC scheme from Fact 12, so adopting it would require per-slot saved states and a careful look at DHCP lease reuse — not a drop-in optimization. + +**16. macOS 15+ Local Network privacy can block host→guest connections, and +there is no reliable way to grant it interactively here.** Per +[TN3179](https://developer.apple.com/documentation/technotes/tn3179-understanding-local-network-privacy) +it is not TCC — the check is a Network Extension packet filter, so it is absent +from `TCC.db`, cannot be queried, cannot be reset, and it "uses your main +executable UUID as part of its implementation". A denial returns `EHOSTUNREACH` +(errno 65), indistinguishable from a genuinely unreachable host. Three things +then conspire against the interactive grant: a LaunchAgent has no UI to show the +prompt in; a run started from a shell is attributed to the **responsible +process**, so the prompt and the System Settings row belong to Terminal rather +than to this app, and granting it to Terminal does not carry to the agent; and +under ad-hoc signing the UUID keying (Fact 10a) withdraws the grant on the next +rebuild. +→ *Consequence:* the deterministic fix is the subnet allowlist +(`com.apple.network.local-network`, keys `AllowedEthernetLocalNetworkAddresses` +and `AllowedWiFiLocalNetworkAddresses`), which is keyed on the network rather +than the app and is read at boot — so it needs a reboot, not a service restart. +`Doctor.checkLocalNetwork` reads those keys and requires coverage of +`192.168.64.0/18`, not a single /24, because the NAT subnet is chosen at runtime +and slides to the next free /24. `SSHExec.localNetworkHint` appends the same +guidance to connection failures, since errno 65 gives the operator nothing to go +on by itself. diff --git a/docs/setup.md b/docs/setup.md index ff9c268..20b4136 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -162,8 +162,9 @@ SSH server, so the build appears to hang. See Virtualization.framework refuses to run unless the calling binary carries the `com.apple.security.virtualization` entitlement, and entitlements are only honoured on a signed -binary inside a proper `.app` bundle. Ad-hoc signing (`codesign -s -`) satisfies this — **no paid -Apple developer account is needed.** +binary inside a proper `.app` bundle. That entitlement is not restricted, so ad-hoc signing +(`codesign -s -`) satisfies it — **the runner works with no Apple developer account.** A Developer +ID certificate buys something different and worth having; see [Code signing](#code-signing) below. ```sh git clone && cd gitea-macos-runner @@ -176,7 +177,7 @@ make install | --- | --- | | `make build` | `swift build -c release --arch arm64` | | `make bundle` | Assemble `GiteaMacosRunner.app` around the binary: `Contents/MacOS/gitea-macos-runner`, `Contents/Info.plist`, and `Contents/Resources/` (`provision.sh`, `launchd.plist.template`, `config.example.json`) | -| `make sign` | `codesign --sign - --entitlements …` (ad-hoc) and print the resulting entitlements | +| `make sign` | Sign the bundle with the virtualization entitlement — Developer ID when a matching certificate is in the keychain, ad-hoc otherwise — then print the entitlements and the resulting identity | | `make all` | `build` + `bundle` + `sign`. The default target. | | `make install` | Runs `all`, copies the app to `~/Applications`, and symlinks the CLI to `/usr/local/bin/gitea-macos-runner` | | `make dev` | Debug build + bundle + sign, for fast iteration. Does **not** install. | @@ -192,6 +193,58 @@ code looks in `Contents/Resources` first and only then falls back to repo-relative paths, so an installed `.app` missing them is a working binary with three broken commands. +### Code signing + +`make sign` picks its identity automatically: + +| Keychain state | What you get | +| --- | --- | +| A `Developer ID Application` certificate whose team matches `TEAM_ID` | Developer ID signature, hardened runtime (`--options runtime`), trusted timestamp (`--timestamp`) | +| No matching certificate | Ad-hoc signature (`codesign --sign -`), with a warning | + +Both produce a bundle that boots VMs — the virtualization entitlement is not restricted, and needs +no provisioning profile on either path. What differs is **code identity stability**, and that is the +whole reason to prefer Developer ID: + +- A Developer ID signature carries a designated requirement anchored to your team + (`… and certificate leaf[subject.OU] = L7UDTQ6F5W`). Every subsequent build satisfies it, so macOS + recognises rebuild after rebuild as *the same program*. +- An ad-hoc signature has no anchor, so the system falls back to the main executable's Mach-O UUID — + which the linker regenerates on essentially every link. Each `make install` presents a program + macOS has never seen before. + +The practical consequence is Local Network privacy (§2.6): per +[TN3179](https://developer.apple.com/documentation/technotes/tn3179-understanding-local-network-privacy) +the grant "uses your main executable UUID as part of its implementation", so under ad-hoc signing it +is silently withdrawn by the next rebuild. Under Developer ID it survives. + +The team is baked into the `Makefile` as a default; override it for your own certificate: + +```sh +make install TEAM_ID=ABCDE12345 # your Developer ID team +make install TEAM_ID= # force ad-hoc even if a certificate exists +``` + +Confirm what actually landed — `doctor`'s `code identity` check reports it, or ask `codesign`: + +```sh +codesign -dvv ~/Applications/GiteaMacosRunner.app +# Identifier=xyz.blakeslee.gitea-macos-vm-orchestrator +# CodeDirectory v=20500 … flags=0x10000(runtime) +# Authority=Developer ID Application: Your Name (ABCDE12345) +# TeamIdentifier=ABCDE12345 +``` + +`TeamIdentifier=not set` and `flags=0x2(adhoc)` mean the ad-hoc path was taken. + +Two things this deliberately does **not** do. The bundle is not **notarized**: notarization matters +for software distributed to other Macs, where Gatekeeper checks the quarantine bit, and this app is +built and installed in place. `spctl -a` therefore reports `rejected: Unnotarized Developer ID`, +which is expected and does not stop anything here. Signing does require network access for +`--timestamp`, so an offline host falls back to ad-hoc. And the entitlements list stays minimal: +`com.apple.vm.networking` — needed only for bridged networking, and genuinely restricted — is not +requested. See [DESIGN.md](DESIGN.md). + If `/usr/local/bin` isn't writable, `make install` says so and prints the `sudo ln -sf …` command to run yourself. @@ -420,25 +473,31 @@ a **warning** when an allowlist exists but does not. Both keys are documented by [TN3179](https://developer.apple.com/documentation/technotes/tn3179-understanding-local-network-privacy) and are the workaround [Tart's FAQ](https://tart.run/faq/) recommends for the same problem. -**Approving interactively instead.** The app cannot be pre-approved: it appears under **System -Settings → Privacy & Security → Local Network** only *after* it has actually attempted a connection -to a guest. An empty list is expected on a fresh install and does not mean anything is broken. To -create the entry and answer the prompt, run one boot by hand from a Terminal in the GUI session: +**Why not just approve the prompt?** Because on this host there is usually nothing able to show it, +and when something does, it is attributed to the wrong program. -```sh -gitea-macos-runner vm boot --image default -``` +An app appears under **System Settings → Privacy & Security → Local Network** only *after* it has +attempted a connection to a guest, so an empty list on a fresh install is expected and means +nothing is broken. It cannot be pre-approved. But the two obvious ways to trigger the prompt both +miss: -and click **Allow**. Do not wait for the LaunchAgent to hit it — a background agent has no way to -answer the prompt. +- **From the LaunchAgent.** A background agent has no UI, so the prompt has nowhere to appear. The + connection is simply denied, and it surfaces as `No route to host` (errno 65) — not as a + permission error. +- **By hand from a Terminal**, e.g. `gitea-macos-runner vm boot --image default`. macOS assigns the + privacy decision to the *responsible process*, and a binary exec'd from a shell is Terminal's + responsibility, not its own. So both the prompt and the Settings row belong to **Terminal**, and + approving it there does not carry over to the LaunchAgent. (If you are hunting for a row that + seems missing, look for Terminal rather than for "Gitea macOS Runner".) -> **Caveat with ad-hoc signing.** An interactive grant is not durable for this project's ad-hoc -> signed bundle. Local Network privacy does not use TCC; per TN3179 it "uses your main executable -> UUID as part of its implementation", and the linker mints a fresh `LC_UUID` on essentially every -> rebuild. So `make install` after a code change is liable to present as a new app that must be -> approved again — and macOS offers no way to reset a Local Network decision back to undetermined, -> so the stale entries accumulate. This is why the allowlist above, which is keyed on the subnet -> rather than on the app, is the recommendation for an unattended machine. +> **And under ad-hoc signing it is not durable anyway.** Local Network privacy does not use TCC; per +> TN3179 it "uses your main executable UUID as part of its implementation", and the linker mints a +> fresh `LC_UUID` on essentially every rebuild. So `make install` after a code change presents as a +> new app that must be approved again — and macOS offers no way to reset a Local Network decision +> back to undetermined, so stale entries accumulate. Signing with a Developer ID certificate fixes +> the churn (see [Code signing](#code-signing)); the subnet allowlist above, keyed on the network +> rather than on the app, sidesteps the whole mechanism and is the recommendation for an unattended +> machine either way. --- @@ -463,6 +522,7 @@ The checks, in order: | `host capability` | Apple Silicon, and host macOS ≥ 26 | | `Virtualization.framework` | `VZVirtualMachine.isSupported` | | `virtualization entitlement` | `com.apple.security.virtualization` on the *running* executable — this is the check that catches running from `.build/` instead of the signed `.app` | +| `code identity` | The bundle's signature. Passes naming the identifier, team, and hardened runtime; warns on an ad-hoc signature, because that is what makes Local Network grants evaporate on every rebuild ([Code signing](#code-signing)) | | `configuration` | The config file loads, parses, and passes validation | | `free disk space` | Free space on the `storeDir` volume against `storage.minFreeDiskGB` | | `login.keychain unlocked` | `security show-keychain-info login.keychain` | @@ -470,6 +530,7 @@ The checks, in order: | `registration token` | A static token resolves, or one can be fetched when `fetchRegistrationTokenViaAPI` is on | | `runner download url` | The `gitea-runner` release asset is reachable | | `token file permissions` | Warns — not fails — when a token file is group- or world-readable | +| `guest ssh` | Reachability of the most recent guest lease, when there is one. Warns on a timeout, which is most often a stale 24-hour lease for a guest that is already gone | | `local network access` | Passes when a subnet allowlist covers `192.168.64.0/18`; warns when an allowlist exists but is scoped too narrowly; otherwise an informational note about the macOS 15+ Local Network prompt (§2.6) | If the config file is missing or invalid, the host checks still run and the rest diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 0ca286d..6c6133b 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -24,7 +24,7 @@ gitea-macos-runner service status | VM won't start; entitlement / `com.apple.security.virtualization` error | Running an unsigned binary, or one outside the signed `.app` bundle | `make sign` (or re-run `make install`); invoke the installed bundle, never `.build/release/…` | | `virtualMachineLimitExceeded` at boot | macOS allows at most **2** concurrent macOS VMs | Set `scheduler.maxConcurrentVMs` ≤ 2; kill stray VMs from earlier runs | | VM boots but never gets an IP | DHCP lease not yet written, or Local Network privacy denial (macOS 15+) | Check `/var/db/dhcpd_leases`; grant Local Network permission or pre-authorize the subnet | -| Runner not listed under Privacy & Security → Local Network | Expected — the list is populated only after the app first attempts a local connection; it cannot be pre-approved | Boot one VM by hand from a GUI Terminal to create the entry, or (better on CI) allowlist the subnet with `defaults write com.apple.network.local-network` | +| Runner not listed under Privacy & Security → Local Network | Expected — the list is populated only after the app first attempts a local connection, and a run started from a shell is attributed to Terminal, not to the app | Allowlist the subnet with `defaults write com.apple.network.local-network` and reboot; the interactive grant does not reach the LaunchAgent | | SSH times out on a freshly built image | Guest macOS < 27, so provisioning options were ignored and Setup Assistant is waiting | Rebuild the image from a macOS **27+** IPSW | | VMs boot in a loop; every teardown says `reason=cancelled` and nothing is logged between the lease and the teardown | `scheduler.bootTimeoutSeconds` is below the guest's *worst-case* boot on a contended host, so each clone is killed while still starting — and each replacement makes the next one slower | Raise `scheduler.bootTimeoutSeconds` (default 900) and reduce the number of concurrent guests; see [The daemon boots VMs forever](#the-daemon-boots-vms-forever-and-every-teardown-says-reasoncancelled) | | `ssh failed: cannot connect … No route to host) (errno: 65)` part-way through provisioning | macOS 15+ Local Network privacy blocking the app — the grant is keyed on the executable's UUID, so `make install` withdraws it | Allowlist the subnet (`192.168.64.0/18`) and **reboot**; see [SSH fails with "No route to host" mid-run](#ssh-fails-with-no-route-to-host-errno-65-mid-run) | @@ -63,8 +63,9 @@ codesign -d --entitlements - ~/Applications/GiteaMacosRunner.app # verify The output must list `com.apple.security.virtualization`. Then confirm the command you're running resolves to the installed bundle's binary — `which -a gitea-macos-runner` should point at `~/Applications/GiteaMacosRunner.app/Contents/MacOS/gitea-macos-runner`, not at -`.build/release/gitea-macos-runner`. Ad-hoc signing is sufficient; you do not need a paid developer -account. +`.build/release/gitea-macos-runner`. Ad-hoc signing is sufficient for *this* error; you do not need +a developer account to start VMs. (A Developer ID certificate solves a different problem — Local +Network grants evaporating on rebuild. See [setup.md](setup.md#code-signing).) --- @@ -120,9 +121,10 @@ prompt**: a LaunchAgent that was never granted permission (or was denied) cannot see [Local Network: the app is not listed in System Settings](#local-network-the-app-is-not-listed-in-system-settings) for why the interactive grant is not. -3. Or grant it interactively: run `gitea-macos-runner vm boot --image default` from a Terminal in - the GUI session and click **Allow**. The app is not listed under **System Settings → Privacy & - Security → Local Network** until it has made that first attempt. + Do not reach for the interactive grant instead. Booting a VM by hand from a Terminal makes the + prompt (and the **System Settings → Privacy & Security → Local Network** row) belong to + *Terminal* rather than to this app, and approving it there does not carry over to the + LaunchAgent. --- @@ -142,17 +144,24 @@ privacy controls, Local Network privacy is not stored in TCC — per the checks live "deep in the networking stack" as a Network Extension packet filter, so the permission is absent from `TCC.db` and `tccutil reset` does not apply to it. -**Fix — interactive.** Make the app connect once, from a GUI session where a human can answer: +**Why you will probably never see the app's own row.** macOS assigns a privacy decision to the +*responsible process*, not necessarily to the binary that opened the socket. Launching the runner +the obvious way — ```sh gitea-macos-runner vm boot --image default ``` -Click **Allow**. The entry now exists and can be toggled later. Do not wait for the LaunchAgent to -trigger it; a background agent cannot answer the prompt, so it simply fails to reach the guest. +— execs the bundle's binary from a shell, so the system holds **Terminal** responsible. Both the +prompt and the Settings row belong to Terminal, and allowing it there does **not** carry over to the +LaunchAgent, which is the process that actually needs it. Meanwhile the LaunchAgent itself has no UI +to show a prompt in, so from it the connection is denied outright and surfaces as `No route to host` +(errno 65) rather than as a permission error. Between the two, there is no reliable way to grant +this interactively on an unattended host. -**Fix — deterministic, and what to use on a CI box.** Allowlist the VM subnet instead. It is keyed -on the network rather than on the app, so no prompt is involved and nothing needs redoing: +**Fix — deterministic, and what to use on any host running the LaunchAgent.** Allowlist the VM +subnet instead. It is keyed on the network rather than on the app, so no prompt is involved, and +nothing needs redoing: ```sh sudo defaults write com.apple.network.local-network \ @@ -165,12 +174,18 @@ Reboot afterwards. `doctor` then reports `local network access` as a **pass**. B Apple's, documented in TN3179; the same pair is what [Tart's FAQ](https://tart.run/faq/) recommends for this exact problem on CI hosts. -> **Why the interactive grant does not stick here.** This project ships an **ad-hoc signed** bundle -> (`codesign --sign -`), and TN3179 notes that "local network privacy uses your main executable UUID -> as part of its implementation". The linker writes a new `LC_UUID` on essentially every rebuild, so -> a rebuilt-and-reinstalled runner can read as a *different* program and prompt again — while the -> old row lingers, since macOS provides no way to reset a Local Network decision to undetermined. -> Expect duplicate entries after a few upgrades. The subnet allowlist avoids all of this. +> **And an ad-hoc signed bundle cannot hold the grant anyway.** TN3179 notes that "local network +> privacy uses your main executable UUID as part of its implementation". An ad-hoc signature has no +> team anchor, so that UUID *is* the app's identity — and the linker writes a new `LC_UUID` on +> essentially every rebuild, so a rebuilt-and-reinstalled runner reads as a *different* program and +> prompts again, while the old row lingers (macOS provides no way to reset a Local Network decision +> to undetermined). Expect duplicate entries after a few upgrades. +> +> Signing with a **Developer ID Application** certificate fixes the churn: its designated +> requirement is anchored to your team, so every build is recognised as the same program. `make sign` +> uses one automatically when it is in the keychain — check with `codesign -dvv` or `doctor`'s +> `code identity` line, and see [setup.md](setup.md#code-signing). The subnet allowlist avoids the +> mechanism entirely and is still the right answer for an unattended host. --- @@ -591,23 +606,35 @@ error: provisioning failed: could not upload provision.sh from …/provision.sh: ssh failed: cannot connect to 192.168.65.232:22: … No route to host) (errno: 65) ``` +**First, check whether it is actually failing.** A single errno 65 at `attempt=1 elapsed=0s`, +seconds after `guest leased address`, is normal and not worth chasing. `waitForSSH` logs its first +probe unconditionally and then heartbeats every 30 s, and that first probe usually lands before the +host has an ARP entry for the guest — which is also `EHOSTUNREACH`. What matters is whether the +message *repeats* at `elapsed=30s`, `60s`, `90s`, … If it does not, boot is proceeding normally. If +it does, read on. + **Cause.** On macOS 15 and newer, an app that has not been granted Local Network access does not get a "permission denied": the packet filter answers **`EHOSTUNREACH` — errno 65, "No route to host"**, which is indistinguishable from a guest that is genuinely off the network. Guests here live on a -host-private NAT link that is reachable whenever the VM is up, so on this path errno 65 is far more -often the privacy filter than a routing problem. +host-private NAT link that is reachable whenever the VM is up, so on this path a *persistent* errno +65 is far more often the privacy filter than a routing problem. Two details make it look intermittent rather than like a permission problem: -- **The grant is keyed on the executable's UUID.** Per +- **Under an ad-hoc signature the grant is keyed on the executable's UUID.** Per [TN3179](https://developer.apple.com/documentation/technotes/tn3179-understanding-local-network-privacy), Local Network privacy "uses your main executable UUID as part of its implementation", and the linker mints a fresh `LC_UUID` on essentially every build. A `make install` after a code change therefore presents a program macOS has never seen, whose permission is undetermined again — even - though the binary you ran ten minutes ago worked. + though the binary you ran ten minutes ago worked. A Developer ID signature anchors identity to the + team instead and does not drift; `doctor`'s `code identity` check tells you which you have, and + [setup.md](setup.md#code-signing) covers switching. - **Processes started over SSH are exempt.** Running the same command through `ssh you@host …` succeeds while running it from a GUI Terminal fails. A remote-shell success proves nothing about the interactive path. +- **A shell-launched run is attributed to Terminal.** macOS charges the privacy decision to the + responsible process, so the app's own identity is not what is being evaluated when you launch it + by hand — and a grant given to Terminal does nothing for the LaunchAgent. **Fix.** Allowlist the subnet — it is keyed on the network, not on the app, so no rebuild can withdraw it and no prompt has to be answered: