From 33f299396a5eab572e7fea84f26334ce00b1d338 Mon Sep 17 00:00:00 2001 From: Andrew Moore Date: Fri, 7 Aug 2026 00:44:36 -0700 Subject: [PATCH] Nucleic: Gitea Runner macOS VM Support --- .gitignore | 4 + Makefile | 115 +++ Package.resolved | 87 ++ Package.swift | 75 ++ README.md | 135 +++ Resources/Info.plist | 44 + Resources/config.example.json | 68 ++ Resources/gitea-macos-runner.entitlements | 8 + Resources/launchd.plist.template | 56 ++ Resources/provision.sh | 397 +++++++++ Sources/RunnerCore/Config.swift | 733 ++++++++++++++++ Sources/RunnerCore/CoreErrors.swift | 110 +++ Sources/RunnerCore/DHCPLeases.swift | 246 ++++++ Sources/RunnerCore/GiteaClient.swift | 357 ++++++++ Sources/RunnerCore/GiteaModels.swift | 330 +++++++ Sources/RunnerCore/Labels.swift | 84 ++ Sources/RunnerCore/RunnerNaming.swift | 35 + Sources/RunnerCore/SSHExec.swift | 589 +++++++++++++ Sources/RunnerCore/SchedulerCore.swift | 339 ++++++++ Sources/RunnerCore/Version.swift | 11 + Sources/RunnerHost/Doctor.swift | 597 +++++++++++++ Sources/RunnerHost/GuestProvisioner.swift | 599 +++++++++++++ Sources/RunnerHost/IPSW.swift | 241 +++++ Sources/RunnerHost/ImageBuilder.swift | 706 +++++++++++++++ Sources/RunnerHost/LaunchdService.swift | 354 ++++++++ Sources/RunnerHost/Orchestrator.swift | 821 ++++++++++++++++++ Sources/RunnerHost/VMBundle.swift | 272 ++++++ Sources/RunnerHost/VMInstance.swift | 373 ++++++++ Sources/RunnerHost/VMStore.swift | 366 ++++++++ Sources/RunnerHost/VZConfigFactory.swift | 181 ++++ .../gitea-macos-runner/CommandConfig.swift | 205 +++++ .../gitea-macos-runner/CommandDaemon.swift | 178 ++++ .../gitea-macos-runner/CommandDoctor.swift | 61 ++ Sources/gitea-macos-runner/CommandImage.swift | 274 ++++++ .../gitea-macos-runner/CommandService.swift | 107 +++ Sources/gitea-macos-runner/CommandVM.swift | 195 +++++ Sources/gitea-macos-runner/Main.swift | 162 ++++ Tests/RunnerCoreTests/ConfigTests.swift | 609 +++++++++++++ Tests/RunnerCoreTests/DHCPLeasesTests.swift | 208 +++++ Tests/RunnerCoreTests/GiteaClientTests.swift | 443 ++++++++++ Tests/RunnerCoreTests/GiteaModelsTests.swift | 334 +++++++ Tests/RunnerCoreTests/LabelsTests.swift | 182 ++++ .../RunnerCoreTests/SchedulerCoreTests.swift | 344 ++++++++ docs/DESIGN.md | 581 +++++++++++++ docs/security.md | 150 ++++ docs/setup.md | 479 ++++++++++ docs/troubleshooting.md | 314 +++++++ 47 files changed, 13159 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 Package.resolved create mode 100644 Package.swift create mode 100644 README.md create mode 100644 Resources/Info.plist create mode 100644 Resources/config.example.json create mode 100644 Resources/gitea-macos-runner.entitlements create mode 100644 Resources/launchd.plist.template create mode 100755 Resources/provision.sh create mode 100644 Sources/RunnerCore/Config.swift create mode 100644 Sources/RunnerCore/CoreErrors.swift create mode 100644 Sources/RunnerCore/DHCPLeases.swift create mode 100644 Sources/RunnerCore/GiteaClient.swift create mode 100644 Sources/RunnerCore/GiteaModels.swift create mode 100644 Sources/RunnerCore/Labels.swift create mode 100644 Sources/RunnerCore/RunnerNaming.swift create mode 100644 Sources/RunnerCore/SSHExec.swift create mode 100644 Sources/RunnerCore/SchedulerCore.swift create mode 100644 Sources/RunnerCore/Version.swift create mode 100644 Sources/RunnerHost/Doctor.swift create mode 100644 Sources/RunnerHost/GuestProvisioner.swift create mode 100644 Sources/RunnerHost/IPSW.swift create mode 100644 Sources/RunnerHost/ImageBuilder.swift create mode 100644 Sources/RunnerHost/LaunchdService.swift create mode 100644 Sources/RunnerHost/Orchestrator.swift create mode 100644 Sources/RunnerHost/VMBundle.swift create mode 100644 Sources/RunnerHost/VMInstance.swift create mode 100644 Sources/RunnerHost/VMStore.swift create mode 100644 Sources/RunnerHost/VZConfigFactory.swift create mode 100644 Sources/gitea-macos-runner/CommandConfig.swift create mode 100644 Sources/gitea-macos-runner/CommandDaemon.swift create mode 100644 Sources/gitea-macos-runner/CommandDoctor.swift create mode 100644 Sources/gitea-macos-runner/CommandImage.swift create mode 100644 Sources/gitea-macos-runner/CommandService.swift create mode 100644 Sources/gitea-macos-runner/CommandVM.swift create mode 100644 Sources/gitea-macos-runner/Main.swift create mode 100644 Tests/RunnerCoreTests/ConfigTests.swift create mode 100644 Tests/RunnerCoreTests/DHCPLeasesTests.swift create mode 100644 Tests/RunnerCoreTests/GiteaClientTests.swift create mode 100644 Tests/RunnerCoreTests/GiteaModelsTests.swift create mode 100644 Tests/RunnerCoreTests/LabelsTests.swift create mode 100644 Tests/RunnerCoreTests/SchedulerCoreTests.swift create mode 100644 docs/DESIGN.md create mode 100644 docs/security.md create mode 100644 docs/setup.md create mode 100644 docs/troubleshooting.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..31b9b0c --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.build/ +*.xcodeproj +.DS_Store +.swiftpm diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..b6ec9aa --- /dev/null +++ b/Makefile @@ -0,0 +1,115 @@ +# gitea-macos-runner +# +# Virtualization.framework refuses to start a VM unless the calling process +# carries the `com.apple.security.virtualization` entitlement, and entitlements +# only survive on a signed bundle. So the shipping artifact is not a bare +# executable but a minimal `.app` bundle that we ad-hoc sign. See docs/DESIGN.md +# ("Verified Facts", item 10). + +SHELL := /bin/bash +APP_NAME := GiteaMacosRunner +BIN_NAME := gitea-macos-runner +BUILD_DIR := .build +APP_DIR := $(BUILD_DIR)/$(APP_NAME).app +CONTENTS := $(APP_DIR)/Contents +MACOS_DIR := $(CONTENTS)/MacOS +RES_DIR := $(CONTENTS)/Resources +INFO_PLIST := Resources/Info.plist + +# The entitlements plist grants exactly one entitlement, +# `com.apple.security.virtualization`. Virtualization.framework refuses to +# create a VM without it, and it is granted by ad-hoc signing +# (`codesign --sign -`) -- no Apple developer account required. +# +# Deliberately absent: com.apple.vm.networking, which would be needed for a +# bridged network attachment. That one IS restricted and requires an approved +# provisioning profile. We use NAT instead, which needs nothing extra and has +# the side benefit of putting each guest into /var/db/dhcpd_leases, which is +# how the daemon discovers guest IPs. +# +# Keep that plist free of XML comments. `plutil` accepts them, but codesign +# hands the file to AMFI's stricter parser, which rejects a `` block +# with "AMFIUnserializeXML: syntax error" -- and the bundle then signs with no +# entitlements at all, so every VM start fails at runtime. +ENTITLEMENTS:= Resources/gitea-macos-runner.entitlements + +# Data files the tool reads at runtime. `GuestProvisioner`, `LaunchdService`, +# and `config init` each look in `Contents/Resources` first and only then fall +# back to repo-relative paths, so an installed .app that lacks these is a +# working binary with a broken `image build` / `service install` / `config init`. +APP_RESOURCES := Resources/provision.sh \ + Resources/launchd.plist.template \ + Resources/config.example.json +INSTALL_DIR := $(HOME)/Applications +LINK_PATH := /usr/local/bin/$(BIN_NAME) + +# Release by default; `make dev` overrides to debug. +CONFIG ?= release +BIN_PATH = $(BUILD_DIR)/$(CONFIG)/$(BIN_NAME) + +.PHONY: all build bundle sign dev test install uninstall clean help + +# These targets are a pipeline, not independent work: `bundle` needs the binary +# `build` produced, and `sign` signs the tree `bundle` assembled — a signature +# that overtook the resource copy would not cover Contents/Resources, and the +# bundle would fail to launch. Expressing that as prerequisites is not an option +# because `dev` reuses `bundle` against a debug build it made itself, so serial +# execution is imposed instead. +.NOTPARALLEL: + +all: build bundle sign + +## build: compile the release binary for arm64 +build: + swift build -c release --arch arm64 + +## bundle: assemble the minimal .app around the compiled binary +bundle: + @test -x "$(BIN_PATH)" || { echo "error: $(BIN_PATH) not built; run 'make build' (or 'make dev')"; exit 1; } + mkdir -p "$(MACOS_DIR)" "$(RES_DIR)" + cp "$(BIN_PATH)" "$(MACOS_DIR)/$(BIN_NAME)" + cp "$(INFO_PLIST)" "$(CONTENTS)/Info.plist" + cp $(APP_RESOURCES) "$(RES_DIR)/" + chmod +x "$(RES_DIR)/provision.sh" + +## sign: ad-hoc sign the bundle with the virtualization entitlement +sign: + codesign --sign - --entitlements "$(ENTITLEMENTS)" --force "$(APP_DIR)" + @echo "--- entitlements ---" + @codesign -d --entitlements - "$(APP_DIR)" 2>/dev/null || true + +## dev: debug build + bundle + sign (fast iteration loop) +dev: + swift build --arch arm64 + $(MAKE) CONFIG=debug bundle + $(MAKE) sign + +## test: run the unit test suite +test: + swift test + +## install: copy the signed app to ~/Applications and link the CLI +install: all + mkdir -p "$(INSTALL_DIR)" + rm -rf "$(INSTALL_DIR)/$(APP_NAME).app" + cp -R "$(APP_DIR)" "$(INSTALL_DIR)/$(APP_NAME).app" + @if [ -w "$$(dirname $(LINK_PATH))" ]; then \ + ln -sf "$(INSTALL_DIR)/$(APP_NAME).app/Contents/MacOS/$(BIN_NAME)" "$(LINK_PATH)"; \ + echo "linked $(LINK_PATH)"; \ + else \ + echo "note: $$(dirname $(LINK_PATH)) not writable; skipping symlink."; \ + echo " run: sudo ln -sf $(INSTALL_DIR)/$(APP_NAME).app/Contents/MacOS/$(BIN_NAME) $(LINK_PATH)"; \ + fi + +## uninstall: remove the installed app and symlink +uninstall: + rm -rf "$(INSTALL_DIR)/$(APP_NAME).app" + @if [ -L "$(LINK_PATH)" ]; then rm -f "$(LINK_PATH)"; fi + +## clean: remove all build products +clean: + rm -rf "$(BUILD_DIR)" + +## help: list targets +help: + @grep -E '^## ' $(MAKEFILE_LIST) | sed 's/^## / /' diff --git a/Package.resolved b/Package.resolved new file mode 100644 index 0000000..6bc4d76 --- /dev/null +++ b/Package.resolved @@ -0,0 +1,87 @@ +{ + "originHash" : "b4c3eb0620d177330f884f26ad0d38bc155bb3bf0cc96c36dfbbf45f3f6547b1", + "pins" : [ + { + "identity" : "swift-argument-parser", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-argument-parser.git", + "state" : { + "revision" : "6a52f3251125d74daf04fcbd5e6f08a75d074382", + "version" : "1.8.2" + } + }, + { + "identity" : "swift-asn1", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-asn1.git", + "state" : { + "revision" : "a9a5efd40eaf558a2bcd48d64b1d1646be686008", + "version" : "1.7.1" + } + }, + { + "identity" : "swift-atomics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-atomics.git", + "state" : { + "revision" : "0442cb5a3f98ab802acb777929fdb446bda11a34", + "version" : "1.3.1" + } + }, + { + "identity" : "swift-collections", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-collections.git", + "state" : { + "revision" : "a0cb0954ecb21e4e31b0070e6ed5674e8556685a", + "version" : "1.6.0" + } + }, + { + "identity" : "swift-crypto", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-crypto.git", + "state" : { + "revision" : "47d3869a7291f085c1fb9fb1e6d3b97a793f45c6", + "version" : "4.5.1" + } + }, + { + "identity" : "swift-log", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-log.git", + "state" : { + "revision" : "3ffafb9722d5d918c614feb496c8789a3b59d222", + "version" : "1.15.0" + } + }, + { + "identity" : "swift-nio", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio.git", + "state" : { + "revision" : "0b18836bd8b0162e7e17a995a3fbee20ed8f3b2b", + "version" : "2.101.3" + } + }, + { + "identity" : "swift-nio-ssh", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-ssh.git", + "state" : { + "revision" : "3ec281496f28a3b6581afd946b759e2642f5cd8d", + "version" : "0.15.0" + } + }, + { + "identity" : "swift-system", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-system.git", + "state" : { + "revision" : "704705c5c51156ede21172a38654d522ce487074", + "version" : "1.8.0" + } + } + ], + "version" : 3 +} diff --git a/Package.swift b/Package.swift new file mode 100644 index 0000000..df55793 --- /dev/null +++ b/Package.swift @@ -0,0 +1,75 @@ +// swift-tools-version: 6.0 +// +// Package.swift +// gitea-macos-runner +// +// A single-host daemon for an Apple Silicon Mac that watches a Gitea instance +// for queued Actions jobs requiring macOS, runs each in a fresh ephemeral macOS +// VM via Apple's Virtualization.framework, then destroys the VM. +// +// Layout +// ------ +// * `RunnerCore` — portable, side-effect-light logic (config, Gitea API models +// and client, label matching, DHCP lease parsing, SSH transport, and the pure +// scheduling state machine). Deliberately free of `import Virtualization` so +// it also builds and unit-tests on Linux. +// * `RunnerHost` — everything that touches Apple's Virtualization.framework: +// VM bundles, image building, guest provisioning, and the orchestrator actor. +// macOS-only. +// * `gitea-macos-runner` — the CLI/daemon executable. +// +// SwiftPM auto-globs sources under each target directory; files are never +// enumerated here, so parallel workers can add files without touching this file. + +import PackageDescription + +let package = Package( + name: "gitea-macos-runner", + platforms: [ + // Virtualization's ASIF disk support (`diskutil image create --format ASIF`) + // and the macOS 26 guest tooling are hard requirements. See docs/DESIGN.md. + .macOS("26.0") + ], + products: [ + .executable(name: "gitea-macos-runner", targets: ["gitea-macos-runner"]), + .library(name: "RunnerCore", targets: ["RunnerCore"]), + .library(name: "RunnerHost", targets: ["RunnerHost"]), + ], + dependencies: [ + .package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.5.0"), + .package(url: "https://github.com/apple/swift-log.git", from: "1.6.0"), + .package(url: "https://github.com/apple/swift-nio.git", from: "2.76.0"), + .package(url: "https://github.com/apple/swift-nio-ssh.git", from: "0.9.0"), + ], + targets: [ + .target( + name: "RunnerCore", + dependencies: [ + .product(name: "Logging", package: "swift-log"), + .product(name: "NIOCore", package: "swift-nio"), + .product(name: "NIOPosix", package: "swift-nio"), + .product(name: "NIOSSH", package: "swift-nio-ssh"), + ] + ), + .target( + name: "RunnerHost", + dependencies: [ + "RunnerCore", + .product(name: "Logging", package: "swift-log"), + ] + ), + .executableTarget( + name: "gitea-macos-runner", + dependencies: [ + "RunnerCore", + "RunnerHost", + .product(name: "ArgumentParser", package: "swift-argument-parser"), + .product(name: "Logging", package: "swift-log"), + ] + ), + .testTarget( + name: "RunnerCoreTests", + dependencies: ["RunnerCore"] + ), + ] +) diff --git a/README.md b/README.md new file mode 100644 index 0000000..702b98a --- /dev/null +++ b/README.md @@ -0,0 +1,135 @@ +# gitea-macos-runner + +A Swift daemon that gives a self-hosted Gitea instance on-demand macOS CI capacity from a single +Apple Silicon Mac. It polls Gitea for queued Actions jobs that request macOS, boots a fresh +ephemeral macOS VM through Apple's Virtualization.framework for each one, lets Gitea's own +`gitea-runner` execute exactly one job inside that VM, and then destroys the VM. Nothing from a +job survives into the next: every build starts from an identical, freshly cloned base image. + +## How it works + +``` + Gitea host daemon ephemeral VM + │ │ │ + │ GET /api/v1/admin/actions/ │ │ + │ jobs?status=queued │ │ + │◄────────────────────────────────┤ poll every 5s │ + │ [{id, labels: ["macos-arm64"]}]│ │ + ├────────────────────────────────►│ │ + │ │ APFS clone base image (instant) │ + │ ├──────────────────────────────────►│ + │ │ boot headless, NAT networking │ + │ │ resolve IP (/var/db/dhcpd_leases) │ + │ │ ssh in │ + │ ├──────────────────────────────────►│ + │ │ gitea-runner register --ephemeral │ + │◄────────────────────────────────┼───────────────────────────────────┤ + │ runner appears, job dispatched │ │ + ├─────────────────────────────────┼──────────────────────────────────►│ + │ │ gitea-runner daemon │ + │ job completes; server refuses │ runs ONE job │ + │ a second job and deletes the │ │ + │ ephemeral registration │ │ + │ │ destroy VM + clone │ + │ ├───────────────────────────────► ✗ │ +``` + +A background reconcile loop (every 5 minutes) deletes runner registrations left behind by VMs that +were killed uncleanly, so the Gitea runner list does not accumulate dead entries. + +## Requirements + +- **Apple Silicon Mac.** Virtualization.framework macOS guests are ARM-only. +- **macOS 26 or newer on the host; macOS 27 or newer strongly recommended.** The automated image + builder uses `VZMacGuestProvisioningOptions` (macOS 27) to create the admin user and skip Setup + Assistant. Both host *and* guest must be 27+ — an older guest silently ignores the options and + stalls at Setup Assistant. +- **Disk:** ~60 GB free for a vanilla image (IPSW ~15 GB plus a sparse ASIF disk); 140 GB+ if you + provision Xcode. +- **RAM:** 16 GB minimum. Each guest defaults to 8 GB, and macOS caps the host at **2 concurrent + macOS VMs** regardless of hardware. +- **Gitea 1.25 or newer** (1.26+ recommended). 1.25 added the admin jobs API with the `labels` + field this daemon depends on. +- A code-signed app bundle. The binary must carry the `com.apple.security.virtualization` + entitlement; ad-hoc signing (`codesign -s -`) is sufficient, so no paid Apple developer account + is required. + +## Quickstart + +```sh +git clone && cd gitea-macos-runner + +# Build, bundle (binary + Resources + Info.plist), ad-hoc sign with the +# virtualization entitlement, then copy to ~/Applications and symlink the CLI +# into /usr/local/bin. +make install # = make build bundle sign, then the install step + +# Write a starter config to ~/.config/gitea-macos-runner/config.json +gitea-macos-runner config init + +# Fill in gitea.instanceURL, gitea.adminTokenFile, and the registration token settings. +$EDITOR "$(gitea-macos-runner config path)" + +# Read the config back with secrets redacted, to confirm it parses and validates. +gitea-macos-runner config show + +# Verify entitlements, config, Gitea reachability, disk space, and host/guest versions. +gitea-macos-runner doctor + +# Build a base image from an IPSW (long — installs macOS, then provisions the guest). +# The image is named "default", which is also what `daemon` and `vm boot` look for. +gitea-macos-runner image build --ipsw ~/Downloads/UniversalMac_27.0_*.ipsw + +# Optional: add Xcode to the image. +gitea-macos-runner image provision default --xcode-xip ~/Downloads/Xcode_XX.xip + +# Install and start the LaunchAgent (runs in your GUI login session — not a LaunchDaemon). +gitea-macos-runner service install +gitea-macos-runner service status +``` + +Then push a workflow that targets the runner: + +```yaml +# .gitea/workflows/macos.yml +name: macOS build +on: [push] +jobs: + build: + runs-on: macos-arm64 + steps: + - uses: actions/checkout@v4 + - run: sw_vers && swift build +``` + +The daemon picks the job up within one poll interval, boots a VM, and tears it down when the job +finishes. + +## Documentation + +- [docs/setup.md](docs/setup.md) — full Gitea-side and host-side walkthrough, config reference, + image building, service installation, verification. +- [docs/security.md](docs/security.md) — threat model, isolation boundaries, token handling. +- [docs/troubleshooting.md](docs/troubleshooting.md) — symptom → cause → fix. + +## Command reference + +Every subcommand accepts the global options `--config PATH` (`-c`, default +`~/.config/gitea-macos-runner/config.json`) and `--verbose`. + +| Command | Purpose | +| --- | --- | +| `daemon [--image NAME] [--once]` | The scheduler loop. Normally started by launchd via `service install`. `--image` defaults to `default`; `--once` runs a single scheduling tick and exits. | +| `image build [--name NAME] [--ipsw PATH] [--disk-gb N]` | Install macOS into a new base image and provision it. `--name` defaults to `default`; without `--ipsw` the latest supported restore image is downloaded. `--disk-gb` overrides `guest.diskGB`. | +| `image provision NAME [--xcode-xip PATH]` | Re-run guest provisioning on an existing image; optionally install Xcode from a `.xip`. | +| `image list` | List base images in the store. | +| `image delete NAME [--force]` | Delete a base image and its disk. `--force` (`-f`) skips the confirmation prompt. | +| `vm boot [--image NAME] [--slot N] [--keep]` | Clone an image, boot it, print its IP, and wait for Ctrl-C. `--slot` picks which persistent per-slot MAC to use (default `0`); `--keep` leaves the clone on disk. | +| `vm list` | List ephemeral VM clones on disk. | +| `service install [--executable PATH]` | Write and load `~/Library/LaunchAgents/xyz.blakeslee.gitea-macos-runner.plist`. | +| `service uninstall` | Unload the LaunchAgent and remove its plist. | +| `service status` | Report LaunchAgent installation and run state. | +| `doctor [--json] [--no-fail]` | Preflight checks. `--json` emits machine-readable results; `--no-fail` exits zero even when checks fail. | +| `config init [--force] [--instance-url URL]` | Write the annotated example config. `--force` (`-f`) overwrites an existing file. | +| `config show` | Print the effective configuration with secrets redacted. | +| `config path` | Print the configuration file path. | diff --git a/Resources/Info.plist b/Resources/Info.plist new file mode 100644 index 0000000..c2a7a3a --- /dev/null +++ b/Resources/Info.plist @@ -0,0 +1,44 @@ + + + + + CFBundleIdentifier + xyz.blakeslee.gitea-macos-runner + + CFBundleName + GiteaMacosRunner + + CFBundleDisplayName + Gitea macOS Runner + + CFBundleExecutable + gitea-macos-runner + + CFBundlePackageType + APPL + + CFBundleInfoDictionaryVersion + 6.0 + + CFBundleShortVersionString + 0.1.0 + + CFBundleVersion + 1 + + + LSUIElement + + + LSMinimumSystemVersion + 26.0 + + NSHumanReadableCopyright + + + diff --git a/Resources/config.example.json b/Resources/config.example.json new file mode 100644 index 0000000..2f0b631 --- /dev/null +++ b/Resources/config.example.json @@ -0,0 +1,68 @@ +{ + "_comment": "Example configuration for gitea-macos-runner. Copy to ~/.config/gitea-macos-runner/config.json and edit. Keys beginning with an underscore are comments and are ignored by the loader.", + + "gitea": { + "_comment": "How to reach Gitea and how to authenticate. The token must belong to a Gitea ADMIN: every endpoint used lives under /api/v1/admin/actions/.", + "instanceURL": "https://gitea.example.com", + + "_comment_adminToken": "Admin API token. Set EXACTLY ONE of adminToken and adminTokenFile: setting both, or neither, is rejected at load. Prefer adminTokenFile so the secret is not sitting in a world-readable JSON file.", + "adminTokenFile": "~/.config/gitea-macos-runner/admin-token", + + "_comment_registrationToken": "The shared runner registration token. IMPORTANT: registration tokens are REUSABLE and scope-wide, and minting a new one INVALIDATES every prior token for that scope. Never pre-generate one per VM. The recommended setup is to seed a fixed token server-side with GITEA_RUNNER_REGISTRATION_TOKEN and point registrationTokenFile at a copy of it.", + "registrationTokenFile": "~/.config/gitea-macos-runner/registration-token", + + "_comment_fetchViaAPI": "When no static registration token is configured, fetch one from POST /api/v1/admin/actions/runners/registration-token. Off by default: that endpoint returns the scope's active token, and any behaviour change that made it mint a fresh one would invalidate tokens held by runners elsewhere.", + "fetchRegistrationTokenViaAPI": false + }, + + "runner": { + "_comment": "Identity of the ephemeral runners registered inside each guest.", + + "_comment_labels": "BARE label names, matched case-sensitively against a job's runs-on. The ':host' schema suffix is added only when calling `gitea-runner register --labels`; the server never stores it.", + "labels": ["macos-arm64"], + + "_comment_namePrefix": "Prefix for generated runner names. Each VM registers as , globally unique, which is what lets the reconcile loop identify and delete rows orphaned by an unclean VM death.", + "namePrefix": "macos-vm-", + + "_comment_download": "Release asset for the gitea-runner binary installed into the guest. {version} is substituted. The binary is v3.x, renamed from act_runner and published from gitea.com/gitea/runner.", + "runnerDownloadURL": "https://gitea.com/gitea/runner/releases/download/v{version}/gitea-runner-{version}-darwin-arm64", + "version": "3.0.2" + }, + + "scheduler": { + "_comment": "Polling cadence, concurrency, and the timeouts that bound a stuck VM.", + + "_comment_maxConcurrentVMs": "Hard-clamped to 2. Apple's kernel allows at most two concurrent macOS guests per host; a third start() fails with VZError.virtualMachineLimitExceeded.", + "maxConcurrentVMs": 2, + + "pollIntervalSeconds": 5, + + "_comment_reconcile": "How often to sweep Gitea for orphaned runner rows. Gitea itself only sweeps runner rows at midnight, and never sweeps a runner that claimed no task, so this loop is not optional.", + "reconcileIntervalSeconds": 300, + + "_comment_jobTimeout": "Wall-clock ceiling on a single job before its VM is destroyed. Should be comfortably under Gitea's own ABANDONED_JOB_TIMEOUT (default 24h).", + "jobTimeoutMinutes": 120, + + "_comment_bootTimeout": "Ceiling on clone + boot + DHCP lease + SSH readiness before the slot is declared dead and recycled.", + "bootTimeoutSeconds": 300 + }, + + "guest": { + "_comment": "Shape of each guest VM and the credentials used to reach it over SSH. These credentials only ever traverse the host-private NAT link between this Mac and its own ephemeral guests.", + "username": "admin", + "password": "admin", + "cpuCount": 4, + "memoryGB": 8, + + "_comment_diskGB": "Nominal disk size. With the ASIF sparse format this is a ceiling, not an allocation.", + "diskGB": 64 + }, + + "storage": { + "_comment": "Where images, ephemeral clones, IPSWs, and host state live. Images and clones must share one APFS volume: cloning relies on copy-on-write, which requires the same volume.", + "storeDir": "~/Library/Application Support/gitea-macos-runner", + + "_comment_minFree": "Refuse to clone a VM when the store volume has less than this free. CoW clones start nearly free but grow with every guest write, so keep this well above one clone's nominal size.", + "minFreeDiskGB": 20 + } +} diff --git a/Resources/gitea-macos-runner.entitlements b/Resources/gitea-macos-runner.entitlements new file mode 100644 index 0000000..d7d0d6e --- /dev/null +++ b/Resources/gitea-macos-runner.entitlements @@ -0,0 +1,8 @@ + + + + + com.apple.security.virtualization + + + diff --git a/Resources/launchd.plist.template b/Resources/launchd.plist.template new file mode 100644 index 0000000..e65fab6 --- /dev/null +++ b/Resources/launchd.plist.template @@ -0,0 +1,56 @@ + + + + + + Label + {{LABEL}} + + ProgramArguments + + {{PROGRAM}} +{{ARGUMENTS}} + + + RunAtLoad + + + KeepAlive + + SuccessfulExit + + + + + ThrottleInterval + 30 + + ProcessType + Interactive + + StandardOutPath + {{STDOUT_PATH}} + + StandardErrorPath + {{STDERR_PATH}} + + EnvironmentVariables + + PATH + /usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin + + + diff --git a/Resources/provision.sh b/Resources/provision.sh new file mode 100755 index 0000000..7604e30 --- /dev/null +++ b/Resources/provision.sh @@ -0,0 +1,397 @@ +#!/bin/bash +# +# provision.sh — run once inside a freshly installed macOS guest. +# +# Uploaded to /tmp/provision.sh by GuestProvisioner.runProvisionScript and run +# under sudo. Non-secret values arrive via the environment (GUEST_USER, +# GITEA_HOST) rather than as arguments, since arguments are visible to every +# process on the guest via ps. The account password is never passed here at all: +# it is fed to `sudo -S` on stdin from a mode-0600 file, which this script then +# detaches from (see `exec &2 + exit 1 +fi + +log() { echo "provision.sh: $*"; } +warn() { echo "provision.sh: WARNING: $*" >&2; } + +# macOS ships no timeout(1) — it is GNU coreutils, not BSD. Several steps here +# can block forever (softwareupdate against an unreachable server, ssh-keyscan +# against a firewalled host), and a hung provision looks exactly like a hung VM +# from the host side, so they all get bounded by hand. +# +# Usage: run_with_timeout SECONDS cmd args... → 124 on timeout. +run_with_timeout() { + local secs="$1" + shift + "$@" & + local pid=$! + local waited=0 + while kill -0 "$pid" 2>/dev/null; do + if [ "$waited" -ge "$secs" ]; then + kill -TERM "$pid" 2>/dev/null || true + sleep 2 + kill -KILL "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + return 124 + fi + sleep 1 + waited=$((waited + 1)) + done + wait "$pid" +} + +# Run a command as the runner account, in its own login context. +as_guest_user() { + launchctl asuser "$(id -u "$GUEST_USER")" sudo -u "$GUEST_USER" "$@" 2>/dev/null \ + || sudo -u "$GUEST_USER" "$@" +} + +# -------------------------------------------------------------------------- +# 1. Passwordless sudo for the runner account +# +# This comes first on purpose: every later step in this script and every later +# command GuestProvisioner issues assumes `sudo -n` works. Validated with +# `visudo -cf` on a temporary file BEFORE moving it into place — a syntax error +# in sudoers locks the account out of sudo entirely, and there is no recovery in +# a headless VM. +# -------------------------------------------------------------------------- +log "configuring passwordless sudo for ${GUEST_USER}" +SUDOERS_TMP="$(mktemp /tmp/gmr-sudoers.XXXXXX)" +cat >"$SUDOERS_TMP" </dev/null 2>&1; then + mkdir -p /etc/sudoers.d + chmod 755 /etc/sudoers.d + install -m 0440 -o root -g wheel "$SUDOERS_TMP" /etc/sudoers.d/gitea-macos-runner + rm -f "$SUDOERS_TMP" + log "passwordless sudo installed at /etc/sudoers.d/gitea-macos-runner" +else + rm -f "$SUDOERS_TMP" + echo "provision.sh: generated sudoers drop-in failed validation; refusing to install it" >&2 + exit 1 +fi + +# -------------------------------------------------------------------------- +# 2. /usr/local/bin and a PATH that non-interactive SSH sessions actually see +# +# On a clean arm64 macOS install /usr/local does not exist at all, and +# `installer -pkg node.pkg` plus the gitea-runner binary both land there. +# +# The PATH half matters more than it looks: an `ssh host command` invocation +# runs a NON-login, NON-interactive shell, so /etc/zprofile (which is where +# path_helper injects /usr/local/bin) is never sourced. Without this the +# orchestrator's `gitea-runner …` invocation fails with "command not found" even +# though the binary is installed. /etc/zshenv is the one file zsh reads for +# every invocation, login or not. +# -------------------------------------------------------------------------- +log "ensuring /usr/local/bin exists and is on PATH for non-login shells" +mkdir -p /usr/local/bin +chown root:wheel /usr/local /usr/local/bin +chmod 755 /usr/local /usr/local/bin + +ZSHENV_MARKER="# gitea-macos-runner: ensure /usr/local/bin on PATH" +if [ ! -f /etc/zshenv ] || ! grep -qF "$ZSHENV_MARKER" /etc/zshenv 2>/dev/null; then + cat >>/etc/zshenv </dev/null; then + cat >>/etc/bashrc </dev/null 2>&1 || warn "systemsetup -setsleep failed (continuing; pmset below is authoritative)" +systemsetup -setcomputersleep Off >/dev/null 2>&1 || true +systemsetup -setdisplaysleep Off >/dev/null 2>&1 || true +systemsetup -setharddisksleep Off >/dev/null 2>&1 || true + +pmset -a sleep 0 displaysleep 0 disksleep 0 >/dev/null 2>&1 || warn "pmset sleep settings failed" +# standby/autopoweroff/powernap only exist on some models; ignore failures. +pmset -a standby 0 >/dev/null 2>&1 || true +pmset -a autopoweroff 0 >/dev/null 2>&1 || true +pmset -a powernap 0 >/dev/null 2>&1 || true +pmset -a womp 0 >/dev/null 2>&1 || true + +# Screen saver idle time 0 == never. -currentHost because the screensaver +# domain is per-host, and as the user because it is a per-user preference. +as_guest_user defaults -currentHost write com.apple.screensaver idleTime -int 0 >/dev/null 2>&1 \ + || warn "could not disable the screen saver idle timer" +as_guest_user defaults write com.apple.screensaver askForPassword -int 0 >/dev/null 2>&1 || true +as_guest_user defaults write com.apple.screensaver askForPasswordDelay -int 0 >/dev/null 2>&1 || true + +# Auto-login keeps the guest's GUI session alive after a reboot, which some +# toolchains (simulators, codesign against the login keychain) depend on. +# VZMacGuestProvisioningOptions.logsInAutomatically already sets this on first +# boot; re-asserting it here keeps `image provision` runs consistent. +defaults write /Library/Preferences/com.apple.loginwindow autoLoginUser -string "$GUEST_USER" >/dev/null 2>&1 || true + +# -------------------------------------------------------------------------- +# 4. Disable Spotlight indexing +# +# Indexing a checkout and a build directory is pure waste in a VM that is +# destroyed after one job, and it competes for I/O with the build itself. +# -------------------------------------------------------------------------- +log "disabling Spotlight indexing" +mdutil -a -i off >/dev/null 2>&1 || warn "mdutil -a -i off failed" +# Drop any index that the installer already built. +mdutil -a -E >/dev/null 2>&1 || true + +# -------------------------------------------------------------------------- +# 5. Raise file descriptor limits +# +# The stock 256 soft limit is exhausted by npm installs and by Xcode builds of +# any size, and the failure mode ("EMFILE: too many open files") reads like a +# bug in the job rather than in the image. +# -------------------------------------------------------------------------- +log "raising the maxfiles limit" +cat >/Library/LaunchDaemons/limit.maxfiles.plist <<'EOF' + + + + + Label + limit.maxfiles + ProgramArguments + + launchctl + limit + maxfiles + 65536 + 200000 + + RunAtLoad + + ServiceIPC + + + +EOF +chown root:wheel /Library/LaunchDaemons/limit.maxfiles.plist +chmod 644 /Library/LaunchDaemons/limit.maxfiles.plist +# Already-loaded is not an error on a re-run, hence the `|| true`. +launchctl load -w /Library/LaunchDaemons/limit.maxfiles.plist >/dev/null 2>&1 || true +# Apply now too, so this boot benefits without a restart. +launchctl limit maxfiles 65536 200000 >/dev/null 2>&1 || true + +# -------------------------------------------------------------------------- +# 6. Pre-seed known_hosts +# +# Without this, a git+ssh checkout blocks forever on an interactive host-key +# confirmation that nothing will ever answer — and it blocks *silently*, so the +# job just sits there until jobTimeout. +# +# Seeded system-wide (/etc/ssh/ssh_known_hosts) rather than into the user's +# ~/.ssh, so it survives a job that resets the home directory. +# -------------------------------------------------------------------------- +log "pre-seeding SSH host keys" +KNOWN_HOSTS=/etc/ssh/ssh_known_hosts +mkdir -p /etc/ssh +touch "$KNOWN_HOSTS" +chmod 644 "$KNOWN_HOSTS" + +seed_host_key() { + local host="$1" + [ -n "$host" ] || return 0 + # Already present? Nothing to do — keeps re-runs from growing the file. + if ssh-keygen -F "$host" -f "$KNOWN_HOSTS" >/dev/null 2>&1; then + log "host key for ${host} already present" + return 0 + fi + local tmp + tmp="$(mktemp /tmp/gmr-keyscan.XXXXXX)" + if run_with_timeout 30 ssh-keyscan -t rsa,ecdsa,ed25519 "$host" >"$tmp" 2>/dev/null && [ -s "$tmp" ]; then + cat "$tmp" >>"$KNOWN_HOSTS" + log "seeded host key for ${host}" + else + warn "ssh-keyscan for ${host} failed or timed out; git+ssh checkouts against it may hang" + fi + rm -f "$tmp" +} + +seed_host_key github.com +seed_host_key "$GITEA_HOST" + +# Belt and braces: if a keyscan failed, a checkout should fail fast rather than +# block on a prompt no one can answer. +SSHCONF_MARKER="# gitea-macos-runner: never prompt for unknown host keys" +if [ ! -f /etc/ssh/ssh_config ] || ! grep -qF "$SSHCONF_MARKER" /etc/ssh/ssh_config 2>/dev/null; then + cat >>/etc/ssh/ssh_config </dev/null 2>&1; then + log "Command Line Tools already installed" + return 0 + fi + if [ -x /Applications/Xcode.app/Contents/Developer/usr/bin/git ]; then + log "Xcode is installed; skipping Command Line Tools" + return 0 + fi + + log "installing Command Line Tools (this can take several minutes)" + local sentinel=/tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress + touch "$sentinel" + + local label + label="$(softwareupdate -l 2>/dev/null \ + | sed -n 's/^.*Label: \(Command Line Tools.*\)$/\1/p' \ + | tail -1 || true)" + + local rc=0 + if [ -n "$label" ]; then + log "found update label: ${label}" + run_with_timeout 2700 softwareupdate -i "$label" --verbose || rc=$? + else + warn "softwareupdate listed no Command Line Tools package" + rc=1 + fi + + rm -f "$sentinel" + + if [ "$rc" -eq 124 ]; then + warn "Command Line Tools install timed out" + elif [ "$rc" -ne 0 ]; then + warn "Command Line Tools install failed (exit ${rc})" + fi + + if [ -d /Library/Developer/CommandLineTools ]; then + xcode-select --switch /Library/Developer/CommandLineTools >/dev/null 2>&1 || true + fi + + if pkgutil --pkg-info=com.apple.pkg.CLTools_Executables >/dev/null 2>&1; then + log "Command Line Tools installed" + return 0 + fi + return 1 +} + +if [ "$INSTALL_CLT" != "0" ]; then + if ! install_command_line_tools; then + warn "Command Line Tools are not installed. git will not work in this guest." + warn "Re-run with: image provision --xcode-xip /path/to/Xcode.xip" + fi +else + log "INSTALL_CLT=0; skipping Command Line Tools" +fi + +# -------------------------------------------------------------------------- +# 8. Sanity checks +# +# Node.js and the gitea-runner binary are installed separately by +# GuestProvisioner (host-side download, then upload), not here, so their absence +# at this point is expected and only reported. +# +# NOTE for future edits: do NOT write a gitea-runner config.yaml that sets +# runner.labels. That key silently overrides the --labels passed at +# registration, and the runner would advertise labels the server never matches. +# -------------------------------------------------------------------------- +log "running sanity checks" +export PATH="/usr/local/bin:$PATH" + +if ! command -v bash >/dev/null 2>&1; then + echo "provision.sh: bash is missing — this guest cannot run Gitea Actions" >&2 + exit 1 +fi +log "bash: $(bash --version | head -1)" + +# Guarded by the CLT check so this cannot be the call that hangs on the GUI +# installer dialog. +if pkgutil --pkg-info=com.apple.pkg.CLTools_Executables >/dev/null 2>&1 \ + || [ -x /Applications/Xcode.app/Contents/Developer/usr/bin/git ]; then + if run_with_timeout 60 git --version >/dev/null 2>&1; then + log "git: $(git --version)" + else + warn "git is present but did not respond within 60s" + fi +else + warn "git is unavailable (no Command Line Tools); host-side verifyToolchain will fail the build" +fi + +command -v node >/dev/null 2>&1 && log "node: $(node --version)" || log "node: not installed yet (host installs it next)" +command -v gitea-runner >/dev/null 2>&1 && log "gitea-runner: present" || log "gitea-runner: not installed yet (host installs it next)" + +log "host-side steps remaining: Node.js, gitea-runner binary" +echo "PROVISION_OK" diff --git a/Sources/RunnerCore/Config.swift b/Sources/RunnerCore/Config.swift new file mode 100644 index 0000000..7b308cd --- /dev/null +++ b/Sources/RunnerCore/Config.swift @@ -0,0 +1,733 @@ +import Foundation + +/// The runner's on-disk configuration, loaded from +/// `~/.config/gitea-macos-runner/config.json`. +/// +/// Every section has defaults, and decoding tolerates missing keys, so a minimal +/// config only needs `gitea.instanceURL` plus a way to obtain tokens. See +/// `Resources/config.example.json` for an annotated full example. +public struct RunnerConfig: Codable, Sendable, Equatable { + + // MARK: - Sections + + /// How to reach the Gitea instance and how to authenticate to it. + public struct GiteaSection: Codable, Sendable, Equatable { + /// Base URL of the Gitea instance, e.g. `https://gitea.example.com`. + /// Paths are appended to this, so a trailing slash is harmless. + public var instanceURL: URL + + /// A Gitea admin API token, inline. Used for the admin Actions endpoints + /// (job listing, runner listing/deletion, registration-token minting). + /// Prefer ``adminTokenFile`` so the secret is not world-readable in JSON. + /// + /// - Important: Exactly one of this and ``adminTokenFile`` must be set. + /// ``RunnerConfig/validated()`` rejects both-set and neither-set alike; + /// a stale inline token sitting beside a live token file is exactly the + /// ambiguity that produces a baffling 401 at 3am. + public var adminToken: String? + + /// Path to a file whose (trimmed) contents are the admin API token. + /// Tilde-expanded. + /// + /// - Important: Exactly one of this and ``adminToken`` must be set — see + /// that property. This one does *not* silently win over an inline + /// value; setting both is a validation error. + public var adminTokenFile: String? + + /// The shared runner registration token, inline. + /// + /// - Important: Registration tokens are **reusable** and **scoped**. + /// Minting a new token for a scope invalidates all prior tokens of that + /// scope, so per-VM tokens must never be pre-generated. One shared + /// token serves the whole fleet. See docs/DESIGN.md, Verified Fact 4. + public var registrationToken: String? + + /// Path to a file whose (trimmed) contents are the registration token. + /// Tilde-expanded. Takes precedence over ``registrationToken``. + public var registrationTokenFile: String? + + /// When no static registration token is configured, fetch one from + /// `POST /api/v1/admin/actions/runners/registration-token`. + /// + /// Defaults to `false` because that endpoint effectively returns the + /// *existing* active token for the scope, and any implementation change + /// that made it mint a fresh one would invalidate tokens held by runners + /// registered elsewhere. + public var fetchRegistrationTokenViaAPI: Bool + + public init( + instanceURL: URL, + adminToken: String? = nil, + adminTokenFile: String? = nil, + registrationToken: String? = nil, + registrationTokenFile: String? = nil, + fetchRegistrationTokenViaAPI: Bool = false + ) { + self.instanceURL = instanceURL + self.adminToken = adminToken + self.adminTokenFile = adminTokenFile + self.registrationToken = registrationToken + self.registrationTokenFile = registrationTokenFile + self.fetchRegistrationTokenViaAPI = fetchRegistrationTokenViaAPI + } + } + + /// Identity and provenance of the runners registered inside each guest. + public struct RunnerSection: Codable, Sendable, Equatable { + /// Bare label names this host serves. Matched case-sensitively against a + /// job's `labels` (i.e. its `runs-on:`). The `:host` schema suffix is + /// added only when calling `gitea-runner register`. + public var labels: [String] + + /// Prefix for generated runner names. Must be distinctive enough that the + /// reconcile loop can tell our stale rows from other runners'. + public var namePrefix: String + + /// Template for the `gitea-runner` release asset to install in the guest. + /// `{version}` is substituted with ``version``. + public var runnerDownloadURL: String + + /// The `gitea-runner` version to install (v3.x; the binary was renamed + /// from `act_runner`, and now lives at `gitea.com/gitea/runner`). + public var version: String + + public init( + labels: [String] = ["macos-arm64"], + namePrefix: String = "macos-vm-", + runnerDownloadURL: String = RunnerSection.defaultDownloadURLTemplate, + version: String = "3.0.2" + ) { + self.labels = labels + self.namePrefix = namePrefix + self.runnerDownloadURL = runnerDownloadURL + self.version = version + } + + /// Default release-asset URL template for the darwin/arm64 build. + public static let defaultDownloadURLTemplate = + "https://gitea.com/gitea/runner/releases/download/v{version}/gitea-runner-{version}-darwin-arm64" + + /// ``runnerDownloadURL`` with `{version}` substituted. + public var resolvedDownloadURL: URL { + get throws { + let substituted = runnerDownloadURL.replacingOccurrences(of: "{version}", with: version) + guard let url = URL(string: substituted), url.scheme != nil else { + throw CoreError.configInvalid( + "runner.runnerDownloadURL does not form a valid URL: \(substituted)") + } + return url + } + } + } + + /// Polling cadence, concurrency, and the timeouts that bound a stuck VM. + public struct SchedulerSection: Codable, Sendable, Equatable { + /// How many macOS guests may run at once. + /// + /// - Important: Hard-clamped to 2 by ``RunnerConfig/validated()``. Apple's + /// kernel enforces a limit of two concurrent macOS VMs per host; a third + /// `start()` fails with `VZError.virtualMachineLimitExceeded`. + public var maxConcurrentVMs: Int + + /// Seconds between queued-job polls. + public var pollIntervalSeconds: Int + + /// Seconds between reconcile passes that sweep orphaned runner rows. + public var reconcileIntervalSeconds: Int + + /// Wall-clock ceiling on a single job before its VM is torn down. + public var jobTimeoutMinutes: Int + + /// Ceiling on boot + DHCP lease + SSH readiness before a slot is + /// declared dead and recycled. + public var bootTimeoutSeconds: Int + + public init( + maxConcurrentVMs: Int = 2, + pollIntervalSeconds: Int = 5, + reconcileIntervalSeconds: Int = 300, + jobTimeoutMinutes: Int = 120, + bootTimeoutSeconds: Int = 300 + ) { + self.maxConcurrentVMs = maxConcurrentVMs + self.pollIntervalSeconds = pollIntervalSeconds + self.reconcileIntervalSeconds = reconcileIntervalSeconds + self.jobTimeoutMinutes = jobTimeoutMinutes + self.bootTimeoutSeconds = bootTimeoutSeconds + } + + /// The absolute cap on concurrent macOS guests, enforced by the kernel. + public static let hardMaxConcurrentVMs = 2 + } + + /// Shape of each guest VM and the credentials used to reach it over SSH. + /// + /// - Note: These credentials only ever exist on the NAT network between the + /// host and its own ephemeral guests. They are not secrets in any + /// meaningful sense, but they are also why the NAT attachment (rather than + /// bridged networking) is not optional. + public struct GuestSection: Codable, Sendable, Equatable { + /// The admin account created by Setup Assistant automation. + public var username: String + /// That account's password, also used for SSH password auth. + public var password: String + /// Virtual CPUs per guest. + public var cpuCount: Int + /// RAM per guest, in gibibytes. + public var memoryGB: Int + /// Backing disk size per guest, in gibibytes. Sparse (ASIF) where + /// available, so this is a ceiling rather than an allocation. + public var diskGB: Int + + public init( + username: String = "admin", + password: String = "admin", + cpuCount: Int = 4, + memoryGB: Int = 8, + diskGB: Int = 64 + ) { + self.username = username + self.password = password + self.cpuCount = cpuCount + self.memoryGB = memoryGB + self.diskGB = diskGB + } + } + + /// Where images, clones, IPSWs, and host state live on disk. + public struct StorageSection: Codable, Sendable, Equatable { + /// Root of the store. Tilde-expanded. + /// + /// - Important: Clones are made with APFS copy-on-write, which requires + /// source and destination on the *same volume*. Keep base images and + /// ephemeral clones under one root. + public var storeDir: String + + /// Refuse to clone a new VM when the store volume has less than this + /// much free space. CoW clones start near-free but grow as the guest + /// writes, so a floor well above one clone's nominal size is prudent. + public var minFreeDiskGB: Int + + public init( + storeDir: String = "~/Library/Application Support/gitea-macos-runner", + minFreeDiskGB: Int = 20 + ) { + self.storeDir = storeDir + self.minFreeDiskGB = minFreeDiskGB + } + } + + // MARK: - Stored properties + + public var gitea: GiteaSection + public var runner: RunnerSection + public var scheduler: SchedulerSection + public var guest: GuestSection + public var storage: StorageSection + + public init( + gitea: GiteaSection, + runner: RunnerSection = .init(), + scheduler: SchedulerSection = .init(), + guest: GuestSection = .init(), + storage: StorageSection = .init() + ) { + self.gitea = gitea + self.runner = runner + self.scheduler = scheduler + self.guest = guest + self.storage = storage + } + + // MARK: - Defaults + + /// A configuration with every default applied and a placeholder instance URL. + /// Used by `config init` to seed a new file, and by tests. + public static var `default`: RunnerConfig { + RunnerConfig(gitea: GiteaSection(instanceURL: URL(string: "https://gitea.example.com")!)) + } + + /// The conventional config path, `~/.config/gitea-macos-runner/config.json`, + /// tilde-expanded. + public static var defaultPath: String { + expandTilde("~/.config/gitea-macos-runner/config.json") + } + + // MARK: - Loading & validation + + /// Loads and validates a configuration from a JSON file. + /// + /// - Parameter path: Filesystem path; tilde-expanded. Defaults to + /// ``defaultPath``. + /// - Returns: A validated configuration. + /// - Throws: ``CoreError/configInvalid(_:)`` if the file is missing, + /// unparseable, or fails ``validated()``. + public static func load(from path: String = RunnerConfig.defaultPath) throws -> RunnerConfig { + let expanded = expandTilde(path) + + guard FileManager.default.fileExists(atPath: expanded) else { + throw CoreError.configInvalid("no configuration file at \(expanded)") + } + + let data: Data + do { + data = try Data(contentsOf: URL(fileURLWithPath: expanded)) + } catch { + throw CoreError.configInvalid("cannot read \(expanded): \(error.localizedDescription)") + } + + let decoded: RunnerConfig + do { + decoded = try JSONDecoder().decode(RunnerConfig.self, from: data) + } catch let error as DecodingError { + throw CoreError.configInvalid("\(expanded): \(RunnerConfig.describe(error))") + } catch { + throw CoreError.configInvalid("\(expanded): \(error.localizedDescription)") + } + + return try decoded.validated() + } + + /// Renders a `DecodingError` as something an operator can act on, since the + /// default description is a multi-line dump of the underlying context. + private static func describe(_ error: DecodingError) -> String { + func keyPath(_ context: DecodingError.Context) -> String { + let path = context.codingPath.map(\.stringValue).joined(separator: ".") + return path.isEmpty ? "" : path + } + switch error { + case .keyNotFound(let key, let context): + let parent = keyPath(context) + return "missing required key `\(key.stringValue)`" + + (parent == "" ? "" : " under `\(parent)`") + case .typeMismatch(let type, let context): + return "key `\(keyPath(context))` has the wrong type (expected \(type))" + case .valueNotFound(let type, let context): + return "key `\(keyPath(context))` is null (expected \(type))" + case .dataCorrupted(let context): + let path = keyPath(context) + return path == "" + ? "not valid JSON (\(context.debugDescription))" + : "key `\(path)` is malformed (\(context.debugDescription))" + @unknown default: + return "\(error)" + } + } + + /// Writes this configuration as pretty-printed JSON, creating parent + /// directories as needed. + /// + /// - Parameter path: Destination; tilde-expanded. + public func save(to path: String) throws { + let expanded = RunnerConfig.expandTilde(path) + let url = URL(fileURLWithPath: expanded) + + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes] + + do { + try FileManager.default.createDirectory( + at: url.deletingLastPathComponent(), + withIntermediateDirectories: true) + var data = try encoder.encode(self) + data.append(0x0A) // trailing newline, so the file is diff-friendly + try data.write(to: url, options: .atomic) + } catch { + throw CoreError.configInvalid("cannot write \(expanded): \(error.localizedDescription)") + } + } + + /// Writes the annotated example configuration shipped in `Resources/`, or — + /// when that resource is not reachable — this configuration serialized by + /// ``save(to:)``. + /// + /// `config init` uses this so a fresh install lands an operator on the + /// commented example rather than a bare JSON dump. + /// + /// - Parameters: + /// - path: Destination; tilde-expanded. + /// - exampleContents: The example document, if the caller could load it. + /// - overwrite: When `false` (the default) an existing file is left alone. + /// - Returns: `true` if a file was written, `false` if one already existed. + @discardableResult + public func writeExample( + to path: String, + exampleContents: String? = nil, + overwrite: Bool = false + ) throws -> Bool { + let expanded = RunnerConfig.expandTilde(path) + if !overwrite, FileManager.default.fileExists(atPath: expanded) { + return false + } + guard let example = exampleContents else { + try save(to: expanded) + return true + } + let url = URL(fileURLWithPath: expanded) + do { + try FileManager.default.createDirectory( + at: url.deletingLastPathComponent(), + withIntermediateDirectories: true) + try Data(example.utf8).write(to: url, options: .atomic) + } catch { + throw CoreError.configInvalid("cannot write \(expanded): \(error.localizedDescription)") + } + return true + } + + /// Returns a normalized copy, or throws describing what is wrong. + /// + /// Normalization clamps ``SchedulerSection/maxConcurrentVMs`` into + /// `1...2` and expands tildes in path-bearing fields. Validation rejects a + /// non-http(s) instance URL, an empty label list, an empty name prefix, + /// non-positive intervals or timeouts, a guest with fewer than 1 CPU or less + /// than 1 GB of RAM, and a configuration with no way to obtain either token. + /// + /// - Returns: The normalized configuration. + /// - Throws: ``CoreError/configInvalid(_:)``. + public func validated() throws -> RunnerConfig { + var c = self + + // --- gitea.instanceURL ------------------------------------------------ + let scheme = c.gitea.instanceURL.scheme?.lowercased() + guard scheme == "http" || scheme == "https" else { + throw CoreError.configInvalid( + "gitea.instanceURL must be an http:// or https:// URL, got \"\(c.gitea.instanceURL.absoluteString)\"") + } + guard let host = c.gitea.instanceURL.host, !host.isEmpty else { + throw CoreError.configInvalid( + "gitea.instanceURL has no host: \"\(c.gitea.instanceURL.absoluteString)\"") + } + + // --- admin token: exactly one source ---------------------------------- + // + // Both-set is rejected rather than silently preferring one, because a + // stale inline token sitting next to a live token file is precisely the + // kind of ambiguity that produces a baffling 401 at 3am. + let inlineAdmin = RunnerConfig.nonEmpty(c.gitea.adminToken) + let fileAdmin = RunnerConfig.nonEmpty(c.gitea.adminTokenFile) + switch (inlineAdmin, fileAdmin) { + case (nil, nil): + throw CoreError.configInvalid( + "no admin API token configured: set exactly one of gitea.adminToken or gitea.adminTokenFile") + case (.some, .some): + throw CoreError.configInvalid( + "gitea.adminToken and gitea.adminTokenFile are both set: use exactly one") + default: + break + } + c.gitea.adminToken = inlineAdmin + c.gitea.adminTokenFile = fileAdmin.map(RunnerConfig.expandTilde) + + // --- registration token: at least one source -------------------------- + // + // Unlike the admin token, a file and an inline value are not mutually + // exclusive here (the file wins); what is rejected is having no source + // at all with the API fallback switched off. + let inlineReg = RunnerConfig.nonEmpty(c.gitea.registrationToken) + let fileReg = RunnerConfig.nonEmpty(c.gitea.registrationTokenFile) + if inlineReg == nil, fileReg == nil, !c.gitea.fetchRegistrationTokenViaAPI { + throw CoreError.configInvalid( + "no runner registration token configured: set gitea.registrationTokenFile " + + "(or gitea.registrationToken), or set gitea.fetchRegistrationTokenViaAPI to true") + } + c.gitea.registrationToken = inlineReg + c.gitea.registrationTokenFile = fileReg.map(RunnerConfig.expandTilde) + + // --- runner ----------------------------------------------------------- + let labels = c.runner.labels.map { $0.trimmingCharacters(in: .whitespaces) } + guard !labels.isEmpty else { + throw CoreError.configInvalid("runner.labels must not be empty") + } + if labels.contains(where: \.isEmpty) { + throw CoreError.configInvalid("runner.labels contains an empty label name") + } + // Bare names only: the `:schema` suffix belongs on the `register + // --labels` argument, never in stored config, and Gitea reports bare + // names on jobs — so a configured "macos-arm64:host" would never match. + if let schemed = labels.first(where: { $0.contains(":") }) { + throw CoreError.configInvalid( + "runner.labels must contain bare names only, but \"\(schemed)\" carries a ':schema' suffix; " + + "the schema is appended automatically at registration time") + } + c.runner.labels = labels + + let prefix = c.runner.namePrefix.trimmingCharacters(in: .whitespaces) + guard !prefix.isEmpty else { + throw CoreError.configInvalid("runner.namePrefix must not be empty") + } + c.runner.namePrefix = prefix + + guard !c.runner.version.trimmingCharacters(in: .whitespaces).isEmpty else { + throw CoreError.configInvalid("runner.version must not be empty") + } + c.runner.version = c.runner.version.trimmingCharacters(in: .whitespaces) + _ = try c.runner.resolvedDownloadURL + + // --- scheduler -------------------------------------------------------- + // + // Clamped rather than rejected: Apple's kernel caps concurrent macOS + // guests at two, and that is not a limit a config file gets to negotiate. + c.scheduler.maxConcurrentVMs = min( + max(c.scheduler.maxConcurrentVMs, 1), + SchedulerSection.hardMaxConcurrentVMs) + + guard c.scheduler.pollIntervalSeconds > 0 else { + throw CoreError.configInvalid("scheduler.pollIntervalSeconds must be greater than 0") + } + guard c.scheduler.reconcileIntervalSeconds > 0 else { + throw CoreError.configInvalid("scheduler.reconcileIntervalSeconds must be greater than 0") + } + guard c.scheduler.jobTimeoutMinutes > 0 else { + throw CoreError.configInvalid("scheduler.jobTimeoutMinutes must be greater than 0") + } + guard c.scheduler.bootTimeoutSeconds > 0 else { + throw CoreError.configInvalid("scheduler.bootTimeoutSeconds must be greater than 0") + } + + // --- guest ------------------------------------------------------------ + guard !c.guest.username.trimmingCharacters(in: .whitespaces).isEmpty else { + throw CoreError.configInvalid("guest.username must not be empty") + } + // SSH password auth is the only channel into the guest, and an empty + // password would leave the boot hanging at authentication with no + // diagnostic worth reading. + guard !c.guest.password.isEmpty else { + throw CoreError.configInvalid("guest.password must not be empty") + } + guard c.guest.cpuCount >= 1 else { + throw CoreError.configInvalid("guest.cpuCount must be at least 1") + } + guard c.guest.memoryGB >= 1 else { + throw CoreError.configInvalid("guest.memoryGB must be at least 1") + } + guard c.guest.diskGB >= 1 else { + throw CoreError.configInvalid("guest.diskGB must be at least 1") + } + + // --- storage ---------------------------------------------------------- + let storeDir = c.storage.storeDir.trimmingCharacters(in: .whitespaces) + guard !storeDir.isEmpty else { + throw CoreError.configInvalid("storage.storeDir must not be empty") + } + c.storage.storeDir = RunnerConfig.expandTilde(storeDir) + guard c.storage.minFreeDiskGB >= 0 else { + throw CoreError.configInvalid("storage.minFreeDiskGB must not be negative") + } + + return c + } + + /// Trims a string and maps `""` to `nil`, so an empty JSON value reads as + /// "not configured" rather than as a zero-length token. + private static func nonEmpty(_ value: String?) -> String? { + guard let trimmed = value?.trimmingCharacters(in: .whitespacesAndNewlines), + !trimmed.isEmpty + else { return nil } + return trimmed + } + + /// The admin API token, resolved from ``GiteaSection/adminTokenFile`` (read + /// and trimmed) or ``GiteaSection/adminToken``. + /// + /// On a configuration that has been through ``validated()`` exactly one of + /// those is set, so the file-first order here never actually chooses between + /// two live values. + /// + /// - Returns: The token, or `nil` when neither source is configured. + public func resolveAdminToken() throws -> String? { + if let path = RunnerConfig.nonEmpty(gitea.adminTokenFile) { + return try RunnerConfig.readTokenFile(path, describedAs: "gitea.adminTokenFile") + } + return RunnerConfig.nonEmpty(gitea.adminToken) + } + + /// The registration token from static configuration only — file first, then + /// inline value. Returns `nil` when the caller must fall back to the API + /// (see ``GiteaSection/fetchRegistrationTokenViaAPI``). + public func resolveStaticRegistrationToken() throws -> String? { + if let path = RunnerConfig.nonEmpty(gitea.registrationTokenFile) { + return try RunnerConfig.readTokenFile(path, describedAs: "gitea.registrationTokenFile") + } + return RunnerConfig.nonEmpty(gitea.registrationToken) + } + + /// Reads a secret from a file: tilde-expanded, trimmed of surrounding + /// whitespace and newlines (an `echo`-written token file always has one). + /// + /// - Throws: ``CoreError/configInvalid(_:)`` when the file is missing, + /// unreadable, not UTF-8, or empty once trimmed. + private static func readTokenFile(_ path: String, describedAs key: String) throws -> String { + let expanded = expandTilde(path) + guard FileManager.default.fileExists(atPath: expanded) else { + throw CoreError.configInvalid("\(key): no such file: \(expanded)") + } + let data: Data + do { + data = try Data(contentsOf: URL(fileURLWithPath: expanded)) + } catch { + throw CoreError.configInvalid("\(key): cannot read \(expanded): \(error.localizedDescription)") + } + guard let text = String(data: data, encoding: .utf8) else { + throw CoreError.configInvalid("\(key): \(expanded) is not valid UTF-8") + } + let token = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !token.isEmpty else { + throw CoreError.configInvalid("\(key): \(expanded) is empty") + } + return token + } + + /// Whether a token file is readable by users other than its owner. + /// + /// Permissions are deliberately **not** enforced — refusing to start because + /// a file is `0644` would be a poor trade on a single-user CI Mac — but + /// `doctor` surfaces this as a warning. + /// + /// - Parameter path: Path to check; tilde-expanded. + /// - Returns: `true` when group or other bits are set, `false` when the file + /// is owner-only, and `nil` when the mode cannot be read. + public static func tokenFileIsGroupOrWorldReadable(_ path: String) -> Bool? { + let expanded = expandTilde(path) + guard + let attrs = try? FileManager.default.attributesOfItem(atPath: expanded), + let mode = attrs[.posixPermissions] as? NSNumber + else { return nil } + return (mode.int16Value & 0o077) != 0 + } + + /// Paths of configured token files whose permissions are looser than `0600`. + /// Empty when everything is owner-only or nothing is file-backed. + public var insecureTokenFilePaths: [String] { + [gitea.adminTokenFile, gitea.registrationTokenFile] + .compactMap { RunnerConfig.nonEmpty($0) } + .filter { RunnerConfig.tokenFileIsGroupOrWorldReadable($0) == true } + } + + /// ``StorageSection/storeDir`` with `~` expanded, as a `URL`. + public var storeDirectoryURL: URL { + URL(fileURLWithPath: RunnerConfig.expandTilde(storage.storeDir), isDirectory: true) + } + + /// The label set used for job matching. + public var labelSet: LabelSet { + LabelSet(runner.labels) + } + + // MARK: - Helpers + + /// Expands a leading `~` or `~/` to the current user's home directory. + /// + /// `NSString.expandingTildeInPath` is used rather than `FileManager`'s + /// deprecated home lookup so the behaviour matches the shell. + /// + /// - Parameter path: A possibly tilde-prefixed path. + /// - Returns: An absolute path. + public static func expandTilde(_ path: String) -> String { + (path as NSString).expandingTildeInPath + } + + // MARK: - Codable + + private enum CodingKeys: String, CodingKey { + case gitea, runner, scheduler, guest, storage + } + + /// Decodes a configuration, substituting section defaults for absent keys. + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + self.gitea = try c.decode(GiteaSection.self, forKey: .gitea) + self.runner = try c.decodeIfPresent(RunnerSection.self, forKey: .runner) ?? .init() + self.scheduler = try c.decodeIfPresent(SchedulerSection.self, forKey: .scheduler) ?? .init() + self.guest = try c.decodeIfPresent(GuestSection.self, forKey: .guest) ?? .init() + self.storage = try c.decodeIfPresent(StorageSection.self, forKey: .storage) ?? .init() + } +} + +// MARK: - Tolerant section decoding + +extension RunnerConfig.GiteaSection { + private enum CodingKeys: String, CodingKey { + case instanceURL, adminToken, adminTokenFile + case registrationToken, registrationTokenFile, fetchRegistrationTokenViaAPI + } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + self.instanceURL = try c.decode(URL.self, forKey: .instanceURL) + self.adminToken = try c.decodeIfPresent(String.self, forKey: .adminToken) + self.adminTokenFile = try c.decodeIfPresent(String.self, forKey: .adminTokenFile) + self.registrationToken = try c.decodeIfPresent(String.self, forKey: .registrationToken) + self.registrationTokenFile = try c.decodeIfPresent(String.self, forKey: .registrationTokenFile) + self.fetchRegistrationTokenViaAPI = + try c.decodeIfPresent(Bool.self, forKey: .fetchRegistrationTokenViaAPI) ?? false + } +} + +extension RunnerConfig.RunnerSection { + private enum CodingKeys: String, CodingKey { + case labels, namePrefix, runnerDownloadURL, version + } + + public init(from decoder: Decoder) throws { + let d = RunnerConfig.RunnerSection() + let c = try decoder.container(keyedBy: CodingKeys.self) + self.labels = try c.decodeIfPresent([String].self, forKey: .labels) ?? d.labels + self.namePrefix = try c.decodeIfPresent(String.self, forKey: .namePrefix) ?? d.namePrefix + self.runnerDownloadURL = + try c.decodeIfPresent(String.self, forKey: .runnerDownloadURL) ?? d.runnerDownloadURL + self.version = try c.decodeIfPresent(String.self, forKey: .version) ?? d.version + } +} + +extension RunnerConfig.SchedulerSection { + private enum CodingKeys: String, CodingKey { + case maxConcurrentVMs, pollIntervalSeconds, reconcileIntervalSeconds + case jobTimeoutMinutes, bootTimeoutSeconds + } + + public init(from decoder: Decoder) throws { + let d = RunnerConfig.SchedulerSection() + let c = try decoder.container(keyedBy: CodingKeys.self) + self.maxConcurrentVMs = + try c.decodeIfPresent(Int.self, forKey: .maxConcurrentVMs) ?? d.maxConcurrentVMs + self.pollIntervalSeconds = + try c.decodeIfPresent(Int.self, forKey: .pollIntervalSeconds) ?? d.pollIntervalSeconds + self.reconcileIntervalSeconds = + try c.decodeIfPresent(Int.self, forKey: .reconcileIntervalSeconds) ?? d.reconcileIntervalSeconds + self.jobTimeoutMinutes = + try c.decodeIfPresent(Int.self, forKey: .jobTimeoutMinutes) ?? d.jobTimeoutMinutes + self.bootTimeoutSeconds = + try c.decodeIfPresent(Int.self, forKey: .bootTimeoutSeconds) ?? d.bootTimeoutSeconds + } +} + +extension RunnerConfig.GuestSection { + private enum CodingKeys: String, CodingKey { + case username, password, cpuCount, memoryGB, diskGB + } + + public init(from decoder: Decoder) throws { + let d = RunnerConfig.GuestSection() + let c = try decoder.container(keyedBy: CodingKeys.self) + self.username = try c.decodeIfPresent(String.self, forKey: .username) ?? d.username + self.password = try c.decodeIfPresent(String.self, forKey: .password) ?? d.password + self.cpuCount = try c.decodeIfPresent(Int.self, forKey: .cpuCount) ?? d.cpuCount + self.memoryGB = try c.decodeIfPresent(Int.self, forKey: .memoryGB) ?? d.memoryGB + self.diskGB = try c.decodeIfPresent(Int.self, forKey: .diskGB) ?? d.diskGB + } +} + +extension RunnerConfig.StorageSection { + private enum CodingKeys: String, CodingKey { + case storeDir, minFreeDiskGB + } + + public init(from decoder: Decoder) throws { + let d = RunnerConfig.StorageSection() + let c = try decoder.container(keyedBy: CodingKeys.self) + self.storeDir = try c.decodeIfPresent(String.self, forKey: .storeDir) ?? d.storeDir + self.minFreeDiskGB = + try c.decodeIfPresent(Int.self, forKey: .minFreeDiskGB) ?? d.minFreeDiskGB + } +} diff --git a/Sources/RunnerCore/CoreErrors.swift b/Sources/RunnerCore/CoreErrors.swift new file mode 100644 index 0000000..41ec4d1 --- /dev/null +++ b/Sources/RunnerCore/CoreErrors.swift @@ -0,0 +1,110 @@ +import Foundation + +/// The single error domain shared by every layer of the runner. +/// +/// Host-side (`RunnerHost`) code wraps Virtualization.framework's `VZError` into +/// these cases rather than propagating it, so the CLI only ever has to render one +/// error type. `unimplemented` exists so that skeleton bodies can `throw` instead +/// of trapping in code paths where a trap would take down the daemon. +public enum CoreError: Error, Sendable { + /// A code path that has not been written yet. + case unimplemented + + /// The on-disk configuration is missing, malformed, or internally inconsistent. + /// The payload is a human-readable explanation suitable for printing to stderr. + case configInvalid(String) + + /// The Gitea API returned a non-2xx status. + /// - Parameters: + /// - status: The HTTP status code. + /// - message: The response body (truncated) or a decoded API error message. + case gitea(status: Int, message: String) + + /// An SSH session could not be established, authenticated, or the remote + /// command exited non-zero when a zero exit was required. + case sshFailed(String) + + /// A bounded wait elapsed. The payload names what was being waited on + /// (for example `"dhcp lease for aa:bb:cc:dd:ee:ff"` or `"ssh on 192.168.64.7"`). + case timeout(String) + + /// A required external tool or file was absent (`diskutil`, an IPSW, the + /// `gitea-runner` release asset, …). + case notFound(String) + + /// The host cannot run VMs: wrong architecture, unsupported macOS, missing + /// `com.apple.security.virtualization` entitlement, or a locked login keychain. + case hostUnsupported(String) + + /// Apple's kernel-enforced limit of two concurrent macOS guests was hit. + /// Surfaced distinctly because it is transient and the scheduler retries. + case vmLimitExceeded + + /// A VM bundle on disk is missing files or has an unreadable `config.json`. + case bundleCorrupt(String) + + /// Not enough free space on the store volume to safely clone or grow a VM. + /// - Parameters: + /// - requiredGB: The configured floor. + /// - availableGB: What the volume actually has. + case insufficientDiskSpace(requiredGB: Int, availableGB: Int) + + /// A subprocess (`diskutil`, `codesign`, `security`, …) exited non-zero. + case processFailed(command: String, exitCode: Int32, output: String) + + /// The image build or provisioning pipeline failed at a named stage. + case provisioningFailed(String) +} + +extension CoreError: CustomStringConvertible { + /// A one-line, user-facing rendering of the error. + public var description: String { + switch self { + case .unimplemented: + return "not implemented" + + case .configInvalid(let detail): + return "invalid configuration: \(detail)" + + case .gitea(let status, let message): + let trimmed = message.trimmingCharacters(in: .whitespacesAndNewlines) + return trimmed.isEmpty + ? "gitea API error (HTTP \(status))" + : "gitea API error (HTTP \(status)): \(trimmed)" + + case .sshFailed(let detail): + return "ssh failed: \(detail)" + + case .timeout(let what): + return "timed out waiting for \(what)" + + case .notFound(let what): + return "not found: \(what)" + + case .hostUnsupported(let detail): + return "host cannot run VMs: \(detail)" + + case .vmLimitExceeded: + return "macOS guest limit reached (Apple allows at most 2 concurrent VMs per host)" + + case .bundleCorrupt(let detail): + return "VM bundle is corrupt: \(detail)" + + case .insufficientDiskSpace(let requiredGB, let availableGB): + return "insufficient disk space: need \(requiredGB) GB free, have \(availableGB) GB" + + case .processFailed(let command, let exitCode, let output): + let trimmed = output.trimmingCharacters(in: .whitespacesAndNewlines) + return trimmed.isEmpty + ? "`\(command)` exited \(exitCode)" + : "`\(command)` exited \(exitCode): \(trimmed)" + + case .provisioningFailed(let stage): + return "provisioning failed: \(stage)" + } + } +} + +extension CoreError: LocalizedError { + public var errorDescription: String? { description } +} diff --git a/Sources/RunnerCore/DHCPLeases.swift b/Sources/RunnerCore/DHCPLeases.swift new file mode 100644 index 0000000..98dc91b --- /dev/null +++ b/Sources/RunnerCore/DHCPLeases.swift @@ -0,0 +1,246 @@ +import Foundation + +/// One entry from macOS's `/var/db/dhcpd_leases`. +/// +/// The Virtualization NAT attachment hands guests addresses from the host's +/// built-in `bootpd`, which records each lease in that file. There is no API for +/// this, so parsing the file keyed by the guest's MAC is how we learn a VM's IP. +public struct DHCPLease: Sendable, Equatable { + /// The guest's advertised hostname (`name=` in the lease block). Often the + /// guest's local hostname, sometimes absent. + public let name: String? + /// The leased IPv4 address, e.g. `192.168.64.7`. + public let ipAddress: String + /// The hardware address, **normalized**: lowercase, colon-separated, each + /// octet zero-padded to two hex digits, with the `1,` type prefix stripped. + public let hwAddress: String + /// Lease expiry, parsed from the `lease=` hex epoch, when present. + public let leaseExpiry: Date? + + public init(name: String?, ipAddress: String, hwAddress: String, leaseExpiry: Date?) { + self.name = name + self.ipAddress = ipAddress + self.hwAddress = hwAddress + self.leaseExpiry = leaseExpiry + } +} + +/// Parser for `/var/db/dhcpd_leases`. +/// +/// ## File format +/// +/// A sequence of brace-delimited blocks of `key=value` lines: +/// +/// ``` +/// { +/// name=macos-guest +/// ip_address=192.168.64.7 +/// hw_address=1,aa:bb:c:dd:ee:ff +/// identifier=1,aa:bb:c:dd:ee:ff +/// lease=0x67a1b2c3 +/// } +/// ``` +/// +/// Two details bite: +/// +/// 1. `hw_address` carries a leading hardware-type prefix (`1,` for Ethernet) +/// that is not part of the MAC. +/// 2. Octets are **not zero-padded** — `aa:bb:c:dd:ee:ff` is the same address +/// that `VZMACAddress.string` renders as `aa:bb:0c:dd:ee:ff`. Comparing raw +/// strings silently fails to match; both sides must be normalized. +/// +/// Blocks accumulate: a MAC can appear more than once as leases are renewed or +/// reissued, so lookups take the **newest** lease (latest `leaseExpiry`, falling +/// back to last-in-file when expiry is missing). +/// +/// - Note: macOS's DHCP lease time is 24 hours. That is exactly why clones must +/// reuse a small set of **persistent per-slot MACs** rather than randomizing a +/// MAC per VM: a randomized fleet would fill this file with day-long stale +/// leases and exhaust the NAT subnet. +public enum DHCPLeaseParser { + /// The canonical path of the lease database. + public static let defaultPath = "/var/db/dhcpd_leases" + + /// Parses the whole file. + /// + /// Malformed blocks are skipped rather than throwing — the file is written + /// by another process and may be observed mid-write. + /// + /// - Parameter text: The file's contents. + /// - Returns: Leases in file order. + public static func parse(_ text: String) -> [DHCPLease] { + var leases: [DHCPLease] = [] + var fields: [String: String] = [:] + var inBlock = false + + for rawLine in text.split(separator: "\n", omittingEmptySubsequences: false) { + let line = rawLine.trimmingCharacters(in: .whitespaces) + if line.isEmpty { continue } + + if line.hasPrefix("{") { + // A `{` while already inside a block means the previous one was + // truncated (the file is written by bootpd and can be observed + // mid-write). Drop it and start over rather than merging. + inBlock = true + fields = [:] + continue + } + + if line.hasPrefix("}") { + if inBlock, let lease = makeLease(from: fields) { leases.append(lease) } + inBlock = false + fields = [:] + continue + } + + guard inBlock, let separator = line.firstIndex(of: "=") else { continue } + let key = line[line.startIndex.. DHCPLease? { + guard + let ip = fields["ip_address"], !ip.isEmpty, + let rawMAC = fields["hw_address"] ?? fields["identifier"], + let mac = normalizeMAC(rawMAC) + else { return nil } + + let name = fields["name"].flatMap { $0.isEmpty ? nil : $0 } + return DHCPLease( + name: name, + ipAddress: ip, + hwAddress: mac, + leaseExpiry: fields["lease"].flatMap(parseLeaseTime) + ) + } + + /// Parses a `lease=` value. `bootpd` writes a hex epoch (`0x66b2c0de`), but + /// a plain decimal epoch has been observed too, so both are accepted. + private static func parseLeaseTime(_ raw: String) -> Date? { + let text = raw.trimmingCharacters(in: .whitespaces).lowercased() + guard !text.isEmpty else { return nil } + + let seconds: UInt64? + if text.hasPrefix("0x") { + seconds = UInt64(text.dropFirst(2), radix: 16) + } else { + seconds = UInt64(text, radix: 10) + } + + guard let seconds else { return nil } + return Date(timeIntervalSince1970: TimeInterval(seconds)) + } + + /// Reads and parses the lease database from disk. + /// + /// - Parameter path: Defaults to ``defaultPath``. + /// - Returns: Leases, or `[]` when the file does not exist yet (no guest has + /// ever leased an address). + public static func parseFile(at path: String = DHCPLeaseParser.defaultPath) -> [DHCPLease] { + guard let text = try? String(contentsOfFile: path, encoding: .utf8) else { return [] } + return parse(text) + } + + /// Finds the current IP for a MAC. + /// + /// Both `mac` and each lease's `hwAddress` are normalized before comparison. + /// + /// - Parameters: + /// - mac: The guest's MAC, in any common rendering. + /// - leases: Leases from ``parse(_:)``. + /// - Returns: The newest matching lease's IP, or `nil`. + public static func ipAddress(forMAC mac: String, in leases: [DHCPLease]) -> String? { + lease(forMAC: mac, in: leases)?.ipAddress + } + + /// Finds the newest lease for a MAC. + /// + /// Callers that must distinguish a *fresh* lease from the 24 h-old one the + /// slot's previous guest left behind need the whole record, not just its + /// address — see ``isNewer(_:than:)``. + /// + /// - Parameters: + /// - mac: The guest's MAC, in any common rendering. + /// - leases: Leases from ``parse(_:)``. + /// - Returns: The newest matching lease, or `nil`. + public static func lease(forMAC mac: String, in leases: [DHCPLease]) -> DHCPLease? { + guard let wanted = normalizeMAC(mac) else { return nil } + + var best: DHCPLease? + for lease in leases where lease.hwAddress == wanted { + guard let current = best else { + best = lease + continue + } + // Newest expiry wins; a missing expiry sorts oldest. `>=` means that + // among equally-dated (or equally-undated) duplicates the last block + // in the file wins, which is the one bootpd wrote most recently. + let candidate = lease.leaseExpiry ?? .distantPast + let incumbent = current.leaseExpiry ?? .distantPast + if candidate >= incumbent { best = lease } + } + + return best + } + + /// Whether `candidate` is a lease `bootpd` wrote *after* `previous`. + /// + /// Slot MACs are persistent and macOS leases live 24 h, so a MAC almost + /// always still has its previous guest's entry when the next clone boots. + /// A caller that accepted the first entry it saw would hand out a stale + /// address and then spend the whole boot timeout SSHing at nothing. + /// + /// `bootpd` rewrites the block — bumping `lease=` — whenever it hands the + /// address out again, so a strictly later expiry means a new lease. A + /// changed address means the same thing. With no `previous` (first boot on + /// this MAC) anything counts as new. + /// + /// - Parameters: + /// - candidate: The lease just read from the file. + /// - previous: The lease observed before the guest was started. + /// - Returns: `true` when `candidate` may be used. + public static func isNewer(_ candidate: DHCPLease, than previous: DHCPLease?) -> Bool { + guard let previous else { return true } + if candidate.ipAddress != previous.ipAddress { return true } + guard let previousExpiry = previous.leaseExpiry else { return true } + guard let candidateExpiry = candidate.leaseExpiry else { return false } + return candidateExpiry > previousExpiry + } + + /// Normalizes a MAC to lowercase, colon-separated, zero-padded octets. + /// + /// Accepts an optional `,` prefix (as written by `bootpd`), and + /// tolerates `-` separators. + /// + /// - Parameter raw: For example `1,aa:bb:c:dd:ee:ff` or `AA-BB-0C-DD-EE-FF`. + /// - Returns: For example `aa:bb:0c:dd:ee:ff`, or `nil` if unparseable. + public static func normalizeMAC(_ raw: String) -> String? { + var text = raw.trimmingCharacters(in: .whitespaces) + + // `bootpd` prefixes the hardware type: `1,` for Ethernet. + if let comma = text.lastIndex(of: ",") { + text = String(text[text.index(after: comma)...]) + } + text = text.replacingOccurrences(of: "-", with: ":") + + let octets = text.split(separator: ":", omittingEmptySubsequences: false) + guard octets.count == 6 else { return nil } + + var normalized: [String] = [] + normalized.reserveCapacity(6) + for octet in octets { + guard (1...2).contains(octet.count), octet.allSatisfy(\.isHexDigit) else { return nil } + normalized.append(String(repeating: "0", count: 2 - octet.count) + octet.lowercased()) + } + + return normalized.joined(separator: ":") + } +} diff --git a/Sources/RunnerCore/GiteaClient.swift b/Sources/RunnerCore/GiteaClient.swift new file mode 100644 index 0000000..cc62455 --- /dev/null +++ b/Sources/RunnerCore/GiteaClient.swift @@ -0,0 +1,357 @@ +import Foundation + +#if canImport(FoundationNetworking) +import FoundationNetworking +#endif + +/// The HTTP seam under ``GiteaClient``. +/// +/// Everything network-facing goes through this protocol so tests can supply a +/// canned transport without a live Gitea instance. +public protocol HTTPTransport: Sendable { + /// Performs a request. + /// + /// - Parameter request: A fully-formed request, including auth headers. + /// - Returns: The response body and its HTTP status code. + /// - Throws: Transport-level errors only; a non-2xx status is *not* an error + /// here — ``GiteaClient`` maps that to ``CoreError/gitea(status:message:)``. + func send(_ request: URLRequest) async throws -> (Data, Int) +} + +/// The production transport, backed by `URLSession`. +public struct URLSessionTransport: HTTPTransport { + /// The underlying session. + public let session: URLSession + + /// Creates a transport. + /// + /// - Parameter session: Defaults to an ephemeral session with a 30 s request + /// timeout, so a hung Gitea cannot stall the poll loop. + public init(session: URLSession = URLSessionTransport.makeDefaultSession()) { + self.session = session + } + + /// Builds the default ephemeral session. + public static func makeDefaultSession() -> URLSession { + let cfg = URLSessionConfiguration.ephemeral + cfg.timeoutIntervalForRequest = 30 + cfg.timeoutIntervalForResource = 60 + return URLSession(configuration: cfg) + } + + public func send(_ request: URLRequest) async throws -> (Data, Int) { + // `dataTask` + a continuation rather than `session.data(for:)`, because + // the async URLSession API is not uniformly available in + // swift-corelibs-foundation, and RunnerCore must build on Linux. + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<(Data, Int), Error>) in + let task = session.dataTask(with: request) { data, response, error in + if let error { + continuation.resume(throwing: error) + return + } + guard let http = response as? HTTPURLResponse else { + continuation.resume( + throwing: CoreError.gitea(status: 0, message: "no HTTP response")) + return + } + continuation.resume(returning: (data ?? Data(), http.statusCode)) + } + task.resume() + } + } +} + +/// A thin, typed client for the subset of Gitea's admin Actions API this daemon +/// needs. +/// +/// All endpoints used here are **admin**-scoped, so the token must belong to a +/// Gitea administrator. `doctor` verifies that by calling ``listRunners()``. +public struct GiteaClient: Sendable { + /// Instance base URL, e.g. `https://gitea.example.com`. + public let baseURL: URL + /// Admin API token, sent as `Authorization: token `. + public let token: String + /// The HTTP seam. + public let transport: any HTTPTransport + + /// Creates a client. + /// + /// - Parameters: + /// - baseURL: Instance base URL; a trailing slash is tolerated. + /// - token: Admin API token. + /// - transport: Defaults to ``URLSessionTransport``. + public init(baseURL: URL, token: String, transport: any HTTPTransport = URLSessionTransport()) { + self.baseURL = baseURL + self.token = token + self.transport = transport + } + + // MARK: - Endpoints + + /// Lists jobs currently waiting for a runner. + /// + /// `GET /api/v1/admin/actions/jobs?status=queued&limit=` + /// + /// A queued job stays queued until a matching runner claims it, or until + /// Gitea's `ABANDONED_JOB_TIMEOUT` (default 24 h, swept every 6 h) expires + /// it. There is therefore no urgency risk in a 5-second poll. + /// + /// - Parameter limit: Page size. The scheduler only ever needs a handful. + /// - Returns: The queued jobs, oldest-first as Gitea returns them. + /// - Throws: ``CoreError/gitea(status:message:)`` on a non-2xx response. + public func listQueuedJobs(limit: Int = 50) async throws -> [WorkflowJob] { + // `status=queued` and nothing else. Gitea's `convertToInternal` maps + // "queued" onto StatusWaiting ("ready, waiting for a runner") and maps + // "waiting" onto StatusBlocked ("blocked on a dependency") — so asking + // for "waiting" would return exactly the jobs that must not be booted. + let request = try makeRequest( + method: "GET", + path: "/api/v1/admin/actions/jobs", + query: [ + URLQueryItem(name: "status", value: "queued"), + URLQueryItem(name: "limit", value: String(max(limit, 1))), + ]) + let response = try await send(request, as: WorkflowJobsResponse.self) + return response.items + } + + /// Lists every registered runner on the instance. + /// + /// `GET /api/v1/admin/actions/runners?page=&limit=` + /// + /// Used by the reconcile loop and by `doctor` (as an admin-scope probe). + /// + /// Paginated deliberately: an unpaginated request returns only Gitea's + /// default first page, and reconcile is precisely the thing that stops an + /// instance from accumulating orphan rows. Missing rows past the page + /// boundary would let the leak accelerate — each undeleted row pushes more + /// rows out of view — and it would silently no-op the targeted cleanup that + /// looks a single runner up by name. + /// + /// - Parameters: + /// - limit: Page size. + /// - maxPages: Defensive ceiling, so a server that ignores `page` cannot + /// spin this forever. + /// - Returns: Every runner across all fetched pages, in server order. + public func listRunners(limit: Int = 50, maxPages: Int = 50) async throws -> [ActionRunner] { + let pageSize = max(limit, 1) + var all: [ActionRunner] = [] + + for page in 1...max(maxPages, 1) { + let request = try makeRequest( + method: "GET", + path: "/api/v1/admin/actions/runners", + query: [ + URLQueryItem(name: "page", value: String(page)), + URLQueryItem(name: "limit", value: String(pageSize)), + ]) + let response = try await send(request, as: RunnersResponse.self) + all.append(contentsOf: response.items) + // Terminate on the server's own total rather than on a short page: + // Gitea clamps `limit` to its configured maximum, so a page shorter + // than the one we asked for is not evidence that it is the last. + if response.items.isEmpty { break } + if let total = response.totalCount, all.count >= total { break } + } + + return all + } + + /// Deletes a runner row. + /// + /// `DELETE /api/v1/admin/actions/runners/{id}` + /// + /// Needed because a VM that dies uncleanly leaves its row behind: Gitea only + /// sweeps runner rows at midnight, and never sweeps a runner that claimed no + /// task. A 404 is treated as success (someone else already removed it). + /// + /// - Parameter id: The runner id. + public func deleteRunner(id: Int64) async throws { + let request = try makeRequest( + method: "DELETE", + path: "/api/v1/admin/actions/runners/\(id)") + // Gitea answers 204. 200 is accepted for tolerance, and 404 counts as + // success: the reconcile loop's only goal is that the row be gone, and + // it races with Gitea's own midnight sweep and with `--ephemeral` + // auto-deregistration. + try await sendIgnoringBody(request, acceptingStatuses: [200, 202, 204, 404]) + } + + /// Returns the instance-scoped runner registration token. + /// + /// `POST /api/v1/admin/actions/runners/registration-token` + /// + /// - Warning: In current Gitea this returns the *existing* active token for + /// the scope rather than minting a new one — but the semantics of "mint" + /// are that a new token **invalidates all prior tokens of that scope**. + /// Never call this per VM as a way of getting a throwaway secret; call it + /// once and cache. Prefer seeding the token server-side via + /// `GITEA_RUNNER_REGISTRATION_TOKEN` and configuring it statically. + public func getRegistrationToken() async throws -> String { + let request = try makeRequest( + method: "POST", + path: "/api/v1/admin/actions/runners/registration-token") + let response = try await send(request, as: RegistrationTokenResponse.self) + let token = response.token.trimmingCharacters(in: .whitespacesAndNewlines) + guard !token.isEmpty else { + throw CoreError.gitea(status: 200, message: "registration-token response carried an empty token") + } + return token + } + + /// Cheap reachability + auth probe used by `doctor`. + /// + /// - Throws: ``CoreError/gitea(status:message:)`` when the instance is + /// reachable but rejects the token. + public func ping() async throws { + // The runners list rather than /api/v1/version: version is anonymously + // readable on most instances, so it would report "reachable" for a token + // that is expired, wrong, or simply not an admin's — which is the exact + // failure `doctor` exists to catch. + _ = try await listRunners() + } + + // MARK: - Request plumbing + + /// Builds an authenticated request against an API path. + /// + /// - Parameters: + /// - method: HTTP method. + /// - path: API path relative to the instance root, e.g. + /// `/api/v1/admin/actions/runners`. + /// - query: Optional query items. + /// - body: Optional request body; sets `Content-Type: application/json`. + /// - Returns: A request carrying `Authorization` and `Accept` headers. + public func makeRequest( + method: String, + path: String, + query: [URLQueryItem] = [], + body: Data? = nil + ) throws -> URLRequest { + // Built by string-joining rather than `URL(string:relativeTo:)`, which + // would discard any path component of `baseURL` — instances served under + // a subpath (https://example.com/gitea) are common enough to matter. + var base = baseURL.absoluteString + while base.hasSuffix("/") { base.removeLast() } + let suffix = path.hasPrefix("/") ? path : "/" + path + + guard var components = URLComponents(string: base + suffix) else { + throw CoreError.configInvalid("cannot form a request URL from \(base + suffix)") + } + if !query.isEmpty { + components.queryItems = query + } + guard let url = components.url else { + throw CoreError.configInvalid("cannot form a request URL from \(base + suffix)") + } + + var request = URLRequest(url: url) + request.httpMethod = method + // Gitea's PAT scheme. `Bearer` also works on recent versions, but + // `token` is the documented form and works on every 1.x. + request.setValue("token \(token)", forHTTPHeaderField: "Authorization") + request.setValue("application/json", forHTTPHeaderField: "Accept") + request.setValue( + "gitea-macos-runner/\(RunnerVersion.current)", forHTTPHeaderField: "User-Agent") + if let body { + request.httpBody = body + request.setValue("application/json", forHTTPHeaderField: "Content-Type") + } + return request + } + + /// Sends a request and decodes a JSON body, mapping non-2xx to + /// ``CoreError/gitea(status:message:)``. + public func send(_ request: URLRequest, as type: T.Type) async throws -> T { + let (data, status) = try await transport.send(request) + guard (200..<300).contains(status) else { + throw CoreError.gitea(status: status, message: GiteaClient.errorMessage(from: data)) + } + do { + return try GiteaClient.makeDecoder().decode(T.self, from: data) + } catch { + throw CoreError.gitea( + status: status, + message: "could not decode \(T.self): \(error) — body: \(GiteaClient.excerpt(data))") + } + } + + /// Sends a request that is expected to have no useful body. + public func sendIgnoringBody(_ request: URLRequest, acceptingStatuses: Set) async throws { + let (data, status) = try await transport.send(request) + guard acceptingStatuses.contains(status) || (200..<300).contains(status) else { + throw CoreError.gitea(status: status, message: GiteaClient.errorMessage(from: data)) + } + } + + /// Gitea's error bodies are `{"message": "...", "url": "..."}`. Prefer that + /// message; fall back to a truncated raw body so nothing is ever reported as + /// an empty error. + private static func errorMessage(from data: Data) -> String { + struct APIError: Decodable { + let message: String? + let errors: [String]? + } + if let decoded = try? JSONDecoder().decode(APIError.self, from: data) { + if let message = decoded.message?.trimmingCharacters(in: .whitespacesAndNewlines), + !message.isEmpty + { + return message + } + if let errors = decoded.errors, !errors.isEmpty { + return errors.joined(separator: "; ") + } + } + return excerpt(data) + } + + /// At most `limit` characters of a response body, for error messages. + private static func excerpt(_ data: Data, limit: Int = 512) -> String { + guard !data.isEmpty else { return "" } + let text = String(decoding: data, as: UTF8.self) + .trimmingCharacters(in: .whitespacesAndNewlines) + guard text.count > limit else { return text } + return String(text.prefix(limit)) + "… (\(data.count) bytes)" + } + + /// A `JSONDecoder` configured for Gitea's timestamps (RFC 3339 / ISO 8601 + /// with an offset). + /// + /// Go's `time.Time` marshals as RFC 3339 **Nano**: the fractional-seconds + /// part is present only when non-zero, so a single strict formatter fails + /// intermittently on real traffic. Both spellings are tried, plus a plain + /// `YYYY-MM-DD` for good measure. + public static func makeDecoder() -> JSONDecoder { + let d = JSONDecoder() + d.dateDecodingStrategy = .custom { decoder in + let raw = try decoder.singleValueContainer().decode(String.self) + if let date = parseTimestamp(raw) { return date } + throw DecodingError.dataCorrupted( + DecodingError.Context( + codingPath: decoder.codingPath, + debugDescription: "not an RFC 3339 timestamp: \"\(raw)\"")) + } + return d + } + + /// Parses an RFC 3339 timestamp with or without fractional seconds. + /// + /// - Parameter raw: The timestamp string. + /// - Returns: The instant, or `nil` if it is in no recognized form. + public static func parseTimestamp(_ raw: String) -> Date? { + // Formatters are built per call rather than cached in a `static let`: + // `ISO8601DateFormatter` is a non-Sendable reference type, and this is + // called a handful of times per poll — not a hot path. + let withFractional = ISO8601DateFormatter() + withFractional.formatOptions = [.withInternetDateTime, .withFractionalSeconds] + if let date = withFractional.date(from: raw) { return date } + + let plain = ISO8601DateFormatter() + plain.formatOptions = [.withInternetDateTime] + if let date = plain.date(from: raw) { return date } + + let dateOnly = ISO8601DateFormatter() + dateOnly.formatOptions = [.withFullDate] + return dateOnly.date(from: raw) + } +} diff --git a/Sources/RunnerCore/GiteaModels.swift b/Sources/RunnerCore/GiteaModels.swift new file mode 100644 index 0000000..c5b06e8 --- /dev/null +++ b/Sources/RunnerCore/GiteaModels.swift @@ -0,0 +1,330 @@ +import Foundation + +/// A single job within a workflow run, as reported by +/// `GET /api/v1/admin/actions/jobs` (Gitea 1.25+). +/// +/// - Note: `status` values are the *external* strings. `queued` is the one we +/// act on: it maps to Gitea's internal `StatusWaiting`, meaning "ready and +/// waiting for a matching runner". The string `waiting` means something quite +/// different — the job is **blocked** on a dependency — and must never be +/// treated as schedulable. +public struct WorkflowJob: Codable, Sendable, Equatable, Identifiable { + /// Job id. Unique across the instance; the scheduler dedups on this. + public let id: Int64 + /// The workflow run this job belongs to. + public let runID: Int64 + /// The job's display name. + public let name: String + /// One of `queued`, `waiting`, `running`, `success`, `failure`, `cancelled`, + /// `skipped`, `blocked`. + public let status: String + /// The job's `runs-on:` values, as **bare** label names. + public let labels: [String] + /// The runner that claimed the job, if any. + public let runnerID: Int64? + /// That runner's name, if any. Lets the reconcile loop tie a Gitea runner + /// row back to one of our VMs. + public let runnerName: String? + /// When the job was created. + public let createdAt: Date? + /// When a runner picked it up. + public let startedAt: Date? + /// When it finished. + public let completedAt: Date? + + public init( + id: Int64, + runID: Int64, + name: String, + status: String, + labels: [String], + runnerID: Int64? = nil, + runnerName: String? = nil, + createdAt: Date? = nil, + startedAt: Date? = nil, + completedAt: Date? = nil + ) { + self.id = id + self.runID = runID + self.name = name + self.status = status + self.labels = labels + self.runnerID = runnerID + self.runnerName = runnerName + self.createdAt = createdAt + self.startedAt = startedAt + self.completedAt = completedAt + } + + private enum CodingKeys: String, CodingKey { + case id + case runID = "run_id" + case name + case status + case labels + case runnerID = "runner_id" + case runnerName = "runner_name" + case createdAt = "created_at" + case startedAt = "started_at" + case completedAt = "completed_at" + } + + /// Whether this job is waiting for a runner right now. + public var isQueued: Bool { status == "queued" } + + /// ``status`` as a case, with an ``JobStatus/unknown(_:)`` catch-all. + public var jobStatus: JobStatus { JobStatus(rawValue: status) } + + /// Decodes tolerantly: `runner_id` / `runner_name` carry `omitempty` in + /// Gitea, and the timestamps are Go `time.Time` values that serialize as + /// `0001-01-01T00:00:00Z` when unset (a queued job has no `started_at`). + /// Those zero instants are surfaced as `nil` rather than as a year-1 date. + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + self.id = try c.decode(Int64.self, forKey: .id) + self.runID = try c.decodeIfPresent(Int64.self, forKey: .runID) ?? 0 + self.name = try c.decodeIfPresent(String.self, forKey: .name) ?? "" + self.status = try c.decodeIfPresent(String.self, forKey: .status) ?? "" + self.labels = try c.decodeIfPresent([String].self, forKey: .labels) ?? [] + self.runnerID = try c.decodeIfPresent(Int64.self, forKey: .runnerID) + self.runnerName = try c.decodeIfPresent(String.self, forKey: .runnerName) + self.createdAt = WorkflowJob.nonZero(try c.decodeIfPresent(Date.self, forKey: .createdAt)) + self.startedAt = WorkflowJob.nonZero(try c.decodeIfPresent(Date.self, forKey: .startedAt)) + self.completedAt = WorkflowJob.nonZero(try c.decodeIfPresent(Date.self, forKey: .completedAt)) + } + + /// Maps Go's zero `time.Time` (year 1) to `nil`. + private static func nonZero(_ date: Date?) -> Date? { + guard let date else { return nil } + // 0001-01-01T00:00:00Z is ~62.1e9 seconds before the reference date. + return date.timeIntervalSinceReferenceDate <= -62_135_596_800 ? nil : date + } +} + +/// The *external* status strings Gitea reports for a workflow job. +/// +/// Gitea maps its internal statuses onto GitHub's vocabulary in +/// `convert.ToActionsStatus`: `StatusWaiting → "queued"`, +/// `StatusBlocked → "waiting"`, `StatusRunning → "in_progress"`, and every +/// terminal status → `"completed"` (the detail moves to a separate `conclusion` +/// field). The `unknown` case exists because that mapping is Gitea's to change: +/// a closed enum that threw on an unrecognized string would turn a new server +/// version into a decode failure and stop the poll loop dead. +public enum JobStatus: RawRepresentable, Sendable, Equatable, Hashable { + /// Ready and waiting for a matching runner — Gitea's internal `StatusWaiting`. + /// This is the only status that is schedulable. + case queued + /// **Blocked** on a dependency — Gitea's internal `StatusBlocked`. Despite + /// the name, this is *not* a job waiting for a runner. + case waiting + /// Claimed by a runner and executing. + case inProgress + /// Terminal, in any of success / failure / cancelled / skipped. + case completed + /// A status string this build does not know about. + case unknown(String) + + public init(rawValue: String) { + switch rawValue { + case "queued": self = .queued + case "waiting": self = .waiting + case "in_progress": self = .inProgress + case "completed": self = .completed + default: self = .unknown(rawValue) + } + } + + public var rawValue: String { + switch self { + case .queued: return "queued" + case .waiting: return "waiting" + case .inProgress: return "in_progress" + case .completed: return "completed" + case .unknown(let raw): return raw + } + } +} + +/// Envelope returned by `GET /api/v1/admin/actions/jobs`. +/// +/// - Important: The array key has been observed as `jobs`, which is what is +/// decoded here; some Gitea builds/OpenAPI revisions have used `workflow_jobs` +/// for the equivalent repo-scoped endpoint. ``CodingKeys`` is written out +/// explicitly so that adding a fallback is a one-line change, and +/// ``jobs`` is optional so an empty response body decodes rather than throwing. +public struct WorkflowJobsResponse: Codable, Sendable, Equatable { + /// Total matching jobs server-side, ignoring `limit`. + public let totalCount: Int? + /// The page of jobs. `nil` and `[]` both mean "nothing queued". + public let jobs: [WorkflowJob]? + + public init(totalCount: Int?, jobs: [WorkflowJob]?) { + self.totalCount = totalCount + self.jobs = jobs + } + + private enum CodingKeys: String, CodingKey { + case totalCount = "total_count" + case jobs + case workflowJobs = "workflow_jobs" + case entries + } + + /// The jobs, never `nil`. + public var items: [WorkflowJob] { jobs ?? [] } + + /// Decodes the array under `jobs`, falling back to `workflow_jobs` and + /// `entries`. + /// + /// Gitea 1.25's `ActionWorkflowJobsResponse` tags its slice `json:"jobs"` + /// (the Go field is named `Entries`), which is what the fallbacks guard + /// against: a future rename of the tag, or a proxy that reserializes from + /// the Go field name. + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + self.totalCount = try c.decodeIfPresent(Int.self, forKey: .totalCount) + if let jobs = try c.decodeIfPresent([WorkflowJob].self, forKey: .jobs) { + self.jobs = jobs + } else if let jobs = try c.decodeIfPresent([WorkflowJob].self, forKey: .workflowJobs) { + self.jobs = jobs + } else { + self.jobs = try c.decodeIfPresent([WorkflowJob].self, forKey: .entries) + } + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encodeIfPresent(totalCount, forKey: .totalCount) + try c.encodeIfPresent(jobs, forKey: .jobs) + } +} + +/// A registered Actions runner, from `GET /api/v1/admin/actions/runners`. +public struct ActionRunner: Codable, Sendable, Equatable, Identifiable { + /// Runner id, used for `DELETE /api/v1/admin/actions/runners/{id}`. + public let id: Int64 + /// Runner name. Ours always start with the configured `namePrefix`. + public let name: String + /// Bare label names the runner advertises. + public let labels: [String] + /// Server-side liveness, e.g. `online` / `offline`. + public let status: String? + /// Whether the runner is currently executing a task. + public let busy: Bool? + /// Whether the runner registered with `--ephemeral`, i.e. the server will + /// hand it exactly one task and then auto-deregister it (Gitea 1.24+). + public let ephemeral: Bool? + + public init( + id: Int64, + name: String, + labels: [String], + status: String? = nil, + busy: Bool? = nil, + ephemeral: Bool? = nil + ) { + self.id = id + self.name = name + self.labels = labels + self.status = status + self.busy = busy + self.ephemeral = ephemeral + } + + private enum CodingKeys: String, CodingKey { + case id, name, labels, status, busy, ephemeral + } + + /// A single entry of `ActionRunner.labels` as Gitea actually serializes it: + /// an object, not a string. + /// + /// Verified against `modules/structs/repo_actions.go` at tag `v1.25.0`, + /// where `ActionRunner.Labels` is `[]*ActionRunnerLabel` and + /// `ActionRunnerLabel` is `{id int64, name string, type string}`. Only + /// ``name`` is of any use here. + private struct LabelObject: Decodable { + let name: String? + } + + /// Decodes `labels` from either shape. + /// + /// Gitea's job payload gives labels as plain strings, its runner payload + /// gives them as objects. Both are accepted so that this one model keeps + /// working if a version, a proxy, or a hand-written fixture disagrees. + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + self.id = try c.decode(Int64.self, forKey: .id) + self.name = try c.decodeIfPresent(String.self, forKey: .name) ?? "" + self.status = try c.decodeIfPresent(String.self, forKey: .status) + self.busy = try c.decodeIfPresent(Bool.self, forKey: .busy) + self.ephemeral = try c.decodeIfPresent(Bool.self, forKey: .ephemeral) + + if let strings = try? c.decode([String].self, forKey: .labels) { + self.labels = strings + } else if let objects = try? c.decode([LabelObject].self, forKey: .labels) { + self.labels = objects.compactMap(\.name) + } else { + // Absent or explicitly null. Gitea does emit `"labels": null` for a + // runner registered without any. + self.labels = [] + } + } + + /// `busy` treated as `false` when the server omits it. + public var isBusy: Bool { busy ?? false } + + /// `ephemeral` treated as `false` when the server omits it. The reconcile + /// loop only ever deletes rows it is sure are ephemeral. + public var isEphemeral: Bool { ephemeral ?? false } +} + +/// Envelope returned by `GET /api/v1/admin/actions/runners`. +public struct RunnersResponse: Codable, Sendable, Equatable { + public let totalCount: Int? + public let runners: [ActionRunner]? + + public init(totalCount: Int?, runners: [ActionRunner]?) { + self.totalCount = totalCount + self.runners = runners + } + + private enum CodingKeys: String, CodingKey { + case totalCount = "total_count" + case runners + case entries + } + + /// The runners, never `nil`. + public var items: [ActionRunner] { runners ?? [] } + + /// Decodes the array under `runners` (Gitea 1.25's tag), falling back to + /// `entries` (the Go field name). + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + self.totalCount = try c.decodeIfPresent(Int.self, forKey: .totalCount) + if let runners = try c.decodeIfPresent([ActionRunner].self, forKey: .runners) { + self.runners = runners + } else { + self.runners = try c.decodeIfPresent([ActionRunner].self, forKey: .entries) + } + } + + public func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encodeIfPresent(totalCount, forKey: .totalCount) + try c.encodeIfPresent(runners, forKey: .runners) + } +} + +/// Response from `POST /api/v1/admin/actions/runners/registration-token`. +/// +/// - Warning: This is scope-wide and **reusable**. Treat the returned value as +/// "the current token for this scope", not as a freshly minted per-VM secret. +public struct RegistrationTokenResponse: Codable, Sendable, Equatable { + /// The registration token. + public let token: String + + public init(token: String) { + self.token = token + } +} diff --git a/Sources/RunnerCore/Labels.swift b/Sources/RunnerCore/Labels.swift new file mode 100644 index 0000000..b4d325d --- /dev/null +++ b/Sources/RunnerCore/Labels.swift @@ -0,0 +1,84 @@ +import Foundation + +/// The set of labels this host's runners advertise, used to decide whether a +/// queued Gitea job is ours to pick up. +/// +/// ## Bare names only +/// +/// Gitea's label syntax at *registration* time is `name:schema` (for example +/// `macos-arm64:host`), where the schema defaults to `host` when omitted. The +/// schema is a runner-side execution hint — it tells `gitea-runner` to run the +/// job directly on the machine instead of inside a container. **The server only +/// ever stores and reports the bare name.** A workflow's `runs-on:` value, and +/// therefore the `labels` array on a queued job, likewise contains bare names. +/// +/// So: pass `macos-arm64:host` to `gitea-runner register --labels`, but match +/// against `macos-arm64` here. Matching is case-sensitive, because Gitea's own +/// comparison is. +/// +/// - Warning: If the guest's `.runner`/`config.yaml` sets `runner.labels`, it +/// silently overrides whatever `--labels` was passed at registration. The +/// guest must therefore never ship a config file containing labels. +public struct LabelSet: Sendable, Equatable, Hashable { + /// The bare label names this host serves, e.g. `["macos-arm64", "macos"]`. + public let names: Set + + /// Creates a label set from bare names. + /// + /// Any `:schema` suffix present in `names` is stripped, so it is safe to + /// hand this the same array that is written into the config file. + /// + /// - Parameter names: Label names, with or without a `:schema` suffix. + public init(_ names: [String]) { + self.names = Set( + names + .map(LabelSet.bareName) + .filter { !$0.isEmpty } + ) + } + + /// Whether a queued job's `labels` array can be satisfied by this host. + /// + /// Returns `true` if and only if `jobLabels` is non-empty *and* every entry + /// is a member of ``names``. An empty job label array is treated as "no + /// declared requirement" and is deliberately **not** matched — a job that + /// asks for nothing must not be scheduled onto a scarce macOS VM. + /// + /// The job side is put through ``bareName(_:)`` too. The server normally + /// stores bare names, so this changes nothing in the common case — but a + /// workflow that writes `runs-on: [macos-arm64:host]` would otherwise never + /// match anything and its job would be skipped with no log line at all. + /// + /// - Parameter jobLabels: The `labels` array from a `WorkflowJob`. + /// - Returns: `true` when this host should boot a VM for the job. + public func matches(jobLabels: [String]) -> Bool { + guard !jobLabels.isEmpty else { return false } + let wanted = Set(jobLabels.map(LabelSet.bareName).filter { !$0.isEmpty }) + guard !wanted.isEmpty else { return false } + return wanted.isSubset(of: names) + } + + /// The value to pass to `gitea-runner register --labels`, i.e. each bare + /// name suffixed with the given schema and joined by commas. + /// + /// - Parameter schema: The execution schema; `host` for a bare-metal guest. + /// - Returns: For example `"macos-arm64:host,macos:host"`. + public func registrationArgument(schema: String = "host") -> String { + // Sorted so the argument is stable across process runs — a `Set` has no + // inherent order, and an unstable registration argument would make the + // guest command line (and its logs) needlessly non-reproducible. + names.sorted() + .map { schema.isEmpty ? $0 : "\($0):\(schema)" } + .joined(separator: ",") + } + + /// Strips an optional `:schema` suffix from a single label token. + /// + /// - Parameter label: A label such as `macos-arm64:host` or `macos-arm64`. + /// - Returns: The bare name. + public static func bareName(_ label: String) -> String { + let trimmed = label.trimmingCharacters(in: .whitespaces) + guard let colon = trimmed.firstIndex(of: ":") else { return trimmed } + return String(trimmed[trimmed.startIndex.. String { + prefix + UUID().uuidString.lowercased() + } + + /// Whether a runner name reported by Gitea was minted by this host. + /// + /// - Parameters: + /// - name: A runner name from `GET /api/v1/admin/actions/runners`. + /// - prefix: The configured prefix. + /// - Returns: `true` when the reconcile loop may consider deleting it. + public static func hasPrefix(_ name: String, prefix: String) -> Bool { + // An empty prefix would match every runner on the instance, including + // other people's. The reconcile loop deletes what this returns true for, + // so refuse rather than match everything. `validated()` also rejects an + // empty `namePrefix`; this is the second line of defence. + guard !prefix.isEmpty else { return false } + return name.hasPrefix(prefix) + } +} diff --git a/Sources/RunnerCore/SSHExec.swift b/Sources/RunnerCore/SSHExec.swift new file mode 100644 index 0000000..1232c02 --- /dev/null +++ b/Sources/RunnerCore/SSHExec.swift @@ -0,0 +1,589 @@ +import Foundation +import NIOCore +import NIOPosix +import NIOSSH + +/// The outcome of a command run inside a guest. +public struct SSHCommandResult: Sendable, Equatable { + /// The remote process's exit status. `0` on success. + public let exitCode: Int32 + /// Captured stdout, UTF-8 decoded with lossy replacement. + public let stdout: String + /// Captured stderr, UTF-8 decoded with lossy replacement. + public let stderr: String + + public init(exitCode: Int32, stdout: String, stderr: String) { + self.exitCode = exitCode + self.stdout = stdout + self.stderr = stderr + } + + /// Whether the command exited zero. + public var succeeded: Bool { exitCode == 0 } + + /// Throws ``CoreError/sshFailed(_:)`` unless the command exited zero. + /// + /// - Parameter command: Echoed into the error message for context. + public func throwIfFailed(command: String) throws { + guard exitCode != 0 else { return } + let detail = stderr.isEmpty ? stdout : stderr + let trimmed = detail.trimmingCharacters(in: .whitespacesAndNewlines) + throw CoreError.sshFailed( + "command failed (exit \(exitCode)): \(command)" + (trimmed.isEmpty ? "" : "\n\(trimmed)") + ) + } +} + +/// The seam for talking to a guest. +/// +/// The orchestrator and the provisioner are written against this rather than +/// against ``SSHExecutor`` so that provisioning logic can be unit-tested with a +/// recording fake, and so a future vsock-based transport could be dropped in +/// without touching callers. +public protocol GuestExecutor: Sendable { + /// Runs a shell command in the guest and waits for it to exit. + /// + /// - Parameters: + /// - command: A `/bin/sh`-compatible command line. + /// - timeout: Wall-clock ceiling; exceeding it throws + /// ``CoreError/timeout(_:)`` and closes the channel. + /// - Returns: Exit status and captured output. + func run(_ command: String, timeout: Duration) async throws -> SSHCommandResult + + /// Copies a local file into the guest. + /// + /// - Parameters: + /// - localPath: Source path on the host. + /// - remotePath: Destination path in the guest. + func upload(localPath: String, remotePath: String) async throws + + /// Writes bytes to a guest file with an explicit mode. + /// + /// Used for secrets — notably the registration token, which is written with + /// mode `0600` and deleted immediately after `gitea-runner register` reads + /// it, so it never appears in a process argument list. + /// + /// - Parameters: + /// - data: File contents. + /// - remotePath: Destination path in the guest. + /// - mode: Octal mode string, e.g. `"0600"`. + func uploadData(_ data: Data, remotePath: String, mode: String) async throws +} + +extension GuestExecutor { + /// ``run(_:timeout:)`` with a two-minute default ceiling. + public func run(_ command: String) async throws -> SSHCommandResult { + try await run(command, timeout: .seconds(120)) + } + + /// Runs a command and throws unless it exits zero. + /// + /// - Returns: The successful result. + @discardableResult + public func runChecked(_ command: String, timeout: Duration = .seconds(120)) async throws -> SSHCommandResult { + let result = try await run(command, timeout: timeout) + try result.throwIfFailed(command: command) + return result + } +} + +/// SSH client over swift-nio-ssh using password authentication. +/// +/// Password auth (rather than keys) is deliberate: the guest is a throwaway VM +/// on a host-private NAT network whose credentials come from the same config +/// that created it, and injecting a key would mean another provisioning step +/// during the window before SSH is up. +/// +/// - Important: Host keys are **not** verified. The peer is a VM this process +/// just booted, on a link no other host shares; there is no trust-on-first-use +/// story that would add security here, and pinning would break on every clone. +public final class SSHExecutor: GuestExecutor, @unchecked Sendable { + /// Guest IP, as learned from ``DHCPLeaseParser``. + public let host: String + /// SSH port; `22` for a stock guest with Remote Login enabled. + public let port: Int + /// Guest account name. + public let username: String + /// Guest account password. + public let password: String + + /// Creates an executor. No connection is made until the first command. + /// + /// - Parameters: + /// - host: Guest IP address. + /// - port: SSH port. Defaults to `22`. + /// - username: Guest account. + /// - password: Guest password. + public init(host: String, port: Int = 22, username: String, password: String) { + self.host = host + self.port = port + self.username = username + self.password = password + } + + public func run(_ command: String, timeout: Duration) async throws -> SSHCommandResult { + do { + return try await execute(command, stdin: nil, timeout: timeout) + } catch let error as SSHTransportError { + throw error.asCoreError + } + } + + public func upload(localPath: String, remotePath: String) async throws { + let url = URL(fileURLWithPath: localPath) + guard let data = try? Data(contentsOf: url) else { + throw CoreError.notFound("local file for upload: \(localPath)") + } + try await uploadData(data, remotePath: remotePath, mode: "0644") + } + + public func uploadData(_ data: Data, remotePath: String, mode: String) async throws { + // Deliberately not SFTP or SCP: a stock macOS guest runs an sshd whose + // subsystem set we do not control at this point in provisioning, and an + // exec channel with the payload as stdin needs nothing beyond what we + // already use for every other command. + let quotedPath = Self.shellQuote(remotePath) + guard mode.allSatisfy(\.isNumber), !mode.isEmpty else { + throw CoreError.sshFailed("invalid file mode \(mode.debugDescription) for \(remotePath)") + } + + let command = """ + mkdir -p "$(dirname \(quotedPath))" && cat > \(quotedPath) && chmod \(mode) \(quotedPath) + """ + + let result: SSHCommandResult + do { + result = try await execute(command, stdin: data, timeout: .seconds(300)) + } catch let error as SSHTransportError { + throw error.asCoreError + } + try result.throwIfFailed(command: "upload to \(remotePath)") + } + + /// Releases any pooled connection and event loop resources. + /// + /// Connections are not pooled — each command opens and closes its own — and + /// the event loop group is NIO's process-wide singleton, so there is nothing + /// to release. Kept so callers can be written against a lifecycle that a + /// future pooling or vsock transport may need. + public func close() async {} + + // MARK: - Transport + + /// Opens a connection, runs one exec channel, and tears both down. + /// + /// - Parameters: + /// - command: The `/bin/sh` command line to exec. + /// - stdin: Bytes to stream as the command's standard input. Standard + /// input is closed (channel EOF) either way, so a command that would + /// otherwise read from the terminal exits instead of hanging. + /// - timeout: Wall-clock ceiling on the whole exchange. + /// - Throws: ``SSHTransportError`` for connect/auth problems (which + /// ``waitForSSH(host:port:username:password:timeout:pollInterval:)`` needs + /// to tell apart), or ``CoreError/timeout(_:)`` when the ceiling elapses. + func execute(_ command: String, stdin: Data?, timeout: Duration) async throws -> SSHCommandResult { + let group = MultiThreadedEventLoopGroup.singleton + let auth = SSHAuthOutcome() + let username = self.username + let password = self.password + let host = self.host + let port = self.port + + let bootstrap = ClientBootstrap(group: group) + .channelOption(ChannelOptions.socketOption(.tcp_nodelay), value: 1) + .channelInitializer { channel in + channel.eventLoop.makeCompletedFuture { + let configuration = SSHClientConfiguration( + userAuthDelegate: PasswordOnlyAuthDelegate( + username: username, + password: password, + outcome: auth + ), + serverAuthDelegate: AcceptAnyHostKeyDelegate() + ) + try channel.pipeline.syncOperations.addHandler( + NIOSSHHandler( + role: .client(configuration), + allocator: channel.allocator, + inboundChildChannelInitializer: nil + ) + ) + } + } + + let channel: Channel + do { + channel = try await bootstrap.connect(host: host, port: port).get() + } catch { + // No TCP connection at all: sshd is not listening yet (or the guest + // is unreachable). Recoverable — this is what waitForSSH retries on. + throw SSHTransportError.connectFailed(host: host, port: port, underlying: error) + } + + let loop = channel.eventLoop + let resultPromise = loop.makePromise(of: SSHCommandResult.self) + let stdinBuffer = stdin.map { ByteBuffer(bytes: $0) } + let description = command + + let timeoutTask = loop.scheduleTask(in: .nanoseconds(Self.nanoseconds(timeout))) { + resultPromise.fail(CoreError.timeout("ssh command on \(host): \(description)")) + channel.close(promise: nil) + } + + // Completing an already-completed NIO promise is a no-op, so these + // racing completions are safe: whichever fires first wins. + channel.closeFuture.whenComplete { _ in + if auth.wasRejected { + resultPromise.fail( + SSHTransportError.authenticationFailed(host: host, username: username) + ) + } else { + resultPromise.fail( + CoreError.sshFailed("ssh connection to \(host):\(port) closed before the command finished") + ) + } + } + + channel.pipeline.handler(type: NIOSSHHandler.self).flatMap { sshHandler -> EventLoopFuture in + let childPromise = loop.makePromise(of: Channel.self) + sshHandler.createChannel(childPromise, channelType: .session) { child, channelType in + guard channelType == .session else { + return child.eventLoop.makeFailedFuture( + CoreError.sshFailed("unexpected SSH channel type \(channelType)") + ) + } + return child.eventLoop.makeCompletedFuture { + try child.pipeline.syncOperations.addHandler( + ExecChannelHandler( + command: description, + stdin: stdinBuffer, + promise: resultPromise + ) + ) + } + // Without this the guest's EOF would close the channel before the + // exit-status request arrives. + .flatMap { child.setOption(ChannelOptions.allowRemoteHalfClosure, value: true) } + } + return childPromise.futureResult + }.whenFailure { error in + resultPromise.fail(error) + channel.close(promise: nil) + } + + do { + let result = try await resultPromise.futureResult.get() + timeoutTask.cancel() + channel.close(promise: nil) + return result + } catch { + timeoutTask.cancel() + channel.close(promise: nil) + throw error + } + } + + /// Wraps a path (or any argument) so `/bin/sh` sees it literally. + static func shellQuote(_ value: String) -> String { + "'" + value.replacingOccurrences(of: "'", with: "'\\''") + "'" + } + + private static func nanoseconds(_ duration: Duration) -> Int64 { + let components = duration.components + let seconds = components.seconds.multipliedReportingOverflow(by: 1_000_000_000) + guard !seconds.overflow else { return .max } + let sum = seconds.partialValue.addingReportingOverflow( + Int64(components.attoseconds / 1_000_000_000) + ) + return sum.overflow ? .max : sum.partialValue + } +} + +// MARK: - Transport failures + +/// Connection-level failures, kept distinct from ``CoreError`` so that +/// ``waitForSSH(host:port:username:password:timeout:pollInterval:)`` can tell +/// "sshd is not up yet" (retry) from "the password is wrong" (give up now). +enum SSHTransportError: Error { + /// No TCP connection could be established. + case connectFailed(host: String, port: Int, underlying: Error) + /// The server rejected our credentials. + case authenticationFailed(host: String, username: String) + + var asCoreError: CoreError { + switch self { + case .connectFailed(let host, let port, let underlying): + return .sshFailed("cannot connect to \(host):\(port): \(underlying)") + case .authenticationFailed(let host, let username): + return .sshFailed("authentication failed for \(username)@\(host)") + } + } +} + +/// Shared, thread-safe record of whether the server rejected our password. +/// +/// The auth delegate runs on the event loop; the value is read from the async +/// caller, hence the lock. +final class SSHAuthOutcome: @unchecked Sendable { + private let lock = NSLock() + private var rejected = false + + var wasRejected: Bool { + lock.lock() + defer { lock.unlock() } + return rejected + } + + func markRejected() { + lock.lock() + defer { lock.unlock() } + rejected = true + } +} + +// MARK: - Delegates + +/// Accepts every host key. +/// +/// The peer is a VM this process booted seconds ago, on a host-private NAT link +/// no other machine shares, from an image that is destroyed after one job. Its +/// host key is freshly generated per clone, so there is nothing to pin: +/// trust-on-first-use would accept whatever the first connection presented — +/// exactly what this does — while a pinned key would reject every legitimate +/// guest. See docs/DESIGN.md §6. +final class AcceptAnyHostKeyDelegate: NIOSSHClientServerAuthenticationDelegate { + func validateHostKey(hostKey: NIOSSHPublicKey, validationCompletePromise: EventLoopPromise) { + validationCompletePromise.succeed(()) + } +} + +/// Offers the configured password once, then reports rejection. +/// +/// NIOSSH asks again after a failed attempt; a second ask means the server +/// refused the first, which is a provisioning bug rather than a transient +/// condition, so it is recorded for the caller to fail fast on. +final class PasswordOnlyAuthDelegate: NIOSSHClientUserAuthenticationDelegate { + private let username: String + private let password: String + private let outcome: SSHAuthOutcome + private var offered = false + + init(username: String, password: String, outcome: SSHAuthOutcome) { + self.username = username + self.password = password + self.outcome = outcome + } + + func nextAuthenticationType( + availableMethods: NIOSSHAvailableUserAuthenticationMethods, + nextChallengePromise: EventLoopPromise + ) { + guard !offered, availableMethods.contains(.password) else { + // Either the server refused our password, or it never offered + // password auth at all. Both mean this guest will not let us in. + outcome.markRejected() + nextChallengePromise.succeed(nil) + return + } + + offered = true + nextChallengePromise.succeed( + NIOSSHUserAuthenticationOffer( + username: username, + serviceName: "", + offer: .password(.init(password: password)) + ) + ) + } +} + +// MARK: - Exec channel + +/// Drives one exec channel: sends the request, streams stdin, splits stdout from +/// stderr, and captures the exit status. +final class ExecChannelHandler: ChannelInboundHandler { + typealias InboundIn = SSHChannelData + typealias OutboundOut = SSHChannelData + + /// SSH channel data is framed into packets; keep writes comfortably under + /// the 128 KiB default maximum packet size. + private static let chunkSize = 32 * 1024 + + private let command: String + private var stdin: ByteBuffer? + private var promise: EventLoopPromise? + + private var stdout = ByteBufferAllocator().buffer(capacity: 0) + private var stderr = ByteBufferAllocator().buffer(capacity: 0) + private var exitCode: Int32? + + init(command: String, stdin: ByteBuffer?, promise: EventLoopPromise) { + self.command = command + self.stdin = stdin + self.promise = promise + } + + func channelActive(context: ChannelHandlerContext) { + let request = SSHChannelRequestEvent.ExecRequest(command: command, wantReply: true) + let sent = context.eventLoop.makePromise(of: Void.self) + // Capture only Sendable values: the handler and its context must not + // escape onto another thread. + let resultPromise = promise + let channel = context.channel + sent.futureResult.whenFailure { error in + resultPromise?.fail(error) + channel.close(promise: nil) + } + context.triggerUserOutboundEvent(request, promise: sent) + context.fireChannelActive() + } + + func userInboundEventTriggered(context: ChannelHandlerContext, event: Any) { + switch event { + case is ChannelSuccessEvent: + sendStandardInput(context: context) + + case is ChannelFailureEvent: + fail(context: context, error: CoreError.sshFailed("guest refused to exec: \(command)")) + + case let status as SSHChannelRequestEvent.ExitStatus: + exitCode = Int32(truncatingIfNeeded: status.exitStatus) + + case let signal as SSHChannelRequestEvent.ExitSignal: + // A signalled process has no exit status; report it the way a shell + // would, and keep the signal name in stderr so it is not lost. + exitCode = 128 + var note = ByteBuffer(string: "\nterminated by SIG\(signal.signalName): \(signal.errorMessage)\n") + stderr.writeBuffer(¬e) + + default: + context.fireUserInboundEventTriggered(event) + } + } + + func channelRead(context: ChannelHandlerContext, data: NIOAny) { + let channelData = unwrapInboundIn(data) + guard case .byteBuffer(var bytes) = channelData.data else { return } + + switch channelData.type { + case .channel: stdout.writeBuffer(&bytes) + case .stdErr: stderr.writeBuffer(&bytes) + default: break // An extended data type we did not ask for. + } + } + + func channelInactive(context: ChannelHandlerContext) { + complete() + context.fireChannelInactive() + } + + func handlerRemoved(context: ChannelHandlerContext) { + complete() + } + + func errorCaught(context: ChannelHandlerContext, error: Error) { + fail(context: context, error: error) + } + + private func sendStandardInput(context: ChannelHandlerContext) { + if var payload = stdin { + stdin = nil + while payload.readableBytes > 0 { + let slice = payload.readSlice(length: min(Self.chunkSize, payload.readableBytes))! + context.write( + wrapOutboundOut(SSHChannelData(type: .channel, data: .byteBuffer(slice))), + promise: nil + ) + } + context.flush() + } + + // EOF either way: `cat > file` needs it to finish, and a command that + // would otherwise block reading stdin gets an immediate end of input. + context.close(mode: .output, promise: nil) + } + + private func complete() { + guard let promise else { return } + self.promise = nil + + if let exitCode { + promise.succeed( + SSHCommandResult( + exitCode: exitCode, + stdout: String(buffer: stdout), + stderr: String(buffer: stderr) + ) + ) + } else { + promise.fail( + CoreError.sshFailed("guest closed the channel without an exit status: \(command)") + ) + } + } + + private func fail(context: ChannelHandlerContext, error: Error) { + if let promise { + self.promise = nil + promise.fail(error) + } + context.close(promise: nil) + } +} + +/// Blocks until a guest accepts an authenticated SSH session, or the deadline +/// passes. +/// +/// Called after a DHCP lease appears but before any provisioning: a fresh guest +/// answers on port 22 only once `launchd` has started `sshd`, which lags the +/// lease by tens of seconds. +/// +/// - Parameters: +/// - host: Guest IP. +/// - port: SSH port. Defaults to `22`. +/// - username: Guest account. +/// - password: Guest password. +/// - timeout: Overall ceiling. +/// - pollInterval: Delay between attempts. Defaults to 2 s. +/// - Throws: ``CoreError/timeout(_:)`` if the guest never answers. +public func waitForSSH( + host: String, + port: Int = 22, + username: String, + password: String, + timeout: Duration, + pollInterval: Duration = .seconds(2) +) async throws { + let executor = SSHExecutor(host: host, port: port, username: username, password: password) + let started = ContinuousClock.now + var lastError: Error? + + while true { + do { + // A real authenticated session running a trivial command, not a bare + // TCP probe: sshd binds the port before it is ready to authenticate, + // so a connect that succeeds proves very little. + _ = try await executor.execute("true", stdin: nil, timeout: .seconds(20)) + return + } catch let error as SSHTransportError { + if case .authenticationFailed = error { + // Wrong credentials will not become right by waiting: the guest + // was provisioned with a different account or password, which is + // a build failure, not a boot delay. + throw error.asCoreError + } + lastError = error + } catch { + // Timeouts and mid-handshake closures are what a guest that is still + // starting `sshd` looks like. Keep waiting. + lastError = error + } + + guard ContinuousClock.now - started < timeout else { break } + try await Task.sleep(for: pollInterval) + guard ContinuousClock.now - started < timeout else { break } + } + + let detail = lastError.map { "; last error: \($0)" } ?? "" + throw CoreError.timeout("ssh on \(host):\(port)\(detail)") +} diff --git a/Sources/RunnerCore/SchedulerCore.swift b/Sources/RunnerCore/SchedulerCore.swift new file mode 100644 index 0000000..ae11e93 --- /dev/null +++ b/Sources/RunnerCore/SchedulerCore.swift @@ -0,0 +1,339 @@ +import Foundation + +/// What a VM slot is doing. +/// +/// Slots are fixed in number (two, matching both the kernel's concurrent-VM cap +/// and our two persistent MAC addresses) and are recycled, never created. +public enum SlotState: Sendable, Equatable { + /// No VM. Available to boot. + case idle + + /// A VM is being cloned/booted/provisioned; not yet registered with Gitea. + /// - Parameter since: When the transition happened, for boot-timeout checks. + case provisioning(since: Date) + + /// A VM is up with `gitea-runner daemon` attached. + /// + /// - Parameters: + /// - jobHint: The queued job whose presence motivated this boot, if known. + /// **Only a hint** — the server, not us, decides which job this runner + /// actually claims. + /// - since: When the VM went live, for job-timeout checks. + case running(jobHint: Int64?, since: Date) + + /// Whether the slot currently holds a VM (booting or live). + public var isOccupied: Bool { + if case .idle = self { return false } + return true + } + + /// When the slot entered its current state, or `nil` when idle. + public var since: Date? { + switch self { + case .idle: return nil + case .provisioning(let t): return t + case .running(_, let t): return t + } + } +} + +/// One recyclable VM slot. +public struct VMSlot: Sendable, Equatable, Identifiable { + /// Stable index, `0.. + + /// Creates a state with `count` idle slots and an empty ledger. + public init(slotCount: Int) { + self.slots = (0.. = []) { + self.slots = slots + self.dispatchedJobIDs = dispatchedJobIDs + } + + /// Slots not currently holding a VM. + public var idleSlots: [VMSlot] { slots.filter { !$0.state.isOccupied } } + + /// Slots holding a VM. + public var occupiedSlots: [VMSlot] { slots.filter { $0.state.isOccupied } } +} + +/// A side effect the orchestrator should perform. +/// +/// The planner returns these; it never performs I/O itself, which is what makes +/// the whole scheduling policy unit-testable against a fixed `now`. +public enum SchedulerAction: Sendable, Equatable { + /// Clone, boot, provision, and register a VM in the given slot. + /// - Parameters: + /// - slot: Slot id. + /// - jobHint: The queued job that motivated the boot. + case bootVM(slot: Int, jobHint: Int64) + + /// Stop and delete the VM in the given slot. + /// - Parameters: + /// - slot: Slot id. + /// - reason: Human-readable cause, logged and used in tests. + case teardownVM(slot: Int, reason: String) + + /// Explicit no-op. Returned so a caller can distinguish "planner ran and + /// chose to do nothing" from "planner returned an empty list". + case none +} + +/// The pure scheduling state machine. +/// +/// ## Capacity, not assignment +/// +/// A booted VM is **capacity**, not a promise to run a specific job. We register +/// an ephemeral runner and the *server* decides which queued job it claims — +/// possibly not the one that triggered the boot. That is fine and in fact +/// desirable: it means we never have to reimplement Gitea's matching rules. The +/// `jobHint` carried through ``SchedulerAction/bootVM(slot:jobHint:)`` and +/// ``SlotState/running(jobHint:since:)`` exists purely for logs and for the +/// dedup ledger. +/// +/// Because `--ephemeral` makes the server hand each runner exactly one task and +/// then deregister it, a slot's life is: boot → register → claim one job → the +/// `gitea-runner daemon` process exits → we tear down. There is no reuse, which +/// is what makes the VM genuinely disposable. +/// +/// ## Rules +/// +/// 1. Only jobs whose labels ``LabelSet/matches(jobLabels:)`` are considered. +/// 2. A job id already in ``SchedulerState/dispatchedJobIDs`` never boots a +/// second VM. +/// 3. At most `maxVMs` slots may be occupied (hard-clamped to 2 — the kernel +/// fails a third `start()` with `VZError.virtualMachineLimitExceeded`). +/// 4. Ledger entries for jobs no longer visible as queued are expired, so a +/// slot freed by a completed job can be re-earned by a genuinely new job. +/// 5. A slot in ``SlotState/provisioning(since:)`` longer than `bootTimeout`, or +/// ``SlotState/running(jobHint:since:)`` longer than `jobTimeout`, is torn +/// down. +public enum SchedulerCore { + /// Computes the next state and the actions to reach it. + /// + /// Deterministic and side-effect free: same inputs, same outputs. `now` is + /// injected rather than read so timeout behaviour is testable. + /// + /// - Parameters: + /// - state: Current state. + /// - queuedJobs: Jobs Gitea currently reports as `queued`. Callers must + /// not include `waiting` (blocked) jobs. + /// - labels: This host's label set. + /// - maxVMs: Concurrency cap; values above 2 are clamped. + /// - now: Reference time for timeout arithmetic. + /// - jobTimeout: Ceiling on ``SlotState/running(jobHint:since:)``. + /// - bootTimeout: Ceiling on ``SlotState/provisioning(since:)``. + /// - Returns: The updated state and the actions to execute, teardowns first + /// so a freed slot can be reused within the same pass. + public static func plan( + state: SchedulerState, + queuedJobs: [WorkflowJob], + labels: LabelSet, + maxVMs: Int, + now: Date, + jobTimeout: TimeInterval, + bootTimeout: TimeInterval + ) -> (SchedulerState, [SchedulerAction]) { + // The kernel fails a third concurrent guest, so the config never gets to + // negotiate this. Clamped here as well as in `RunnerConfig.validated()`. + let cap = min(max(maxVMs, 0), 2) + + var newState = state + var teardowns: [SchedulerAction] = [] + var boots: [SchedulerAction] = [] + + // 1. Which of the queued jobs are ours to serve, in the order Gitea + // reported them (so the plan is a deterministic function of input). + let matching = queuedJobs.filter { labels.matches(jobLabels: $0.labels) } + let queuedIDs = Set(queuedJobs.map(\.id)) + + // 2. Expire the dedup ledger against reality rather than against a + // timer: an id that is no longer queued was either claimed or + // cancelled, and dedup only matters while a job is still waiting. + newState.dispatchedJobIDs.formIntersection(queuedIDs) + + // 3. Timeouts, emitted before any boot so a slot freed here can be + // reused in this same pass. + for index in newState.slots.indices { + let slot = newState.slots[index] + switch slot.state { + case .idle: + continue + + case .provisioning(let since): + let age = now.timeIntervalSince(since) + guard age > bootTimeout else { continue } + teardowns.append( + .teardownVM( + slot: slot.id, + reason: "boot timeout: provisioning for \(Int(age))s (limit \(Int(bootTimeout))s)" + ) + ) + newState.slots[index].state = .idle + // Losing a boot must not permanently strand the job that + // motivated it. `SlotState.provisioning` deliberately carries no + // jobHint (the hint is a log/dedup detail, not an assignment), so + // there is no specific id to drop here. Instead we release one + // ledger entry — the lowest still-queued dispatched id, i.e. the + // oldest such job, since Gitea's ids increase monotonically. + // That is deterministic, releases exactly the capacity we lost, + // and lets a replacement VM boot (possibly on this very tick). + if let oldest = newState.dispatchedJobIDs.min() { + newState.dispatchedJobIDs.remove(oldest) + } + + case .running(let jobHint, let since): + let age = now.timeIntervalSince(since) + guard age > jobTimeout else { continue } + teardowns.append( + .teardownVM( + slot: slot.id, + reason: "job timeout: running for \(Int(age))s (limit \(Int(jobTimeout))s)" + ) + ) + newState.slots[index].state = .idle + // Same reasoning as above, except here we do know the hint. It is + // usually gone from the ledger already (a claimed job stops being + // queued), so this is normally a no-op. + if let jobHint { newState.dispatchedJobIDs.remove(jobHint) } + } + } + + // 4. Boot capacity for jobs we have not already booted for. + // + // A booted VM is CAPACITY, not an assignment: the ephemeral runner we + // register may legally claim a DIFFERENT matching job than the one + // whose presence motivated the boot. The counting still works out — + // one queued matching job earns one VM, and whichever job that VM + // claims stops being queued and drops out of the ledger. + for job in matching { + guard !newState.dispatchedJobIDs.contains(job.id) else { continue } + guard newState.occupiedSlots.count < cap else { break } + guard let free = newState.slots.firstIndex(where: { !$0.state.isOccupied }) else { break } + + boots.append(.bootVM(slot: newState.slots[free].id, jobHint: job.id)) + newState.slots[free].state = .provisioning(since: now) + newState.dispatchedJobIDs.insert(job.id) + } + + // Teardowns first, boots second. An empty list is the no-op; `.none` is + // never emitted, so callers never have to filter it out of a real plan. + return (newState, teardowns + boots) + } + + /// Records that a slot began booting for a job. + /// + /// Called by the orchestrator once it has actually started the clone/boot, + /// so that a failed `plan` execution does not leave a phantom occupied slot. + /// + /// - Returns: The updated state. + public static func markProvisioning( + state: SchedulerState, + slot: Int, + jobHint: Int64, + now: Date + ) -> SchedulerState { + var newState = state + guard let index = newState.slots.firstIndex(where: { $0.id == slot }) else { return newState } + newState.slots[index].state = .provisioning(since: now) + newState.dispatchedJobIDs.insert(jobHint) + return newState + } + + /// Promotes a slot from provisioning to running. + public static func markRunning( + state: SchedulerState, + slot: Int, + now: Date + ) -> SchedulerState { + var newState = state + guard let index = newState.slots.firstIndex(where: { $0.id == slot }) else { return newState } + // The hint, if any, is carried over purely so logs and the job-timeout + // teardown reason can name a job. It is never an assignment. + let hint: Int64? + if case .running(let existing, _) = newState.slots[index].state { + hint = existing + } else { + hint = nil + } + newState.slots[index].state = .running(jobHint: hint, since: now) + return newState + } + + /// Promotes a slot to running while recording the job that motivated its + /// boot, which ``SlotState/provisioning(since:)`` does not carry. + /// + /// Additive convenience over ``markRunning(state:slot:now:)``; the hint is + /// still only ever used for logging and the job-timeout reason string. + public static func markRunning( + state: SchedulerState, + slot: Int, + jobHint: Int64?, + now: Date + ) -> SchedulerState { + var newState = state + guard let index = newState.slots.firstIndex(where: { $0.id == slot }) else { return newState } + newState.slots[index].state = .running(jobHint: jobHint, since: now) + return newState + } + + /// Drops a job id from the dedup ledger. + /// + /// The ledger's only automatic expiry is "the job stopped being queued" + /// (``plan(state:queuedJobs:labels:maxVMs:now:jobTimeout:bootTimeout:)``, + /// step 2), which is exactly wrong for a boot that never happened: the job + /// is *still* queued, so its entry is retained and no further VM is ever + /// booted for it. Every failure path — a refused boot, a clone error, a lost + /// lease, a dead SSH channel — must call this, or the job waits out Gitea's + /// 24 h `ABANDONED_JOB_TIMEOUT` for nothing. + /// + /// Safe to call for an id that was never dispatched, or twice. + /// + /// - Parameters: + /// - state: Current state. + /// - jobID: The job to release. + /// - Returns: The updated state. + public static func releaseJob( + state: SchedulerState, + jobID: Int64 + ) -> SchedulerState { + var newState = state + newState.dispatchedJobIDs.remove(jobID) + return newState + } + + /// Returns a slot to ``SlotState/idle`` after teardown. + public static func markIdle( + state: SchedulerState, + slot: Int + ) -> SchedulerState { + var newState = state + guard let index = newState.slots.firstIndex(where: { $0.id == slot }) else { return newState } + newState.slots[index].state = .idle + return newState + } +} diff --git a/Sources/RunnerCore/Version.swift b/Sources/RunnerCore/Version.swift new file mode 100644 index 0000000..f5a03e5 --- /dev/null +++ b/Sources/RunnerCore/Version.swift @@ -0,0 +1,11 @@ +import Foundation + +/// Version stamp for the runner host tool itself. +/// +/// This is *not* the version of the `gitea-runner` binary installed into the +/// guest — that one lives in ``RunnerConfig/RunnerSection/version``. +public enum RunnerVersion { + /// The semantic version of this build, reported by `--version` and by + /// `doctor`. + public static let current = "0.1.0" +} diff --git a/Sources/RunnerHost/Doctor.swift b/Sources/RunnerHost/Doctor.swift new file mode 100644 index 0000000..eced3de --- /dev/null +++ b/Sources/RunnerHost/Doctor.swift @@ -0,0 +1,597 @@ +import Foundation +import RunnerCore +import Virtualization + +/// The outcome of one preflight check. +public struct DoctorCheck: Sendable, Equatable { + /// How a check turned out. + public enum Result: Sendable, Equatable { + /// Requirement satisfied. + case pass + /// Requirement not satisfied; the daemon will not work. + case fail + /// Not a hard requirement, but worth knowing about. + case warn + /// Informational only. + case info + } + + /// Short check name, e.g. `virtualization entitlement`. + public let name: String + /// Outcome. + public let result: Result + /// What was observed. + public let detail: String + /// What to do about it, when the outcome is not ``Result/pass``. + public let remediation: String? + + public init(name: String, result: Result, detail: String, remediation: String? = nil) { + self.name = name + self.result = result + self.detail = detail + self.remediation = remediation + } + + /// Whether this check blocks the daemon from working. + public var isBlocking: Bool { result == .fail } +} + +extension DoctorCheck.Result { + /// Lowercase name, for JSON output and log lines. + public var label: String { + switch self { + case .pass: return "pass" + case .fail: return "fail" + case .warn: return "warn" + case .info: return "info" + } + } + + /// Single-character marker used by ``Doctor/format(_:)``. + public var symbol: String { + switch self { + case .pass: return "✓" + case .fail: return "✗" + case .warn: return "!" + case .info: return "·" + } + } +} + +/// Preflight checks for a host that is supposed to run macOS guests. +/// +/// Each of these corresponds to a failure mode that is otherwise diagnosed only +/// by a confusing runtime error deep inside the boot path, so `doctor` exists to +/// surface them all at once, before anything is installed. +public enum Doctor { + + /// Runs every check. + /// + /// Checks performed: + /// + /// 1. **Architecture is arm64.** Virtualization cannot run macOS guests on + /// Intel at all. + /// 2. **Host macOS ≥ 26.** Required for ASIF disks; the guest-provisioning + /// automation additionally wants 27. + /// 3. **`VZVirtualMachine.isSupported`.** The framework's own verdict. + /// 4. **`com.apple.security.virtualization` entitlement present** on the + /// running binary, read with `codesign -d --entitlements - `. + /// Running from `.build/` instead of the signed `.app` is the single most + /// common setup mistake, and this is what catches it. + /// 5. **Free disk ≥ `storage.minFreeDiskGB`.** CoW clones grow as guests + /// write. + /// 6. **`login.keychain` unlocked**, via `security show-keychain-info + /// login.keychain`. macOS 15+ refuses to start a VM otherwise — the + /// reason the daemon must be a LaunchAgent in a logged-in session. + /// 7. **Gitea reachable and the token has admin scope**, probed with + /// ``GiteaClient/listRunners()``. A non-admin token fails here rather + /// than at the first poll. + /// 8. **Registration token resolvable** from file, inline value, or (if + /// enabled) the API. + /// 9. **Runner download URL is live**, via a `HEAD` expecting 200. Catches a + /// version bump that no longer has a darwin-arm64 asset. + /// 10. **Local Network privacy note** (informational). On macOS 15+ the + /// first attempt to reach a guest over the NAT link can be blocked by + /// the Local Network permission prompt, which a background agent cannot + /// answer; the operator must approve the app once. + /// + /// - Parameter config: Validated configuration. Gitea-dependent checks are + /// skipped with a ``DoctorCheck/Result/warn`` when no admin token is set. + /// - Returns: Checks in the order above. + public static func runChecks(config: RunnerConfig) async -> [DoctorCheck] { + var checks = hostChecks() + checks.append(checkDiskSpace(config: config)) + checks.append(checkLoginKeychain()) + checks.append(contentsOf: await checkGitea(config: config)) + checks.append(await checkRunnerDownloadURL(config: config)) + checks.append(localNetworkNote()) + return checks + } + + /// Runs every check, loading configuration from `path` first. + /// + /// The host checks still run when the configuration is missing or invalid, + /// which is the state a first-time operator is actually in. + /// + /// - Parameter configPath: Path to the configuration file; tilde-expanded. + /// - Returns: Checks, with configuration loading itself reported as a check. + public static func runChecks(configPath: String) async -> [DoctorCheck] { + var checks = hostChecks() + + let loaded: RunnerConfig + do { + loaded = try RunnerConfig.load(from: configPath).validated() + checks.append( + DoctorCheck( + name: "configuration", + result: .pass, + detail: "loaded and validated \(RunnerConfig.expandTilde(configPath))" + ) + ) + } catch { + checks.append( + DoctorCheck( + name: "configuration", + result: .fail, + detail: "\(error)", + remediation: "run `gitea-macos-runner config init` and edit \(RunnerConfig.expandTilde(configPath))" + ) + ) + checks.append(localNetworkNote()) + return checks + } + + checks.append(checkDiskSpace(config: loaded)) + checks.append(checkLoginKeychain()) + checks.append(contentsOf: await checkGitea(config: loaded)) + checks.append(await checkRunnerDownloadURL(config: loaded)) + checks.append(contentsOf: checkTokenFilePermissions(config: loaded)) + checks.append(localNetworkNote()) + return checks + } + + /// The configuration-independent host checks: architecture, OS version, + /// framework support, entitlement. + public static func hostChecks() -> [DoctorCheck] { + [ + checkHostCapability(), + checkVirtualizationSupported(), + checkVirtualizationEntitlement(), + ] + } + + /// Whether the running binary carries `com.apple.security.virtualization`. + /// + /// - Parameter binaryPath: Defaults to the current executable. + /// - Returns: The check result. + public static func checkVirtualizationEntitlement( + binaryPath: String = CommandLine.arguments.first ?? "" + ) -> DoctorCheck { + let name = "virtualization entitlement" + let remediation = """ + build and install the signed bundle: `make install`, then run \ + ~/Applications/GiteaMacosRunner.app/Contents/MacOS/gitea-macos-runner + """ + + guard let executable = resolveExecutablePath(binaryPath) else { + return DoctorCheck( + name: name, + result: .warn, + detail: "could not locate the running executable to inspect", + remediation: remediation + ) + } + + // The entitlement lives on the signature, so a bare binary copied out of + // the bundle loses it. Report where we are as well as what we found. + let inAppBundle = executable.contains(".app/Contents/MacOS/") + let signedTarget = inAppBundle + ? String(executable.prefix(upTo: executable.range(of: ".app/Contents/MacOS/")!.upperBound) + .dropLast("/Contents/MacOS/".count)) + : executable + + let result = DoctorShell.run( + "/usr/bin/codesign", + ["-d", "--entitlements", "-", "--xml", signedTarget] + ) + let hasEntitlement = result.output.contains("com.apple.security.virtualization") + + if hasEntitlement { + return DoctorCheck( + name: name, + result: .pass, + detail: "present on \(signedTarget)" + ) + } + + if inAppBundle { + return DoctorCheck( + name: name, + result: .fail, + detail: "\(signedTarget) is not signed with com.apple.security.virtualization", + remediation: "re-sign the bundle: `make sign` (or `make install`)" + ) + } + + // Running the plain SwiftPM binary is normal for `doctor`, `config`, and + // `service`; it only becomes fatal when a VM is actually started. + return DoctorCheck( + name: name, + result: .warn, + detail: "running an unsigned binary at \(executable); VM starts will fail", + remediation: remediation + ) + } + + /// Whether `login.keychain` is currently unlocked. + public static func checkLoginKeychain() -> DoctorCheck { + let name = "login.keychain unlocked" + let result = DoctorShell.run("/usr/bin/security", ["show-keychain-info", "login.keychain"]) + + if result.exitCode == 0 { + return DoctorCheck(name: name, result: .pass, detail: "unlocked") + } + + return DoctorCheck( + name: name, + result: .warn, + detail: "locked or unavailable (security exited \(result.exitCode))", + remediation: """ + macOS 15+ refuses to start a VM while login.keychain is locked. Configure \ + automatic login, and do not lock the session — the daemon runs as a \ + LaunchAgent inside it. + """ + ) + } + + /// Whether the host architecture and macOS version can run macOS guests. + public static func checkHostCapability() -> DoctorCheck { + let name = "host capability" + let version = ProcessInfo.processInfo.operatingSystemVersion + let versionString = "\(version.majorVersion).\(version.minorVersion).\(version.patchVersion)" + + // Emitted straight from the preprocessor branch rather than via a `let + // isAppleSilicon` flag: with the flag, one arm is a compile-time + // constant and the compiler warns that the other is dead code. + #if !arch(arm64) + return DoctorCheck( + name: name, + result: .fail, + detail: "not an Apple silicon host; macOS guests require arm64", + remediation: "run this daemon on an Apple silicon Mac" + ) + #else + + guard version.majorVersion >= 26 else { + return DoctorCheck( + name: name, + result: .fail, + detail: "macOS \(versionString); this daemon requires macOS 26 or newer", + remediation: "upgrade the host: ASIF sparse disks need macOS 26+" + ) + } + + if version.majorVersion < 27 { + return DoctorCheck( + name: name, + result: .warn, + detail: "arm64, macOS \(versionString)", + remediation: """ + automated guest provisioning (VZMacGuestProvisioningOptions) needs macOS 27 \ + on both host and guest; on 26 the first boot's Setup Assistant must be \ + completed by hand once per image + """ + ) + } + + return DoctorCheck(name: name, result: .pass, detail: "arm64, macOS \(versionString)") + #endif + } + + /// The framework's own verdict on this host. + public static func checkVirtualizationSupported() -> DoctorCheck { + let name = "Virtualization.framework" + if VZVirtualMachine.isSupported { + return DoctorCheck(name: name, result: .pass, detail: "VZVirtualMachine.isSupported == true") + } + return DoctorCheck( + name: name, + result: .fail, + detail: "VZVirtualMachine.isSupported == false", + remediation: "this host cannot run virtual machines" + ) + } + + /// Free space on the store volume against `storage.minFreeDiskGB`. + public static func checkDiskSpace(config: RunnerConfig) -> DoctorCheck { + let name = "free disk space" + let store = VMStore(config: config) + do { + let free = try store.freeDiskSpace() + let freeGB = Double(free) / 1_073_741_824 + let detail = String( + format: "%.1f GB free at %@ (minimum %d GB)", + freeGB, config.storeDirectoryURL.path, config.storage.minFreeDiskGB + ) + if freeGB < Double(config.storage.minFreeDiskGB) { + return DoctorCheck( + name: name, + result: .fail, + detail: detail, + remediation: "free space, or lower storage.minFreeDiskGB" + ) + } + return DoctorCheck(name: name, result: .pass, detail: detail) + } catch { + return DoctorCheck( + name: name, + result: .warn, + detail: "could not measure free space: \(error)", + remediation: "check that \(config.storeDirectoryURL.path) exists and is readable" + ) + } + } + + /// Reachability, admin scope, and registration-token availability. + public static func checkGitea(config: RunnerConfig) async -> [DoctorCheck] { + var checks: [DoctorCheck] = [] + + let adminToken: String? + do { + adminToken = try config.resolveAdminToken() + } catch { + checks.append( + DoctorCheck( + name: "gitea admin token", + result: .fail, + detail: "\(error)", + remediation: "check gitea.adminTokenFile / gitea.adminToken" + ) + ) + return checks + } + + guard let adminToken, !adminToken.isEmpty else { + checks.append( + DoctorCheck( + name: "gitea admin api", + result: .warn, + detail: "no admin token configured; skipped", + remediation: "set gitea.adminTokenFile to a file holding an ADMIN user's API token" + ) + ) + return checks + } + + let client = GiteaClient(baseURL: config.gitea.instanceURL, token: adminToken) + do { + let runners = try await client.listRunners() + checks.append( + DoctorCheck( + name: "gitea admin api", + result: .pass, + detail: "\(config.gitea.instanceURL.absoluteString) reachable; \(runners.count) runner(s) registered" + ) + ) + } catch { + checks.append( + DoctorCheck( + name: "gitea admin api", + result: .fail, + detail: "\(error)", + remediation: """ + every endpoint used lives under /api/v1/admin/actions/ — the token must \ + belong to a Gitea administrator, and the instance must be reachable + """ + ) + ) + } + + checks.append(await checkRegistrationToken(config: config, client: client)) + return checks + } + + /// Whether a registration token can be obtained at all. + private static func checkRegistrationToken(config: RunnerConfig, client: GiteaClient) async -> DoctorCheck { + let name = "registration token" + do { + if let staticToken = try config.resolveStaticRegistrationToken(), !staticToken.isEmpty { + return DoctorCheck(name: name, result: .pass, detail: "resolved from configuration") + } + } catch { + return DoctorCheck( + name: name, + result: .fail, + detail: "\(error)", + remediation: "check gitea.registrationTokenFile" + ) + } + + guard config.gitea.fetchRegistrationTokenViaAPI else { + return DoctorCheck( + name: name, + result: .fail, + detail: "no static token configured and gitea.fetchRegistrationTokenViaAPI is off", + remediation: """ + seed a fixed token server-side (GITEA_RUNNER_REGISTRATION_TOKEN) and point \ + gitea.registrationTokenFile at a copy of it + """ + ) + } + + do { + let token = try await client.getRegistrationToken() + guard !token.isEmpty else { + return DoctorCheck( + name: name, + result: .fail, + detail: "the API returned an empty token", + remediation: "configure gitea.registrationTokenFile instead" + ) + } + return DoctorCheck(name: name, result: .pass, detail: "fetched from the admin API") + } catch { + return DoctorCheck( + name: name, + result: .fail, + detail: "\(error)", + remediation: "configure gitea.registrationTokenFile instead" + ) + } + } + + /// Whether the configured `gitea-runner` asset still exists. + public static func checkRunnerDownloadURL(config: RunnerConfig) async -> DoctorCheck { + let name = "runner download url" + let url: URL + do { + url = try config.runner.resolvedDownloadURL + } catch { + return DoctorCheck( + name: name, + result: .fail, + detail: "\(error)", + remediation: "check runner.runnerDownloadURL and runner.version" + ) + } + + var request = URLRequest(url: url) + request.httpMethod = "HEAD" + request.timeoutInterval = 15 + + do { + let (_, response) = try await URLSession.shared.data(for: request) + let status = (response as? HTTPURLResponse)?.statusCode ?? 0 + if status <= 399 { + return DoctorCheck(name: name, result: .pass, detail: "\(url.absoluteString) → \(status)") + } + return DoctorCheck( + name: name, + result: .warn, + detail: "\(url.absoluteString) → \(status)", + remediation: "check runner.version and runner.runnerDownloadURL for a darwin-arm64 asset" + ) + } catch { + // Best effort: a proxy or offline build host is not a reason to + // block the daemon. + return DoctorCheck( + name: name, + result: .warn, + detail: "could not reach \(url.absoluteString): \(error.localizedDescription)", + remediation: nil + ) + } + } + + /// Warns about token files readable by other users on this Mac. + public static func checkTokenFilePermissions(config: RunnerConfig) -> [DoctorCheck] { + let insecure = config.insecureTokenFilePaths + guard !insecure.isEmpty else { return [] } + return [ + DoctorCheck( + name: "token file permissions", + result: .warn, + detail: "group/world readable: \(insecure.joined(separator: ", "))", + remediation: "chmod 600 \(insecure.joined(separator: " "))" + ) + ] + } + + /// The macOS 15+ Local Network permission note. + public static func localNetworkNote() -> DoctorCheck { + DoctorCheck( + name: "local network access", + result: .info, + detail: "guests are reached over the host-private NAT link", + remediation: """ + on macOS 15+ the first connection to a guest can be blocked by the Local Network \ + privacy prompt, which a background LaunchAgent cannot answer. Approve the app once \ + under System Settings → Privacy & Security → Local Network. + """ + ) + } + + /// Renders checks as aligned, human-readable lines for the CLI. + public static func format(_ checks: [DoctorCheck]) -> String { + let width = checks.map(\.name.count).max() ?? 0 + var lines: [String] = [] + + for check in checks { + let padded = check.name.padding(toLength: max(width, check.name.count), withPad: " ", startingAt: 0) + lines.append("\(check.result.symbol) \(padded) \(check.detail)") + if check.result != .pass, let remediation = check.remediation { + for (index, wrapped) in wrap(remediation, width: 76).enumerated() { + let prefix = index == 0 ? "→ " : " " + lines.append(String(repeating: " ", count: width + 4) + prefix + wrapped) + } + } + } + + let failures = checks.filter(\.isBlocking).count + let warnings = checks.filter { $0.result == .warn }.count + lines.append("") + lines.append("\(checks.count) checks, \(failures) failed, \(warnings) warned") + return lines.joined(separator: "\n") + } + + /// Greedy word wrap for remediation text. + private static func wrap(_ text: String, width: Int) -> [String] { + var lines: [String] = [] + var current = "" + for word in text.split(whereSeparator: { $0 == " " || $0 == "\n" }) { + if current.isEmpty { + current = String(word) + } else if current.count + 1 + word.count <= width { + current += " " + word + } else { + lines.append(current) + current = String(word) + } + } + if !current.isEmpty { lines.append(current) } + return lines + } + + /// Resolves the running executable, preferring the bundle's own record of it + /// over `argv[0]`, which may be a symlink or a bare command name. + private static func resolveExecutablePath(_ candidate: String) -> String? { + let fm = FileManager.default + if !candidate.isEmpty, candidate.hasPrefix("/"), fm.fileExists(atPath: candidate) { + return URL(fileURLWithPath: candidate).resolvingSymlinksInPath().path + } + if let executableURL = Bundle.main.executableURL { + return executableURL.resolvingSymlinksInPath().path + } + return nil + } +} + +/// Minimal synchronous process runner for the tools `doctor` shells out to. +private enum DoctorShell { + struct Output { + let exitCode: Int32 + let output: String + } + + static func run(_ launchPath: String, _ arguments: [String]) -> Output { + let process = Process() + process.executableURL = URL(fileURLWithPath: launchPath) + process.arguments = arguments + + let pipe = Pipe() + process.standardOutput = pipe + // codesign and security both report on stderr; merge so callers can grep + // one stream. + process.standardError = pipe + + do { + try process.run() + } catch { + return Output(exitCode: 127, output: "\(error)") + } + + let data = pipe.fileHandleForReading.readDataToEndOfFile() + process.waitUntilExit() + return Output(exitCode: process.terminationStatus, output: String(data: data, encoding: .utf8) ?? "") + } +} diff --git a/Sources/RunnerHost/GuestProvisioner.swift b/Sources/RunnerHost/GuestProvisioner.swift new file mode 100644 index 0000000..03daf94 --- /dev/null +++ b/Sources/RunnerHost/GuestProvisioner.swift @@ -0,0 +1,599 @@ +import Foundation +import RunnerCore + +/// Turns a bare macOS guest into something that can execute Gitea Actions jobs. +/// +/// ## What a guest actually needs +/// +/// Gitea Actions in `host` schema does not containerize anything: it shells out +/// on the guest. The hard requirements are therefore small but non-negotiable: +/// +/// * **`gitea-runner`** — the runner binary itself (v3.x; renamed from +/// `act_runner`, now published from `gitea.com/gitea/runner`). +/// * **`node`** — not optional. JavaScript actions such as `actions/checkout` +/// are executed by spawning `node` directly; without it, essentially every +/// real workflow fails at its first step. Installed from Apple's official +/// arm64 `.pkg` via `installer -pkg`. +/// * **`git`** and **`bash`** — present on stock macOS, but `git` only after the +/// Command Line Tools are materialized, so presence is verified rather than +/// assumed. +/// * **A writable `$HOME`** — the runner writes its registration and workspace +/// under the guest account's home directory. +/// +/// ## What else provisioning does +/// +/// Everything in `Resources/provision.sh`: a passwordless-sudo drop-in +/// installed through `visudo -cf` (validated before it is moved into place, so a +/// syntax error cannot lock the account out), disabling sleep/screensaver so a +/// long job is not interrupted, disabling Spotlight indexing of build +/// directories, raising `maxfiles` (Xcode and npm both exhaust the stock 256), +/// and pre-seeding `github.com` into `known_hosts` so a checkout does not stall +/// on host-key confirmation. +/// +/// - Note: The guest must **never** ship a `gitea-runner` `config.yaml` that +/// sets `runner.labels`: that key silently overrides the `--labels` passed at +/// registration, so the runner would advertise the wrong labels and never be +/// matched. +public struct GuestProvisioner: Sendable { + + /// Creates a provisioner. + public init() {} + + /// Runs the full provisioning sequence against a booted guest. + /// + /// Steps, in order: + /// 1. Upload `Resources/provision.sh` to `/tmp/provision.sh`, `chmod +x`, + /// and run it under `sudo` with the guest username and password passed + /// via the environment (never as arguments, which are world-visible in + /// `ps`). + /// 2. Download the Node.js arm64 `.pkg` on the **host**, upload it, and + /// `installer -pkg … -target /`. Downloading host-side keeps the guest + /// off the public internet for this step and makes the version pinnable. + /// 3. Verify `git`, `bash`, and `node` all resolve. + /// 4. Download the `gitea-runner` darwin-arm64 release asset on the host + /// (URL from ``RunnerConfig/RunnerSection/resolvedDownloadURL``), upload + /// it to `/usr/local/bin/gitea-runner`, and `chmod +x`. + /// 5. Verify `gitea-runner --version` runs. + /// + /// - Parameters: + /// - executor: A connected guest executor. + /// - config: Supplies guest credentials and the runner download URL. + /// - progress: Optional per-step callback, for `image build` output. + /// - Throws: ``CoreError/provisioningFailed(_:)`` naming the failed step. + public func provision( + executor: any GuestExecutor, + config: RunnerConfig, + progress: (@Sendable (String) -> Void)? = nil + ) async throws { + progress?("system configuration (provision.sh)") + try await runProvisionScript(executor: executor, config: config) + + progress?("Node.js \(Self.defaultNodeVersion)") + try await installNode(executor: executor) + + progress?("verifying toolchain") + try await verifyToolchain(executor: executor) + + progress?("gitea-runner \(config.runner.version)") + try await installGiteaRunner(executor: executor, config: config) + } + + /// Uploads and executes `Resources/provision.sh`. + /// + /// - Parameters: + /// - executor: A connected guest executor. + /// - config: Guest credentials. + public func runProvisionScript( + executor: any GuestExecutor, + config: RunnerConfig + ) async throws { + let scriptURL = try Self.provisionScriptURL() + + do { + try await executor.upload(localPath: scriptURL.path, remotePath: "/tmp/provision.sh") + } catch { + throw CoreError.provisioningFailed( + "could not upload provision.sh from \(scriptURL.path): \(error)" + ) + } + + // The account password has to reach `sudo -S` somehow, and every obvious + // route leaks it: as an argument it is visible in `ps` to any process on + // the guest, and `echo pw | sudo -S` puts it in the shell's own argv, + // which is exactly the same exposure. A mode-0600 file read via stdin + // redirection is the one form that never appears in an argument list; it + // is removed in the same command, so it does not outlive the call even if + // the script fails. + try await executor.uploadData( + Data((config.guest.password + "\n").utf8), + remotePath: "/tmp/.gmr-auth", + mode: "0600" + ) + + let giteaHost = config.gitea.instanceURL.host ?? "" + // `sudo VAR=value cmd` is how variables survive sudo's env_reset; a + // plain `VAR=value sudo cmd` would be stripped. Neither the username nor + // the Gitea hostname is secret, so argv exposure is fine for these. + let command = """ + sudo -S -p '' \ + GUEST_USER=\(Self.shellQuote(config.guest.username)) \ + GITEA_HOST=\(Self.shellQuote(giteaHost)) \ + /bin/bash /tmp/provision.sh < /tmp/.gmr-auth; \ + rc=$?; rm -f /tmp/.gmr-auth /tmp/provision.sh; exit $rc + """ + + // Generous: the Command Line Tools download inside the script is the + // long pole and is itself bounded at 45 minutes. + let result = try await executor.run(command, timeout: .seconds(3600)) + + guard result.succeeded else { + throw CoreError.provisioningFailed( + "provision.sh failed (exit \(result.exitCode))\n" + + Self.tail(result.stderr.isEmpty ? result.stdout : result.stderr) + ) + } + + // The script warns rather than aborts on best-effort steps, so a zero + // exit alone does not prove it ran to the end — a truncated SSH channel + // would also look like success. The marker is the actual proof. + guard result.stdout.contains("PROVISION_OK") else { + throw CoreError.provisioningFailed( + "provision.sh exited 0 but never printed PROVISION_OK; it did not run to completion\n" + + Self.tail(result.stdout) + ) + } + } + + /// Installs Node.js from the official arm64 package. + /// + /// - Parameters: + /// - executor: A connected guest executor. + /// - version: Node major/minor/patch, e.g. `22.11.0`. + /// - packageURL: Overrides the derived download URL entirely. Used by + /// ``resolveLatestLTSNodeVersion()`` callers and by air-gapped setups + /// pointing at an internal mirror. + public func installNode( + executor: any GuestExecutor, + version: String = GuestProvisioner.defaultNodeVersion, + packageURL: URL? = nil + ) async throws { + guard let url = packageURL ?? Self.nodePackageURL(version: version) else { + throw CoreError.configInvalid("cannot form a Node.js package URL for version \(version)") + } + + // Downloaded host-side rather than by the guest: the version is then + // pinned by the host's config, the guest needs no outbound access for + // this step, and a rebuild of ten images hits the host's cache instead of + // nodejs.org ten times. + let local = try await Self.downloadToTemporaryFile(url: url, suggestedName: "node.pkg") + defer { try? FileManager.default.removeItem(at: local) } + + do { + try await executor.upload(localPath: local.path, remotePath: "/tmp/node.pkg") + } catch { + throw CoreError.provisioningFailed("could not upload the Node.js package: \(error)") + } + + let install = try await executor.run( + "sudo -n /usr/sbin/installer -pkg /tmp/node.pkg -target /; rc=$?; rm -f /tmp/node.pkg; exit $rc", + timeout: .seconds(900) + ) + guard install.succeeded else { + throw CoreError.provisioningFailed( + "installing Node.js failed (exit \(install.exitCode))\n" + + Self.tail(install.stderr.isEmpty ? install.stdout : install.stderr) + ) + } + + let check = try await executor.run(Self.withGuestPath("node --version"), timeout: .seconds(120)) + guard check.succeeded else { + throw CoreError.provisioningFailed( + "Node.js installed but `node --version` failed (exit \(check.exitCode)). " + + "Gitea's JavaScript actions spawn `node` directly, so this image would fail " + + "every workflow that uses actions/checkout.\n" + + Self.tail(check.stderr.isEmpty ? check.stdout : check.stderr) + ) + } + } + + /// Verifies that `git`, `bash`, and `node` are all present and executable. + /// + /// - Throws: ``CoreError/provisioningFailed(_:)`` naming what is missing. + public func verifyToolchain(executor: any GuestExecutor) async throws { + // Order matters. On a vanilla guest `/usr/bin/git` is a shim that pops a + // GUI "install command line developer tools" dialog and blocks until + // someone clicks it — which, headless, is never. So the presence of a + // real git is established from the *package receipt* first, and `git` + // itself is only invoked once that check passes. + let hasTools = try await executor.run( + "pkgutil --pkg-info=com.apple.pkg.CLTools_Executables >/dev/null 2>&1 " + + "|| [ -x /Applications/Xcode.app/Contents/Developer/usr/bin/git ]", + timeout: .seconds(120) + ) + guard hasTools.succeeded else { + throw CoreError.provisioningFailed( + """ + the guest has no Command Line Tools, so `git` is only a stub that blocks on a \ + GUI installer dialog. provision.sh attempted a non-interactive install and it \ + did not take. Install a real toolchain instead: + + gitea-macos-runner image provision --xcode-xip /path/to/Xcode.xip + + (Shipping the image without git would fail every checkout at job time rather \ + than here, so the build stops now.) + """ + ) + } + + for (tool, command) in [ + ("git", "git --version"), + ("bash", "bash --version"), + ("node", "node --version"), + ] { + let result = try await executor.run(Self.withGuestPath(command), timeout: .seconds(120)) + guard result.succeeded else { + throw CoreError.provisioningFailed( + "required tool `\(tool)` is not usable in the guest (`\(command)` exited \(result.exitCode))\n" + + Self.tail(result.stderr.isEmpty ? result.stdout : result.stderr) + ) + } + } + } + + /// Downloads the `gitea-runner` release asset on the host and installs it + /// into the guest at `/usr/local/bin/gitea-runner`. + /// + /// - Parameters: + /// - executor: A connected guest executor. + /// - config: Supplies the download URL template and version. + public func installGiteaRunner( + executor: any GuestExecutor, + config: RunnerConfig + ) async throws { + let url = try config.runner.resolvedDownloadURL + + let local = try await Self.downloadToTemporaryFile(url: url, suggestedName: "gitea-runner") + defer { try? FileManager.default.removeItem(at: local) } + + do { + try await executor.upload(localPath: local.path, remotePath: "/tmp/gitea-runner") + } catch { + throw CoreError.provisioningFailed("could not upload the gitea-runner binary: \(error)") + } + + try await executor.runChecked( + "sudo -n /usr/bin/install -o root -g wheel -m 755 /tmp/gitea-runner /usr/local/bin/gitea-runner " + + "&& rm -f /tmp/gitea-runner", + timeout: .seconds(300) + ) + + // arm64 macOS refuses to exec a binary with no code signature at all + // (SIGKILL, no diagnostic). Release tarballs are usually ad-hoc signed + // already, in which case re-signing is a no-op; when they are not, this + // is what keeps the runner from being killed on its first invocation. + // Best-effort: `codesign` needs the Command Line Tools, and a signature + // that was already valid does not need replacing. + _ = try? await executor.run( + "sudo -n /usr/bin/codesign --force --sign - /usr/local/bin/gitea-runner", + timeout: .seconds(300) + ) + + let check = try await executor.run( + Self.withGuestPath("gitea-runner --version"), + timeout: .seconds(120) + ) + guard check.succeeded else { + throw CoreError.provisioningFailed( + "gitea-runner installed from \(url.absoluteString) but `gitea-runner --version` " + + "failed (exit \(check.exitCode))\n" + + Self.tail(check.stderr.isEmpty ? check.stdout : check.stderr) + ) + } + } + + /// Installs Xcode from a `.xip` into the guest — the optional heavy step. + /// + /// Xcode is not installed by default: the `.xip` is ~8 GB and expanding it + /// roughly triples the base image, which is a poor default for a workflow + /// that only needs `swift build`. Operators who need it run + /// `image provision NAME --xcode-xip PATH` once, after which every clone + /// inherits it. + /// + /// Expansion uses `xip --expand` inside the guest, followed by + /// `xcode-select -s` and `xcodebuild -license accept`, and finishes by + /// running `xcodebuild -runFirstLaunch` so the first job does not pay for + /// component installation. + /// + /// - Parameters: + /// - executor: A connected guest executor. + /// - xipPath: Path to the `.xip` **on the host**; it is uploaded. + public func installXcode(executor: any GuestExecutor, xipPath: String) async throws { + let localURL = URL(fileURLWithPath: (xipPath as NSString).expandingTildeInPath) + guard FileManager.default.fileExists(atPath: localURL.path) else { + throw CoreError.notFound("Xcode .xip not found at \(localURL.path)") + } + + let remoteXIP = "/tmp/Xcode.xip" + // Uploads go over an SSH exec channel with the payload as stdin, and + // `GuestExecutor.upload` reads the whole local file into memory first — + // fine for a 90 MB pkg, ruinous for a 12 GB xip. So this streams the file + // in bounded chunks and appends them guest-side instead. It is still slow + // (an exec channel is not SCP), but it is functional and its host memory + // use is capped at one chunk. + try await executor.runChecked("rm -f \(Self.shellQuote(remoteXIP))", timeout: .seconds(120)) + try await Self.uploadLargeFile(executor: executor, localURL: localURL, remotePath: remoteXIP) + + // Free the disk the old copy occupies before expanding into ~40 GB more. + _ = try? await executor.run("sudo -n rm -rf /Applications/Xcode.app", timeout: .seconds(600)) + + let staging = "/tmp/xcode-expand" + // `xip --expand` writes into the current directory and needs no sudo, but + // /tmp is small on some layouts; staging under /tmp keeps it beside the + // archive so the later move is a rename within one volume where possible. + try await executor.runChecked( + "rm -rf \(Self.shellQuote(staging)) && mkdir -p \(Self.shellQuote(staging))", + timeout: .seconds(300) + ) + + // Expansion of a full Xcode takes 20–45 minutes on VM-backed storage. + try await executor.runChecked( + "cd \(Self.shellQuote(staging)) && sudo -n /usr/bin/xip --expand \(Self.shellQuote(remoteXIP))", + timeout: .seconds(5400) + ) + + // Writing into /Applications needs root. + try await executor.runChecked( + "sudo -n mv \(Self.shellQuote(staging + "/Xcode.app")) /Applications/Xcode.app " + + "&& sudo -n rm -rf \(Self.shellQuote(staging)) \(Self.shellQuote(remoteXIP))", + timeout: .seconds(1800) + ) + + // xcode-select writes /var/db/xcode_select_link — root only. + try await executor.runChecked( + "sudo -n /usr/bin/xcode-select -s /Applications/Xcode.app/Contents/Developer", + timeout: .seconds(300) + ) + + // Both of these write under /Library and must run as root; -runFirstLaunch + // installs the bundled packages (simulators, device support) that would + // otherwise be installed lazily during the first job. + try await executor.runChecked( + "sudo -n /usr/bin/xcodebuild -license accept", + timeout: .seconds(600) + ) + try await executor.runChecked( + "sudo -n /usr/bin/xcodebuild -runFirstLaunch", + timeout: .seconds(3600) + ) + + let check = try await executor.run("/usr/bin/xcodebuild -version", timeout: .seconds(300)) + guard check.succeeded else { + throw CoreError.provisioningFailed( + "Xcode installed but `xcodebuild -version` failed (exit \(check.exitCode))\n" + + Self.tail(check.stderr.isEmpty ? check.stdout : check.stderr) + ) + } + } + + /// The Node.js version installed when none is specified. + /// + /// Pinned rather than resolved at build time so that two images built weeks + /// apart are identical unless someone changes this line. Use + /// ``resolveLatestLTSNodeVersion()`` to look up a newer LTS deliberately. + public static let defaultNodeVersion = "24.19.0" + + /// The official Node.js macOS arm64 package URL for a version. + public static func nodePackageURL(version: String) -> URL? { + URL(string: "https://nodejs.org/dist/v\(version)/node-v\(version).pkg") + } + + /// Looks up the current Node.js LTS version from nodejs.org. + /// + /// Best-effort and deliberately not called by ``provision(executor:config:progress:)``: + /// an image build that silently picks up a different Node depending on the + /// day it ran is not reproducible. Callers that want the newest LTS pass the + /// result to ``installNode(executor:version:packageURL:)`` explicitly. + /// + /// - Returns: The version string without the leading `v`, or `nil` if the + /// index could not be read. + public static func resolveLatestLTSNodeVersion() async -> String? { + guard let indexURL = URL(string: "https://nodejs.org/dist/index.json") else { return nil } + var request = URLRequest(url: indexURL) + request.timeoutInterval = 30 + + guard let (data, response) = try? await URLSession.shared.data(for: request), + let http = response as? HTTPURLResponse, (200..<300).contains(http.statusCode), + let entries = try? JSONSerialization.jsonObject(with: data) as? [[String: Any]] + else { return nil } + + // The index is newest-first, and `lts` is `false` for non-LTS releases + // and the codename string ("Krypton") for LTS ones. + for entry in entries { + guard let version = entry["version"] as? String else { continue } + if entry["lts"] is String { + return String(version.dropFirst()) // "v24.19.0" -> "24.19.0" + } + } + return nil + } + + // MARK: - Locating provision.sh + + /// Finds `Resources/provision.sh`. + /// + /// The package declares no SwiftPM `resources:`, so `Bundle.module` does not + /// exist and the script has to be located by hand. Three deployments matter: + /// the signed `.app` the daemon actually runs from (`Contents/Resources`), a + /// bare `swift build` binary in `.build/debug`, and a `swift run` from the + /// checkout. Each is tried in turn, and the error names every path searched + /// so a packaging mistake is diagnosable from the message alone. + /// + /// - Returns: URL of the script. + /// - Throws: ``CoreError/notFound(_:)`` listing the searched paths. + public static func provisionScriptURL() throws -> URL { + let fileManager = FileManager.default + var searched: [URL] = [] + + func check(_ url: URL) -> URL? { + searched.append(url) + return fileManager.isReadableFile(atPath: url.path) ? url : nil + } + + // 1. The .app's own resources, via the bundle API and by hand (the API + // returns nil for a bare executable with no Info.plist). + if let url = Bundle.main.url(forResource: "provision", withExtension: "sh") { + searched.append(url) + if fileManager.isReadableFile(atPath: url.path) { return url } + } + + var roots: [URL] = [Bundle.main.bundleURL] + if let executableDirectory = Bundle.main.executableURL? + .resolvingSymlinksInPath() + .deletingLastPathComponent() + { + roots.append(executableDirectory) + } + roots.append(URL(fileURLWithPath: fileManager.currentDirectoryPath)) + // The checkout this file was compiled from: Sources/RunnerHost/ → + // three levels up is the package root. Only useful for `swift run` during + // development, hence last. + roots.append( + URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + ) + + for root in roots { + var candidate = root.resolvingSymlinksInPath() + // Walk upward: `.build/debug/gitea-macos-runner` is four levels below + // the checkout root, and an .app nested in a staging directory is + // similar. + for _ in 0..<6 { + if let found = check(candidate.appendingPathComponent("Contents/Resources/provision.sh")) { + return found + } + if let found = check(candidate.appendingPathComponent("Resources/provision.sh")) { + return found + } + let parent = candidate.deletingLastPathComponent() + if parent.path == candidate.path { break } + candidate = parent + } + } + + let list = searched.map { " \($0.path)" }.joined(separator: "\n") + throw CoreError.notFound( + "provision.sh could not be located. Searched:\n\(list)\n" + + "When running from a bundled .app, Resources/provision.sh must be copied into " + + "Contents/Resources/ by the build." + ) + } + + // MARK: - Helpers + + /// A PATH that includes `/usr/local/bin`. + /// + /// `ssh host command` runs a non-login, non-interactive shell, which never + /// sources the file where `path_helper` adds `/usr/local/bin`. Both `node` + /// and `gitea-runner` install there, so every command that names one is + /// wrapped in this. (`provision.sh` also writes `/etc/zshenv` to fix this for + /// everything else that talks to the guest.) + static func withGuestPath(_ command: String) -> String { + "export PATH=/usr/local/bin:/opt/homebrew/bin:$PATH; " + command + } + + /// Wraps a value so `/bin/sh` sees it literally. + static func shellQuote(_ value: String) -> String { + "'" + value.replacingOccurrences(of: "'", with: "'\\''") + "'" + } + + /// Trims captured output to something a terminal error can carry. + static func tail(_ output: String, lines: Int = 30) -> String { + let all = output.split(separator: "\n", omittingEmptySubsequences: false) + return all.suffix(lines).joined(separator: "\n") + } + + /// Downloads a URL to a unique temporary file. + /// + /// - Parameters: + /// - url: Source. + /// - suggestedName: File name within the temporary directory. + /// - Returns: The local file, which the caller owns and must delete. + static func downloadToTemporaryFile(url: URL, suggestedName: String) async throws -> URL { + let configuration = URLSessionConfiguration.ephemeral + configuration.timeoutIntervalForRequest = 60 + configuration.timeoutIntervalForResource = 60 * 60 + let session = URLSession(configuration: configuration) + defer { session.finishTasksAndInvalidate() } + + let temporary: URL + let response: URLResponse + do { + (temporary, response) = try await session.download(from: url) + } catch { + throw CoreError.provisioningFailed( + "download failed for \(url.absoluteString): \(error.localizedDescription)" + ) + } + + if let http = response as? HTTPURLResponse, !(200..<300).contains(http.statusCode) { + try? FileManager.default.removeItem(at: temporary) + throw CoreError.provisioningFailed( + "download failed: HTTP \(http.statusCode) for \(url.absoluteString)" + ) + } + + let destination = FileManager.default.temporaryDirectory + .appendingPathComponent("gmr-\(UUID().uuidString)-\(suggestedName)") + do { + try FileManager.default.moveItem(at: temporary, to: destination) + } catch { + try? FileManager.default.removeItem(at: temporary) + throw CoreError.provisioningFailed( + "could not stage the download from \(url.absoluteString): \(error.localizedDescription)" + ) + } + return destination + } + + /// Uploads a file too large to hold in memory, one chunk at a time. + /// + /// ``GuestExecutor/upload(localPath:remotePath:)`` slurps the whole file, so + /// a multi-gigabyte Xcode archive would exhaust host memory before a byte + /// moved. Each chunk is written to a scratch path and appended guest-side, + /// which keeps both ends bounded. + static func uploadLargeFile( + executor: any GuestExecutor, + localURL: URL, + remotePath: String, + chunkBytes: Int = 128 * 1024 * 1024, + progress: (@Sendable (Double) -> Void)? = nil + ) async throws { + let handle = try FileHandle(forReadingFrom: localURL) + defer { try? handle.close() } + + let attributes = try? FileManager.default.attributesOfItem(atPath: localURL.path) + let totalBytes = attributes?[.size] as? Int + let quotedRemote = shellQuote(remotePath) + let scratch = remotePath + ".part" + let quotedScratch = shellQuote(scratch) + + var sent = 0 + while true { + let chunk = try handle.read(upToCount: chunkBytes) ?? Data() + if chunk.isEmpty { break } + + try await executor.uploadData(chunk, remotePath: scratch, mode: "0644") + try await executor.runChecked( + "cat \(quotedScratch) >> \(quotedRemote) && rm -f \(quotedScratch)", + timeout: .seconds(600) + ) + + sent += chunk.count + if let totalBytes, totalBytes > 0 { + progress?(min(Double(sent) / Double(totalBytes), 1)) + } + } + } +} diff --git a/Sources/RunnerHost/IPSW.swift b/Sources/RunnerHost/IPSW.swift new file mode 100644 index 0000000..d272d83 --- /dev/null +++ b/Sources/RunnerHost/IPSW.swift @@ -0,0 +1,241 @@ +import Foundation +import RunnerCore +import Virtualization + +/// Locates, downloads, and opens macOS restore images (IPSWs). +/// +/// Two distinct notions of "restore image" get conflated easily, so this type +/// keeps them apart: +/// +/// * `VZMacOSRestoreImage.latestSupported` returns an image whose `url` is a +/// **network** URL on Apple's CDN. It cannot be handed to `VZMacOSInstaller`. +/// * `VZMacOSRestoreImage.image(from:)` (or `load(from:)`) opens a **local +/// file** URL. That is what the installer needs. +/// +/// So the pipeline is always: discover → download → load. +public struct IPSWProvider: Sendable { + /// Where downloads are written, typically `/ipsw`. + public let downloadDirectory: URL + + /// Creates a provider. + /// + /// - Parameter downloadDirectory: Destination directory for downloads. + public init(downloadDirectory: URL) { + self.downloadDirectory = downloadDirectory + } + + /// Asks Apple for the newest restore image this host can run. + /// + /// - Returns: The CDN URL to download and the image's build version (e.g. + /// `25A354`), recorded into ``VMBundleConfig/macOSVersion``. + /// - Throws: ``CoreError/notFound(_:)`` when Apple reports no supported + /// image (which also happens with no network). + public func latestSupported() async throws -> (url: URL, buildVersion: String) { + let image: VZMacOSRestoreImage + do { + image = try await VZMacOSRestoreImage.latestSupported + } catch { + // The framework reports "no supported image" and "could not reach + // the CDN" identically, so the message has to cover both. + throw CoreError.notFound( + "no supported macOS restore image available: \(error.localizedDescription) " + + "(check network connectivity, or pass --ipsw with a local file)" + ) + } + return (image.url, image.buildVersion) + } + + /// Downloads a restore image to ``downloadDirectory``. + /// + /// IPSWs are ~15 GB, so this reports progress and resumes nothing — a failed + /// download is retried from scratch. The file is written to a `.partial` + /// name and renamed on completion so an interrupted run never leaves a + /// truncated file that looks valid. + /// + /// - Parameters: + /// - remoteURL: The CDN URL from ``latestSupported()``. + /// - progress: Called with a fraction in `0...1`. May be called from an + /// arbitrary thread. + /// - Returns: The local file URL. + public func download( + from remoteURL: URL, + progress: (@Sendable (Double) -> Void)? = nil + ) async throws -> URL { + // A file URL is already local; nothing to do. + if remoteURL.isFileURL { + progress?(1.0) + return remoteURL + } + + let fileManager = FileManager.default + try fileManager.createDirectory(at: downloadDirectory, withIntermediateDirectories: true) + + let fileName = IPSWProvider.localFileName(for: remoteURL) + let finalURL = downloadDirectory.appendingPathComponent(fileName) + + // A previously completed download is reused: only fully-written files + // ever get the final name. + if fileManager.fileExists(atPath: finalURL.path) { + progress?(1.0) + return finalURL + } + + let partialURL = downloadDirectory.appendingPathComponent(fileName + ".partial") + try? fileManager.removeItem(at: partialURL) + + let configuration = URLSessionConfiguration.default + // The default 7-day resource timeout is useless as a failure signal and + // the default 60 s request timeout only bounds the *response start*. + // Six hours is generous for 15 GB on a slow link and still finite. + configuration.timeoutIntervalForRequest = 120 + configuration.timeoutIntervalForResource = 6 * 60 * 60 + configuration.waitsForConnectivity = true + + let delegate = IPSWDownloadProgressDelegate(onProgress: progress) + let session = URLSession(configuration: configuration) + defer { session.finishTasksAndInvalidate() } + + let temporaryURL: URL + let response: URLResponse + do { + (temporaryURL, response) = try await session.download(from: remoteURL, delegate: delegate) + } catch { + throw CoreError.notFound( + "restore image download failed for \(remoteURL.absoluteString): \(error.localizedDescription)" + ) + } + + if let http = response as? HTTPURLResponse, !(200..<300).contains(http.statusCode) { + try? fileManager.removeItem(at: temporaryURL) + throw CoreError.notFound( + "restore image download failed: HTTP \(http.statusCode) for \(remoteURL.absoluteString)" + ) + } + + // Move into `.partial` first, then rename: the final name is the + // "this file is complete" marker that the reuse check above trusts. + do { + try fileManager.moveItem(at: temporaryURL, to: partialURL) + try fileManager.moveItem(at: partialURL, to: finalURL) + } catch { + try? fileManager.removeItem(at: temporaryURL) + try? fileManager.removeItem(at: partialURL) + throw CoreError.provisioningFailed( + "could not store the downloaded restore image at \(finalURL.path): \(error.localizedDescription)" + ) + } + + progress?(1.0) + return finalURL + } + + /// Opens a local IPSW. + /// + /// Symlinks are resolved first: `VZMacOSRestoreImage` rejects a symlinked + /// path, and `~/Downloads` paths handed in by users are frequently symlinked + /// through `/Users` → `/System/Volumes/Data/Users`. + /// + /// - Parameter localPath: Path to an `.ipsw` file. + /// - Returns: The loaded restore image. + /// - Throws: ``CoreError/notFound(_:)`` when the path does not exist, or + /// ``CoreError/configInvalid(_:)`` when it is not a local file URL. + public func load(localPath: String) async throws -> VZMacOSRestoreImage { + let expanded = (localPath as NSString).expandingTildeInPath + var url = URL(fileURLWithPath: expanded) + // Must happen before the framework ever sees the URL. + url.resolveSymlinksInPath() + + guard url.isFileURL else { + throw CoreError.configInvalid( + "restore image path must be a local file, got \(url.absoluteString)" + ) + } + + // `VZMacOSRestoreImage.image(from:)` raises an Objective-C exception — + // not a Swift error — when handed a non-file or missing path, and an + // ObjC exception cannot be caught here. So the existence check is not + // politeness; it is the only thing standing between a typo and a crash. + var isDirectory: ObjCBool = false + guard FileManager.default.fileExists(atPath: url.path, isDirectory: &isDirectory), + !isDirectory.boolValue + else { + throw CoreError.notFound("restore image not found at \(url.path)") + } + + do { + return try await VZMacOSRestoreImage.image(from: url) + } catch { + throw CoreError.provisioningFailed( + "could not read restore image at \(url.path): \(error.localizedDescription)" + ) + } + } + + /// Convenience: discover, download if not already present, and load. + /// + /// - Parameter progress: Download progress callback. + /// - Returns: The loaded image and the local file it came from. + public func fetchLatest( + progress: (@Sendable (Double) -> Void)? = nil + ) async throws -> (image: VZMacOSRestoreImage, localURL: URL) { + let (remoteURL, _) = try await latestSupported() + let localURL = try await download(from: remoteURL, progress: progress) + // Deliberately reloaded from the local file: the image returned by + // `latestSupported` carries a network URL, and the installer needs one + // whose `url` is on disk. + let image = try await load(localPath: localURL.path) + return (image, localURL) + } + + // MARK: - Helpers + + /// The on-disk name for a remote restore image. + /// + /// Apple's CDN names are already unique (`UniversalMac_15.2_24C101_Restore.ipsw`); + /// anything else falls back to a name derived from the URL so two different + /// sources cannot collide. + static func localFileName(for remoteURL: URL) -> String { + let candidate = remoteURL.lastPathComponent + if candidate.lowercased().hasSuffix(".ipsw"), candidate.count > ".ipsw".count { + return candidate + } + let digest = abs(remoteURL.absoluteString.hashValue) + return "restore-\(String(digest, radix: 16)).ipsw" + } +} + +/// Reports `URLSession` download progress as a fraction. +/// +/// A task-scoped delegate is the only way to observe byte progress from the +/// `async` download API; the `didFinishDownloadingTo` callback is deliberately +/// *not* implemented, because the `async` variant owns the temporary file. +private final class IPSWDownloadProgressDelegate: NSObject, URLSessionDownloadDelegate, @unchecked Sendable { + private let onProgress: (@Sendable (Double) -> Void)? + + init(onProgress: (@Sendable (Double) -> Void)?) { + self.onProgress = onProgress + } + + func urlSession( + _ session: URLSession, + downloadTask: URLSessionDownloadTask, + didWriteData bytesWritten: Int64, + totalBytesWritten: Int64, + totalBytesExpectedToWrite: Int64 + ) { + // A chunked response reports -1 for the expected length; report nothing + // rather than a nonsense fraction. + guard totalBytesExpectedToWrite > 0 else { return } + let fraction = Double(totalBytesWritten) / Double(totalBytesExpectedToWrite) + onProgress?(min(max(fraction, 0), 1)) + } + + func urlSession( + _ session: URLSession, + downloadTask: URLSessionDownloadTask, + didFinishDownloadingTo location: URL + ) { + // Intentionally empty. `URLSession.download(from:delegate:)` moves the + // file itself; doing anything here would race with it. + } +} diff --git a/Sources/RunnerHost/ImageBuilder.swift b/Sources/RunnerHost/ImageBuilder.swift new file mode 100644 index 0000000..b2e3c61 --- /dev/null +++ b/Sources/RunnerHost/ImageBuilder.swift @@ -0,0 +1,706 @@ +import Foundation +import RunnerCore +import Virtualization + +/// A coarse progress report from ``ImageBuilder``. +public enum ImageBuildStage: Sendable, Equatable { + /// Downloading the IPSW. + case downloadingIPSW(fraction: Double) + /// Reading the restore image and deriving a hardware configuration. + case preparing + /// Creating the disk, NVRAM, and bundle metadata. + case creatingBundle + /// `VZMacOSInstaller` is writing macOS onto the disk. + case installing(fraction: Double) + /// First boot; waiting for Setup Assistant, a DHCP lease, and SSH. + case firstBoot + /// Running guest provisioning over SSH. + case provisioning(step: String) + /// Shutting the guest down cleanly and sealing the bundle. + case finalizing + /// Done. + case done +} + +/// Builds a base macOS image from an IPSW, end to end. +/// +/// ## Pipeline +/// +/// 1. **Load restore image** — `VZMacOSRestoreImage` from a local `.ipsw` +/// (downloaded first if the caller did not supply one). +/// 2. **Derive hardware** — `mostFeaturefulSupportedConfiguration`. A `nil` +/// here means this host cannot run this image at all; fail loudly rather +/// than trying to guess a configuration. +/// 3. **Create the bundle** — persist `hardwareModel.dataRepresentation` and a +/// fresh `VZMacMachineIdentifier`; create NVRAM with +/// `VZMacAuxiliaryStorage(creatingStorageAt:hardwareModel:)`; create the +/// disk, preferring sparse ASIF via +/// `/usr/sbin/diskutil image create blank --fs none --format ASIF --size G ` +/// (macOS 26+) and falling back to a `truncate`-style sparse RAW file. +/// 4. **Install** — `VZMacOSInstaller` against a *stopped* VM built from that +/// configuration, observing its `Progress` via KVO. +/// 5. **First boot with Setup Assistant automation** — see +/// ``firstBootAndProvision(bundle:config:progress:)``. +/// 6. **Provision** — ``GuestProvisioner`` over SSH. +/// 7. **Finalize** — clean guest shutdown, then set +/// ``VMBundleConfig/provisioned`` to `true`. Only then is the image clonable. +public struct ImageBuilder: Sendable { + /// The store this image is built into. + public let store: VMStore + + /// Creates a builder. + public init(store: VMStore) { + self.store = store + } + + /// Runs the whole pipeline. + /// + /// - Parameters: + /// - name: Image name under `/images/`, e.g. `default`. + /// - ipswPath: A local `.ipsw`. When `nil`, the latest supported image is + /// discovered and downloaded. + /// - config: Supplies guest shape (CPU/RAM/disk), credentials, and the + /// `gitea-runner` download URL. + /// - progress: Stage callback. May be invoked from arbitrary threads. + /// - Throws: ``CoreError/provisioningFailed(_:)`` naming the failed stage. + public func build( + name: String, + ipswPath: String?, + config: RunnerConfig, + progress: (@Sendable (ImageBuildStage) -> Void)? = nil + ) async throws { + try store.ensureLayout() + + // Checked against the filesystem rather than `store.image(named:)`: a + // half-built bundle from a previous failed run is exactly the thing this + // needs to catch, and it would not read back as a valid image. + let bundleURL = store.imagesDir.appendingPathComponent(name, isDirectory: true) + if FileManager.default.fileExists(atPath: bundleURL.path) { + throw CoreError.configInvalid( + "image '\(name)' already exists at \(bundleURL.path). " + + "Delete it first (`image delete \(name)`), or build under a different --name." + ) + } + + // The IPSW alone is ~15 GB and the installed disk grows to tens more. + try store.ensureFreeSpace(minGB: ipswPath == nil ? 60 : 45) + + // 1. Restore image. + let provider = IPSWProvider(downloadDirectory: store.ipswDir) + let restoreImage: VZMacOSRestoreImage + if let ipswPath { + progress?(.downloadingIPSW(fraction: 1.0)) + restoreImage = try await provider.load(localPath: ipswPath) + } else { + progress?(.downloadingIPSW(fraction: 0)) + let (image, _) = try await provider.fetchLatest { fraction in + progress?(.downloadingIPSW(fraction: fraction)) + } + restoreImage = image + } + + // 2/3. Hardware model and bundle. + progress?(.preparing) + progress?(.creatingBundle) + let bundle = try await createBundle(name: name, restoreImage: restoreImage, config: config) + + // 4. Install. + progress?(.installing(fraction: 0)) + try await install(bundle: bundle, restoreImage: restoreImage) { fraction in + progress?(.installing(fraction: fraction)) + } + + // 5/6/7. First boot, provisioning, seal. + try await firstBootAndProvision(bundle: bundle, config: config, progress: progress) + + progress?(.done) + } + + /// Creates the bundle directory, disk, NVRAM, and `config.json`. + /// + /// - Parameters: + /// - name: Image name. + /// - restoreImage: The loaded IPSW, used for its + /// `mostFeaturefulSupportedConfiguration` and `buildVersion`. + /// - config: Guest shape and credentials. + /// - Returns: The new, uninstalled bundle. + public func createBundle( + name: String, + restoreImage: VZMacOSRestoreImage, + config: RunnerConfig + ) async throws -> VMBundle { + // `mostFeaturefulSupportedConfiguration` is nil when this host cannot run + // this image at all — an Intel host, or a restore image newer than the + // host's Virtualization stack. There is nothing to fall back to, and + // guessing a hardware model produces a VM that fails to boot much later + // with a far less useful message. + guard let requirements = restoreImage.mostFeaturefulSupportedConfiguration else { + let version = restoreImage.operatingSystemVersion + throw CoreError.hostUnsupported( + "this host cannot virtualize macOS \(version.majorVersion).\(version.minorVersion) " + + "(build \(restoreImage.buildVersion)). The restore image reports no supported " + + "configuration — the host is either not Apple silicon or is older than the guest." + ) + } + + let hardwareModel = requirements.hardwareModel + guard hardwareModel.isSupported else { + throw CoreError.hostUnsupported( + "the hardware model required by build \(restoreImage.buildVersion) is not supported on this host" + ) + } + + // The image's own minimums win over the configured shape: a guest below + // them will not boot, and silently honouring a too-small config would + // produce that failure at first boot instead of here. + let cpuCount = max(config.guest.cpuCount, requirements.minimumSupportedCPUCount) + let minimumMemoryGB = Int( + (requirements.minimumSupportedMemorySize + (1 << 30) - 1) / (1 << 30) + ) + let memoryGB = max(config.guest.memoryGB, minimumMemoryGB) + + let bundle = VMBundle(rootURL: store.imagesDir.appendingPathComponent(name, isDirectory: true)) + try bundle.createDirectory() + + do { + let diskFormat = try createDisk(bundle: bundle, sizeGB: config.guest.diskGB) + + // NVRAM must be created against the *same* hardware model that goes + // into config.json and into VZMacPlatformConfiguration. A mismatch is + // undefined behaviour in the framework, not a validation error. + _ = try VZMacAuxiliaryStorage( + creatingStorageAt: bundle.auxiliaryStorageURL, + hardwareModel: hardwareModel, + options: [] + ) + + let osVersion = restoreImage.operatingSystemVersion + let bundleConfig = VMBundleConfig( + hardwareModelData: hardwareModel.dataRepresentation, + machineIdentifierData: VZMacMachineIdentifier().dataRepresentation, + // A placeholder: the base image is never booted on a slot. Every + // clone rewrites this with its slot's persistent MAC, which is + // what DHCP lease discovery keys on. It still has to be a valid + // locally-administered address, because the base image *is* + // booted once, here, for provisioning. + macAddress: VZMACAddress.randomLocallyAdministered().string, + diskFormat: diskFormat, + cpuCount: cpuCount, + memoryGB: memoryGB, + guestUsername: config.guest.username, + macOSVersion: + "\(osVersion.majorVersion).\(osVersion.minorVersion).\(osVersion.patchVersion) " + + "(\(restoreImage.buildVersion))", + provisioned: false + ) + try bundle.saveConfig(bundleConfig) + } catch { + // A bundle that got partway through creation is not something a later + // run can recover from, and leaving it behind would make `build` with + // the same name fail on the "already exists" check for the wrong + // reason. + try? bundle.destroy() + throw error + } + + return bundle + } + + /// Creates the backing disk, preferring sparse ASIF. + /// + /// ASIF (`diskutil image create blank --fs none --format ASIF`) is available + /// from macOS 26 and is the right choice here: it is sparse, so a 64 GB + /// nominal disk costs what the guest actually writes, and it CoW-clones + /// cleanly on APFS. If `diskutil` fails for any reason, a sparse RAW file is + /// created instead and the format recorded in the bundle config so + /// ``VZConfigFactory`` attaches the right file. + /// + /// - Parameters: + /// - bundle: Destination bundle. + /// - sizeGB: Nominal disk size. + /// - Returns: The format that was actually used. + public func createDisk(bundle: VMBundle, sizeGB: Int) throws -> VMBundleConfig.DiskFormat { + guard sizeGB > 0 else { + throw CoreError.configInvalid("guest.diskGB must be positive, got \(sizeGB)") + } + + let asifURL = bundle.asifDiskURL + try? FileManager.default.removeItem(at: asifURL) + + // No `#available` guard: the package's deployment target is already + // macOS 26, so the compiler would reject the check as redundant. The + // runtime feature check that matters is whether *this* diskutil + // understands `--format ASIF`, which the exit status answers directly — + // that also covers early 26 builds where the format was still landing. + do { + try ImageBuilder.runProcess( + "/usr/sbin/diskutil", + [ + "image", "create", "blank", + "--fs", "none", + "--format", "ASIF", + "--size", "\(sizeGB)G", + asifURL.path, + ] + ) + + // diskutil occasionally appends its own extension; accept either + // spelling rather than failing on a cosmetic difference. + if !FileManager.default.fileExists(atPath: asifURL.path) { + let suffixed = URL(fileURLWithPath: asifURL.path + ".asif") + if FileManager.default.fileExists(atPath: suffixed.path) { + try FileManager.default.moveItem(at: suffixed, to: asifURL) + } + } + + if FileManager.default.fileExists(atPath: asifURL.path) { + return .asif + } + } catch { + // Fall through to RAW. + } + + try? FileManager.default.removeItem(at: asifURL) + + // RAW fallback: an empty file extended to the nominal size. APFS keeps it + // sparse, so this costs nothing until the guest writes. Sizes are decimal + // GB (1000³) to match what `diskutil … --size NG` produces, so switching + // formats does not silently change the guest's disk size. + let rawURL = bundle.rawDiskURL + try? FileManager.default.removeItem(at: rawURL) + guard FileManager.default.createFile(atPath: rawURL.path, contents: nil) else { + throw CoreError.provisioningFailed("could not create the disk image at \(rawURL.path)") + } + let handle = try FileHandle(forWritingTo: rawURL) + defer { try? handle.close() } + do { + try handle.truncate(atOffset: UInt64(sizeGB) * 1_000_000_000) + } catch { + throw CoreError.provisioningFailed( + "could not size the disk image at \(rawURL.path) to \(sizeGB) GB: \(error.localizedDescription)" + ) + } + return .raw + } + + /// Runs `VZMacOSInstaller` to completion. + /// + /// - Parameters: + /// - bundle: The bundle to install into. + /// - restoreImage: The loaded IPSW. + /// - progress: Called with the installer's completed fraction. + public func install( + bundle: VMBundle, + restoreImage: VZMacOSRestoreImage, + progress: (@Sendable (Double) -> Void)? = nil + ) async throws { + let imageURL = restoreImage.url + guard imageURL.isFileURL else { + // The image returned by `VZMacOSRestoreImage.latestSupported` carries + // a CDN URL. Handing that to the installer fails deep inside the + // framework; catching it here names the actual mistake. + throw CoreError.configInvalid( + "VZMacOSInstaller needs a local restore image, but this one points at " + + "\(imageURL.absoluteString). Download it first with IPSWProvider.download." + ) + } + + let configuration = try VZConfigFactory.makeConfiguration(bundle: bundle, headless: true) + + // Everything about VZMacOSInstaller is queue-bound: the VM must be + // created on a queue, the installer must be *constructed* on that same + // queue with the VM stopped, and `install` must be *called* on it too. + // The VM is also created here rather than through VMInstance because the + // installer needs the VZVirtualMachine object itself, and because the VM + // must never be started, paused, or stopped while installing — behaviour + // VMInstance exists to provide and which would be actively harmful here. + let queue = DispatchQueue(label: "gitea-macos-runner.install.\(bundle.name)") + let session = InstallSession() + let boxedConfiguration = UncheckedBox(configuration) + + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + queue.async { + let virtualMachine = VZVirtualMachine( + configuration: boxedConfiguration.value, + queue: queue + ) + let installer = VZMacOSInstaller( + virtualMachine: virtualMachine, + restoringFromImageAt: imageURL + ) + // Held for the duration: the completion handler is the only other + // strong reference, and dropping the VM mid-install would be a + // use-after-free rather than a cancellation. + session.virtualMachine = virtualMachine + session.installer = installer + + if let progress { + session.observation = installer.progress.observe( + \.fractionCompleted, + options: [.initial, .new] + ) { observed, _ in + progress(observed.fractionCompleted) + } + } + + installer.install { result in + session.observation = nil + session.installer = nil + session.virtualMachine = nil + switch result { + case .success: + progress?(1.0) + continuation.resume() + case .failure(let error): + continuation.resume(throwing: VMInstance.mapVZError(error)) + } + } + } + } + } + + /// Boots the freshly installed guest, gets it onto the network, and hands it + /// to ``GuestProvisioner``. + /// + /// ## Setup Assistant + /// + /// A newly installed macOS sits at Setup Assistant with no account and no + /// SSH. On a **macOS 27+ host with a macOS 27+ guest**, Virtualization can + /// automate that: build a `VZMacGuestProvisioningOptions` carrying the + /// configured username, password, and full name, with + /// `logsInAutomatically = true` and `enablesRemoteLogin = true`, and attach + /// it to the start options via + /// `VZMacOSVirtualMachineStartOptions.setGuestProvisioning(_:)` — the ObjC + /// selector is `setGuestProvisioningOptions:error:`, but Swift imports it + /// under the shorter name. The guest then creates the account and enables + /// SSH unattended. + /// + /// - Important: An **older guest silently ignores** these options — no + /// error, no account, no SSH, and this method will simply time out waiting + /// for a lease or a login. When that happens the only recovery is a + /// manual, GUI-driven first boot, which is deliberately **out of v1 + /// scope**: this method fails with an explanatory + /// ``CoreError/provisioningFailed(_:)`` telling the operator that a + /// `--manual-setup` flow is not implemented and that the IPSW must be + /// macOS 27 or newer. + /// + /// The API itself is gated at `#available(macOS 27.0, *)`, so a macOS 26 + /// host takes the same explanatory failure path. + /// + /// - Parameters: + /// - bundle: The installed bundle. + /// - config: Guest credentials and timeouts. + /// - progress: Stage callback. + public func firstBootAndProvision( + bundle: VMBundle, + config: RunnerConfig, + progress: (@Sendable (ImageBuildStage) -> Void)? = nil + ) async throws { + guard #available(macOS 27.0, *) else { + throw CoreError.hostUnsupported( + """ + automating Setup Assistant requires macOS 27 or newer on the host; this host is older. \ + Without it the freshly installed guest sits at the setup screen forever, with no \ + account and no SSH. A manual, GUI-driven first boot is not implemented in v1. + """ + ) + } + + let provisioningOptions = VZMacGuestProvisioningOptions() + provisioningOptions.username = config.guest.username + provisioningOptions.password = config.guest.password + provisioningOptions.fullName = ImageBuilder.guestAccountFullName + // Auto-login keeps a GUI session alive, which codesign against the login + // keychain and the simulators both need. + provisioningOptions.logsInAutomatically = true + // This is what turns on sshd — the only channel provisioning has. + provisioningOptions.enablesRemoteLogin = true + + let startOptions = VZMacOSVirtualMachineStartOptions() + do { + // The validating setter: it rejects, for instance, a password that + // the guest's account policy will not accept, here rather than by + // quietly producing a guest with no usable account. + try startOptions.setGuestProvisioning(provisioningOptions) + } catch { + throw CoreError.configInvalid( + "guest provisioning options were rejected (check guest.username / guest.password): " + + error.localizedDescription + ) + } + + try await bootProvisionAndSeal( + bundle: bundle, + config: config, + startOptions: startOptions, + isFirstBoot: true, + xcodeXIPPath: nil, + progress: progress + ) + } + + /// Re-runs guest provisioning against an already-installed image. + /// + /// Backs `image provision NAME`, which exists so that bumping the + /// `gitea-runner` version or adding Xcode does not require a 15 GB + /// reinstall. + /// + /// - Parameters: + /// - name: Image name. + /// - config: Guest credentials and download URLs. + /// - xcodeXIPPath: Optional Xcode `.xip` to install as well. + /// - progress: Stage callback. + public func reprovision( + name: String, + config: RunnerConfig, + xcodeXIPPath: String? = nil, + progress: (@Sendable (ImageBuildStage) -> Void)? = nil + ) async throws { + let bundleURL = store.imagesDir.appendingPathComponent(name, isDirectory: true) + guard FileManager.default.fileExists(atPath: bundleURL.path) else { + throw CoreError.notFound("image '\(name)' at \(bundleURL.path)") + } + let bundle = VMBundle(rootURL: bundleURL) + + if let xcodeXIPPath { + let expanded = (xcodeXIPPath as NSString).expandingTildeInPath + guard FileManager.default.fileExists(atPath: expanded) else { + throw CoreError.notFound("Xcode .xip at \(expanded)") + } + } + + // No start options: the account already exists, so there is nothing for + // Setup Assistant automation to do, and re-applying it on a guest that is + // already past first boot has no effect anyway (macOS only evaluates + // guest provisioning on the first boot after a restore). + try await bootProvisionAndSeal( + bundle: bundle, + config: config, + startOptions: nil, + isFirstBoot: false, + xcodeXIPPath: xcodeXIPPath, + progress: progress + ) + } + + // MARK: - Boot, provision, seal + + /// The shared tail of ``firstBootAndProvision(bundle:config:progress:)`` and + /// ``reprovision(name:config:xcodeXIPPath:progress:)``: boot, find the guest + /// on the network, provision it, shut it down cleanly, mark it provisioned. + private func bootProvisionAndSeal( + bundle: VMBundle, + config: RunnerConfig, + startOptions: VZMacOSVirtualMachineStartOptions?, + isFirstBoot: Bool, + xcodeXIPPath: String?, + progress: (@Sendable (ImageBuildStage) -> Void)? + ) async throws { + var bundleConfig = try bundle.loadConfig() + let macAddress = bundleConfig.macAddress + let bootTimeout = Duration.seconds(max(60, config.scheduler.bootTimeoutSeconds)) + + progress?(.firstBoot) + let instance = try VMInstance(bundle: bundle, label: "image:\(bundle.name)", headless: true) + + do { + try await instance.start(options: startOptions) + } catch { + throw CoreError.provisioningFailed( + "could not boot image '\(bundle.name)': \(error)" + ) + } + + let address: String + do { + // A DHCP lease is the first observable sign of life: the guest has + // booted far enough to bring up its NIC. SSH comes tens of seconds + // later, once launchd has started sshd. + address = try await ImageBuilder.waitForDHCPLease(macAddress: macAddress, timeout: bootTimeout) + try await waitForSSH( + host: address, + username: config.guest.username, + password: config.guest.password, + timeout: bootTimeout + ) + } catch { + _ = await instance.requestStopThenForce() + if isFirstBoot { + // The most likely cause by far, and the one with no diagnostic of + // its own: a pre-27 guest accepts the provisioning options and + // ignores them, so it sits at Setup Assistant with no account and + // no sshd while we wait for a login that will never be possible. + throw CoreError.provisioningFailed( + """ + the guest never became reachable over SSH within \(config.scheduler.bootTimeoutSeconds)s. + + The usual cause is a guest older than macOS 27: earlier versions do not implement \ + the automated setup protocol and silently ignore the provisioning options, leaving \ + the VM parked at Setup Assistant with no account and no Remote Login. Rebuild with \ + a macOS 27 or newer restore image. + + A manual, GUI-driven first boot is not implemented in v1. + + Underlying error: \(error) + """ + ) + } + throw CoreError.provisioningFailed( + "image '\(bundle.name)' booted but never became reachable over SSH: \(error)" + ) + } + + let executor = SSHExecutor( + host: address, + username: config.guest.username, + password: config.guest.password + ) + + do { + let provisioner = GuestProvisioner() + try await provisioner.provision(executor: executor, config: config) { step in + progress?(.provisioning(step: step)) + } + + if let xcodeXIPPath { + progress?(.provisioning(step: "Xcode")) + try await provisioner.installXcode( + executor: executor, + xipPath: (xcodeXIPPath as NSString).expandingTildeInPath + ) + } + } catch { + await executor.close() + _ = await instance.requestStopThenForce() + throw error + } + + // Shut down from inside. A forced stop is a power cut: it leaves the + // guest's filesystem in whatever state it was in, and every clone would + // inherit that state, so the graceful path is worth waiting for. + progress?(.finalizing) + _ = try? await executor.run("sudo -n /sbin/shutdown -h now", timeout: .seconds(30)) + await executor.close() + + let stopped = await ImageBuilder.withTimeout(.seconds(180)) { + await instance.waitUntilStopped() + } + if stopped == nil { + _ = await instance.requestStopThenForce() + } + + bundleConfig.provisioned = true + try bundle.saveConfig(bundleConfig) + } + + // MARK: - Helpers + + /// Full name for the account Setup Assistant automation creates. + static let guestAccountFullName = "Gitea Runner" + + /// Polls `/var/db/dhcpd_leases` until the guest's MAC appears. + /// + /// - Parameters: + /// - macAddress: The bundle's MAC, in any common formatting. + /// - timeout: Overall ceiling. + /// - pollInterval: Delay between reads. Defaults to 2 s. + /// - Returns: The leased IP address. + /// - Throws: ``CoreError/timeout(_:)`` if no lease appears in time. + static func waitForDHCPLease( + macAddress: String, + timeout: Duration, + pollInterval: Duration = .seconds(2) + ) async throws -> String { + let started = ContinuousClock.now + while true { + let leases = DHCPLeaseParser.parseFile() + if let address = DHCPLeaseParser.ipAddress(forMAC: macAddress, in: leases) { + return address + } + guard ContinuousClock.now - started < timeout else { break } + try await Task.sleep(for: pollInterval) + guard ContinuousClock.now - started < timeout else { break } + } + throw CoreError.timeout("no DHCP lease for \(macAddress) in /var/db/dhcpd_leases") + } + + /// Runs an async operation with a ceiling, returning `nil` if it elapses. + /// + /// Used for the graceful-shutdown wait, which otherwise has no bound: + /// `waitUntilStopped()` waits forever, and a guest that hangs on shutdown + /// would hang the build with it. + static func withTimeout( + _ duration: Duration, + operation: @escaping @Sendable () async -> T + ) async -> T? { + await withTaskGroup(of: Optional.self) { group in + group.addTask { await operation() } + group.addTask { + try? await Task.sleep(for: duration) + return nil + } + let first = await group.next() ?? nil + group.cancelAll() + return first + } + } + + /// Runs a host process and throws with its output if it exits non-zero. + @discardableResult + static func runProcess(_ executablePath: String, _ arguments: [String]) throws -> String { + let process = Process() + process.executableURL = URL(fileURLWithPath: executablePath) + process.arguments = arguments + + let pipe = Pipe() + process.standardOutput = pipe + process.standardError = pipe + + do { + try process.run() + } catch { + throw CoreError.processFailed( + command: "\(executablePath) \(arguments.joined(separator: " "))", + exitCode: -1, + output: error.localizedDescription + ) + } + + // Drained before waiting: a command that outfills the pipe buffer would + // block forever otherwise. + let data = pipe.fileHandleForReading.readDataToEndOfFile() + process.waitUntilExit() + + let output = String(decoding: data, as: UTF8.self) + guard process.terminationStatus == 0 else { + throw CoreError.processFailed( + command: "\(executablePath) \(arguments.joined(separator: " "))", + exitCode: process.terminationStatus, + output: output + ) + } + return output + } +} + +// MARK: - Install plumbing + +/// Carries a non-`Sendable` Virtualization object onto the VM's serial queue. +/// +/// The framework's configuration objects are not `Sendable` and never will be, +/// but handing one to the queue that will own the VM is exactly the transfer the +/// framework itself prescribes. +private final class UncheckedBox: @unchecked Sendable { + let value: T + init(_ value: T) { self.value = value } +} + +/// Owns the VM, installer, and KVO observation for one install. +/// +/// Every field is read and written only on the install queue, which is what +/// makes the unchecked conformance sound. +private final class InstallSession: @unchecked Sendable { + var virtualMachine: VZVirtualMachine? + var installer: VZMacOSInstaller? + var observation: NSKeyValueObservation? +} diff --git a/Sources/RunnerHost/LaunchdService.swift b/Sources/RunnerHost/LaunchdService.swift new file mode 100644 index 0000000..bfb8ed2 --- /dev/null +++ b/Sources/RunnerHost/LaunchdService.swift @@ -0,0 +1,354 @@ +import Foundation +import RunnerCore + +/// Whether the LaunchAgent is installed and running. +public struct ServiceStatus: Sendable, Equatable { + /// Whether the plist exists at ``LaunchdService/agentPlistURL``. + public let installed: Bool + /// Whether `launchctl` reports the label as loaded. + public let loaded: Bool + /// The running PID, when loaded and alive. + public let pid: Int? + /// The last exit status `launchctl` reported, when not running. + public let lastExitStatus: Int? + /// Path to the plist, whether or not it exists. + public let plistPath: String + + public init( + installed: Bool, + loaded: Bool, + pid: Int? = nil, + lastExitStatus: Int? = nil, + plistPath: String + ) { + self.installed = installed + self.loaded = loaded + self.pid = pid + self.lastExitStatus = lastExitStatus + self.plistPath = plistPath + } +} + +/// Installs, removes, and inspects the daemon's `launchd` job. +/// +/// ## LaunchAgent, never LaunchDaemon +/// +/// This is not a stylistic choice. Two hard constraints force it: +/// +/// * Virtualization.framework needs a **GUI login session**. A LaunchDaemon runs +/// in the system context with no session, and VM startup fails there. +/// * From macOS 15, starting a VM requires an **unlocked `login.keychain`**. +/// That keychain unlocks when a user logs in graphically; a LaunchDaemon never +/// sees it. +/// +/// So the daemon runs as a LaunchAgent in the logged-in user's session, and the +/// host must be configured for automatic login with the screen allowed to sleep +/// but the session never locked. `doctor` checks the keychain state precisely +/// because this is the failure people hit first. +public enum LaunchdService { + /// The `launchd` label, matching `CFBundleIdentifier`. + public static let label = "xyz.blakeslee.gitea-macos-runner" + + /// `~/Library/LaunchAgents/xyz.blakeslee.gitea-macos-runner.plist`. + public static var agentPlistURL: URL { + URL(fileURLWithPath: RunnerConfig.expandTilde("~/Library/LaunchAgents/\(label).plist")) + } + + /// The default install location of the signed app's executable. + /// + /// `make install` puts the bundle here; the entitlement only exists on the + /// signed bundle, so this — not a bare binary — is what `launchd` must run. + public static let defaultExecutablePath = + "~/Applications/GiteaMacosRunner.app/Contents/MacOS/gitea-macos-runner" + + /// The GUI domain target for this user, e.g. `gui/501`. + public static var domainTarget: String { "gui/\(getuid())" } + + /// The service target for this user's agent, e.g. `gui/501/xyz.blakeslee…`. + public static var serviceTarget: String { "\(domainTarget)/\(label)" } + + /// Writes the plist and loads the job. + /// + /// `ProgramArguments` is the **installed app bundle's** executable followed + /// by `daemon` — not `.build/…` and not a bare binary, because the + /// entitlement only exists on the signed bundle. `RunAtLoad` and `KeepAlive` + /// are both set so the daemon survives crashes and logins. + /// + /// - Parameters: + /// - executablePath: Absolute path to the installed binary, e.g. + /// `~/Applications/GiteaMacosRunner.app/Contents/MacOS/gitea-macos-runner`. + /// - configPath: Optional `--config` argument for a non-default location. + /// - Throws: ``CoreError/notFound(_:)`` when the executable or the template + /// is missing, ``CoreError/processFailed(command:exitCode:output:)`` when + /// `launchctl` refuses the job. + public static func install(executablePath: String, configPath: String? = nil) throws { + let executable = RunnerConfig.expandTilde(executablePath) + guard FileManager.default.isExecutableFile(atPath: executable) else { + throw CoreError.notFound( + """ + no executable at \(executable) — run `make install` to build, sign, \ + and install the app bundle first + """ + ) + } + + var arguments = ["daemon"] + if let configPath { + arguments += ["--config", RunnerConfig.expandTilde(configPath)] + } + + let xml = try renderPlist(executablePath: executable, arguments: arguments) + + let fm = FileManager.default + try fm.createDirectory(at: logDirectoryURL, withIntermediateDirectories: true) + try fm.createDirectory( + at: agentPlistURL.deletingLastPathComponent(), + withIntermediateDirectories: true + ) + + // A reinstall over a loaded job is the common case (upgrade, config + // change), so unload before rewriting rather than failing on "already + // bootstrapped". + if fm.fileExists(atPath: agentPlistURL.path) { + _ = try? uninstallJobOnly() + } + + do { + try Data(xml.utf8).write(to: agentPlistURL, options: .atomic) + } catch { + throw CoreError.processFailed( + command: "write \(agentPlistURL.path)", + exitCode: 1, + output: error.localizedDescription + ) + } + + let bootstrap = LaunchdShell.run("/bin/launchctl", ["bootstrap", domainTarget, agentPlistURL.path]) + if bootstrap.exitCode != 0 { + // `bootstrap` is the modern verb but is unavailable in some session + // contexts (and returns 5 for "input/output error" on odd domains); + // the legacy loader still works there. + let legacy = LaunchdShell.run("/bin/launchctl", ["load", "-w", agentPlistURL.path]) + if legacy.exitCode != 0 { + throw CoreError.processFailed( + command: "launchctl bootstrap \(domainTarget) \(agentPlistURL.path)", + exitCode: bootstrap.exitCode, + output: (bootstrap.output + "\n" + legacy.output).trimmingCharacters(in: .whitespacesAndNewlines) + ) + } + } + } + + /// Unloads the job and removes the plist. Safe when not installed. + public static func uninstall() throws { + _ = try? uninstallJobOnly() + if FileManager.default.fileExists(atPath: agentPlistURL.path) { + try FileManager.default.removeItem(at: agentPlistURL) + } + } + + /// Unloads the job but leaves the plist on disk. + private static func uninstallJobOnly() throws { + let bootout = LaunchdShell.run("/bin/launchctl", ["bootout", serviceTarget]) + if bootout.exitCode != 0 { + _ = LaunchdShell.run("/bin/launchctl", ["unload", "-w", agentPlistURL.path]) + } + } + + /// Reports installation and run state. + public static func status() throws -> ServiceStatus { + let installed = FileManager.default.fileExists(atPath: agentPlistURL.path) + let printed = LaunchdShell.run("/bin/launchctl", ["print", serviceTarget]) + + guard printed.exitCode == 0 else { + // 113 (EAGAIN-ish "Could not find service") and 36 are both "not + // loaded"; anything else is still, for our purposes, not loaded. + return ServiceStatus(installed: installed, loaded: false, plistPath: agentPlistURL.path) + } + + return ServiceStatus( + installed: installed, + loaded: true, + pid: firstInteger(in: printed.output, key: "pid"), + lastExitStatus: firstInteger(in: printed.output, key: "last exit code"), + plistPath: agentPlistURL.path + ) + } + + /// Extracts `key = ` from `launchctl print` output. + private static func firstInteger(in output: String, key: String) -> Int? { + for line in output.split(separator: "\n") { + let trimmed = line.trimmingCharacters(in: .whitespaces) + guard trimmed.hasPrefix(key) else { continue } + guard let equals = trimmed.firstIndex(of: "=") else { continue } + let value = trimmed[trimmed.index(after: equals)...].trimmingCharacters(in: .whitespaces) + return Int(value) + } + return nil + } + + /// Renders `Resources/launchd.plist.template` with the given substitutions. + /// + /// Placeholders: `{{LABEL}}`, `{{PROGRAM}}`, `{{ARGUMENTS}}`, + /// `{{STDOUT_PATH}}`, `{{STDERR_PATH}}`. + /// + /// - Parameters: + /// - executablePath: Absolute path to the installed binary. + /// - arguments: Arguments after the executable, e.g. `["daemon"]`. + /// - Returns: The plist XML. + public static func renderPlist(executablePath: String, arguments: [String]) throws -> String { + let template = try loadTemplate() + + let argumentXML = arguments + .map { "\t\t\(xmlEscape($0))" } + .joined(separator: "\n") + + return template + .replacingOccurrences(of: "{{LABEL}}", with: xmlEscape(label)) + .replacingOccurrences(of: "{{PROGRAM}}", with: xmlEscape(executablePath)) + .replacingOccurrences(of: "{{ARGUMENTS}}", with: argumentXML) + .replacingOccurrences( + of: "{{STDOUT_PATH}}", + with: xmlEscape(logDirectoryURL.appendingPathComponent("daemon.out.log").path) + ) + .replacingOccurrences( + of: "{{STDERR_PATH}}", + with: xmlEscape(logDirectoryURL.appendingPathComponent("daemon.err.log").path) + ) + } + + /// Locates the plist template. + /// + /// The template is not an SPM resource bundle and `make bundle` copies only + /// `Info.plist` into the app, so there is no single reliable location: this + /// walks the plausible ones and falls back to a built-in copy so + /// `service install` works from the installed app, from `swift run`, and from + /// a checkout. + private static func loadTemplate() throws -> String { + var candidates: [URL] = [] + + if let resourceURL = Bundle.main.url(forResource: "launchd.plist", withExtension: "template") { + candidates.append(resourceURL) + } + candidates.append( + Bundle.main.bundleURL + .appendingPathComponent("Contents/Resources/launchd.plist.template") + ) + if let executableURL = Bundle.main.executableURL?.resolvingSymlinksInPath() { + let directory = executableURL.deletingLastPathComponent() + candidates.append(directory.appendingPathComponent("Resources/launchd.plist.template")) + candidates.append( + directory.deletingLastPathComponent() + .appendingPathComponent("Resources/launchd.plist.template") + ) + } + // Sources/RunnerHost/LaunchdService.swift → repository root. + let repositoryRoot = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + candidates.append(repositoryRoot.appendingPathComponent("Resources/launchd.plist.template")) + candidates.append( + URL(fileURLWithPath: FileManager.default.currentDirectoryPath) + .appendingPathComponent("Resources/launchd.plist.template") + ) + + for candidate in candidates { + if let contents = try? String(contentsOf: candidate, encoding: .utf8) { + return contents + } + } + return embeddedTemplate + } + + /// Escapes a string for an XML text node. + private static func xmlEscape(_ value: String) -> String { + value + .replacingOccurrences(of: "&", with: "&") + .replacingOccurrences(of: "<", with: "<") + .replacingOccurrences(of: ">", with: ">") + } + + /// Directory for the agent's stdout/stderr logs, + /// `~/Library/Logs/gitea-macos-runner`. + public static var logDirectoryURL: URL { + URL(fileURLWithPath: RunnerConfig.expandTilde("~/Library/Logs/gitea-macos-runner"), isDirectory: true) + } + + /// Byte-for-byte fallback copy of `Resources/launchd.plist.template`, used + /// when the file cannot be found next to the running binary. + private static let embeddedTemplate = """ + + + + + \tLabel + \t{{LABEL}} + + \tProgramArguments + \t + \t\t{{PROGRAM}} + {{ARGUMENTS}} + \t + + \tRunAtLoad + \t + + \tKeepAlive + \t + \t\tSuccessfulExit + \t\t + \t + + \tThrottleInterval + \t30 + + \tProcessType + \tInteractive + + \tStandardOutPath + \t{{STDOUT_PATH}} + + \tStandardErrorPath + \t{{STDERR_PATH}} + + \tEnvironmentVariables + \t + \t\tPATH + \t\t/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin + \t + + + """ +} + +/// Minimal synchronous process runner for `launchctl`. +private enum LaunchdShell { + struct Output { + let exitCode: Int32 + let output: String + } + + static func run(_ launchPath: String, _ arguments: [String]) -> Output { + let process = Process() + process.executableURL = URL(fileURLWithPath: launchPath) + process.arguments = arguments + + let pipe = Pipe() + process.standardOutput = pipe + process.standardError = pipe + + do { + try process.run() + } catch { + return Output(exitCode: 127, output: "\(error)") + } + + let data = pipe.fileHandleForReading.readDataToEndOfFile() + process.waitUntilExit() + return Output( + exitCode: process.terminationStatus, + output: String(data: data, encoding: .utf8) ?? "" + ) + } +} diff --git a/Sources/RunnerHost/Orchestrator.swift b/Sources/RunnerHost/Orchestrator.swift new file mode 100644 index 0000000..f4f964d --- /dev/null +++ b/Sources/RunnerHost/Orchestrator.swift @@ -0,0 +1,821 @@ +import Foundation +import Logging +import RunnerCore +import Virtualization + +/// Everything the orchestrator tracks about one live slot. +public struct LiveVM: Sendable { + /// Slot index. + public let slot: Int + /// The ephemeral clone backing it. + public let bundle: VMBundle + /// The runner name registered with Gitea. Globally unique, prefixed with + /// ``RunnerConfig/RunnerSection/namePrefix`` — this is what lets the + /// reconcile loop tell a stale row apart from a live one. + public let runnerName: String + /// The guest's IP, once its DHCP lease appears. + public var ipAddress: String? + /// When the boot started. + public let startedAt: Date + + public init( + slot: Int, + bundle: VMBundle, + runnerName: String, + ipAddress: String? = nil, + startedAt: Date + ) { + self.slot = slot + self.bundle = bundle + self.runnerName = runnerName + self.ipAddress = ipAddress + self.startedAt = startedAt + } +} + +/// The daemon: watches Gitea, boots ephemeral macOS VMs, and cleans up after +/// them. +/// +/// ## Loop +/// +/// Every `pollIntervalSeconds`: +/// 1. Fetch queued jobs (`status=queued` only — `waiting` means *blocked*). +/// 2. Ask ``SchedulerCore/plan(state:queuedJobs:labels:maxVMs:now:jobTimeout:bootTimeout:)`` +/// what to do. The planner is pure; all I/O happens here. +/// 3. Execute the returned actions. +/// +/// Every `reconcileIntervalSeconds`, additionally run ``reconcileOnce()``. +/// +/// ## Booting a slot +/// +/// `ensureFreeSpace` → `cloneImage(named:slotMAC:)` → ``VMInstance/start(options:)`` +/// → poll `/var/db/dhcpd_leases` for the slot MAC until `bootTimeout` → +/// ``waitForSSH(host:port:username:password:timeout:pollInterval:)`` → write the +/// registration token into a guest file with mode `0600` → over SSH: +/// +/// ```sh +/// gitea-runner register --no-interactive \ +/// --instance --token-file \ +/// --name --labels "macos-arm64:host" --ephemeral \ +/// && rm -f \ +/// && gitea-runner daemon +/// ``` +/// +/// The token goes through a file rather than `--token` because arguments are +/// visible to every process on the guest, and it is deleted the instant +/// registration returns. `--ephemeral` is **server-enforced** (Gitea 1.24+): the +/// server hands this runner exactly one task and then deregisters it. The weaker +/// `--once` is runner-side only and is not used. +/// +/// When the `gitea-runner daemon` SSH command returns — which it does after the +/// single job completes — or when `jobTimeout` elapses, the slot is torn down: +/// force-stop the VM, delete the clone, mark the slot idle. +/// +/// ## Reconcile +/// +/// A VM that dies uncleanly leaves a runner row behind, and Gitea only sweeps +/// rows at midnight — and *never* sweeps a runner that never claimed a task. So +/// every reconcile pass lists runners and deletes any that are `ephemeral`, not +/// `busy`, carry our name prefix, and have no live VM. The associated Running +/// task is reaped separately by Gitea's zombie sweep (~10–15 minutes); that part +/// is not ours to fix. +public actor Orchestrator { + + /// Effective configuration. + public let config: RunnerConfig + /// Gitea admin API client. + public let client: GiteaClient + /// On-disk store. + public let store: VMStore + /// Base image name to clone for each job. + public let imageName: String + + /// Structured logger. + private let logger: Logger + + /// The pure scheduler's state. Every mutation goes through `SchedulerCore`. + private var state: SchedulerState + + /// Slots that currently hold a VM, keyed by slot index. + private var live: [Int: LiveVM] = [:] + + /// The `VZVirtualMachine` wrapper for each live slot. + private var instances: [Int: VMInstance] = [:] + + /// The supervising task per slot: clone → boot → register → run → teardown. + private var slotTasks: [Int: Task] = [:] + + /// The "the guest stopped on its own" watcher per slot. + private var deathWatchTasks: [Int: Task] = [:] + + /// Bumped every time a slot starts a new VM, so a stale watcher from a + /// previous occupant of the same slot cannot trigger a teardown of the + /// current one. + private var slotGeneration: [Int: Int] = [:] + + /// Slots whose teardown is in flight. Guards against the SSH command + /// returning, the death watcher firing, and the scheduler's timeout all + /// racing to tear the same slot down. + private var tearingDown: Set = [] + + /// Runner names minted but not yet visible in ``live`` (the window between + /// deciding to boot and the clone finishing). The reconcile loop must not + /// delete a row that one of these is about to create. + private var reservedRunnerNames: Set = [] + + /// Runner names whose `gitea-runner daemon` exited cleanly, meaning Gitea + /// already deregistered them (`--ephemeral`). Teardown skips the belt-and- + /// braces row deletion for these. + private var completedRunnerNames: Set = [] + + /// The shared registration token, resolved once and cached for the process + /// lifetime. Never minted per VM — see ``registrationToken()``. + private var cachedRegistrationToken: String? + + /// The in-flight fetch of ``cachedRegistrationToken``, if any. + /// + /// Caching the *value* alone is not enough: `Orchestrator` is an actor, so a + /// second slot booting during the `await` on the API call would see an empty + /// cache and mint a second token — and minting invalidates every prior token + /// for the scope, including the one the first VM is about to use. Memoizing + /// the task instead makes concurrent callers share one request. + private var registrationTokenTask: Task? + + /// Set once ``shutdown()`` has begun; stops new work being accepted. + private var isShuttingDown = false + + /// Creates an orchestrator. + /// + /// - Parameters: + /// - config: Validated configuration. + /// - client: Admin-scoped Gitea client. + /// - store: The VM store. + /// - imageName: Base image to clone. Defaults to `default`. + /// - logger: Structured logger. + public init( + config: RunnerConfig, + client: GiteaClient, + store: VMStore, + imageName: String = "default", + logger: Logger = Logger(label: "orchestrator") + ) { + self.config = config + self.client = client + self.store = store + self.imageName = imageName + self.logger = logger + self.state = SchedulerState(slotCount: Orchestrator.slotCount(for: config)) + } + + /// The fixed slot count: the configured concurrency, hard-clamped to the + /// kernel's two-guest limit. + private static func slotCount(for config: RunnerConfig) -> Int { + min( + max(1, config.scheduler.maxConcurrentVMs), + RunnerConfig.SchedulerSection.hardMaxConcurrentVMs + ) + } + + /// This orchestrator's slot count. + public var slotCount: Int { state.slots.count } + + // MARK: - Lifecycle + + /// Runs the poll/reconcile loop until the task is cancelled. + /// + /// On entry it purges clones orphaned by a previous crash and runs one + /// reconcile pass, so a restart converges before it schedules anything new. + /// + /// On cancellation it stops accepting work and calls ``shutdown()``, so + /// `SIGTERM` from `launchd` results in guests being asked to stop rather + /// than being killed with their filesystems dirty. + /// + /// - Throws: Only unrecoverable errors; transient Gitea or VM failures are + /// logged and retried on the next tick. + public func runForever() async throws { + try store.ensureLayout() + + // Clones left behind by a crash are garbage: their guests are gone and + // their runner rows, if any, are handled by the reconcile pass below. + do { + try store.purgeClones() + } catch { + logger.warning("could not purge orphaned clones", metadata: ["error": "\(error)"]) + } + + logger.info( + "orchestrator starting", + metadata: [ + "image": .string(imageName), + "slots": .stringConvertible(slotCount), + "labels": .string(config.runner.labels.joined(separator: ",")), + "instance": .string(config.gitea.instanceURL.absoluteString), + ] + ) + + await reconcileOnce() + + await withTaskGroup(of: Void.self) { group in + group.addTask { [pollInterval = config.scheduler.pollIntervalSeconds] in + while !Task.isCancelled { + await self.tick() + do { + try await Task.sleep(for: .seconds(max(1, pollInterval))) + } catch { + break + } + } + } + group.addTask { [reconcileInterval = config.scheduler.reconcileIntervalSeconds] in + while !Task.isCancelled { + do { + try await Task.sleep(for: .seconds(max(1, reconcileInterval))) + } catch { + break + } + await self.reconcileOnce() + } + } + } + + await shutdown() + logger.info("orchestrator stopped") + } + + /// Tears down every live VM and deletes their clones. Idempotent. + public func shutdown() async { + guard !isShuttingDown else { return } + isShuttingDown = true + logger.info("shutting down", metadata: ["liveVMs": .stringConvertible(live.count)]) + + // Cancel the supervising tasks first so they stop waiting on SSH, then + // let them run their own teardown; whatever they miss we clean up below. + let tasks = slotTasks + slotTasks.removeAll() + for (_, task) in tasks { task.cancel() } + for (_, task) in tasks { await task.value } + + for slot in live.keys.sorted() { + await teardownSlot(slot, reason: "daemon shutdown") + } + } + + /// One iteration of the poll loop: fetch, plan, execute. + /// + /// Exposed separately so tests and `vm boot` can drive a single tick. + /// + /// - Parameter now: Reference time, injected for testability. + public func tick(now: Date = Date()) async { + guard !isShuttingDown else { return } + + let jobs: [WorkflowJob] + do { + jobs = try await client.listQueuedJobs() + } catch { + // A Gitea outage must never take the daemon down: queued jobs wait + // up to ABANDONED_JOB_TIMEOUT (24 h), so a missed tick costs nothing. + logger.warning("listQueuedJobs failed", metadata: ["error": .string("\(error)")]) + return + } + + // `waiting` means *blocked on a dependency* in Gitea's external + // vocabulary and must never be scheduled; only `queued` is schedulable. + let queued = jobs.filter { $0.isQueued } + + let (newState, actions) = SchedulerCore.plan( + state: state, + queuedJobs: queued, + labels: config.labelSet, + maxVMs: slotCount, + now: now, + jobTimeout: TimeInterval(config.scheduler.jobTimeoutMinutes * 60), + bootTimeout: TimeInterval(config.scheduler.bootTimeoutSeconds) + ) + state = newState + + for action in actions { + switch action { + case .none: + continue + + case .teardownVM(let slot, let reason): + // Retire the slot's lifecycle task *before* tearing down, and + // wait for it: the planner deliberately emits teardowns before + // boots so a timed-out slot can be recycled in this same pass, + // and `bootSlot` refuses a slot whose `slotTasks` entry is still + // populated. The task is parked in `executor.run` on a channel we + // are about to kill, so it would otherwise clear that entry only + // after the boot had already been refused. + if let task = slotTasks.removeValue(forKey: slot) { + task.cancel() + // Its own teardown runs to completion here, which also means + // it cannot race a successor booted later in this pass. + await task.value + } + await teardownSlot(slot, reason: reason) + + case .bootVM(let slot, let jobHint): + do { + try await bootSlot(slot, jobHint: jobHint) + } catch { + logger.error( + "boot refused", + metadata: [ + "slot": .stringConvertible(slot), + "job": .stringConvertible(jobHint), + "error": .string("\(error)"), + ] + ) + state = SchedulerCore.markIdle(state: state, slot: slot) + // The job is still queued, so the ledger would never expire + // its entry on its own and the job would never boot again. + state = SchedulerCore.releaseJob(state: state, jobID: jobHint) + } + } + } + } + + /// One reconcile pass over Gitea's runner rows. + /// + /// Deletes runners that are ephemeral, idle, ours by name prefix, and not + /// backed by a live VM. Conservative by construction: a row we are unsure + /// about is left alone, because deleting a live runner would fail a job. + public func reconcileOnce() async { + let runners: [ActionRunner] + do { + runners = try await client.listRunners() + } catch { + logger.warning("listRunners failed", metadata: ["error": .string("\(error)")]) + return + } + + let ours = Set(live.values.map(\.runnerName)).union(reservedRunnerNames) + + for runner in runners { + guard runner.isEphemeral else { continue } + guard !runner.isBusy else { continue } + guard RunnerNaming.hasPrefix(runner.name, prefix: config.runner.namePrefix) else { continue } + guard !ours.contains(runner.name) else { continue } + + do { + try await client.deleteRunner(id: runner.id) + logger.info( + "reconcile: deleted orphaned runner", + metadata: ["name": .string(runner.name), "id": .stringConvertible(runner.id)] + ) + } catch { + logger.warning( + "reconcile: could not delete runner", + metadata: ["name": .string(runner.name), "error": .string("\(error)")] + ) + } + } + } + + // MARK: - Slot operations + + /// Boots, provisions, registers, and then supervises one slot. + /// + /// Returns once the slot has been handed off to its supervising task; the + /// job itself runs asynchronously and teardown is triggered by the SSH + /// command returning or by `jobTimeout`. + /// + /// - Parameters: + /// - slot: Slot index. + /// - jobHint: The queued job that motivated this boot — a **hint** only; + /// the server chooses which job the runner actually claims. + public func bootSlot(_ slot: Int, jobHint: Int64) async throws { + guard !isShuttingDown else { + throw CoreError.provisioningFailed("shutting down; refusing to boot slot \(slot)") + } + guard slot >= 0, slot < slotCount else { + throw CoreError.provisioningFailed("slot \(slot) out of range") + } + guard slotTasks[slot] == nil, live[slot] == nil else { + throw CoreError.provisioningFailed("slot \(slot) is already occupied") + } + + // Fail fast, on the caller's turn, for the conditions that make a boot + // pointless: no space, no image, no token source. + try store.ensureFreeSpace(minGB: config.storage.minFreeDiskGB) + + let runnerName = RunnerNaming.makeRunnerName(prefix: config.runner.namePrefix) + reservedRunnerNames.insert(runnerName) + + let generation = (slotGeneration[slot] ?? 0) + 1 + slotGeneration[slot] = generation + state = SchedulerCore.markProvisioning(state: state, slot: slot, jobHint: jobHint, now: Date()) + + logger.info( + "booting VM", + metadata: [ + "slot": .stringConvertible(slot), + "job": .stringConvertible(jobHint), + "runner": .string(runnerName), + ] + ) + + slotTasks[slot] = Task { [weak self] in + guard let self else { return } + await self.runSlotLifecycle(slot: slot, jobHint: jobHint, runnerName: runnerName, generation: generation) + } + } + + /// The whole life of one slot, from clone to teardown. + /// + /// Every failure path funnels into the same teardown, because a slot that is + /// neither live nor idle is a slot leaked for the process's lifetime. + private func runSlotLifecycle(slot: Int, jobHint: Int64, runnerName: String, generation: Int) async { + let bootTimeout = Duration.seconds(max(30, config.scheduler.bootTimeoutSeconds)) + let jobTimeout = Duration.seconds(max(60, config.scheduler.jobTimeoutMinutes * 60)) + var teardownReason = "job finished" + + do { + let mac = try store.macAddress(forSlot: slot, slotCount: slotCount) + // Whatever lease this MAC already holds belongs to the *previous* + // guest on this slot — the MACs are persistent and macOS leases last + // 24 h. `waitForLease` must not hand that address back before the new + // guest has even brought its NIC up. + let priorLease = DHCPLeaseParser.lease( + forMAC: mac, + in: DHCPLeaseParser.parseFile() + ) + let bundle = try store.cloneImage(named: imageName, slotMAC: mac) + live[slot] = LiveVM(slot: slot, bundle: bundle, runnerName: runnerName, startedAt: Date()) + + let instance = try VMInstance(bundle: bundle, label: "slot-\(slot)") + instances[slot] = instance + try await instance.start() + + // A guest that panics, or is shut down from inside the job, must + // land in the same teardown path as a clean finish. + deathWatchTasks[slot] = Task { [weak self] in + let reason = await instance.waitUntilStopped() + await self?.vmStoppedUnexpectedly(slot: slot, generation: generation, reason: reason) + } + + let ip = try await waitForLease(mac: mac, timeout: bootTimeout, replacing: priorLease) + live[slot]?.ipAddress = ip + logger.info("guest leased address", metadata: ["slot": .stringConvertible(slot), "ip": .string(ip)]) + + try await waitForSSH( + host: ip, + username: config.guest.username, + password: config.guest.password, + timeout: bootTimeout + ) + + let token = try await registrationToken() + let executor = SSHExecutor( + host: ip, + username: config.guest.username, + password: config.guest.password + ) + + // With the hint: the slot is `.provisioning` right now, which carries + // no hint to inherit, so the hintless overload would drop it — and + // with it both the job-timeout ledger release and every log line + // naming which job a running slot is serving. + state = SchedulerCore.markRunning(state: state, slot: slot, jobHint: jobHint, now: Date()) + logger.info( + "registering ephemeral runner", + metadata: [ + "slot": .stringConvertible(slot), + "runner": .string(runnerName), + "job": .stringConvertible(jobHint), + ] + ) + + let result = try await registerAndRun( + executor: executor, + runnerName: runnerName, + token: token, + timeout: jobTimeout + ) + await executor.close() + + if result.succeeded { + // A clean exit means the ephemeral runner claimed its one task, + // finished it, and was deregistered by the server. + completedRunnerNames.insert(runnerName) + logger.info("job finished", metadata: ["slot": .stringConvertible(slot), "runner": .string(runnerName)]) + } else { + teardownReason = "runner exited \(result.exitCode)" + logger.warning( + "runner exited non-zero", + metadata: [ + "slot": .stringConvertible(slot), + "exit": .stringConvertible(result.exitCode), + "stderr": .string(String(result.stderr.suffix(500))), + ] + ) + } + } catch is CancellationError { + teardownReason = "cancelled" + } catch { + teardownReason = "\(error)" + logger.error( + "slot failed", + metadata: [ + "slot": .stringConvertible(slot), + "runner": .string(runnerName), + "error": .string("\(error)"), + ] + ) + } + + await teardownSlot(slot, reason: teardownReason) + + // The clone may never have been adopted into `live` (a MAC or clone + // failure throws before that), in which case teardown's own removal — + // which lives inside `if let info` — never ran. Left behind, the name + // would sit in the reconcile loop's "ours" set for the process lifetime. + reservedRunnerNames.remove(runnerName) + + // A slot that did not finish a job leaves its motivating job queued, and + // the ledger expires entries only when a job *stops* being queued — so + // without this the job is never booted for again. + if teardownReason != "job finished" { + state = SchedulerCore.releaseJob(state: state, jobID: jobHint) + } + + slotTasks[slot] = nil + } + + /// Called by the death watcher when a guest stops without us asking. + private func vmStoppedUnexpectedly(slot: Int, generation: Int, reason: VMStopReason) async { + guard slotGeneration[slot] == generation else { return } + guard !tearingDown.contains(slot), live[slot] != nil else { return } + + logger.warning( + "guest stopped unexpectedly", + metadata: ["slot": .stringConvertible(slot), "reason": .string("\(reason)")] + ) + slotTasks[slot]?.cancel() + await teardownSlot(slot, reason: "guest stopped: \(reason)") + } + + /// Stops the VM in a slot, deletes its clone, and marks the slot idle. + /// + /// Best-effort and never throws: teardown that could fail would leak a slot. + /// + /// - Parameters: + /// - slot: Slot index. + /// - reason: Logged cause. + public func teardownSlot(_ slot: Int, reason: String) async { + guard !tearingDown.contains(slot) else { return } + + let vm = instances.removeValue(forKey: slot) + let info = live.removeValue(forKey: slot) + guard vm != nil || info != nil else { + state = SchedulerCore.markIdle(state: state, slot: slot) + return + } + + tearingDown.insert(slot) + slotGeneration[slot] = (slotGeneration[slot] ?? 0) + 1 + deathWatchTasks.removeValue(forKey: slot)?.cancel() + + logger.info("tearing down slot", metadata: ["slot": .stringConvertible(slot), "reason": .string(reason)]) + + if let vm { + // requestStop first: the guest gets a power-button press and a + // chance to flush before we pull the plug. + _ = await vm.requestStopThenForce(gracePeriod: .seconds(30)) + } + + if let info { + do { + try store.deleteClone(info.bundle) + } catch { + logger.warning( + "could not delete clone", + metadata: ["path": .string(info.bundle.rootURL.path), "error": .string("\(error)")] + ) + } + + reservedRunnerNames.remove(info.runnerName) + + // If the runner never completed a job, Gitea will keep its row + // forever: rows are swept at midnight, and never at all for a + // runner that claimed no task. Delete it ourselves. + if !completedRunnerNames.contains(info.runnerName) { + await deleteRunnerRow(named: info.runnerName) + } + completedRunnerNames.remove(info.runnerName) + } + + state = SchedulerCore.markIdle(state: state, slot: slot) + tearingDown.remove(slot) + } + + /// Best-effort deletion of a runner row by name. + private func deleteRunnerRow(named name: String) async { + do { + let runners = try await client.listRunners() + guard let row = runners.first(where: { $0.name == name }) else { return } + guard !row.isBusy else { + // Deleting a busy runner would fail whatever job it is running; + // leave it for the next reconcile pass. + logger.info("runner still busy; leaving row for reconcile", metadata: ["name": .string(name)]) + return + } + try await client.deleteRunner(id: row.id) + logger.info("deleted runner row", metadata: ["name": .string(name)]) + } catch { + logger.warning( + "could not delete runner row", + metadata: ["name": .string(name), "error": .string("\(error)")] + ) + } + } + + /// Polls `/var/db/dhcpd_leases` until the slot's MAC has an address. + /// + /// Slot MACs are persistent and macOS leases last 24 h, so the previous + /// guest's entry for this MAC is normally still in the file when a new clone + /// boots. `replacing` is that entry, sampled before the guest was started; + /// the poll holds out for a lease `bootpd` wrote afterwards rather than + /// returning an address that belongs to a VM that no longer exists. + /// + /// The gate is deliberately soft: if no newer lease appears within half the + /// timeout but a stale one is present, that address is used with a warning. + /// `bootpd` overwhelmingly reissues the same address to the same MAC, and + /// failing a boot outright over a lease record that was merely not rewritten + /// would be worse than the stale read this guards against. + /// + /// - Parameters: + /// - mac: The slot's persistent MAC. + /// - timeout: Ceiling, from ``RunnerConfig/SchedulerSection/bootTimeoutSeconds``. + /// - replacing: The lease seen for `mac` before the guest was started. + /// - Returns: The guest's IPv4 address. + /// - Throws: ``CoreError/timeout(_:)``. + public func waitForLease( + mac: String, + timeout: Duration, + replacing previous: DHCPLease? = nil + ) async throws -> String { + let start = Date() + let deadline = start.addingTimeInterval(timeout.seconds) + let staleFallbackAfter = start.addingTimeInterval(timeout.seconds / 2) + + while Date() < deadline { + try Task.checkCancellation() + if let lease = DHCPLeaseParser.lease(forMAC: mac, in: DHCPLeaseParser.parseFile()) { + if DHCPLeaseParser.isNewer(lease, than: previous) { + return lease.ipAddress + } + if Date() >= staleFallbackAfter { + logger.warning( + "no fresh dhcp lease; using the previous one for this MAC", + metadata: ["mac": .string(mac), "ip": .string(lease.ipAddress)] + ) + return lease.ipAddress + } + } + try await Task.sleep(for: .seconds(2)) + } + throw CoreError.timeout("dhcp lease for \(mac)") + } + + /// Registers an ephemeral runner in the guest and starts its daemon. + /// + /// Blocks until the daemon exits, which — because the runner is ephemeral — + /// happens after exactly one job. + /// + /// - Parameters: + /// - executor: A connected guest executor. + /// - runnerName: The unique name to register under. + /// - token: The shared registration token. + /// - Returns: The daemon's exit result. + public func registerAndRun( + executor: any GuestExecutor, + runnerName: String, + token: String + ) async throws -> SSHCommandResult { + try await registerAndRun( + executor: executor, + runnerName: runnerName, + token: token, + timeout: .seconds(max(60, config.scheduler.jobTimeoutMinutes * 60)) + ) + } + + /// ``registerAndRun(executor:runnerName:token:)`` with an explicit ceiling on + /// how long the runner daemon may live. + public func registerAndRun( + executor: any GuestExecutor, + runnerName: String, + token: String, + timeout: Duration + ) async throws -> SSHCommandResult { + // Mode 0600, and removed in the same && chain below. A registration + // token is fleet-wide; passing it as --token would publish it to every + // process on a guest that is about to run arbitrary repository code. + try await executor.uploadData(Data(token.utf8), remotePath: Orchestrator.tokenPath, mode: "0600") + + // Only bare names are stored server-side; `:host` is a register-time + // execution hint. The guest must ship no config.yaml `runner.labels`, + // which would silently override this. + let labels = config.labelSet.registrationArgument(schema: "host") + let instance = config.gitea.instanceURL.absoluteString.hasSuffix("/") + ? String(config.gitea.instanceURL.absoluteString.dropLast()) + : config.gitea.instanceURL.absoluteString + + let command = """ + export PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin; \ + gitea-runner register --no-interactive \ + --instance \(Orchestrator.shellQuote(instance)) \ + --token-file \(Orchestrator.tokenPath) \ + --name \(Orchestrator.shellQuote(runnerName)) \ + --labels \(Orchestrator.shellQuote(labels)) \ + --ephemeral \ + && rm -f \(Orchestrator.tokenPath) \ + && gitea-runner daemon + """ + + return try await executor.run(command, timeout: timeout) + } + + /// Where the registration token is staged inside the guest. + private static let tokenPath = "/tmp/.reg-token" + + /// Single-quotes a value for `/bin/sh`. + private static func shellQuote(_ value: String) -> String { + "'" + value.replacingOccurrences(of: "'", with: "'\\''") + "'" + } + + // MARK: - Tokens + + /// Resolves the registration token, caching it for the process lifetime. + /// + /// Order: `registrationTokenFile`, then `registrationToken`, then — only if + /// ``RunnerConfig/GiteaSection/fetchRegistrationTokenViaAPI`` is set — + /// ``GiteaClient/getRegistrationToken()``. + /// + /// - Important: Never called per VM as a way of minting a throwaway secret. + /// Registration tokens are reusable and scope-wide, and minting a new one + /// invalidates every prior token for that scope — including tokens held by + /// runners registered from other hosts. One token, cached, shared. + /// - Throws: ``CoreError/configInvalid(_:)`` when no source is available. + public func registrationToken() async throws -> String { + if let cachedRegistrationToken { return cachedRegistrationToken } + + if let staticToken = try config.resolveStaticRegistrationToken(), + !staticToken.isEmpty { + cachedRegistrationToken = staticToken + return staticToken + } + + guard config.gitea.fetchRegistrationTokenViaAPI else { + throw CoreError.configInvalid( + """ + no registration token available: set gitea.registrationTokenFile or \ + gitea.registrationToken, or enable gitea.fetchRegistrationTokenViaAPI + """ + ) + } + + // Share one in-flight mint between concurrent boots. Assigned before the + // first suspension point, so the second caller cannot miss it. + if let inFlight = registrationTokenTask { + return try await inFlight.value + } + let task = Task { [client] () -> String in + let fetched = try await client.getRegistrationToken() + guard !fetched.isEmpty else { + throw CoreError.configInvalid("Gitea returned an empty registration token") + } + return fetched + } + registrationTokenTask = task + + do { + let fetched = try await task.value + cachedRegistrationToken = fetched + return fetched + } catch { + // A failed mint must not poison every later boot. + registrationTokenTask = nil + throw error + } + } + + /// A snapshot of live slot state, for `vm list` and diagnostics. + public func liveVMs() -> [LiveVM] { + live.keys.sorted().compactMap { live[$0] } + } + + /// A snapshot of the scheduler's slot table. + public func slotStates() -> [VMSlot] { + state.slots + } +} + +extension Duration { + /// This duration as a floating-point number of seconds. + var seconds: TimeInterval { + let components = self.components + return TimeInterval(components.seconds) + TimeInterval(components.attoseconds) / 1e18 + } +} diff --git a/Sources/RunnerHost/VMBundle.swift b/Sources/RunnerHost/VMBundle.swift new file mode 100644 index 0000000..6d6ccda --- /dev/null +++ b/Sources/RunnerHost/VMBundle.swift @@ -0,0 +1,272 @@ +import Foundation +import RunnerCore +import Virtualization + +/// The persisted description of a VM, stored alongside its disk in a bundle +/// directory. +/// +/// Virtualization.framework requires that a macOS guest be recreated with +/// *exactly* the hardware model and machine identifier it was installed with — +/// change either and the guest will not boot. Both are opaque blobs the +/// framework hands us at install time, so they are stored verbatim here. +/// `Data` encodes to base64 in JSON, which keeps `config.json` human-inspectable. +public struct VMBundleConfig: Codable, Sendable, Equatable { + /// How the backing disk was created. + public enum DiskFormat: String, Codable, Sendable { + /// Sparse Apple System Image Format, via `diskutil image create` + /// (macOS 26+). Preferred: clones and grows lazily. + case asif + /// A plain sparse file created with `truncate`. Fallback. + case raw + } + + /// `VZMacHardwareModel.dataRepresentation` from the restore image's + /// `mostFeaturefulSupportedConfiguration`. + public var hardwareModelData: Data + + /// `VZMacMachineIdentifier.dataRepresentation`. Uniquely identifies the + /// "machine"; the guest's Setup Assistant state is tied to it. + public var machineIdentifierData: Data + + /// The NIC MAC, e.g. `aa:bb:0c:dd:ee:ff`. + /// + /// For a **clone** this is one of the two persistent per-slot MACs, not a + /// fresh random address — see ``VMStore`` and docs/DESIGN.md, Verified + /// Fact 12. + public var macAddress: String + + /// Backing disk format. + public var diskFormat: DiskFormat + + /// Virtual CPU count. + public var cpuCount: Int + + /// RAM in gibibytes. + public var memoryGB: Int + + /// The guest admin account created during installation. + public var guestUsername: String + + /// Bundle creation timestamp. + public var createdAt: Date + + /// The installed macOS version/build, when known (from + /// `VZMacOSRestoreImage.buildVersion`). + public var macOSVersion: String? + + /// Whether guest provisioning (Node.js, `gitea-runner`, sudoers, power + /// settings) has completed. A base image is only clonable once this is true. + public var provisioned: Bool + + public init( + hardwareModelData: Data, + machineIdentifierData: Data, + macAddress: String, + diskFormat: DiskFormat, + cpuCount: Int, + memoryGB: Int, + guestUsername: String, + createdAt: Date = Date(), + macOSVersion: String? = nil, + provisioned: Bool = false + ) { + self.hardwareModelData = hardwareModelData + self.machineIdentifierData = machineIdentifierData + self.macAddress = macAddress + self.diskFormat = diskFormat + self.cpuCount = cpuCount + self.memoryGB = memoryGB + self.guestUsername = guestUsername + self.createdAt = createdAt + self.macOSVersion = macOSVersion + self.provisioned = provisioned + } +} + +/// A directory holding everything needed to boot one VM. +/// +/// ``` +/// / +/// disk.asif (or disk.img for the RAW fallback) +/// nvram.bin VZMacAuxiliaryStorage — the guest's NVRAM +/// config.json VMBundleConfig +/// ``` +/// +/// Base images live under `/images//`; ephemeral clones under +/// `/vms//`. A clone is byte-identical except for `config.json`, +/// which is rewritten with the slot's MAC. +public struct VMBundle: Sendable, Equatable { + /// The bundle directory. + public let rootURL: URL + + /// Wraps an existing directory path. Does not touch the filesystem. + public init(rootURL: URL) { + self.rootURL = rootURL + } + + // MARK: - Paths + + /// Path to `config.json`. + public var configURL: URL { rootURL.appendingPathComponent("config.json") } + + /// Path to `nvram.bin`, the `VZMacAuxiliaryStorage` backing file. + public var auxiliaryStorageURL: URL { rootURL.appendingPathComponent("nvram.bin") } + + /// Path to the ASIF disk, used when ``VMBundleConfig/DiskFormat/asif``. + public var asifDiskURL: URL { rootURL.appendingPathComponent("disk.asif") } + + /// Path to the RAW disk, used when ``VMBundleConfig/DiskFormat/raw``. + public var rawDiskURL: URL { rootURL.appendingPathComponent("disk.img") } + + /// The disk file for a given format. + public func diskURL(format: VMBundleConfig.DiskFormat) -> URL { + switch format { + case .asif: return asifDiskURL + case .raw: return rawDiskURL + } + } + + /// The bundle's directory name — the image name, or the clone's UUID. + public var name: String { rootURL.lastPathComponent } + + /// The disk file this bundle actually uses, per its recorded + /// ``VMBundleConfig/diskFormat``. + /// + /// Reads `config.json`, so the format is never re-probed from the + /// filesystem — the builder recorded which of ASIF/RAW it managed to create + /// and that record is authoritative. + public func diskURL() throws -> URL { + diskURL(format: try loadConfig().diskFormat) + } + + // MARK: - Lifecycle + + /// Creates the bundle directory, failing if it already exists. + /// + /// - Throws: ``CoreError/bundleCorrupt(_:)`` if the path exists as a file. + public func createDirectory() throws { + let fm = FileManager.default + var isDir: ObjCBool = false + if fm.fileExists(atPath: rootURL.path, isDirectory: &isDir) { + if isDir.boolValue { + throw CoreError.bundleCorrupt("bundle directory already exists: \(rootURL.path)") + } + throw CoreError.bundleCorrupt("bundle path exists but is a file: \(rootURL.path)") + } + do { + try fm.createDirectory(at: rootURL, withIntermediateDirectories: true) + } catch { + throw CoreError.bundleCorrupt( + "could not create bundle directory \(rootURL.path): \(error.localizedDescription)") + } + } + + /// Reads and decodes `config.json`. + /// + /// - Throws: ``CoreError/bundleCorrupt(_:)`` when absent or undecodable. + public func loadConfig() throws -> VMBundleConfig { + let data: Data + do { + data = try Data(contentsOf: configURL) + } catch { + throw CoreError.bundleCorrupt( + "cannot read \(configURL.path): \(error.localizedDescription)") + } + do { + return try Self.decoder.decode(VMBundleConfig.self, from: data) + } catch { + throw CoreError.bundleCorrupt( + "cannot decode \(configURL.path): \(error.localizedDescription)") + } + } + + /// Encodes and atomically writes `config.json`. + public func saveConfig(_ config: VMBundleConfig) throws { + let data: Data + do { + data = try Self.encoder.encode(config) + } catch { + throw CoreError.bundleCorrupt( + "cannot encode config for \(rootURL.path): \(error.localizedDescription)") + } + do { + // .atomic writes to a temporary sibling and renames, so a crash + // mid-write can never leave a half-written config behind. + try data.write(to: configURL, options: .atomic) + } catch { + throw CoreError.bundleCorrupt( + "cannot write \(configURL.path): \(error.localizedDescription)") + } + } + + /// Whether `config.json`, `nvram.bin`, and the disk all exist. + public func isComplete() -> Bool { + let fm = FileManager.default + guard fm.fileExists(atPath: configURL.path), + fm.fileExists(atPath: auxiliaryStorageURL.path), + let config = try? loadConfig() + else { + return false + } + return fm.fileExists(atPath: diskURL(format: config.diskFormat).path) + } + + /// Total on-disk size of the bundle in bytes, following sparse allocation + /// (i.e. blocks actually used, not the disk's nominal size). + public func diskUsageBytes() throws -> Int64 { + let fm = FileManager.default + let keys: [URLResourceKey] = [.isRegularFileKey, .totalFileAllocatedSizeKey, .fileAllocatedSizeKey] + guard + let enumerator = fm.enumerator( + at: rootURL, + includingPropertiesForKeys: keys, + options: [], + errorHandler: nil + ) + else { + throw CoreError.bundleCorrupt("cannot enumerate \(rootURL.path)") + } + var total: Int64 = 0 + for case let url as URL in enumerator { + guard let values = try? url.resourceValues(forKeys: Set(keys)), + values.isRegularFile == true + else { continue } + // totalFileAllocatedSize is the blocks actually committed, which for + // a sparse ASIF/RAW disk is far below its nominal size. + if let allocated = values.totalFileAllocatedSize ?? values.fileAllocatedSize { + total += Int64(allocated) + } + } + return total + } + + /// Recursively removes the bundle directory. + public func destroy() throws { + let fm = FileManager.default + guard fm.fileExists(atPath: rootURL.path) else { return } + do { + try fm.removeItem(at: rootURL) + } catch { + throw CoreError.bundleCorrupt( + "cannot remove \(rootURL.path): \(error.localizedDescription)") + } + } + + // MARK: - Coding + + /// Shared coders. ISO-8601 dates keep `config.json` readable by humans and + /// by `jq`; `Data` still encodes as base64, which is what the two opaque + /// Virtualization blobs need. + private static let decoder: JSONDecoder = { + let d = JSONDecoder() + d.dateDecodingStrategy = .iso8601 + return d + }() + + private static let encoder: JSONEncoder = { + let e = JSONEncoder() + e.dateEncodingStrategy = .iso8601 + e.outputFormatting = [.prettyPrinted, .sortedKeys] + return e + }() +} diff --git a/Sources/RunnerHost/VMInstance.swift b/Sources/RunnerHost/VMInstance.swift new file mode 100644 index 0000000..884996a --- /dev/null +++ b/Sources/RunnerHost/VMInstance.swift @@ -0,0 +1,373 @@ +import Foundation +import RunnerCore +import Virtualization + +/// Why a VM stopped. +public enum VMStopReason: Sendable, Equatable { + /// The guest shut itself down (our normal path: the SSH session runs + /// `shutdown`, or `gitea-runner daemon` exits and provisioning halts it). + case guestInitiated + /// We asked it to stop and it complied. + case requested + /// The framework reported an error. + case failed(String) +} + +/// Owns one live `VZVirtualMachine` and exposes it as an `async` API. +/// +/// ## Threading +/// +/// `VZVirtualMachine` is not thread-safe and must be used only from the queue it +/// was created with. This class creates it with +/// `VZVirtualMachine(configuration:queue:)` on a **private serial queue** and +/// funnels every call through that queue, bridging the framework's +/// completion-handler API to `async` with continuations. That is why the daemon +/// can drive two VMs from an actor without ever touching the main queue for VM +/// control — though the process still needs a running `NSApplication` main loop +/// for the framework itself (see ``CommandDaemon``). +public final class VMInstance: @unchecked Sendable { + + /// The bundle this instance was created from. + public let bundle: VMBundle + + /// A caller-supplied label used in log messages, typically `slot-0`. + public let label: String + + /// The private serial queue every `VZVirtualMachine` call and every delegate + /// callback runs on. `VZVirtualMachine` is not thread-safe; this queue *is* + /// its thread-safety. + private let queue: DispatchQueue + + /// Only ever touched on ``queue``. + private let vm: VZVirtualMachine + + /// Retained explicitly: `VZVirtualMachine.delegate` is a weak reference. + private let vmDelegate: VMInstanceDelegate + + /// Guards ``stopReason`` and ``activityToken``. A plain lock rather than an + /// actor so the delegate callback — which arrives on ``queue`` and must not + /// block on an await — can publish the stop synchronously. + private let lock = NSLock() + private var stopReason: VMStopReason? + private var activityToken: (any NSObjectProtocol)? + + /// Wraps a non-`Sendable` value so it can cross into a `@Sendable` closure + /// that immediately hops onto ``queue``, which is the only place it is used. + private struct Unchecked: @unchecked Sendable { + let value: T + } + + /// Creates an instance and its underlying `VZVirtualMachine`. + /// + /// - Parameters: + /// - bundle: The VM bundle to boot. Usually an ephemeral clone. + /// - label: Log label. + /// - headless: Passed through to ``VZConfigFactory``. + /// - Throws: Configuration or validation failures. + public init(bundle: VMBundle, label: String, headless: Bool = true) throws { + self.bundle = bundle + self.label = label + let vmQueue = DispatchQueue(label: "vm.\(label).\(bundle.name)", qos: .userInitiated) + self.queue = vmQueue + + let configuration = try VZConfigFactory.makeConfiguration(bundle: bundle, headless: headless) + let delegate = VMInstanceDelegate() + self.vmDelegate = delegate + + // Constructed on the queue it will be driven from, so no VZ object is + // ever created on one thread and used from another. + self.vm = vmQueue.sync { + let machine = VZVirtualMachine(configuration: configuration, queue: vmQueue) + machine.delegate = delegate + return machine + } + + delegate.onStop = { [weak self] reason in + self?.finishStop(reason) + } + } + + /// The framework's current state, read on the VM queue. + public var state: VZVirtualMachine.State { + get async { + // The raw value crosses the concurrency boundary rather than the + // enum, so no assumption is made about the imported type's Sendable + // conformance. + let raw: Int = await withCheckedContinuation { continuation in + queue.async { + continuation.resume(returning: self.vm.state.rawValue) + } + } + return VZVirtualMachine.State(rawValue: raw) ?? .stopped + } + } + + /// Whether the VM is running or in a transitional state. + public var isActive: Bool { + get async { + switch await state { + case .stopped, .error: + return false + default: + return true + } + } + } + + /// Starts the VM. + /// + /// - Parameter options: Optional start options. The install/provision path + /// passes a `VZMacOSVirtualMachineStartOptions` — on macOS 27+ hosts that + /// is also where Setup Assistant automation is attached (see + /// ``GuestProvisioner`` and docs/DESIGN.md, Verified Fact 9). Pass `nil` + /// for a normal boot of an already-provisioned clone. + /// - Throws: ``CoreError/vmLimitExceeded`` when Apple's kernel-enforced cap + /// of **two** concurrent macOS guests is hit — the framework raises + /// `VZError.virtualMachineLimitExceeded` from `start()`, and that case is + /// translated here rather than propagated, because the scheduler treats it + /// as transient back-pressure rather than a failure. + public func start(options: VZMacOSVirtualMachineStartOptions? = nil) async throws { + clearStopReason() + + let boxed = Unchecked(value: options) + do { + try await withCheckedThrowingContinuation { + (continuation: CheckedContinuation) in + queue.async { + if let options = boxed.value { + // The install/provision path: on a macOS 27+ host these + // options carry the Setup Assistant automation. Note the + // options-taking overload reports failure as an optional + // Error, not a Result. + self.vm.start(options: options) { error in + if let error { + continuation.resume(throwing: error) + } else { + continuation.resume() + } + } + } else { + self.vm.start { result in + switch result { + case .success: + continuation.resume() + case .failure(let error): + continuation.resume(throwing: error) + } + } + } + } + } + } catch { + throw Self.mapVZError(error) + } + + // Hold a power assertion for the VM's lifetime: a CI guest that is + // building for twenty minutes over SSH looks completely idle to the host, + // and letting the Mac sleep underneath it would stall the job. + beginActivityAssertion() + } + + /// NSLock's `lock`/`unlock` are unavailable from an async context, so every + /// critical section lives in a synchronous helper. + private func clearStopReason() { + lock.lock() + defer { lock.unlock() } + stopReason = nil + } + + /// Takes the power assertion, unless the VM already stopped in the meantime. + private func beginActivityAssertion() { + let token = ProcessInfo.processInfo.beginActivity( + options: [.userInitiated, .idleSystemSleepDisabled], + reason: "running macOS CI guest \(label) (\(bundle.name))" + ) + lock.lock() + let alreadyStopped = stopReason != nil + if !alreadyStopped { + activityToken = token + } + lock.unlock() + if alreadyStopped { + // Raced with an immediate stop; don't strand the assertion. + ProcessInfo.processInfo.endActivity(token) + } + } + + /// Publishes a terminal stop and releases the power assertion. Idempotent: + /// the first reason wins, so a `didStopWithError` following a `requestStop` + /// cannot overwrite an already-recorded outcome. + private func finishStop(_ reason: VMStopReason) { + lock.lock() + if stopReason == nil { + stopReason = reason + } + let token = activityToken + activityToken = nil + lock.unlock() + if let token { + ProcessInfo.processInfo.endActivity(token) + } + } + + /// The recorded stop reason, if the VM has already stopped. + private var recordedStopReason: VMStopReason? { + lock.lock() + defer { lock.unlock() } + return stopReason + } + + /// Asks the guest to shut down, then force-stops if it does not. + /// + /// Tries `requestStop()` first — that delivers an ACPI-equivalent power + /// button press, giving the guest a chance to flush its filesystem — and + /// falls back to `stop()` after `gracePeriod`. Never throws: teardown must + /// always complete so the slot can be recycled. + /// + /// - Parameter gracePeriod: How long to wait for a graceful stop. + /// - Returns: Why the VM ended up stopped. + @discardableResult + public func requestStopThenForce(gracePeriod: Duration = .seconds(30)) async -> VMStopReason { + if let reason = recordedStopReason { return reason } + if await !isActive { + // Stopped without a delegate callback ever landing (for example a + // start() that failed outright). Record it so waiters unblock. + finishStop(.requested) + return recordedStopReason ?? .requested + } + + // Guest-cooperative first: requestStop() is the equivalent of a power + // button press, which lets the guest flush its filesystem. + _ = await withCheckedContinuation { (continuation: CheckedContinuation) in + queue.async { + guard self.vm.canRequestStop else { + continuation.resume(returning: false) + return + } + do { + try self.vm.requestStop() + continuation.resume(returning: true) + } catch { + // "not running", or the guest refused. Force is next either way. + continuation.resume(returning: false) + } + } + } + + if let reason = await waitForStop(within: gracePeriod) { + return reason + } + + // Grace elapsed — pull the plug. Teardown must always complete so the + // slot can be recycled, so every failure here is swallowed. + await withCheckedContinuation { (continuation: CheckedContinuation) in + queue.async { + guard self.vm.canStop else { + continuation.resume() + return + } + self.vm.stop { _ in + continuation.resume() + } + } + } + + if let reason = await waitForStop(within: .seconds(10)) { + return reason + } + // The framework never told us; treat it as stopped regardless rather + // than leaving the caller blocked on a dead slot. + finishStop(.requested) + return recordedStopReason ?? .requested + } + + /// Polls for a recorded stop for at most `limit`. Returns `nil` on timeout. + /// + /// Polling rather than a parked continuation keeps this cancellable and + /// leak-free: a continuation registered for a VM that never stops would be + /// stranded forever. + private func waitForStop(within limit: Duration) async -> VMStopReason? { + let deadline = ContinuousClock.now.advanced(by: limit) + while true { + if let reason = recordedStopReason { return reason } + if ContinuousClock.now >= deadline { return nil } + do { + try await Task.sleep(for: .milliseconds(200)) + } catch { + return recordedStopReason + } + } + } + + /// Suspends until the VM stops for any reason. + /// + /// - Returns: Why it stopped. + public func waitUntilStopped() async -> VMStopReason { + while true { + if let reason = recordedStopReason { return reason } + // A VM that reaches .stopped or .error without a delegate callback + // (an unusual but observed path) must not hang the caller. + if await !isActive { + finishStop(.guestInitiated) + return recordedStopReason ?? .guestInitiated + } + do { + try await Task.sleep(for: .milliseconds(500)) + } catch { + return recordedStopReason ?? .requested + } + } + } + + /// Translates a Virtualization error into a ``CoreError``. + /// + /// `VZError.Code.virtualMachineLimitExceeded` becomes + /// ``CoreError/vmLimitExceeded``; everything else becomes + /// ``CoreError/provisioningFailed(_:)`` carrying the framework's message. + public static func mapVZError(_ error: any Error) -> CoreError { + if let coreError = error as? CoreError { return coreError } + + // Apple's kernel-enforced cap of two concurrent macOS guests + // (docs/DESIGN.md, Verified Fact 8). The scheduler treats this as + // transient back-pressure, so it must stay distinguishable. + if let vzError = error as? VZError, vzError.code == .virtualMachineLimitExceeded { + return .vmLimitExceeded + } + let nsError = error as NSError + if nsError.domain == VZErrorDomain, + nsError.code == VZError.Code.virtualMachineLimitExceeded.rawValue + { + return .vmLimitExceeded + } + return .provisioningFailed(nsError.localizedDescription) + } +} + +/// Bridges `VZVirtualMachineDelegate` callbacks back into ``VMInstance``. +/// +/// Kept as a separate object so ``VMInstance`` need not inherit `NSObject`, and +/// so the delegate's lifetime is explicitly owned rather than accidentally +/// retained by the framework. +final class VMInstanceDelegate: NSObject, VZVirtualMachineDelegate { + /// Invoked on the VM queue whenever the machine stops. + var onStop: (@Sendable (VMStopReason) -> Void)? + + func guestDidStop(_ virtualMachine: VZVirtualMachine) { + onStop?(.guestInitiated) + } + + func virtualMachine(_ virtualMachine: VZVirtualMachine, didStopWithError error: any Error) { + onStop?(.failed((error as NSError).localizedDescription)) + } + + func virtualMachine( + _ virtualMachine: VZVirtualMachine, + networkDevice: VZNetworkDevice, + attachmentWasDisconnectedWithError error: any Error + ) { + // NAT attachments do drop transiently. The VM keeps running and the + // guest's DHCP client recovers, so this is deliberately not treated as a + // stop — the boot/job timeouts are what catch a guest that never comes + // back onto the network. + } +} diff --git a/Sources/RunnerHost/VMStore.swift b/Sources/RunnerHost/VMStore.swift new file mode 100644 index 0000000..6e39ec3 --- /dev/null +++ b/Sources/RunnerHost/VMStore.swift @@ -0,0 +1,366 @@ +import Foundation +import RunnerCore +import Virtualization + +/// Host-level state persisted across daemon restarts. +/// +/// The only thing in it today is the pair of per-slot MAC addresses, but it is +/// versioned so future fields (saved-state handles, image pins) can be added. +public struct HostState: Codable, Sendable, Equatable { + /// Schema version of this file. + public var version: Int + + /// One MAC per VM slot, generated once with + /// `VZMACAddress.randomLocallyAdministered()` and then **never changed**. + /// + /// Reusing a small fixed set of MACs is deliberate. macOS's `bootpd` hands + /// out 24-hour leases and records each in `/var/db/dhcpd_leases`; a fleet + /// that randomized a MAC per ephemeral VM would leave a day's worth of dead + /// leases behind and eventually exhaust the NAT subnet. Two persistent MACs + /// mean each slot simply renews the same lease forever. + public var slotMACAddresses: [String] + + public init(version: Int = 1, slotMACAddresses: [String] = []) { + self.version = version + self.slotMACAddresses = slotMACAddresses + } +} + +/// Owns the on-disk layout of images, ephemeral clones, IPSWs, and host state. +/// +/// ``` +/// / +/// images// base VM bundles (installed + provisioned) +/// vms// ephemeral clones, destroyed after each job +/// ipsw/ downloaded restore images +/// state.json HostState +/// ``` +public struct VMStore: Sendable { + /// Root directory, tilde-expanded by the caller. + public let storeDir: URL + + /// Creates a store rooted at `storeDir`. Does not touch the filesystem; + /// call ``ensureLayout()`` first. + public init(storeDir: URL) { + self.storeDir = storeDir + } + + /// Convenience initializer reading ``RunnerConfig/storeDirectoryURL``. + public init(config: RunnerConfig) { + self.init(storeDir: config.storeDirectoryURL) + } + + // MARK: - Paths + + /// `/images`. + public var imagesDir: URL { storeDir.appendingPathComponent("images", isDirectory: true) } + /// `/vms`. + public var clonesDir: URL { storeDir.appendingPathComponent("vms", isDirectory: true) } + /// `/ipsw`. + public var ipswDir: URL { storeDir.appendingPathComponent("ipsw", isDirectory: true) } + /// `/state.json`. + public var stateURL: URL { storeDir.appendingPathComponent("state.json") } + + /// Creates every directory in the layout if missing. + public func ensureLayout() throws { + let fm = FileManager.default + for dir in [storeDir, imagesDir, clonesDir, ipswDir] { + do { + try fm.createDirectory(at: dir, withIntermediateDirectories: true) + } catch { + throw CoreError.bundleCorrupt( + "cannot create \(dir.path): \(error.localizedDescription)") + } + } + } + + // MARK: - Images + + /// Names of every base image, sorted. + public func listImages() throws -> [String] { + let fm = FileManager.default + guard fm.fileExists(atPath: imagesDir.path) else { return [] } + let entries: [URL] + do { + entries = try fm.contentsOfDirectory( + at: imagesDir, + includingPropertiesForKeys: [.isDirectoryKey], + options: [.skipsHiddenFiles] + ) + } catch { + throw CoreError.bundleCorrupt( + "cannot list \(imagesDir.path): \(error.localizedDescription)") + } + return + entries + .filter { (try? $0.resourceValues(forKeys: [.isDirectoryKey]))?.isDirectory == true } + // A directory without a decodable config.json is not an image — it is + // a half-finished build or somebody's scratch folder. Skip silently. + .filter { (try? VMBundle(rootURL: $0).loadConfig()) != nil } + .map { $0.lastPathComponent } + .sorted() + } + + /// The bundle for a named base image. + /// + /// - Parameter name: Image name, e.g. `default`. + /// - Returns: The bundle, or `nil` when no such directory exists. + public func image(named name: String) throws -> VMBundle? { + let url = imagesDir.appendingPathComponent(name, isDirectory: true) + var isDir: ObjCBool = false + guard FileManager.default.fileExists(atPath: url.path, isDirectory: &isDir), + isDir.boolValue + else { return nil } + return VMBundle(rootURL: url) + } + + /// Deletes a base image and everything in it. + public func deleteImage(named name: String) throws { + guard let bundle = try image(named: name) else { + throw CoreError.notFound("image '\(name)'") + } + try bundle.destroy() + } + + // MARK: - Clones + + /// Copy-on-write clones a base image into a fresh ephemeral bundle. + /// + /// Cloning is done with `FileManager.copyItem` **per file**, which on APFS + /// performs a copy-on-write clone: the new disk costs almost nothing until + /// the guest writes to it. Two constraints follow, and both are enforced + /// here: + /// + /// * Source and destination must be on the **same APFS volume**, so images + /// and clones both live under `storeDir`. + /// * A CoW clone's *apparent* size is the full disk size while its real cost + /// grows with guest writes, so ``ensureFreeSpace(minGB:)`` must be called + /// before cloning and the floor kept generous. + /// + /// The clone's `config.json` is rewritten with `slotMAC` so the VM comes up + /// on its slot's persistent address; everything else is inherited. + /// + /// - Parameters: + /// - name: Base image name. Must be ``VMBundleConfig/provisioned``. + /// - slotMAC: The persistent MAC for the slot this clone will occupy. + /// - Returns: The new clone bundle under `/vms//`. + /// - Throws: ``CoreError/notFound(_:)`` if the image is missing, + /// ``CoreError/bundleCorrupt(_:)`` if it is unprovisioned or incomplete. + public func cloneImage(named name: String, slotMAC: String) throws -> VMBundle { + guard let source = try image(named: name) else { + throw CoreError.notFound("base image '\(name)' under \(imagesDir.path)") + } + let sourceConfig = try source.loadConfig() + guard sourceConfig.provisioned else { + throw CoreError.bundleCorrupt( + "base image '\(name)' is not provisioned; run `image build` to completion first") + } + guard source.isComplete() else { + throw CoreError.bundleCorrupt( + "base image '\(name)' is missing its disk, nvram.bin, or config.json") + } + + try ensureLayout() + + let fm = FileManager.default + let destination = VMBundle( + rootURL: clonesDir.appendingPathComponent(UUID().uuidString, isDirectory: true)) + try destination.createDirectory() + + // Anything that fails past this point leaves a partial clone behind, and + // a partial clone is worse than none: it would be counted by + // `listClones` and booted by nobody. + func abort(_ error: any Error) -> any Error { + try? destination.destroy() + return error + } + + do { + // Per-file `copyItem`, NOT a directory copy: APFS performs a + // copy-on-write clone for a regular file copied within the same + // volume, so this is effectively instantaneous and costs no space + // until the guest writes. It is *only* copy-on-write when source and + // destination share a volume — which is why images/ and vms/ both + // live under storeDir (docs/DESIGN.md, Verified Fact 13). Cloning + // across volumes silently degrades to a full byte copy of a + // multi-gigabyte disk. + let diskName = source.diskURL(format: sourceConfig.diskFormat) + try fm.copyItem( + at: diskName, + to: destination.diskURL(format: sourceConfig.diskFormat)) + try fm.copyItem(at: source.auxiliaryStorageURL, to: destination.auxiliaryStorageURL) + try fm.copyItem(at: source.configURL, to: destination.configURL) + } catch { + throw abort( + CoreError.bundleCorrupt( + "cannot clone image '\(name)': \(error.localizedDescription)")) + } + + do { + // Rewrite only the MAC. The machine identifier is deliberately + // SHARED with the base image: the guest's Setup Assistant state and + // its installed system are tied to it, regenerating it would present + // the guest with new hardware, and a future save/restore path + // (docs/DESIGN.md §9) forbids changing the ECID anyway. + var cloneConfig = sourceConfig + cloneConfig.macAddress = slotMAC + cloneConfig.provisioned = true + try destination.saveConfig(cloneConfig) + } catch { + throw abort(error) + } + + return destination + } + + /// Removes an ephemeral clone. Safe to call twice. + /// + /// - Parameter bundle: A bundle previously returned by + /// ``cloneImage(named:slotMAC:)``. Refuses to delete anything outside + /// ``clonesDir``. + public func deleteClone(_ bundle: VMBundle) throws { + // `rm -rf` driven by a path that came from elsewhere deserves a guard. + let root = clonesDir.standardizedFileURL.resolvingSymlinksInPath().path + let target = bundle.rootURL.standardizedFileURL.resolvingSymlinksInPath().path + guard target.hasPrefix(root.hasSuffix("/") ? root : root + "/"), target != root else { + throw CoreError.bundleCorrupt( + "refusing to delete \(bundle.rootURL.path): not inside \(clonesDir.path)") + } + try bundle.destroy() + } + + /// Every ephemeral clone currently on disk. + /// + /// Used at startup to garbage-collect clones orphaned by a crash. + public func listClones() throws -> [VMBundle] { + let fm = FileManager.default + guard fm.fileExists(atPath: clonesDir.path) else { return [] } + let entries: [URL] + do { + entries = try fm.contentsOfDirectory( + at: clonesDir, + includingPropertiesForKeys: [.isDirectoryKey], + options: [.skipsHiddenFiles] + ) + } catch { + throw CoreError.bundleCorrupt( + "cannot list \(clonesDir.path): \(error.localizedDescription)") + } + return + entries + .filter { (try? $0.resourceValues(forKeys: [.isDirectoryKey]))?.isDirectory == true } + .sorted { $0.lastPathComponent < $1.lastPathComponent } + .map { VMBundle(rootURL: $0) } + } + + /// Deletes every clone. Called on daemon startup, before any VM is booted. + public func purgeClones() throws { + // Best-effort per clone: one undeletable directory must not stop the + // daemon from starting, so the first failure is remembered and rethrown + // only after every other clone has been tried. + var firstError: (any Error)? + for clone in try listClones() { + do { + try deleteClone(clone) + } catch { + if firstError == nil { firstError = error } + } + } + if let firstError { throw firstError } + } + + // MARK: - Host state + + /// Reads `state.json`, returning a fresh ``HostState`` when absent. + public func loadState() throws -> HostState { + guard FileManager.default.fileExists(atPath: stateURL.path) else { + return HostState() + } + do { + let data = try Data(contentsOf: stateURL) + return try JSONDecoder().decode(HostState.self, from: data) + } catch { + throw CoreError.bundleCorrupt( + "cannot read \(stateURL.path): \(error.localizedDescription)") + } + } + + /// Atomically writes `state.json`. + public func saveState(_ state: HostState) throws { + try ensureLayout() + do { + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + try encoder.encode(state).write(to: stateURL, options: .atomic) + } catch { + throw CoreError.bundleCorrupt( + "cannot write \(stateURL.path): \(error.localizedDescription)") + } + } + + /// Returns the persistent MAC for a slot, generating and persisting the + /// whole table the first time. + /// + /// - Parameters: + /// - slot: Slot index. + /// - slotCount: How many slots to provision addresses for. + /// - Returns: A MAC string such as `aa:bb:0c:dd:ee:ff`. + public func macAddress(forSlot slot: Int, slotCount: Int) throws -> String { + guard slot >= 0, slot < slotCount else { + throw CoreError.configInvalid( + "slot \(slot) is out of range for \(slotCount) slot(s)") + } + var state = try loadState() + if state.slotMACAddresses.count < slotCount { + // Generated exactly once and then persisted forever. See HostState's + // doc comment and docs/DESIGN.md, Verified Fact 12: randomizing a MAC + // per ephemeral clone would strand a 24-hour bootpd lease per boot + // and eventually exhaust the NAT subnet. + while state.slotMACAddresses.count < slotCount { + state.slotMACAddresses.append( + VZMACAddress.randomLocallyAdministered().string) + } + try saveState(state) + } + return state.slotMACAddresses[slot] + } + + // MARK: - Disk space + + /// Free space on the store's volume, in bytes. + /// + /// Uses the *important usage* resource key so the number matches what Finder + /// reports and accounts for purgeable space. + public func freeDiskSpace() throws -> Int64 { + // The volume keys only resolve for a path that exists, and the daemon may + // call this before anything has been created. + try ensureLayout() + do { + let values = try storeDir.resourceValues(forKeys: [ + .volumeAvailableCapacityForImportantUsageKey + ]) + guard let available = values.volumeAvailableCapacityForImportantUsage else { + throw CoreError.notFound( + "free-space information for the volume holding \(storeDir.path)") + } + return available + } catch let error as CoreError { + throw error + } catch { + throw CoreError.notFound( + "free space for \(storeDir.path): \(error.localizedDescription)") + } + } + + /// Throws unless the store volume has at least `minGB` free. + /// + /// - Throws: ``CoreError/insufficientDiskSpace(requiredGB:availableGB:)``. + public func ensureFreeSpace(minGB: Int) throws { + guard minGB > 0 else { return } + let availableBytes = try freeDiskSpace() + let availableGB = Int(availableBytes / 1_073_741_824) + guard availableGB >= minGB else { + throw CoreError.insufficientDiskSpace(requiredGB: minGB, availableGB: availableGB) + } + } +} diff --git a/Sources/RunnerHost/VZConfigFactory.swift b/Sources/RunnerHost/VZConfigFactory.swift new file mode 100644 index 0000000..bd9bcb7 --- /dev/null +++ b/Sources/RunnerHost/VZConfigFactory.swift @@ -0,0 +1,181 @@ +import Foundation +import RunnerCore +import Virtualization + +/// Builds a `VZVirtualMachineConfiguration` from a ``VMBundle``. +/// +/// The configuration is assembled the same way for base-image installs and for +/// ephemeral clones; only the bundle differs. Devices are chosen for the minimum +/// that a headless CI guest needs while still satisfying macOS's own +/// requirements. +public enum VZConfigFactory { + + /// Assembles and validates a configuration. + /// + /// Composition: + /// + /// * **Platform** — `VZMacPlatformConfiguration` with `hardwareModel` and + /// `machineIdentifier` restored from the bundle's stored blobs, and + /// `auxiliaryStorage` opened from `nvram.bin`. These three must match the + /// install exactly or the guest will not boot. + /// * **Boot loader** — `VZMacOSBootLoader`. + /// * **CPU / memory** — `max(4, config.cpuCount)` clamped into the + /// framework's supported range; memory likewise clamped. + /// * **Storage** — `VZVirtioBlockDeviceConfiguration` over a + /// `VZDiskImageStorageDeviceAttachment` on the bundle's disk. + /// * **Network** — `VZVirtioNetworkDeviceConfiguration` with a + /// `VZNATNetworkDeviceAttachment` and the bundle's MAC. NAT, not bridged: + /// bridged networking requires the restricted + /// `com.apple.vm.networking` entitlement, which Apple does not grant for + /// ad-hoc signing, whereas NAT needs nothing beyond + /// `com.apple.security.virtualization`. NAT is also what puts the guest in + /// `/var/db/dhcpd_leases`, which is how we discover its IP. + /// * **Graphics** — a `VZMacGraphicsDeviceConfiguration` with a single + /// 1920×1200 @ 72 ppi display, configured **always**, even headless. macOS + /// guests misbehave without a display device; we simply never attach a + /// `VZVirtualMachineView` to it. + /// * **Input** — `VZMacKeyboardConfiguration` and a pointing device, needed + /// for Setup Assistant automation to have something to talk to. + /// * **Entropy** — `VZVirtioEntropyDeviceConfiguration`, so the guest's RNG + /// seeds promptly instead of blocking early boot. + /// * **Socket** — `VZVirtioSocketDeviceConfiguration`, reserved for a future + /// vsock control channel that would replace SSH. + /// + /// - Parameters: + /// - bundle: The VM to configure. + /// - headless: When `true`, no view will be attached. Retained as a + /// parameter because `vm boot` may later want a window; it does **not** + /// change whether the graphics device is present. + /// - Returns: A configuration that has passed `validate()`. + /// - Throws: ``CoreError/bundleCorrupt(_:)`` when the bundle's blobs cannot + /// be restored, or the framework's own validation error. + public static func makeConfiguration( + bundle: VMBundle, + headless: Bool = true + ) throws -> VZVirtualMachineConfiguration { + let bundleConfig = try bundle.loadConfig() + + let configuration = VZVirtualMachineConfiguration() + configuration.platform = try makePlatform(bundle: bundle) + configuration.bootLoader = VZMacOSBootLoader() + configuration.cpuCount = clampedCPUCount(bundleConfig.cpuCount) + configuration.memorySize = clampedMemorySize(gigabytes: bundleConfig.memoryGB) + + // Storage. The bundle records which of ASIF/RAW the builder produced, so + // the right file is attached without probing the filesystem. + let diskURL = bundle.diskURL(format: bundleConfig.diskFormat) + guard FileManager.default.fileExists(atPath: diskURL.path) else { + throw CoreError.bundleCorrupt("missing disk image at \(diskURL.path)") + } + let attachment: VZDiskImageStorageDeviceAttachment + do { + attachment = try VZDiskImageStorageDeviceAttachment(url: diskURL, readOnly: false) + } catch { + throw CoreError.bundleCorrupt( + "cannot attach disk \(diskURL.path): \(error.localizedDescription)") + } + configuration.storageDevices = [VZVirtioBlockDeviceConfiguration(attachment: attachment)] + + // Network: NAT, with the bundle's MAC. NAT is what puts the guest into + // /var/db/dhcpd_leases, which is the only way we learn its IP. + guard let mac = VZMACAddress(string: bundleConfig.macAddress) else { + throw CoreError.bundleCorrupt( + "malformed MAC address '\(bundleConfig.macAddress)' in \(bundle.configURL.path)") + } + let network = VZVirtioNetworkDeviceConfiguration() + network.attachment = VZNATNetworkDeviceAttachment() + network.macAddress = mac + configuration.networkDevices = [network] + + // Graphics: always present, even headless, and never sized from + // NSScreen — the daemon runs as a LaunchAgent that may have no attached + // display at all, and a nil main screen there would be fatal. `headless` + // only decides whether a VZVirtualMachineView is ever bound to this + // device; the device itself is unconditional because macOS guests + // misbehave without one. + _ = headless + let graphics = VZMacGraphicsDeviceConfiguration() + graphics.displays = [ + VZMacGraphicsDisplayConfiguration( + widthInPixels: 1920, + heightInPixels: 1200, + pixelsPerInch: 72 + ) + ] + configuration.graphicsDevices = [graphics] + + // Input: Setup Assistant automation needs something to talk to. + configuration.keyboards = [VZMacKeyboardConfiguration()] + configuration.pointingDevices = [VZMacTrackpadConfiguration()] + + // Entropy, so the guest's RNG seeds promptly rather than blocking early boot. + configuration.entropyDevices = [VZVirtioEntropyDeviceConfiguration()] + + // Exactly one socket device — the framework permits no more. Reserved for + // the vsock control channel that would eventually replace SSH. + configuration.socketDevices = [VZVirtioSocketDeviceConfiguration()] + + try configuration.validate() + return configuration + } + + /// Builds only the platform configuration, so the installer path can share it. + /// + /// - Parameter bundle: The VM whose hardware model, machine identifier, and + /// auxiliary storage should be restored. + public static func makePlatform(bundle: VMBundle) throws -> VZMacPlatformConfiguration { + let bundleConfig = try bundle.loadConfig() + let platform = VZMacPlatformConfiguration() + + guard + let hardwareModel = VZMacHardwareModel( + dataRepresentation: bundleConfig.hardwareModelData) + else { + throw CoreError.bundleCorrupt( + "hardwareModelData in \(bundle.configURL.path) is not a valid VZMacHardwareModel") + } + guard hardwareModel.isSupported else { + throw CoreError.hostUnsupported( + "this host does not support the hardware model recorded in \(bundle.configURL.path)" + ) + } + guard + let machineIdentifier = VZMacMachineIdentifier( + dataRepresentation: bundleConfig.machineIdentifierData) + else { + throw CoreError.bundleCorrupt( + "machineIdentifierData in \(bundle.configURL.path) is not a valid VZMacMachineIdentifier" + ) + } + + // The *existing*-storage initializer. Using + // VZMacAuxiliaryStorage(creatingStorageAt:hardwareModel:) here would + // blank the guest's NVRAM and it would no longer boot. + guard FileManager.default.fileExists(atPath: bundle.auxiliaryStorageURL.path) else { + throw CoreError.bundleCorrupt("missing nvram.bin at \(bundle.auxiliaryStorageURL.path)") + } + platform.auxiliaryStorage = VZMacAuxiliaryStorage(url: bundle.auxiliaryStorageURL) + platform.hardwareModel = hardwareModel + platform.machineIdentifier = machineIdentifier + return platform + } + + /// Clamps a requested CPU count into the framework's supported range, with a + /// floor of 4 — Xcode builds are miserable below that. + public static func clampedCPUCount(_ requested: Int) -> Int { + let lowerBound = max(VZVirtualMachineConfiguration.minimumAllowedCPUCount, 4) + let upperBound = VZVirtualMachineConfiguration.maximumAllowedCPUCount + // On a host whose maximum is below our floor, the maximum wins. + guard lowerBound <= upperBound else { return upperBound } + return min(max(requested, lowerBound), upperBound) + } + + /// Clamps a requested memory size (in gibibytes) into the framework's + /// supported range, returning bytes. + public static func clampedMemorySize(gigabytes: Int) -> UInt64 { + let lowerBound = VZVirtualMachineConfiguration.minimumAllowedMemorySize + let upperBound = VZVirtualMachineConfiguration.maximumAllowedMemorySize + let requested = UInt64(max(gigabytes, 0)) * 1_073_741_824 + return min(max(requested, lowerBound), upperBound) + } +} diff --git a/Sources/gitea-macos-runner/CommandConfig.swift b/Sources/gitea-macos-runner/CommandConfig.swift new file mode 100644 index 0000000..7850979 --- /dev/null +++ b/Sources/gitea-macos-runner/CommandConfig.swift @@ -0,0 +1,205 @@ +import ArgumentParser +import Foundation +import RunnerCore + +/// `gitea-macos-runner config …` — create and inspect configuration. +struct ConfigCommand: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "config", + abstract: "Create and inspect the runner configuration.", + subcommands: [Init.self, Show.self, Path.self] + ) + + /// `config init` — write a commented example config. + struct Init: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "init", + abstract: "Write an example config.json, creating parent directories.", + discussion: """ + Writes to ~/.config/gitea-macos-runner/config.json unless --config says \ + otherwise. Refuses to overwrite an existing file without --force. The \ + written file carries "_comment" keys explaining each section; they are \ + ignored when the config is read back. + """ + ) + + @OptionGroup var options: GlobalOptions + + /// Overwrite an existing file. + @Flag(name: .shortAndLong, help: "Overwrite an existing config file.") + var force: Bool = false + + /// Seed `gitea.instanceURL` instead of the placeholder. + @Option(name: .long, help: "Gitea instance URL to seed into the config.") + var instanceURL: String? + + func run() async throws { + var config = RunnerConfig.default + if let instanceURL { + guard let url = URL(string: instanceURL), url.scheme != nil, url.host != nil else { + throw ValidationError("not a valid absolute URL: \(instanceURL)") + } + config.gitea.instanceURL = url + } + + var example = ConfigCommand.loadExampleDocument() + if let instanceURL, example != nil { + example = example?.replacingOccurrences( + of: "https://gitea.example.com", + with: instanceURL + ) + } + + let path = RunnerConfig.expandTilde(options.configPath) + let written = try config.writeExample(to: path, exampleContents: example, overwrite: force) + + guard written else { + CLI.error("\(path) already exists; pass --force to overwrite") + throw ExitCode(1) + } + + print("wrote \(path)") + print("") + if let contents = try? String(contentsOfFile: path, encoding: .utf8) { + print(contents) + } + print("edit it, then run: gitea-macos-runner doctor") + } + } + + /// `config show` — print the effective, validated configuration. + /// + /// Token values are redacted; token *sources* are shown, which is what you + /// actually need when debugging "why does it say no registration token". + struct Show: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "show", + abstract: "Print the effective configuration with secrets redacted." + ) + + @OptionGroup var options: GlobalOptions + + func run() async throws { + let config = try options.loadConfig() + + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + let encoded = try encoder.encode(config) + + var object = (try JSONSerialization.jsonObject(with: encoded)) as? [String: Any] ?? [:] + if var gitea = object["gitea"] as? [String: Any] { + if gitea["adminToken"] != nil { gitea["adminToken"] = "" } + if gitea["registrationToken"] != nil { gitea["registrationToken"] = "" } + object["gitea"] = gitea + } + + let redacted = try JSONSerialization.data( + withJSONObject: object, + options: [.prettyPrinted, .sortedKeys] + ) + print(String(data: redacted, encoding: .utf8) ?? "{}") + + // The sources matter more than the values: "no registration token" + // is almost always a path problem, not a secret problem. + print("") + print("config path: \(RunnerConfig.expandTilde(options.configPath))") + print("store directory: \(config.storeDirectoryURL.path)") + print("labels: \(config.runner.labels.joined(separator: ", "))") + print("register --labels: \(config.labelSet.registrationArgument())") + let downloadURL = (try? config.runner.resolvedDownloadURL)?.absoluteString ?? "" + print("runner download: \(downloadURL)") + let adminSource = ConfigCommand.describeSource( + inline: config.gitea.adminToken, + file: config.gitea.adminTokenFile, + resolved: (try? config.resolveAdminToken()) ?? nil + ) + let registrationSource = ConfigCommand.describeSource( + inline: config.gitea.registrationToken, + file: config.gitea.registrationTokenFile, + resolved: (try? config.resolveStaticRegistrationToken()) ?? nil, + fallback: config.gitea.fetchRegistrationTokenViaAPI + ? "admin API (fetchRegistrationTokenViaAPI)" + : nil + ) + print("admin token: \(adminSource)") + print("registration token: \(registrationSource)") + + let insecure = config.insecureTokenFilePaths + if !insecure.isEmpty { + print("") + CLI.note("warning: group/world readable token files: \(insecure.joined(separator: ", "))") + } + } + } + + /// `config path` — print the config path being used. + struct Path: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "path", + abstract: "Print the configuration file path." + ) + + @OptionGroup var options: GlobalOptions + + func run() async throws { + print(RunnerConfig.expandTilde(options.configPath)) + } + } + + /// Describes where a secret comes from, without printing it. + static func describeSource( + inline: String?, + file: String?, + resolved: String?, + fallback: String? = nil + ) -> String { + let value = resolved + if let file, !file.isEmpty { + let expanded = RunnerConfig.expandTilde(file) + let readable = (value?.isEmpty == false) + return "\(expanded) (\(readable ? "readable" : "MISSING or empty"))" + } + if let inline, !inline.isEmpty { + return "inline value in config.json (prefer a file)" + } + return fallback ?? "not configured" + } + + /// Finds `Resources/config.example.json` next to the binary or in a checkout. + /// + /// The example is not an SPM resource bundle and `make bundle` does not copy + /// it into the app, so several plausible locations are tried; `writeExample` + /// falls back to a plain serialization when none is found. + static func loadExampleDocument() -> String? { + var candidates: [URL] = [] + + if let resource = Bundle.main.url(forResource: "config.example", withExtension: "json") { + candidates.append(resource) + } + candidates.append(Bundle.main.bundleURL.appendingPathComponent("Contents/Resources/config.example.json")) + if let executableURL = Bundle.main.executableURL?.resolvingSymlinksInPath() { + let directory = executableURL.deletingLastPathComponent() + candidates.append(directory.appendingPathComponent("Resources/config.example.json")) + candidates.append( + directory.deletingLastPathComponent().appendingPathComponent("Resources/config.example.json") + ) + } + // Sources/gitea-macos-runner/CommandConfig.swift → repository root. + let repositoryRoot = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + candidates.append(repositoryRoot.appendingPathComponent("Resources/config.example.json")) + candidates.append( + URL(fileURLWithPath: FileManager.default.currentDirectoryPath) + .appendingPathComponent("Resources/config.example.json") + ) + + for candidate in candidates { + if let contents = try? String(contentsOf: candidate, encoding: .utf8) { + return contents + } + } + return nil + } +} diff --git a/Sources/gitea-macos-runner/CommandDaemon.swift b/Sources/gitea-macos-runner/CommandDaemon.swift new file mode 100644 index 0000000..04b0c48 --- /dev/null +++ b/Sources/gitea-macos-runner/CommandDaemon.swift @@ -0,0 +1,178 @@ +import AppKit +import ArgumentParser +import Foundation +import Logging +import RunnerCore +import RunnerHost + +/// `gitea-macos-runner daemon` — the long-running service. +/// +/// ## Why there is an `NSApplication` here +/// +/// Virtualization.framework requires a running main run loop in an application +/// context; a plain command-line process that blocks in `await` never services +/// it, and VM startup either hangs or fails. The fix is to start a real +/// `NSApplication` but suppress every trace of a GUI: +/// +/// ```swift +/// NSApplication.shared.setActivationPolicy(.prohibited) // no Dock icon, no menu bar +/// // spawn the orchestrator Task +/// NSApplication.shared.run() // never returns +/// ``` +/// +/// `.prohibited` (mirrored by `LSUIElement` in `Info.plist`) is what makes this +/// invisible. The orchestrator runs in a detached `Task`; `run()` owns the main +/// thread from then on. +/// +/// `SIGTERM` and `SIGINT` are trapped with `DispatchSourceSignal` — not +/// `signal(2)` handlers, which cannot safely touch Swift concurrency — and +/// trigger ``Orchestrator/shutdown()`` before the process leaves, so guests get +/// a chance to stop cleanly instead of having their disks yanked. +struct DaemonCommand: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "daemon", + abstract: "Watch Gitea for queued macOS jobs and run each in a fresh VM." + ) + + @OptionGroup var options: GlobalOptions + + /// Base image to clone for each job. + @Option(name: .long, help: "Base image to clone for each job.") + var image: String = "default" + + /// Run one poll/reconcile tick and exit. Useful for debugging without + /// installing the service. + @Flag(name: .long, help: "Run a single scheduling tick, then exit.") + var once: Bool = false + + func run() async throws { + CLI.bootstrapLogging(verbose: options.verbose) + let logger = Logger(label: "daemon") + + let config = try options.loadConfig() + + guard let adminToken = try config.resolveAdminToken(), !adminToken.isEmpty else { + throw ValidationError( + """ + no Gitea admin token: set gitea.adminTokenFile (preferred) or gitea.adminToken \ + in \(RunnerConfig.expandTilde(options.configPath)) + """ + ) + } + + for path in config.insecureTokenFilePaths { + logger.warning("token file is group/world readable", metadata: ["path": .string(path)]) + } + + let store = VMStore(config: config) + try store.ensureLayout() + + guard try store.image(named: image) != nil else { + throw ValidationError( + "no base image named '\(image)' — build one with `gitea-macos-runner image build --name \(image)`" + ) + } + + let client = GiteaClient(baseURL: config.gitea.instanceURL, token: adminToken) + let orchestrator = Orchestrator( + config: config, + client: client, + store: store, + imageName: image, + logger: Logger(label: "orchestrator") + ) + + let singleTick = once + let jobTimeout = TimeInterval(config.scheduler.jobTimeoutMinutes * 60) + + // Even a single tick can start a VM, and a VM needs the run loop — so + // both modes go through NSApplication. + await VZAppRuntime.run( + onSignal: { await orchestrator.shutdown() }, + body: { + do { + if singleTick { + await orchestrator.reconcileOnce() + await orchestrator.tick() + // Let whatever the tick started run to completion rather + // than tearing a just-booted guest down mid-boot. + let deadline = Date().addingTimeInterval(jobTimeout) + var pending = await orchestrator.liveVMs().count + while pending > 0, Date() < deadline { + try? await Task.sleep(for: .seconds(5)) + pending = await orchestrator.liveVMs().count + } + await orchestrator.shutdown() + } else { + try await orchestrator.runForever() + } + } catch is CancellationError { + // Expected on shutdown. + } catch { + logger.critical("daemon stopped", metadata: ["error": .string("\(error)")]) + // Fully qualified: inside a ParsableCommand a bare `exit` + // resolves to ParsableCommand.exit(withError:). + await MainActor.run { Foundation.exit(1) } + } + } + ) + } +} + +/// Hosts an `NSApplication` run loop so Virtualization.framework has the main +/// run loop it requires, while the real work runs in a `Task`. +/// +/// Shared by `daemon` and `vm boot`: any command that starts a VM needs this. +@MainActor +enum VZAppRuntime { + /// Signal sources have to outlive the call that creates them or they are + /// cancelled on deinit and the signals go nowhere. + private static var signalSources: [DispatchSourceSignal] = [] + private static var isTerminating = false + + /// Starts the run loop and runs `body` alongside it. Never returns. + /// + /// - Parameters: + /// - onSignal: Cleanup to perform on `SIGINT`/`SIGTERM` before exiting. + /// - body: The work to run. When it returns, the process exits zero. + static func run( + onSignal: @escaping @Sendable () async -> Void, + body: @escaping @Sendable () async -> Void + ) -> Never { + let app = NSApplication.shared + // No Dock icon, no menu bar, no activation: this is a background agent + // that merely needs to be an application as far as the kernel is + // concerned. + app.setActivationPolicy(.prohibited) + + for signalNumber in [SIGINT, SIGTERM] { + // DispatchSourceSignal only observes; the default disposition still + // kills the process unless it is ignored first. + signal(signalNumber, SIG_IGN) + let source = DispatchSource.makeSignalSource(signal: signalNumber, queue: .main) + source.setEventHandler { + Task { @MainActor in + guard !isTerminating else { return } + isTerminating = true + CLI.note("received signal; shutting down…") + await onSignal() + NSApp.terminate(nil) + exit(0) + } + } + source.resume() + signalSources.append(source) + } + + Task { + await body() + await MainActor.run { + NSApp.terminate(nil) + exit(0) + } + } + + app.run() + exit(0) + } +} diff --git a/Sources/gitea-macos-runner/CommandDoctor.swift b/Sources/gitea-macos-runner/CommandDoctor.swift new file mode 100644 index 0000000..2adc3bd --- /dev/null +++ b/Sources/gitea-macos-runner/CommandDoctor.swift @@ -0,0 +1,61 @@ +import ArgumentParser +import Foundation +import RunnerCore +import RunnerHost + +/// `gitea-macos-runner doctor` — verify the host before anything else. +/// +/// Every check corresponds to a failure that would otherwise show up as an +/// opaque error deep inside a VM boot: wrong architecture, unsigned binary, +/// locked keychain, non-admin Gitea token, dead download URL. Run this first, +/// and again after `service install`. +struct DoctorCommand: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "doctor", + abstract: "Check that this host can build and run macOS guests." + ) + + @OptionGroup var options: GlobalOptions + + /// Emit machine-readable JSON instead of aligned text. + @Flag(name: .long, help: "Emit results as JSON.") + var json: Bool = false + + /// By default `doctor` exits non-zero when any check fails, so it can gate a + /// setup script. This makes it always exit zero. + @Flag(name: .customLong("no-fail"), help: "Exit zero even when checks fail.") + var noFail: Bool = false + + func run() async throws { + // Deliberately does not use options.loadConfig(): a broken or missing + // config is exactly the state doctor exists to diagnose, so it is + // reported as a check rather than thrown as an error. + let checks = await Doctor.runChecks(configPath: options.configPath) + + if json { + let payload: [[String: Any]] = checks.map { check in + var entry: [String: Any] = [ + "name": check.name, + "result": check.result.label, + "detail": check.detail, + "blocking": check.isBlocking, + ] + if let remediation = check.remediation { + entry["remediation"] = remediation + } + return entry + } + let data = try JSONSerialization.data( + withJSONObject: payload, + options: [.prettyPrinted, .sortedKeys] + ) + print(String(data: data, encoding: .utf8) ?? "[]") + } else { + print(Doctor.format(checks)) + } + + if !noFail, checks.contains(where: \.isBlocking) { + throw ExitCode(1) + } + } +} diff --git a/Sources/gitea-macos-runner/CommandImage.swift b/Sources/gitea-macos-runner/CommandImage.swift new file mode 100644 index 0000000..472d54e --- /dev/null +++ b/Sources/gitea-macos-runner/CommandImage.swift @@ -0,0 +1,274 @@ +import ArgumentParser +import Foundation +import RunnerCore +import RunnerHost + +/// `gitea-macos-runner image …` — manage base VM images. +/// +/// A base image is installed and provisioned once and then cloned per job. +/// Building one takes the better part of an hour, most of it downloading a +/// ~15 GB IPSW; cloning one takes milliseconds. +struct ImageCommand: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "image", + abstract: "Build, list, provision, and delete base VM images.", + subcommands: [Build.self, List.self, Delete.self, Provision.self] + ) + + /// `image build` — install macOS from an IPSW and provision it. + struct Build: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "build", + abstract: "Install macOS into a new base image and provision it.", + discussion: """ + Downloads the latest supported restore image unless --ipsw is given, \ + installs it, automates Setup Assistant, then installs Node.js and the \ + gitea-runner binary over SSH. The guest must be macOS 27 or newer for \ + unattended Setup Assistant automation to work. + """ + ) + + @OptionGroup var options: GlobalOptions + + /// Image name under `/images/`. + @Option(name: .long, help: "Image name.") + var name: String = "default" + + /// A local `.ipsw`; omit to download the latest supported image. + @Option(name: .long, help: "Path to a local .ipsw (default: download the latest supported).") + var ipsw: String? + + /// Nominal guest disk size, overriding `guest.diskGB`. + @Option(name: .customLong("disk-gb"), help: "Guest disk size in GB (overrides config).") + var diskGB: Int? + + func run() async throws { + CLI.bootstrapLogging(verbose: options.verbose) + var config = try options.loadConfig() + if let diskGB { + config.guest.diskGB = diskGB + } + + let store = VMStore(config: config) + try store.ensureLayout() + + if try store.image(named: name) != nil { + throw ValidationError( + "image '\(name)' already exists — delete it first with `image delete \(name)`" + ) + } + try store.ensureFreeSpace(minGB: max(config.storage.minFreeDiskGB, 40)) + + CLI.note("building image '\(name)' (this takes a while; the IPSW alone is ~15 GB)") + + let printer = ProgressPrinter() + let builder = ImageBuilder(store: store) + let imageName = name + let ipswPath = ipsw + let frozenConfig = config + + // `image build` runs `VZMacOSInstaller` and then boots the guest, so + // it needs the same `NSApplication` main run loop `daemon` and + // `vm boot` do — without it Virtualization.framework's callbacks are + // never serviced and the install hangs. See `VZAppRuntime`. + await VZAppRuntime.run( + onSignal: {}, + body: { + do { + try await builder.build( + name: imageName, + ipswPath: ipswPath, + config: frozenConfig, + progress: { stage in printer.update(ImageCommand.describe(stage)) } + ) + } catch { + printer.finish() + CLI.error("\(error)") + // Fully qualified: inside a ParsableCommand a bare `exit` + // resolves to ParsableCommand.exit(withError:). + await MainActor.run { Foundation.exit(1) } + } + printer.finish("done") + + print("built image '\(imageName)'") + print("next: gitea-macos-runner vm boot --image \(imageName)") + } + ) + } + } + + /// `image list` — show base images and whether they are provisioned. + struct List: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "list", + abstract: "List base images." + ) + + @OptionGroup var options: GlobalOptions + + func run() async throws { + let config = try options.loadConfig() + let store = VMStore(config: config) + try store.ensureLayout() + + let names = try store.listImages() + guard !names.isEmpty else { + print("no images (build one with `gitea-macos-runner image build`)") + return + } + + print("NAME MACOS PROVISIONED DISK SIZE") + for name in names { + guard let bundle = try store.image(named: name) else { continue } + let bundleConfig = try? bundle.loadConfig() + let size = (try? bundle.diskUsageBytes()).map(CLI.formatBytes) ?? "-" + print( + pad(name, 20) + + pad(bundleConfig?.macOSVersion ?? "-", 12) + + pad((bundleConfig?.provisioned ?? false) ? "yes" : "no", 13) + + pad(bundleConfig?.diskFormat.rawValue ?? "-", 11) + + size + ) + } + } + + private func pad(_ value: String, _ width: Int) -> String { + value.count >= width + ? value + " " + : value + String(repeating: " ", count: width - value.count) + } + } + + /// `image delete NAME` — remove a base image. + struct Delete: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "delete", + abstract: "Delete a base image and its disk." + ) + + @OptionGroup var options: GlobalOptions + + /// Image name. + @Argument(help: "Image name.") + var name: String + + /// Skip the confirmation prompt. + @Flag(name: .shortAndLong, help: "Do not prompt for confirmation.") + var force: Bool = false + + func run() async throws { + let config = try options.loadConfig() + let store = VMStore(config: config) + + guard let bundle = try store.image(named: name) else { + throw ValidationError("no image named '\(name)'") + } + + if !force { + let size = (try? bundle.diskUsageBytes()).map(CLI.formatBytes) ?? "unknown size" + guard CLI.confirm("delete image '\(name)' (\(size))?") else { + print("cancelled") + throw ExitCode(1) + } + } + + try store.deleteImage(named: name) + print("deleted image '\(name)'") + } + } + + /// `image provision NAME` — re-run guest provisioning on an existing image. + /// + /// Exists so that bumping the `gitea-runner` version, or adding Xcode, does + /// not require reinstalling macOS. + struct Provision: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "provision", + abstract: "Re-run guest provisioning against an existing image.", + discussion: """ + Boots the BASE image bundle itself — not a clone — runs provisioning, and \ + shuts it down. This deliberately mutates the golden image in place, which \ + is the point: every clone made afterwards inherits the change. Nothing \ + else may be using the image while this runs, so stop the daemon first. + """ + ) + + @OptionGroup var options: GlobalOptions + + /// Image name. + @Argument(help: "Image name.") + var name: String + + /// Optional Xcode `.xip` to install into the guest. Adds tens of + /// gigabytes; omitted by default. + @Option(name: .customLong("xcode-xip"), help: "Path to an Xcode .xip to install into the guest.") + var xcodeXIP: String? + + func run() async throws { + CLI.bootstrapLogging(verbose: options.verbose) + let config = try options.loadConfig() + let store = VMStore(config: config) + + guard try store.image(named: name) != nil else { + throw ValidationError("no image named '\(name)'") + } + if let xcodeXIP, !FileManager.default.fileExists(atPath: RunnerConfig.expandTilde(xcodeXIP)) { + throw ValidationError("no file at \(RunnerConfig.expandTilde(xcodeXIP))") + } + + CLI.note("provisioning base image '\(name)' in place — stop the daemon before doing this") + + let printer = ProgressPrinter() + let builder = ImageBuilder(store: store) + let imageName = name + let frozenConfig = config + let xipPath = xcodeXIP.map(RunnerConfig.expandTilde) + + // Boots the image to run provision.sh in it, so it needs the run + // loop for exactly the reason `image build` does. + await VZAppRuntime.run( + onSignal: {}, + body: { + do { + try await builder.reprovision( + name: imageName, + config: frozenConfig, + xcodeXIPPath: xipPath, + progress: { stage in printer.update(ImageCommand.describe(stage)) } + ) + } catch { + printer.finish() + CLI.error("\(error)") + // Fully qualified: inside a ParsableCommand a bare `exit` + // resolves to ParsableCommand.exit(withError:). + await MainActor.run { Foundation.exit(1) } + } + printer.finish("done") + print("provisioned image '\(imageName)'") + } + ) + } + } + + /// Renders a build stage as one status line. + static func describe(_ stage: ImageBuildStage) -> String { + switch stage { + case .downloadingIPSW(let fraction): + return "downloading IPSW " + CLI.progressBar(fraction) + case .preparing: + return "preparing" + case .creatingBundle: + return "creating bundle" + case .installing(let fraction): + return "installing macOS " + CLI.progressBar(fraction) + case .firstBoot: + return "first boot (Setup Assistant)" + case .provisioning(let step): + return "provisioning: \(step)" + case .finalizing: + return "finalizing" + case .done: + return "done" + } + } +} diff --git a/Sources/gitea-macos-runner/CommandService.swift b/Sources/gitea-macos-runner/CommandService.swift new file mode 100644 index 0000000..df2f037 --- /dev/null +++ b/Sources/gitea-macos-runner/CommandService.swift @@ -0,0 +1,107 @@ +import ArgumentParser +import Foundation +import RunnerCore +import RunnerHost + +/// `gitea-macos-runner service …` — manage the `launchd` LaunchAgent. +/// +/// - Important: This installs a **LaunchAgent** in the logged-in user's session, +/// never a LaunchDaemon. Virtualization needs a GUI session, and macOS 15+ +/// additionally needs an unlocked `login.keychain` to start a VM — neither of +/// which exists in the system context. The host should be set to log in +/// automatically. +struct ServiceCommand: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "service", + abstract: "Install, remove, or inspect the launchd LaunchAgent.", + subcommands: [Install.self, Uninstall.self, Status.self] + ) + + /// `service install` — write the plist and load the job. + struct Install: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "install", + abstract: "Write ~/Library/LaunchAgents/xyz.blakeslee.gitea-macos-runner.plist and load it.", + discussion: """ + Points the agent at the installed, signed .app bundle — not at a bare \ + binary. The com.apple.security.virtualization entitlement only survives \ + on the signed bundle, so a daemon started from .build/ cannot start VMs. + """ + ) + + @OptionGroup var options: GlobalOptions + + /// Path to the installed executable inside the signed `.app`. + @Option(name: .long, help: "Path to the installed executable (default: ~/Applications/GiteaMacosRunner.app/Contents/MacOS/gitea-macos-runner).") + var executable: String? + + func run() async throws { + let executablePath = executable ?? LaunchdService.defaultExecutablePath + + // Only pass --config when it is not the default; a plist that + // hard-codes the default path is one more thing to keep in sync. + let configPath = options.configPath == RunnerConfig.defaultPath ? nil : options.configPath + + if (try? options.loadConfig()) == nil { + CLI.note("warning: \(RunnerConfig.expandTilde(options.configPath)) is missing or invalid; the agent will fail to start until it is fixed") + } + + try LaunchdService.install(executablePath: executablePath, configPath: configPath) + + print("installed \(LaunchdService.agentPlistURL.path)") + print("program: \(RunnerConfig.expandTilde(executablePath)) daemon") + print("logs: \(LaunchdService.logDirectoryURL.path)") + print("") + print("check it with: gitea-macos-runner service status") + } + } + + /// `service uninstall` — unload and remove the plist. + struct Uninstall: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "uninstall", + abstract: "Unload the LaunchAgent and remove its plist." + ) + + @OptionGroup var options: GlobalOptions + + func run() async throws { + let path = LaunchdService.agentPlistURL.path + let existed = FileManager.default.fileExists(atPath: path) + try LaunchdService.uninstall() + print(existed ? "removed \(path)" : "not installed (\(path))") + } + } + + /// `service status` — report whether the agent is installed and running. + struct Status: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "status", + abstract: "Report LaunchAgent installation and run state." + ) + + @OptionGroup var options: GlobalOptions + + func run() async throws { + let status = try LaunchdService.status() + + print("label: \(LaunchdService.label)") + print("plist: \(status.plistPath)") + print("installed: \(status.installed ? "yes" : "no")") + print("loaded: \(status.loaded ? "yes" : "no")") + if let pid = status.pid { + print("pid: \(pid)") + } + if let lastExitStatus = status.lastExitStatus { + print("last exit: \(lastExitStatus)") + } + print("logs: \(LaunchdService.logDirectoryURL.path)") + + if status.installed, !status.loaded { + print("") + CLI.note("installed but not loaded — reinstall with `service install`, or check the logs above") + throw ExitCode(1) + } + } + } +} diff --git a/Sources/gitea-macos-runner/CommandVM.swift b/Sources/gitea-macos-runner/CommandVM.swift new file mode 100644 index 0000000..4616cc9 --- /dev/null +++ b/Sources/gitea-macos-runner/CommandVM.swift @@ -0,0 +1,195 @@ +import ArgumentParser +import Foundation +import RunnerCore +import RunnerHost + +/// `gitea-macos-runner vm …` — debugging helpers that operate on VMs directly, +/// without any Gitea involvement. +struct VMCommand: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "vm", + abstract: "Boot and inspect VMs directly (debugging).", + subcommands: [Boot.self, List.self] + ) + + /// `vm boot --image NAME` — clone an image, boot it, print its IP, wait. + /// + /// The fastest way to answer "is the image itself broken, or is it the + /// Gitea integration?". Clones the image onto slot 0's MAC, boots it, waits + /// for a DHCP lease, prints the address and an `ssh` line, then blocks until + /// Ctrl-C — at which point the VM is stopped and the clone deleted, exactly + /// as the daemon would. + struct Boot: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "boot", + abstract: "Clone an image, boot it, print its IP, and wait for Ctrl-C." + ) + + @OptionGroup var options: GlobalOptions + + /// Base image to clone. + @Option(name: .long, help: "Base image to clone.") + var image: String = "default" + + /// Which slot's persistent MAC to use. + @Option(name: .long, help: "Slot index whose persistent MAC the clone should use.") + var slot: Int = 0 + + /// Leave the clone on disk after exit, for post-mortem inspection. + @Flag(name: .long, help: "Do not delete the clone on exit.") + var keep: Bool = false + + func run() async throws { + CLI.bootstrapLogging(verbose: options.verbose) + let config = try options.loadConfig() + let store = VMStore(config: config) + try store.ensureLayout() + + guard try store.image(named: image) != nil else { + throw ValidationError("no image named '\(image)'") + } + try store.ensureFreeSpace(minGB: config.storage.minFreeDiskGB) + + let session = BootSession(store: store, keepClone: keep) + let slotIndex = slot + let imageName = image + let bootTimeout = Duration.seconds(max(30, config.scheduler.bootTimeoutSeconds)) + let username = config.guest.username + + await VZAppRuntime.run( + onSignal: { await session.teardown() }, + body: { + do { + let mac = try store.macAddress( + forSlot: slotIndex, + slotCount: RunnerConfig.SchedulerSection.hardMaxConcurrentVMs + ) + let bundle = try store.cloneImage(named: imageName, slotMAC: mac) + let instance = try VMInstance(bundle: bundle, label: "vm-boot") + await session.adopt(bundle: bundle, instance: instance) + + CLI.note("booting clone \(bundle.name) (mac \(mac))…") + try await instance.start() + + let ip = try await VMCommand.waitForLease(mac: mac, timeout: bootTimeout) + print("ip: \(ip)") + print("ssh: ssh \(username)@\(ip)") + print("") + CLI.note("press Ctrl-C to stop the VM and delete the clone") + + // Whichever happens first: the guest shuts itself down, + // or the operator interrupts (handled by onSignal). + let reason = await instance.waitUntilStopped() + CLI.note("guest stopped: \(reason)") + await session.teardown() + } catch { + CLI.error("\(error)") + await session.teardown() + // Fully qualified: inside a ParsableCommand a bare `exit` + // resolves to ParsableCommand.exit(withError:). + await MainActor.run { Foundation.exit(1) } + } + } + ) + } + } + + /// `vm list` — show ephemeral clones currently on disk. + /// + /// Under normal operation this is empty between jobs; anything listed after + /// the daemon has settled is an orphan from an unclean shutdown. + struct List: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "list", + abstract: "List ephemeral VM clones on disk." + ) + + @OptionGroup var options: GlobalOptions + + func run() async throws { + let config = try options.loadConfig() + let store = VMStore(config: config) + try store.ensureLayout() + + let clones = try store.listClones() + guard !clones.isEmpty else { + print("no ephemeral clones on disk") + return + } + + let leases = DHCPLeaseParser.parseFile() + print("CLONE MAC IP SIZE") + for clone in clones { + let bundleConfig = try? clone.loadConfig() + let mac = bundleConfig?.macAddress ?? "-" + let ip = bundleConfig.flatMap { DHCPLeaseParser.ipAddress(forMAC: $0.macAddress, in: leases) } ?? "-" + let size = (try? clone.diskUsageBytes()).map(CLI.formatBytes) ?? "-" + print(pad(clone.name, 31) + pad(mac, 19) + pad(ip, 17) + size) + } + print("") + CLI.note("clones left behind after the daemon has settled are orphans; `purge` happens at daemon start") + } + + private func pad(_ value: String, _ width: Int) -> String { + value.count >= width + ? value + " " + : value + String(repeating: " ", count: width - value.count) + } + } + + /// Polls `/var/db/dhcpd_leases` for a MAC, as the orchestrator does. + static func waitForLease(mac: String, timeout: Duration) async throws -> String { + let deadline = Date().addingTimeInterval( + TimeInterval(timeout.components.seconds) + ) + while Date() < deadline { + if let ip = DHCPLeaseParser.ipAddress(forMAC: mac, in: DHCPLeaseParser.parseFile()) { + return ip + } + try await Task.sleep(for: .seconds(2)) + } + throw CoreError.timeout("dhcp lease for \(mac)") + } +} + +/// Holds the VM and clone `vm boot` created, so the signal handler can tear them +/// down from outside the task that made them. +actor BootSession { + private let store: VMStore + private let keepClone: Bool + private var bundle: VMBundle? + private var instance: VMInstance? + private var finished = false + + init(store: VMStore, keepClone: Bool) { + self.store = store + self.keepClone = keepClone + } + + func adopt(bundle: VMBundle, instance: VMInstance) { + self.bundle = bundle + self.instance = instance + } + + /// Stops the VM and removes the clone. Idempotent. + func teardown() async { + guard !finished else { return } + finished = true + + if let instance { + _ = await instance.requestStopThenForce(gracePeriod: .seconds(30)) + } + guard let bundle else { return } + + if keepClone { + CLI.note("keeping clone at \(bundle.rootURL.path)") + } else { + do { + try store.deleteClone(bundle) + CLI.note("deleted clone \(bundle.name)") + } catch { + CLI.error("could not delete clone: \(error)") + } + } + } +} diff --git a/Sources/gitea-macos-runner/Main.swift b/Sources/gitea-macos-runner/Main.swift new file mode 100644 index 0000000..b251c79 --- /dev/null +++ b/Sources/gitea-macos-runner/Main.swift @@ -0,0 +1,162 @@ +import ArgumentParser +import Foundation +import Logging +import RunnerCore + +/// Options every subcommand accepts. +struct GlobalOptions: ParsableArguments { + /// Path to `config.json`. Tilde-expanded. + @Option(name: [.customLong("config"), .customShort("c")], + help: "Path to config.json (default: ~/.config/gitea-macos-runner/config.json)") + var configPath: String = RunnerConfig.defaultPath + + /// Emit debug-level logs. + @Flag(name: .long, help: "Verbose logging.") + var verbose: Bool = false + + /// Loads and validates the configuration named by ``configPath``. + func loadConfig() throws -> RunnerConfig { + try RunnerConfig.load(from: configPath).validated() + } +} + +/// Root command. +/// +/// The tool is both the daemon and its own admin CLI: `daemon` is what +/// `launchd` starts, and everything else is operator-facing. +@main +struct GiteaMacOSRunner: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "gitea-macos-runner", + abstract: "Run Gitea Actions macOS jobs in fresh, ephemeral Virtualization.framework VMs.", + discussion: """ + Each queued job that matches this host's labels gets a brand-new macOS VM \ + cloned from a base image, an ephemeral runner registered with Gitea, and a \ + teardown as soon as the job finishes. Nothing is reused between jobs. + + Start with `doctor` to verify the host, then `config init`, then \ + `image build`, then `service install`. + """, + version: RunnerVersion.current, + subcommands: [ + DaemonCommand.self, + ImageCommand.self, + VMCommand.self, + ServiceCommand.self, + DoctorCommand.self, + ConfigCommand.self, + ], + defaultSubcommand: nil + ) +} + +/// Shared helpers for command bodies. +enum CLI { + /// Prints to stderr. + static func error(_ message: String) { + FileHandle.standardError.write(Data(("error: " + message + "\n").utf8)) + } + + /// Prints a note to stderr, so it does not pollute pipeable stdout. + static func note(_ message: String) { + FileHandle.standardError.write(Data((message + "\n").utf8)) + } + + /// Prints an "unimplemented" notice and exits non-zero. + static func unimplemented(_ what: String) throws -> Never { + error("\(what): unimplemented") + throw ExitCode(1) + } + + /// Routes swift-log to stderr, leaving stdout for command output. + /// + /// Only the first call has any effect: `LoggingSystem.bootstrap` traps when + /// called twice, and subcommands are free to call this independently. + static func bootstrapLogging(verbose: Bool) { + loggingBootstrap.once { + let level: Logger.Level = verbose ? .debug : .info + LoggingSystem.bootstrap { label in + var handler = StreamLogHandler.standardError(label: label) + handler.logLevel = level + return handler + } + } + } + + private static let loggingBootstrap = OnceFlag() + + /// Asks a yes/no question on stderr. Answers `false` when stdin is not a + /// terminal, so a piped invocation never blocks forever. + static func confirm(_ question: String) -> Bool { + guard isatty(fileno(stdin)) == 1 else { return false } + FileHandle.standardError.write(Data((question + " [y/N] ").utf8)) + guard let answer = readLine(strippingNewline: true)?.lowercased() else { return false } + return answer == "y" || answer == "yes" + } + + /// Formats a byte count as a human-readable size. + static func formatBytes(_ bytes: Int64) -> String { + let units = ["B", "KB", "MB", "GB", "TB"] + var value = Double(bytes) + var unit = 0 + while value >= 1024, unit < units.count - 1 { + value /= 1024 + unit += 1 + } + return unit == 0 + ? "\(Int(value)) \(units[unit])" + : String(format: "%.1f %@", value, units[unit]) + } + + /// Renders a fixed-width progress bar, e.g. `[####------] 40%`. + static func progressBar(_ fraction: Double, width: Int = 30) -> String { + let clamped = min(max(fraction, 0), 1) + let filled = Int((Double(width) * clamped).rounded()) + let bar = String(repeating: "#", count: filled) + String(repeating: "-", count: width - filled) + return String(format: "[%@] %3d%%", bar, Int((clamped * 100).rounded())) + } +} + +/// A thread-safe "run this exactly once" latch. +final class OnceFlag: @unchecked Sendable { + private let lock = NSLock() + private var done = false + + func once(_ body: () -> Void) { + lock.lock() + defer { lock.unlock() } + guard !done else { return } + done = true + body() + } +} + +/// Serializes progress output arriving from arbitrary threads and keeps it on a +/// single rewritten stderr line. +final class ProgressPrinter: @unchecked Sendable { + private let lock = NSLock() + private var lastLine = "" + + /// Rewrites the current line. + func update(_ line: String) { + lock.lock() + defer { lock.unlock() } + guard line != lastLine else { return } + lastLine = line + let padding = String(repeating: " ", count: max(0, 78 - line.count)) + FileHandle.standardError.write(Data(("\r" + line + padding).utf8)) + } + + /// Ends the line so subsequent output starts cleanly. + func finish(_ line: String? = nil) { + lock.lock() + defer { lock.unlock() } + if let line { + let padding = String(repeating: " ", count: max(0, 78 - line.count)) + FileHandle.standardError.write(Data(("\r" + line + padding + "\n").utf8)) + } else if !lastLine.isEmpty { + FileHandle.standardError.write(Data("\n".utf8)) + } + lastLine = "" + } +} diff --git a/Tests/RunnerCoreTests/ConfigTests.swift b/Tests/RunnerCoreTests/ConfigTests.swift new file mode 100644 index 0000000..d9c5608 --- /dev/null +++ b/Tests/RunnerCoreTests/ConfigTests.swift @@ -0,0 +1,609 @@ +import Foundation +import Testing + +@testable import RunnerCore + +/// Tests for ``RunnerConfig`` decoding, normalization, validation, and token +/// resolution. +@Suite("RunnerConfig") +struct ConfigTests { + + // MARK: - Fixtures + + /// A configuration that passes ``RunnerConfig/validated()`` unmodified, so + /// each test can break exactly one thing. + private func validConfig() -> RunnerConfig { + RunnerConfig( + gitea: .init( + instanceURL: URL(string: "https://gitea.example.com")!, + adminToken: "abc123", + registrationToken: "REG123")) + } + + /// Writes `contents` to a unique file under a fresh temporary directory and + /// returns its path. The directory is left for the OS to reap; these are a + /// handful of bytes per test. + private func temporaryFile(named name: String = "token", contents: String) throws -> String { + let dir = URL(fileURLWithPath: NSTemporaryDirectory()) + .appendingPathComponent("gmr-tests-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + let file = dir.appendingPathComponent(name) + try Data(contents.utf8).write(to: file) + return file.path + } + + /// Runs `body`, requiring it to throw ``CoreError/configInvalid(_:)``, and + /// returns the detail message so a test can assert *which* rule fired. + @discardableResult + private func configInvalidDetail( + _ body: () throws -> Void, + sourceLocation: SourceLocation = #_sourceLocation + ) -> String { + do { + try body() + Issue.record("expected CoreError.configInvalid", sourceLocation: sourceLocation) + return "" + } catch let CoreError.configInvalid(detail) { + return detail + } catch { + Issue.record("expected CoreError.configInvalid, got \(error)", sourceLocation: sourceLocation) + return "" + } + } + + // MARK: - Defaults + + @Test("the default configuration carries every documented default") + func defaultsArePresent() { + let c = RunnerConfig.default + + #expect(c.runner.labels == ["macos-arm64"]) + #expect(c.runner.namePrefix == "macos-vm-") + #expect(c.runner.version == "3.0.2") + #expect(c.scheduler.maxConcurrentVMs == 2) + #expect(c.scheduler.pollIntervalSeconds == 5) + #expect(c.scheduler.reconcileIntervalSeconds == 300) + #expect(c.scheduler.jobTimeoutMinutes == 120) + #expect(c.scheduler.bootTimeoutSeconds == 300) + #expect(c.guest.username == "admin") + #expect(c.guest.cpuCount == 4) + #expect(c.guest.memoryGB == 8) + #expect(c.guest.diskGB == 64) + #expect(c.storage.minFreeDiskGB == 20) + // No token of either kind: `config init` writes a template an operator + // must still fill in, and `validated()` says so rather than starting. + #expect(c.gitea.adminToken == nil) + #expect(c.gitea.adminTokenFile == nil) + } + + @Test("the default download URL substitutes the configured version") + func downloadURLSubstitutesVersion() throws { + var section = RunnerConfig.RunnerSection() + section.version = "3.1.0" + let url = try section.resolvedDownloadURL + #expect(url.absoluteString.contains("v3.1.0/")) + #expect(url.absoluteString.hasSuffix("gitea-runner-3.1.0-darwin-arm64")) + #expect(!url.absoluteString.contains("{version}")) + } + + @Test("a download URL template with no scheme is rejected") + func downloadURLWithoutSchemeIsRejected() { + var section = RunnerConfig.RunnerSection() + section.runnerDownloadURL = "gitea.com/runner-{version}" + configInvalidDetail { _ = try section.resolvedDownloadURL } + } + + @Test("the default config path lives under the user's home directory") + func defaultPathIsExpanded() { + #expect(!RunnerConfig.defaultPath.hasPrefix("~")) + #expect(RunnerConfig.defaultPath.hasSuffix("/.config/gitea-macos-runner/config.json")) + } + + // MARK: - Decoding + + @Test("a minimal config decodes with every other section defaulted") + func minimalConfigDecodes() throws { + let json = """ + {"gitea": {"instanceURL": "https://gitea.example.com", + "adminToken": "abc", "registrationToken": "reg"}} + """ + let c = try JSONDecoder().decode(RunnerConfig.self, from: Data(json.utf8)) + + #expect(c.gitea.instanceURL.absoluteString == "https://gitea.example.com") + #expect(c.runner.labels == ["macos-arm64"]) + #expect(c.scheduler.pollIntervalSeconds == 5) + #expect(c.guest.username == "admin") + #expect(c.gitea.fetchRegistrationTokenViaAPI == false) + } + + @Test("a partial section keeps defaults for the keys it omits") + func partialSectionKeepsDefaults() throws { + let json = """ + {"gitea": {"instanceURL": "https://g.example.com", "adminToken": "a", "registrationToken": "r"}, + "guest": {"cpuCount": 8}} + """ + let c = try JSONDecoder().decode(RunnerConfig.self, from: Data(json.utf8)) + #expect(c.guest.cpuCount == 8) + #expect(c.guest.memoryGB == 8) // untouched default + #expect(c.guest.username == "admin") + } + + @Test("a config with no gitea section is rejected by name") + func missingGiteaSectionIsReported() throws { + let path = try temporaryFile(named: "config.json", contents: #"{"guest": {"cpuCount": 2}}"#) + let detail = configInvalidDetail { _ = try RunnerConfig.load(from: path) } + #expect(detail.contains("gitea")) + } + + @Test("loading a file that is not JSON reports it as malformed, not missing") + func malformedJSONIsReported() throws { + let path = try temporaryFile(named: "config.json", contents: "not json {") + let detail = configInvalidDetail { _ = try RunnerConfig.load(from: path) } + #expect(detail.contains(path)) + #expect(!detail.contains("no configuration file")) + } + + @Test("loading a missing file names the expanded path") + func missingFileIsReported() { + let detail = configInvalidDetail { + _ = try RunnerConfig.load(from: "/nonexistent/gmr/config.json") + } + #expect(detail.contains("/nonexistent/gmr/config.json")) + } + + // MARK: - load / save round trip + + @Test("load applies validation, so the loaded value is already normalized") + func loadNormalizes() throws { + let json = """ + {"gitea": {"instanceURL": "https://g.example.com", "adminToken": "a", "registrationToken": "r"}, + "scheduler": {"maxConcurrentVMs": 16}, + "storage": {"storeDir": "~/gmr-store"}} + """ + let path = try temporaryFile(named: "config.json", contents: json) + let c = try RunnerConfig.load(from: path) + + #expect(c.scheduler.maxConcurrentVMs == 2) + #expect(!c.storage.storeDir.hasPrefix("~")) + } + + @Test("a saved config reloads equal to what was saved") + func saveRoundTrips() throws { + let dir = URL(fileURLWithPath: NSTemporaryDirectory()) + .appendingPathComponent("gmr-tests-\(UUID().uuidString)", isDirectory: true) + // Nested and not yet created: `save` is expected to make the parents. + let path = dir.appendingPathComponent("nested/config.json").path + + let original = try validConfig().validated() + try original.save(to: path) + + #expect(FileManager.default.fileExists(atPath: path)) + #expect(try RunnerConfig.load(from: path) == original) + } + + @Test("writeExample does not clobber an existing file unless told to") + func writeExampleRespectsExistingFile() throws { + let path = try temporaryFile(named: "config.json", contents: "ORIGINAL") + let c = try validConfig().validated() + + #expect(try c.writeExample(to: path, exampleContents: "EXAMPLE") == false) + #expect(try String(contentsOfFile: path, encoding: .utf8) == "ORIGINAL") + + #expect(try c.writeExample(to: path, exampleContents: "EXAMPLE", overwrite: true) == true) + #expect(try String(contentsOfFile: path, encoding: .utf8) == "EXAMPLE") + } + + // MARK: - Tilde expansion + + @Test("expandTilde resolves a leading tilde and leaves other paths alone") + func expandTildeBehaviour() { + let home = NSHomeDirectory() + #expect(RunnerConfig.expandTilde("~/x") == home + "/x") + #expect(RunnerConfig.expandTilde("/absolute/x") == "/absolute/x") + #expect(RunnerConfig.expandTilde("relative/x") == "relative/x") + // A tilde anywhere but the front is an ordinary character. + #expect(RunnerConfig.expandTilde("/a/~/b") == "/a/~/b") + } + + @Test("validation expands tildes in every path-bearing field") + func validationExpandsPaths() throws { + var c = validConfig() + c.gitea.adminToken = nil + c.gitea.adminTokenFile = "~/admin.token" + c.gitea.registrationToken = nil + c.gitea.registrationTokenFile = "~/reg.token" + c.storage.storeDir = "~/gmr" + + let v = try c.validated() + let home = NSHomeDirectory() + #expect(v.gitea.adminTokenFile == home + "/admin.token") + #expect(v.gitea.registrationTokenFile == home + "/reg.token") + #expect(v.storage.storeDir == home + "/gmr") + #expect(v.storeDirectoryURL.path == home + "/gmr") + } + + // MARK: - Validation: instance URL + + @Test("a valid configuration validates unchanged") + func validConfigurationSurvivesValidation() throws { + let c = try validConfig().validated() + #expect(c.gitea.instanceURL.absoluteString == "https://gitea.example.com") + #expect(c.gitea.adminToken == "abc123") + } + + @Test("a plain http instance URL is allowed") + func httpInstanceURLIsAllowed() throws { + var c = validConfig() + c.gitea.instanceURL = URL(string: "http://gitea.lan:3000")! + #expect(throws: Never.self) { try c.validated() } + } + + @Test("a non-http scheme is rejected") + func nonHTTPSchemeIsRejected() { + var c = validConfig() + c.gitea.instanceURL = URL(string: "ssh://gitea.example.com")! + #expect(configInvalidDetail { _ = try c.validated() }.contains("instanceURL")) + } + + @Test("an instance URL with no host is rejected") + func hostlessInstanceURLIsRejected() throws { + // `#require` rather than a force-unwrap: whether an empty authority + // parses at all is a Foundation detail, and a nil here should fail this + // one test rather than trap the whole suite. + var c = validConfig() + c.gitea.instanceURL = try #require(URL(string: "https:///path")) + #expect(configInvalidDetail { _ = try c.validated() }.contains("instanceURL")) + } + + // MARK: - Validation: admin token + + @Test("no admin token at all names the keys to set") + func missingAdminTokenIsRejected() { + var c = validConfig() + c.gitea.adminToken = nil + let detail = configInvalidDetail { _ = try c.validated() } + #expect(detail.contains("gitea.adminToken")) + #expect(detail.contains("gitea.adminTokenFile")) + } + + @Test("both admin token sources set is rejected as ambiguous") + func bothAdminTokenSourcesRejected() { + // A stale inline token beside a live token file is the shape of a + // baffling 401; better to fail at load with the reason spelled out. + var c = validConfig() + c.gitea.adminTokenFile = "/tmp/admin.token" + let detail = configInvalidDetail { _ = try c.validated() } + #expect(detail.contains("both")) + } + + @Test("a whitespace-only admin token counts as absent") + func whitespaceAdminTokenIsAbsent() { + var c = validConfig() + c.gitea.adminToken = " \n " + #expect(configInvalidDetail { _ = try c.validated() }.contains("adminToken")) + } + + @Test("a surrounding-whitespace admin token is trimmed rather than rejected") + func adminTokenIsTrimmed() throws { + var c = validConfig() + c.gitea.adminToken = " abc123\n" + #expect(try c.validated().gitea.adminToken == "abc123") + } + + // MARK: - Validation: registration token + + @Test("no registration source at all is rejected") + func missingRegistrationTokenIsRejected() { + var c = validConfig() + c.gitea.registrationToken = nil + let detail = configInvalidDetail { _ = try c.validated() } + #expect(detail.contains("registration")) + } + + @Test("the API fallback alone satisfies the registration requirement") + func apiFallbackSatisfiesRegistration() throws { + var c = validConfig() + c.gitea.registrationToken = nil + c.gitea.fetchRegistrationTokenViaAPI = true + #expect(throws: Never.self) { try c.validated() } + } + + @Test("a file and an inline registration token together are allowed") + func bothRegistrationSourcesAllowed() throws { + // Unlike the admin token: the file simply wins, and the redundancy is + // how operators migrate from inline to file without downtime. + var c = validConfig() + c.gitea.registrationTokenFile = "/tmp/reg.token" + #expect(throws: Never.self) { try c.validated() } + } + + // MARK: - Validation: labels + + @Test("an empty label list is rejected") + func emptyLabelsRejected() { + var c = validConfig() + c.runner.labels = [] + #expect(configInvalidDetail { _ = try c.validated() }.contains("runner.labels")) + } + + @Test("an empty label name is rejected") + func emptyLabelNameRejected() { + var c = validConfig() + c.runner.labels = ["macos-arm64", " "] + #expect(configInvalidDetail { _ = try c.validated() }.contains("empty label")) + } + + @Test("a ':schema' suffix in configured labels is rejected") + func schemedLabelRejected() { + // Gitea reports bare names on jobs, so "macos-arm64:host" in config + // would silently match nothing at all — a config error, not a runtime + // mystery. + var c = validConfig() + c.runner.labels = ["macos-arm64:host"] + let detail = configInvalidDetail { _ = try c.validated() } + #expect(detail.contains("bare names")) + } + + @Test("labels are trimmed by validation") + func labelsAreTrimmed() throws { + var c = validConfig() + c.runner.labels = [" macos-arm64 ", "macos"] + #expect(try c.validated().runner.labels == ["macos-arm64", "macos"]) + } + + @Test("the label set derived from config drives job matching") + func labelSetMatchesJobs() throws { + var c = validConfig() + c.runner.labels = ["macos-arm64", "macos"] + let set = try c.validated().labelSet + #expect(set.matches(jobLabels: ["macos-arm64"])) + #expect(!set.matches(jobLabels: ["ubuntu-latest"])) + } + + @Test("an empty name prefix is rejected") + func emptyNamePrefixRejected() { + // An empty prefix would make the reconcile loop treat every runner on + // the instance as ours. + var c = validConfig() + c.runner.namePrefix = " " + #expect(configInvalidDetail { _ = try c.validated() }.contains("namePrefix")) + } + + @Test("an empty runner version is rejected") + func emptyVersionRejected() { + var c = validConfig() + c.runner.version = "" + #expect(configInvalidDetail { _ = try c.validated() }.contains("version")) + } + + // MARK: - Validation: scheduler + + @Test("maxConcurrentVMs is clamped to the kernel's limit of 2") + func maxConcurrentVMsClampedHigh() throws { + // Apple's kernel fails the third `start()` with + // VZError.virtualMachineLimitExceeded; that is not negotiable in JSON. + for requested in [3, 8, 64, Int.max] { + var c = validConfig() + c.scheduler.maxConcurrentVMs = requested + #expect(try c.validated().scheduler.maxConcurrentVMs == 2) + } + } + + @Test("maxConcurrentVMs is clamped up to 1") + func maxConcurrentVMsClampedLow() throws { + for requested in [0, -1, Int.min] { + var c = validConfig() + c.scheduler.maxConcurrentVMs = requested + #expect(try c.validated().scheduler.maxConcurrentVMs == 1) + } + } + + @Test("an in-range maxConcurrentVMs is left alone") + func maxConcurrentVMsInRange() throws { + var c = validConfig() + c.scheduler.maxConcurrentVMs = 1 + #expect(try c.validated().scheduler.maxConcurrentVMs == 1) + } + + @Test("the hard cap is 2") + func hardCapIsTwo() { + #expect(RunnerConfig.SchedulerSection.hardMaxConcurrentVMs == 2) + } + + @Test("non-positive intervals and timeouts are rejected") + func nonPositiveIntervalsRejected() { + var poll = validConfig() + poll.scheduler.pollIntervalSeconds = 0 + #expect(configInvalidDetail { _ = try poll.validated() }.contains("pollIntervalSeconds")) + + var reconcile = validConfig() + reconcile.scheduler.reconcileIntervalSeconds = -5 + #expect( + configInvalidDetail { _ = try reconcile.validated() }.contains("reconcileIntervalSeconds")) + + var job = validConfig() + job.scheduler.jobTimeoutMinutes = 0 + #expect(configInvalidDetail { _ = try job.validated() }.contains("jobTimeoutMinutes")) + + var boot = validConfig() + boot.scheduler.bootTimeoutSeconds = 0 + #expect(configInvalidDetail { _ = try boot.validated() }.contains("bootTimeoutSeconds")) + } + + // MARK: - Validation: guest + + @Test("an empty guest username is rejected") + func emptyGuestUsernameRejected() { + var c = validConfig() + c.guest.username = " " + #expect(configInvalidDetail { _ = try c.validated() }.contains("guest.username")) + } + + @Test("an empty guest password is rejected") + func emptyGuestPasswordRejected() { + // SSH password auth is the only channel into the guest; an empty + // password turns every boot into an unexplained authentication hang. + var c = validConfig() + c.guest.password = "" + #expect(configInvalidDetail { _ = try c.validated() }.contains("guest.password")) + } + + @Test("a guest password of only whitespace is accepted verbatim") + func whitespaceGuestPasswordIsKept() throws { + // Unlike tokens, the password is not trimmed: whitespace is a legal + // part of a password, and silently trimming it would break SSH. + var c = validConfig() + c.guest.password = " " + #expect(try c.validated().guest.password == " ") + } + + @Test("under-sized guests are rejected") + func undersizedGuestRejected() { + var cpu = validConfig() + cpu.guest.cpuCount = 0 + #expect(configInvalidDetail { _ = try cpu.validated() }.contains("cpuCount")) + + var mem = validConfig() + mem.guest.memoryGB = 0 + #expect(configInvalidDetail { _ = try mem.validated() }.contains("memoryGB")) + + var disk = validConfig() + disk.guest.diskGB = 0 + #expect(configInvalidDetail { _ = try disk.validated() }.contains("diskGB")) + } + + // MARK: - Validation: storage + + @Test("an empty store directory is rejected") + func emptyStoreDirRejected() { + var c = validConfig() + c.storage.storeDir = " " + #expect(configInvalidDetail { _ = try c.validated() }.contains("storeDir")) + } + + @Test("a negative free-space floor is rejected") + func negativeMinFreeDiskRejected() { + var c = validConfig() + c.storage.minFreeDiskGB = -1 + #expect(configInvalidDetail { _ = try c.validated() }.contains("minFreeDiskGB")) + } + + @Test("a zero free-space floor is allowed") + func zeroMinFreeDiskAllowed() throws { + var c = validConfig() + c.storage.minFreeDiskGB = 0 + #expect(throws: Never.self) { try c.validated() } + } + + // MARK: - Token resolution + + @Test("an inline admin token resolves to itself") + func resolveInlineAdminToken() throws { + #expect(try validConfig().resolveAdminToken() == "abc123") + } + + @Test("an admin token file is read and trimmed") + func resolveAdminTokenFromFile() throws { + // `echo secret > token` always leaves a trailing newline; sending that + // in an Authorization header is an instant 401. + let path = try temporaryFile(contents: " file-token-value\n") + var c = validConfig() + c.gitea.adminToken = nil + c.gitea.adminTokenFile = path + + #expect(try c.resolveAdminToken() == "file-token-value") + } + + @Test("the token file wins over an inline value when both are somehow present") + func tokenFileTakesPrecedence() throws { + // `validated()` rejects this combination, but resolution is also called + // on configs assembled in code, so the precedence must be defined. + let path = try temporaryFile(contents: "from-file") + var c = validConfig() + c.gitea.adminTokenFile = path + #expect(try c.resolveAdminToken() == "from-file") + } + + @Test("resolving from a missing token file names the key and the path") + func missingTokenFileIsReported() { + var c = validConfig() + c.gitea.adminToken = nil + c.gitea.adminTokenFile = "/nonexistent/admin.token" + + let detail = configInvalidDetail { _ = try c.resolveAdminToken() } + #expect(detail.contains("gitea.adminTokenFile")) + #expect(detail.contains("/nonexistent/admin.token")) + } + + @Test("an empty token file is rejected rather than yielding an empty token") + func emptyTokenFileIsRejected() throws { + let path = try temporaryFile(contents: "\n\n \n") + var c = validConfig() + c.gitea.adminToken = nil + c.gitea.adminTokenFile = path + + #expect(configInvalidDetail { _ = try c.resolveAdminToken() }.contains("empty")) + } + + @Test("resolving with no admin source configured yields nil, not an error") + func resolveAdminTokenWithNoSource() throws { + var c = validConfig() + c.gitea.adminToken = nil + #expect(try c.resolveAdminToken() == nil) + } + + @Test("a static registration token resolves from file, then inline") + func resolveRegistrationToken() throws { + var inline = validConfig() + #expect(try inline.resolveStaticRegistrationToken() == "REG123") + + let path = try temporaryFile(named: "reg", contents: "REG-FROM-FILE\n") + inline.gitea.registrationTokenFile = path + #expect(try inline.resolveStaticRegistrationToken() == "REG-FROM-FILE") + } + + @Test("no static registration token yields nil so the caller can use the API") + func resolveRegistrationTokenWithNoSource() throws { + var c = validConfig() + c.gitea.registrationToken = nil + c.gitea.fetchRegistrationTokenViaAPI = true + #expect(try c.resolveStaticRegistrationToken() == nil) + } + + // MARK: - Token file permissions + + @Test("a 0600 token file is not reported as insecure") + func ownerOnlyTokenFileIsSecure() throws { + let path = try temporaryFile(contents: "s") + try FileManager.default.setAttributes([.posixPermissions: 0o600], ofItemAtPath: path) + + #expect(RunnerConfig.tokenFileIsGroupOrWorldReadable(path) == false) + + var c = validConfig() + c.gitea.adminToken = nil + c.gitea.adminTokenFile = path + #expect(c.insecureTokenFilePaths.isEmpty) + } + + @Test("a group- or world-readable token file is flagged but not fatal") + func looseTokenFileIsFlagged() throws { + // Deliberately a warning: refusing to start over a 0644 file on a + // single-user CI Mac would be a poor trade. + let path = try temporaryFile(contents: "s") + try FileManager.default.setAttributes([.posixPermissions: 0o644], ofItemAtPath: path) + + #expect(RunnerConfig.tokenFileIsGroupOrWorldReadable(path) == true) + + var c = validConfig() + c.gitea.adminToken = nil + c.gitea.adminTokenFile = path + #expect(c.insecureTokenFilePaths == [path]) + // Still valid: the permission check never blocks startup. + #expect(throws: Never.self) { try c.validated() } + } + + @Test("an unreadable path reports an unknown mode rather than a verdict") + func unknownPermissionsAreNil() { + #expect(RunnerConfig.tokenFileIsGroupOrWorldReadable("/nonexistent/token") == nil) + } +} diff --git a/Tests/RunnerCoreTests/DHCPLeasesTests.swift b/Tests/RunnerCoreTests/DHCPLeasesTests.swift new file mode 100644 index 0000000..e551ebe --- /dev/null +++ b/Tests/RunnerCoreTests/DHCPLeasesTests.swift @@ -0,0 +1,208 @@ +import Foundation +import Testing + +@testable import RunnerCore + +/// Tests for parsing `/var/db/dhcpd_leases`, including the `1,` hardware-type +/// prefix, non-zero-padded octets, and newest-lease-wins on duplicate MACs. +@Suite("DHCP leases") +struct DHCPLeasesTests { + /// A file shaped like the real thing: a `1,` prefix on every `hw_address`, + /// octets that lack zero padding, one block that is missing its MAC, a + /// decimal `lease=` alongside the usual hex ones, a MAC that appears three + /// times with different expiries, and a truncated trailing block of the kind + /// a mid-write read produces. + static let fixture = """ + { + \tname=macos-vm-a + \tip_address=192.168.64.2 + \thw_address=1,aa:bb:c:d:ee:ff + \tidentifier=1,aa:bb:c:d:ee:ff + \tlease=0x66b2c0de + } + { + \tname=macos-vm-b + \tip_address=192.168.64.3 + \thw_address=1,DE:AD:BE:EF:00:01 + \tidentifier=1,de:ad:be:ef:00:01 + \tlease=1723000000 + } + { + \tname=no-hardware-address + \tip_address=192.168.64.4 + \tlease=0x66b2c0de + } + { + \tname=macos-vm-a + \tip_address=192.168.64.9 + \thw_address=1,aa:bb:0c:0d:ee:ff + \tlease=0x66b2ffff + } + { + \tname=macos-vm-a + \tip_address=192.168.64.5 + \thw_address=1,aa:bb:0c:0d:ee:ff + \tlease=0x66b20000 + } + { + \tname=truncated-mid-write + \tip_address=192.168.64.6 + """ + + @Test("a lease carries the fields it was constructed with") + func leaseIsConstructible() { + let lease = DHCPLease( + name: "guest", + ipAddress: "192.168.64.7", + hwAddress: "aa:bb:0c:dd:ee:ff", + leaseExpiry: nil + ) + #expect(lease.ipAddress == "192.168.64.7") + } + + @Test("well-formed blocks parse and malformed ones are skipped") + func parsesWellFormedBlocksOnly() { + let leases = DHCPLeaseParser.parse(Self.fixture) + + // Six blocks in the fixture: one has no hw_address and one is truncated. + #expect(leases.count == 4) + #expect(leases.map(\.ipAddress) == ["192.168.64.2", "192.168.64.3", "192.168.64.9", "192.168.64.5"]) + #expect(!leases.contains { $0.ipAddress == "192.168.64.4" }) + #expect(!leases.contains { $0.ipAddress == "192.168.64.6" }) + #expect(leases[0].name == "macos-vm-a") + } + + @Test("the 1, prefix is stripped and octets are zero-padded") + func normalizesHardwareAddresses() { + let leases = DHCPLeaseParser.parse(Self.fixture) + #expect(leases[0].hwAddress == "aa:bb:0c:0d:ee:ff") + #expect(leases[1].hwAddress == "de:ad:be:ef:00:01") + } + + @Test("lease expiry parses from hex and from decimal") + func parsesHexAndDecimalLeaseTimes() { + let leases = DHCPLeaseParser.parse(Self.fixture) + #expect(leases[0].leaseExpiry == Date(timeIntervalSince1970: TimeInterval(0x66b2_c0de))) + #expect(leases[1].leaseExpiry == Date(timeIntervalSince1970: 1_723_000_000)) + } + + @Test("a duplicate MAC resolves to the newest lease, whatever the file order") + func newestLeaseWinsForDuplicateMACs() { + let leases = DHCPLeaseParser.parse(Self.fixture) + // Three blocks share this MAC: 0x66b2c0de, 0x66b2ffff, then 0x66b20000. + // The newest expiry wins even though it is not the last block. + #expect(DHCPLeaseParser.ipAddress(forMAC: "aa:bb:0c:0d:ee:ff", in: leases) == "192.168.64.9") + } + + @Test("the query MAC is normalized the same way as the file's") + func lookupNormalizesTheQuery() { + let leases = DHCPLeaseParser.parse(Self.fixture) + // Every rendering of the same address must find the same lease. + for query in ["aa:bb:c:d:ee:ff", "AA:BB:0C:0D:EE:FF", "aa-bb-0c-0d-ee-ff", "1,aa:bb:c:d:ee:ff"] { + #expect(DHCPLeaseParser.ipAddress(forMAC: query, in: leases) == "192.168.64.9") + } + #expect(DHCPLeaseParser.ipAddress(forMAC: "de:ad:be:ef:00:01", in: leases) == "192.168.64.3") + } + + @Test("an unknown or unparseable MAC has no address") + func unknownMACHasNoAddress() { + let leases = DHCPLeaseParser.parse(Self.fixture) + #expect(DHCPLeaseParser.ipAddress(forMAC: "00:11:22:33:44:55", in: leases) == nil) + #expect(DHCPLeaseParser.ipAddress(forMAC: "not-a-mac", in: leases) == nil) + } + + @Test("a lease is only newer when bootpd actually rewrote it") + func leaseFreshnessGate() { + // Slot MACs are persistent and leases live 24 h, so the previous guest's + // entry is normally still there when the next clone boots. Only a later + // expiry (or a different address) proves the new guest has leased. + let previous = DHCPLease( + name: nil, + ipAddress: "192.168.64.7", + hwAddress: "aa:bb:0c:dd:ee:ff", + leaseExpiry: Date(timeIntervalSince1970: 1_000) + ) + let same = previous + let renewed = DHCPLease( + name: nil, + ipAddress: "192.168.64.7", + hwAddress: "aa:bb:0c:dd:ee:ff", + leaseExpiry: Date(timeIntervalSince1970: 2_000) + ) + let reassigned = DHCPLease( + name: nil, + ipAddress: "192.168.64.9", + hwAddress: "aa:bb:0c:dd:ee:ff", + leaseExpiry: Date(timeIntervalSince1970: 1_000) + ) + + #expect(!DHCPLeaseParser.isNewer(same, than: previous)) + #expect(DHCPLeaseParser.isNewer(renewed, than: previous)) + #expect(DHCPLeaseParser.isNewer(reassigned, than: previous)) + // First boot on this MAC: there is nothing to be stale against. + #expect(DHCPLeaseParser.isNewer(same, than: nil)) + } + + @Test("lease(forMAC:) returns the same record the address lookup uses") + func leaseLookupAgreesWithAddressLookup() { + let leases = DHCPLeaseParser.parse(Self.fixture) + let lease = DHCPLeaseParser.lease(forMAC: "aa:bb:0c:0d:ee:ff", in: leases) + #expect(lease?.ipAddress == "192.168.64.9") + #expect(lease?.ipAddress == DHCPLeaseParser.ipAddress(forMAC: "aa:bb:0c:0d:ee:ff", in: leases)) + #expect(DHCPLeaseParser.lease(forMAC: "00:11:22:33:44:55", in: leases) == nil) + } + + @Test("normalizeMAC accepts the renderings bootpd and VZMACAddress produce") + func normalizeMACAcceptsCommonForms() { + #expect(DHCPLeaseParser.normalizeMAC("1,aa:bb:c:dd:ee:ff") == "aa:bb:0c:dd:ee:ff") + #expect(DHCPLeaseParser.normalizeMAC("aa:bb:0c:dd:ee:ff") == "aa:bb:0c:dd:ee:ff") + #expect(DHCPLeaseParser.normalizeMAC("AA-BB-0C-DD-EE-FF") == "aa:bb:0c:dd:ee:ff") + #expect(DHCPLeaseParser.normalizeMAC(" 1,0:0:0:0:0:1 ") == "00:00:00:00:00:01") + } + + @Test("normalizeMAC rejects anything that is not six hex octets") + func normalizeMACRejectsGarbage() { + #expect(DHCPLeaseParser.normalizeMAC("") == nil) + #expect(DHCPLeaseParser.normalizeMAC("aa:bb:cc:dd:ee") == nil) + #expect(DHCPLeaseParser.normalizeMAC("aa:bb:cc:dd:ee:ff:00") == nil) + #expect(DHCPLeaseParser.normalizeMAC("aa:bb:cc:dd:ee:gg") == nil) + #expect(DHCPLeaseParser.normalizeMAC("aa:bb:cc:dd:ee:fff") == nil) + #expect(DHCPLeaseParser.normalizeMAC("192.168.64.2") == nil) + } + + @Test("parsing never throws on hostile input") + func parsingIsTotal() { + #expect(DHCPLeaseParser.parse("").isEmpty) + #expect(DHCPLeaseParser.parse("}}}{{{\n=\nname=\n").isEmpty) + #expect(DHCPLeaseParser.parse("{\nip_address=1.2.3.4\n").isEmpty) + // A block interrupted by the start of the next one is dropped, not merged. + let interrupted = """ + { + ip_address=192.168.64.20 + { + ip_address=192.168.64.21 + hw_address=1,2:2:2:2:2:2 + } + """ + let leases = DHCPLeaseParser.parse(interrupted) + #expect(leases.count == 1) + #expect(leases[0].ipAddress == "192.168.64.21") + #expect(leases[0].leaseExpiry == nil) + } + + @Test("a missing lease database reads as no leases") + func missingFileIsEmpty() { + #expect(DHCPLeaseParser.parseFile(at: "/var/db/definitely-not-a-lease-file").isEmpty) + } + + @Test("a lease database on disk round-trips through parseFile") + func parsesFromDisk() throws { + let path = FileManager.default.temporaryDirectory + .appendingPathComponent("dhcpd_leases_test_\(UUID().uuidString)") + try Self.fixture.write(to: path, atomically: true, encoding: .utf8) + defer { try? FileManager.default.removeItem(at: path) } + + let leases = DHCPLeaseParser.parseFile(at: path.path) + #expect(DHCPLeaseParser.ipAddress(forMAC: "aa:bb:c:d:ee:ff", in: leases) == "192.168.64.9") + } +} diff --git a/Tests/RunnerCoreTests/GiteaClientTests.swift b/Tests/RunnerCoreTests/GiteaClientTests.swift new file mode 100644 index 0000000..6e29604 --- /dev/null +++ b/Tests/RunnerCoreTests/GiteaClientTests.swift @@ -0,0 +1,443 @@ +import Foundation +import Testing + +#if canImport(FoundationNetworking) +// `URLRequest` lives in FoundationNetworking on Linux, as it does in RunnerCore. +import FoundationNetworking +#endif + +@testable import RunnerCore + +/// Ways a fixture can be wrong about its own inputs. +private enum FixtureFailure: Error { + case unexpectedRequestCount(Int) + case unusableURL +} + +/// A canned ``HTTPTransport`` that records what it was asked to send. +/// +/// An actor rather than a locked class: ``HTTPTransport/send(_:)`` is `async`, +/// so actor isolation satisfies the requirement directly and the recorded +/// requests need no lock of their own. No `URLProtocol`, no loopback server — +/// the seam is the protocol. +private actor MockTransport: HTTPTransport { + /// Every request handed to ``send(_:)``, in order. + private(set) var requests: [URLRequest] = [] + + private let handler: @Sendable (URLRequest) -> (Data, Int) + + /// - Parameter handler: Produces the canned `(body, status)` for a request. + init(handler: @escaping @Sendable (URLRequest) -> (Data, Int)) { + self.handler = handler + } + + /// Convenience: always answer with one status and body. + init(status: Int, body: String = "") { + self.handler = { (_: URLRequest) in (Data(body.utf8), status) } + } + + func send(_ request: URLRequest) async throws -> (Data, Int) { + requests.append(request) + return handler(request) + } + + /// The single request that was sent, or a failure if the count differs. + func onlyRequest() throws -> URLRequest { + guard requests.count == 1, let only = requests.first else { + throw FixtureFailure.unexpectedRequestCount(requests.count) + } + return only + } +} + +/// Tests for ``GiteaClient`` request shaping and error mapping, driven through a +/// fake ``HTTPTransport``. +@Suite("GiteaClient") +struct GiteaClientTests { + + private let base = URL(string: "https://gitea.example.com")! + + private func components(_ request: URLRequest) throws -> URLComponents { + guard let url = request.url, + let components = URLComponents(url: url, resolvingAgainstBaseURL: false) + else { throw FixtureFailure.unusableURL } + return components + } + + private func queryValue(_ request: URLRequest, _ name: String) throws -> String? { + try components(request).queryItems?.first(where: { $0.name == name })?.value + } + + // MARK: - Construction + + @Test("a client retains the base URL it was constructed with") + func clientRetainsBaseURL() throws { + let url = try #require(URL(string: "https://gitea.example.com")) + let client = GiteaClient(baseURL: url, token: "t") + #expect(client.baseURL == url) + } + + // MARK: - Headers + + @Test("every request carries Gitea's token auth and a JSON Accept header") + func requestHeaders() throws { + let client = GiteaClient(baseURL: base, token: "s3cret") + let request = try client.makeRequest(method: "GET", path: "/api/v1/admin/actions/runners") + + #expect(request.value(forHTTPHeaderField: "Authorization") == "token s3cret") + #expect(request.value(forHTTPHeaderField: "Accept") == "application/json") + // No body, so no Content-Type. + #expect(request.value(forHTTPHeaderField: "Content-Type") == nil) + } + + @Test("a request with a body declares JSON content") + func requestWithBody() throws { + let client = GiteaClient(baseURL: base, token: "t") + let request = try client.makeRequest( + method: "POST", path: "/api/v1/x", body: Data(#"{"a":1}"#.utf8)) + + #expect(request.httpMethod == "POST") + #expect(request.value(forHTTPHeaderField: "Content-Type") == "application/json") + #expect(request.httpBody == Data(#"{"a":1}"#.utf8)) + } + + @Test("a trailing slash on the base URL does not double up") + func baseURLTrailingSlash() throws { + let client = GiteaClient(baseURL: try #require(URL(string: "https://gitea.example.com/")), token: "t") + let request = try client.makeRequest(method: "GET", path: "/api/v1/version") + #expect(request.url?.absoluteString == "https://gitea.example.com/api/v1/version") + } + + @Test("an instance served under a subpath keeps that subpath") + func baseURLWithSubpath() throws { + // `URL(string:relativeTo:)` would drop "/gitea" here; string joining does not. + let client = GiteaClient(baseURL: try #require(URL(string: "https://example.com/gitea")), token: "t") + let request = try client.makeRequest(method: "GET", path: "/api/v1/version") + #expect(request.url?.absoluteString == "https://example.com/gitea/api/v1/version") + } + + // MARK: - listQueuedJobs + + @Test("listQueuedJobs GETs the admin jobs endpoint with status=queued") + func listQueuedJobsRequestShape() async throws { + let body = """ + {"total_count": 1, "jobs": [ + {"id": 4711, "run_id": 12, "name": "build", "labels": ["macos-arm64"], + "status": "queued", "created_at": "2026-08-07T09:15:04Z"} + ]} + """ + let transport = MockTransport(status: 200, body: body) + let client = GiteaClient(baseURL: base, token: "t", transport: transport) + + let jobs = try await client.listQueuedJobs(limit: 25) + #expect(jobs.map(\.id) == [4711]) + #expect(jobs.first?.labels == ["macos-arm64"]) + + let request = try await transport.onlyRequest() + #expect(request.httpMethod == "GET") + #expect(try components(request).path == "/api/v1/admin/actions/jobs") + // "queued" and never "waiting": the latter means blocked on a dependency. + #expect(try queryValue(request, "status") == "queued") + #expect(try queryValue(request, "limit") == "25") + #expect(request.value(forHTTPHeaderField: "Authorization") == "token t") + } + + @Test("listQueuedJobs defaults to a limit of 50") + func listQueuedJobsDefaultLimit() async throws { + let transport = MockTransport(status: 200, body: #"{"total_count": 0, "jobs": []}"#) + let client = GiteaClient(baseURL: base, token: "t", transport: transport) + + #expect(try await client.listQueuedJobs().isEmpty) + #expect(try await queryValue(transport.onlyRequest(), "limit") == "50") + } + + @Test("listQueuedJobs never asks for a limit below 1") + func listQueuedJobsClampsLimit() async throws { + let transport = MockTransport(status: 200, body: #"{"jobs": []}"#) + let client = GiteaClient(baseURL: base, token: "t", transport: transport) + + _ = try await client.listQueuedJobs(limit: 0) + #expect(try await queryValue(transport.onlyRequest(), "limit") == "1") + } + + @Test("listQueuedJobs surfaces a 401 as a gitea error") + func listQueuedJobsUnauthorized() async throws { + let transport = MockTransport(status: 401, body: #"{"message": "token is invalid", "url": "..."}"#) + let client = GiteaClient(baseURL: base, token: "bad", transport: transport) + + await #expect(throws: CoreError.self) { + _ = try await client.listQueuedJobs() + } + + do { + _ = try await client.listQueuedJobs() + Issue.record("expected a CoreError.gitea") + } catch let CoreError.gitea(status, message) { + #expect(status == 401) + // Gitea's own `message` field, not the raw envelope. + #expect(message == "token is invalid") + } + } + + @Test("a 500 with a non-JSON body reports a body excerpt") + func serverErrorReportsExcerpt() async throws { + let transport = MockTransport(status: 500, body: "Internal Server Error") + let client = GiteaClient(baseURL: base, token: "t", transport: transport) + + do { + _ = try await client.listQueuedJobs() + Issue.record("expected a CoreError.gitea") + } catch let CoreError.gitea(status, message) { + #expect(status == 500) + #expect(message.contains("Internal Server Error")) + } + } + + @Test("a 2xx with an undecodable body is an error, not a silent empty list") + func undecodableBodyIsAnError() async throws { + let transport = MockTransport(status: 200, body: "not json at all") + let client = GiteaClient(baseURL: base, token: "t", transport: transport) + + await #expect(throws: CoreError.self) { + _ = try await client.listQueuedJobs() + } + } + + // MARK: - listRunners + + @Test("listRunners GETs the admin runners endpoint and decodes object labels") + func listRunnersRequestShape() async throws { + let body = """ + {"total_count": 1, "runners": [ + {"id": 9, "name": "macos-vm-abc", "status": "online", "busy": false, + "ephemeral": true, "labels": [{"id": 3, "name": "macos-arm64", "type": "custom"}]} + ]} + """ + let transport = MockTransport(status: 200, body: body) + let client = GiteaClient(baseURL: base, token: "t", transport: transport) + + let runners = try await client.listRunners() + #expect(runners.count == 1) + #expect(runners.first?.labels == ["macos-arm64"]) + #expect(runners.first?.isEphemeral == true) + #expect(runners.first?.isBusy == false) + + let request = try await transport.onlyRequest() + #expect(request.httpMethod == "GET") + #expect(try components(request).path == "/api/v1/admin/actions/runners") + // Paginated: `total_count` says there is nothing past this page, so one + // request is all it takes. + let query = try components(request).queryItems ?? [] + #expect(query.contains(URLQueryItem(name: "page", value: "1"))) + #expect(query.contains(URLQueryItem(name: "limit", value: "50"))) + } + + @Test("listRunners walks every page rather than returning only the first") + func listRunnersPaginates() async throws { + // Gitea clamps `limit` to its own maximum, so a page shorter than the + // one asked for does not mean the walk is over — only `total_count` does. + let transport = MockTransport { request in + let page = URLComponents(url: request.url!, resolvingAgainstBaseURL: false)? + .queryItems?.first { $0.name == "page" }?.value ?? "1" + let id = page == "1" ? 1 : 2 + let body = """ + {"total_count": 2, "runners": [ + {"id": \(id), "name": "macos-vm-\(id)", "status": "online", "busy": false, + "ephemeral": true, "labels": ["macos-arm64"]} + ]} + """ + return (Data(body.utf8), 200) + } + let client = GiteaClient(baseURL: base, token: "t", transport: transport) + + let runners = try await client.listRunners() + #expect(runners.map(\.id) == [1, 2]) + await #expect(transport.requests.count == 2) + } + + @Test("listRunners stops on an empty page when the server omits total_count") + func listRunnersStopsOnEmptyPage() async throws { + let transport = MockTransport { request in + let page = URLComponents(url: request.url!, resolvingAgainstBaseURL: false)? + .queryItems?.first { $0.name == "page" }?.value ?? "1" + let body = page == "1" + ? #"{"runners": [{"id": 1, "name": "macos-vm-1", "ephemeral": true, "labels": []}]}"# + : #"{"runners": []}"# + return (Data(body.utf8), 200) + } + let client = GiteaClient(baseURL: base, token: "t", transport: transport) + + let runners = try await client.listRunners() + #expect(runners.map(\.id) == [1]) + await #expect(transport.requests.count == 2) + } + + @Test("listRunners surfaces a 403 as a gitea error") + func listRunnersForbidden() async throws { + let transport = MockTransport(status: 403, body: #"{"message": "not an admin"}"#) + let client = GiteaClient(baseURL: base, token: "t", transport: transport) + + do { + _ = try await client.listRunners() + Issue.record("expected a CoreError.gitea") + } catch let CoreError.gitea(status, message) { + #expect(status == 403) + #expect(message == "not an admin") + } + } + + // MARK: - deleteRunner + + @Test("deleteRunner DELETEs the runner by id and accepts 204") + func deleteRunnerRequestShape() async throws { + let transport = MockTransport(status: 204) + let client = GiteaClient(baseURL: base, token: "t", transport: transport) + + try await client.deleteRunner(id: 9) + + let request = try await transport.onlyRequest() + #expect(request.httpMethod == "DELETE") + #expect(try components(request).path == "/api/v1/admin/actions/runners/9") + } + + @Test("deleteRunner tolerates a 404") + func deleteRunnerTolerates404() async throws { + // The goal is only that the row be gone. We race Gitea's own midnight + // sweep and `--ephemeral` auto-deregistration, so "already absent" is + // success, not a failure worth logging every five minutes. + let transport = MockTransport(status: 404, body: #"{"message": "runner not found"}"#) + let client = GiteaClient(baseURL: base, token: "t", transport: transport) + + try await client.deleteRunner(id: 9) + } + + @Test("deleteRunner accepts a 200 as well as a 204") + func deleteRunnerTolerates200() async throws { + let client = GiteaClient(baseURL: base, token: "t", transport: MockTransport(status: 200)) + try await client.deleteRunner(id: 1) + } + + @Test("deleteRunner still fails on a 500") + func deleteRunnerFailsOn500() async throws { + let transport = MockTransport(status: 500, body: #"{"message": "boom"}"#) + let client = GiteaClient(baseURL: base, token: "t", transport: transport) + + do { + try await client.deleteRunner(id: 9) + Issue.record("expected a CoreError.gitea") + } catch let CoreError.gitea(status, message) { + #expect(status == 500) + #expect(message == "boom") + } + } + + @Test("deleteRunner rejects a 401 rather than treating it as done") + func deleteRunnerFailsOn401() async throws { + let client = GiteaClient( + baseURL: base, token: "t", transport: MockTransport(status: 401, body: "unauthorized")) + + await #expect(throws: CoreError.self) { + try await client.deleteRunner(id: 9) + } + } + + // MARK: - getRegistrationToken + + @Test("getRegistrationToken POSTs and returns the token") + func registrationTokenRequestShape() async throws { + let transport = MockTransport(status: 200, body: #"{"token": "AABBCC00112233"}"#) + let client = GiteaClient(baseURL: base, token: "t", transport: transport) + + #expect(try await client.getRegistrationToken() == "AABBCC00112233") + + let request = try await transport.onlyRequest() + #expect(request.httpMethod == "POST") + #expect(try components(request).path == "/api/v1/admin/actions/runners/registration-token") + #expect(request.value(forHTTPHeaderField: "Authorization") == "token t") + } + + @Test("getRegistrationToken trims surrounding whitespace") + func registrationTokenIsTrimmed() async throws { + let transport = MockTransport(status: 200, body: "{\"token\": \" AABB \\n\"}") + let client = GiteaClient(baseURL: base, token: "t", transport: transport) + #expect(try await client.getRegistrationToken() == "AABB") + } + + @Test("getRegistrationToken rejects an empty token") + func registrationTokenRejectsEmpty() async throws { + // An empty token would be written into the guest and fail at + // `gitea-runner register`, tens of seconds and one VM boot later. + let client = GiteaClient( + baseURL: base, token: "t", transport: MockTransport(status: 200, body: #"{"token": ""}"#)) + + await #expect(throws: CoreError.self) { + _ = try await client.getRegistrationToken() + } + } + + @Test("getRegistrationToken surfaces a 500") + func registrationTokenServerError() async throws { + let client = GiteaClient( + baseURL: base, token: "t", transport: MockTransport(status: 500, body: #"{"message": "nope"}"#)) + + do { + _ = try await client.getRegistrationToken() + Issue.record("expected a CoreError.gitea") + } catch let CoreError.gitea(status, _) { + #expect(status == 500) + } + } + + // MARK: - ping + + @Test("ping probes an admin endpoint, so a non-admin token fails") + func pingUsesAnAdminEndpoint() async throws { + let transport = MockTransport(status: 200, body: #"{"total_count": 0, "runners": []}"#) + let client = GiteaClient(baseURL: base, token: "t", transport: transport) + + try await client.ping() + + let request = try await transport.onlyRequest() + // Deliberately not /api/v1/version, which most instances serve + // anonymously and would therefore pass with a bad token. + #expect(try components(request).path == "/api/v1/admin/actions/runners") + #expect(request.value(forHTTPHeaderField: "Authorization") == "token t") + } + + @Test("ping fails when the token is rejected") + func pingFailsOnBadToken() async throws { + let client = GiteaClient( + baseURL: base, token: "bad", + transport: MockTransport(status: 401, body: #"{"message": "token is invalid"}"#)) + + do { + try await client.ping() + Issue.record("expected a CoreError.gitea") + } catch let CoreError.gitea(status, _) { + #expect(status == 401) + } + } + + // MARK: - Transport-level failures + + @Test("a transport error propagates unchanged") + func transportErrorPropagates() async throws { + // A dropped connection is not an API error; the poll loop logs it and + // retries on the next tick without touching any state. It must not be + // laundered into a `CoreError.gitea` with a made-up status. + let client = GiteaClient(baseURL: base, token: "t", transport: ThrowingTransport()) + + await #expect(throws: ThrowingTransport.Offline.self) { + _ = try await client.listQueuedJobs() + } + } +} + +/// A transport that always fails, standing in for an unreachable instance. +private struct ThrowingTransport: HTTPTransport { + struct Offline: Error {} + + func send(_ request: URLRequest) async throws -> (Data, Int) { + throw Offline() + } +} diff --git a/Tests/RunnerCoreTests/GiteaModelsTests.swift b/Tests/RunnerCoreTests/GiteaModelsTests.swift new file mode 100644 index 0000000..b5dee35 --- /dev/null +++ b/Tests/RunnerCoreTests/GiteaModelsTests.swift @@ -0,0 +1,334 @@ +import Foundation +import Testing + +@testable import RunnerCore + +/// Tests for the Gitea API models' JSON mapping. +/// +/// The fixtures below are written from the shapes Gitea actually serializes, +/// verified against `modules/structs/repo_actions.go` and +/// `routers/api/v1/shared/{action,runners}.go` at tag `v1.25.0`, cross-checked +/// against the published swagger (`ActionWorkflowJobsResponse`, +/// `ActionRunnersResponse`, `ActionRunner`, `ActionRunnerLabel`). Two details +/// are easy to get wrong and are pinned here deliberately: +/// +/// * a job's `labels` is an array of **strings**, but a runner's `labels` is an +/// array of **objects** (`{id, name, type}`); +/// * timestamps are Go `time.Time` values, so fractional seconds appear only +/// when non-zero and an unset time serializes as `0001-01-01T00:00:00Z`. +@Suite("Gitea models") +struct GiteaModelsTests { + + private func decode(_ type: T.Type, _ json: String) throws -> T { + try GiteaClient.makeDecoder().decode(T.self, from: Data(json.utf8)) + } + + // MARK: - Job status + + @Test("a job with status 'queued' is schedulable") + func queuedStatusIsSchedulable() { + let job = WorkflowJob(id: 1, runID: 2, name: "build", status: "queued", labels: ["macos-arm64"]) + #expect(job.isQueued) + #expect(job.jobStatus == .queued) + } + + @Test("'waiting' means blocked and is never schedulable") + func waitingIsNotSchedulable() { + // Gitea maps the external "queued" onto its internal StatusWaiting, and + // the external "waiting" onto StatusBlocked — a job waiting on a + // dependency, not on a runner. Booting a VM for one would be pure waste. + let job = WorkflowJob(id: 1, runID: 2, name: "build", status: "waiting", labels: ["macos-arm64"]) + #expect(!job.isQueued) + #expect(job.jobStatus == .waiting) + } + + @Test("every reported status maps to a case, unknown strings included") + func statusMapping() { + #expect(JobStatus(rawValue: "queued") == .queued) + #expect(JobStatus(rawValue: "waiting") == .waiting) + #expect(JobStatus(rawValue: "in_progress") == .inProgress) + #expect(JobStatus(rawValue: "completed") == .completed) + // A status this build has never heard of must round-trip, not throw: + // the mapping is Gitea's to change, and a decode failure would stop the + // poll loop entirely. + #expect(JobStatus(rawValue: "some_new_status") == .unknown("some_new_status")) + #expect(JobStatus(rawValue: "some_new_status").rawValue == "some_new_status") + } + + // MARK: - Job decoding + + @Test("a realistic queued-job response decodes") + func decodeQueuedJobsResponse() throws { + let json = """ + { + "total_count": 1, + "jobs": [ + { + "id": 4711, + "url": "https://gitea.example.com/api/v1/repos/acme/widget/actions/jobs/4711", + "html_url": "https://gitea.example.com/acme/widget/actions/runs/12/jobs/0", + "run_id": 12, + "run_url": "https://gitea.example.com/api/v1/repos/acme/widget/actions/runs/12", + "name": "build (macos)", + "labels": ["macos-arm64"], + "run_attempt": 1, + "head_sha": "0f1e2d3c4b5a69788796a5b4c3d2e1f001234567", + "head_branch": "main", + "status": "queued", + "steps": [], + "created_at": "2026-08-07T09:15:04Z", + "started_at": "0001-01-01T00:00:00Z", + "completed_at": "0001-01-01T00:00:00Z" + } + ] + } + """ + + let response = try decode(WorkflowJobsResponse.self, json) + #expect(response.totalCount == 1) + #expect(response.items.count == 1) + + let job = try #require(response.items.first) + #expect(job.id == 4711) + #expect(job.runID == 12) + #expect(job.name == "build (macos)") + #expect(job.status == "queued") + #expect(job.isQueued) + #expect(job.labels == ["macos-arm64"]) + // `runner_id` / `runner_name` carry omitempty and are absent while queued. + #expect(job.runnerID == nil) + #expect(job.runnerName == nil) + #expect(job.createdAt != nil) + // Go's zero time must not surface as a year-1 date. + #expect(job.startedAt == nil) + #expect(job.completedAt == nil) + } + + @Test("a running job reports its runner") + func decodeRunningJob() throws { + let json = """ + { + "total_count": 1, + "jobs": [ + { + "id": 4712, + "run_id": 12, + "name": "test", + "labels": ["macos-arm64", "self-hosted"], + "status": "in_progress", + "runner_id": 9, + "runner_name": "macos-vm-3f1c2f8e-0a4b-4c1d-9e2f-5a6b7c8d9e0f", + "created_at": "2026-08-07T09:15:04Z", + "started_at": "2026-08-07T09:16:31.482913Z", + "completed_at": "0001-01-01T00:00:00Z" + } + ] + } + """ + + let job = try #require(try decode(WorkflowJobsResponse.self, json).items.first) + #expect(!job.isQueued) + #expect(job.jobStatus == .inProgress) + #expect(job.runnerID == 9) + #expect(job.runnerName == "macos-vm-3f1c2f8e-0a4b-4c1d-9e2f-5a6b7c8d9e0f") + // Fractional seconds are present here and absent above — both must parse. + #expect(job.startedAt != nil) + #expect(job.completedAt == nil) + } + + @Test("an unknown status string decodes rather than throwing") + func unknownStatusTolerated() throws { + let json = """ + {"total_count": 1, "jobs": [ + {"id": 1, "run_id": 1, "name": "j", "labels": [], "status": "quantum_superposition"} + ]} + """ + let job = try #require(try decode(WorkflowJobsResponse.self, json).items.first) + #expect(job.status == "quantum_superposition") + #expect(job.jobStatus == .unknown("quantum_superposition")) + #expect(!job.isQueued) + } + + @Test("an empty jobs response decodes to no jobs") + func decodeEmptyJobsResponse() throws { + #expect(try decode(WorkflowJobsResponse.self, #"{"total_count": 0, "jobs": []}"#).items.isEmpty) + // A server that omits the array entirely, or nulls it, must not throw: + // "nothing queued" is the overwhelmingly common case in the poll loop. + #expect(try decode(WorkflowJobsResponse.self, #"{"total_count": 0}"#).items.isEmpty) + #expect(try decode(WorkflowJobsResponse.self, #"{"total_count": 0, "jobs": null}"#).items.isEmpty) + #expect(try decode(WorkflowJobsResponse.self, "{}").items.isEmpty) + } + + @Test("the 'workflow_jobs' array key is accepted as a fallback") + func decodeAlternateJobsKey() throws { + let json = """ + {"total_count": 1, "workflow_jobs": [ + {"id": 7, "run_id": 1, "name": "j", "labels": ["macos-arm64"], "status": "queued"} + ]} + """ + let response = try decode(WorkflowJobsResponse.self, json) + #expect(response.items.map(\.id) == [7]) + } + + @Test("a job round-trips through encode and decode") + func jobRoundTrip() throws { + let original = WorkflowJobsResponse( + totalCount: 1, + jobs: [WorkflowJob(id: 1, runID: 2, name: "n", status: "queued", labels: ["macos-arm64"])]) + let encoder = JSONEncoder() + encoder.dateEncodingStrategy = .iso8601 + let data = try encoder.encode(original) + let decoded = try GiteaClient.makeDecoder().decode(WorkflowJobsResponse.self, from: data) + #expect(decoded == original) + } + + // MARK: - Runner decoding + + @Test("a runners response with object-shaped labels decodes") + func decodeRunnersResponse() throws { + // This is the shape that matters most: `ActionRunner.Labels` is + // `[]*ActionRunnerLabel`, NOT `[]string` as on a job. + let json = """ + { + "total_count": 2, + "runners": [ + { + "id": 9, + "name": "macos-vm-3f1c2f8e-0a4b-4c1d-9e2f-5a6b7c8d9e0f", + "status": "online", + "busy": false, + "ephemeral": true, + "labels": [ + {"id": 31, "name": "macos-arm64", "type": "custom"} + ] + }, + { + "id": 10, + "name": "shared-linux-1", + "status": "offline", + "busy": true, + "ephemeral": false, + "labels": [ + {"id": 32, "name": "ubuntu-latest", "type": "custom"}, + {"id": 33, "name": "self-hosted", "type": "custom"} + ] + } + ] + } + """ + + let response = try decode(RunnersResponse.self, json) + #expect(response.totalCount == 2) + #expect(response.items.count == 2) + + let ours = try #require(response.items.first) + #expect(ours.id == 9) + #expect(ours.labels == ["macos-arm64"]) + #expect(ours.status == "online") + #expect(ours.isEphemeral) + #expect(!ours.isBusy) + // All four reconcile conditions are readable from this row. + #expect(RunnerNaming.hasPrefix(ours.name, prefix: "macos-vm-")) + + let theirs = try #require(response.items.last) + #expect(theirs.labels == ["ubuntu-latest", "self-hosted"]) + #expect(!theirs.isEphemeral) + #expect(theirs.isBusy) + #expect(!RunnerNaming.hasPrefix(theirs.name, prefix: "macos-vm-")) + } + + @Test("string-shaped runner labels are also accepted") + func decodeRunnerWithStringLabels() throws { + // Defensive: a proxy, an older build, or a hand-written fixture may use + // the job-style array of strings. + let json = #"{"id": 1, "name": "r", "labels": ["macos-arm64", "macos"]}"# + let runner = try decode(ActionRunner.self, json) + #expect(runner.labels == ["macos-arm64", "macos"]) + } + + @Test("absent, null, and empty runner labels all decode to no labels") + func decodeRunnerWithoutLabels() throws { + #expect(try decode(ActionRunner.self, #"{"id": 1, "name": "r"}"#).labels.isEmpty) + #expect(try decode(ActionRunner.self, #"{"id": 1, "name": "r", "labels": null}"#).labels.isEmpty) + #expect(try decode(ActionRunner.self, #"{"id": 1, "name": "r", "labels": []}"#).labels.isEmpty) + } + + @Test("omitted busy and ephemeral default to false") + func runnerFlagDefaults() throws { + // The reconcile loop only ever deletes a row it is sure is ephemeral and + // idle, so absence must read as "not ephemeral", never as "assume yes". + let runner = try decode(ActionRunner.self, #"{"id": 1, "name": "r", "labels": []}"#) + #expect(runner.busy == nil) + #expect(runner.ephemeral == nil) + #expect(!runner.isBusy) + #expect(!runner.isEphemeral) + } + + @Test("an unknown runner status string is tolerated") + func runnerUnknownStatus() throws { + let json = #"{"id": 1, "name": "r", "labels": [], "status": "hibernating"}"# + #expect(try decode(ActionRunner.self, json).status == "hibernating") + } + + @Test("extra server-side fields are ignored") + func unknownFieldsIgnored() throws { + // Gitea 1.27 adds `disabled` to ActionRunner; newer fields must not + // break a 1.25-era client. + let json = """ + {"id": 1, "name": "r", "labels": [], "disabled": false, "some_future_field": {"a": 1}} + """ + #expect(try decode(ActionRunner.self, json).id == 1) + } + + @Test("an empty runners response decodes to no runners") + func decodeEmptyRunnersResponse() throws { + #expect(try decode(RunnersResponse.self, #"{"total_count": 0, "runners": []}"#).items.isEmpty) + #expect(try decode(RunnersResponse.self, "{}").items.isEmpty) + } + + @Test("the 'entries' array key is accepted as a fallback") + func decodeAlternateRunnersKey() throws { + let json = #"{"total_count": 1, "entries": [{"id": 5, "name": "r", "labels": []}]}"# + #expect(try decode(RunnersResponse.self, json).items.map(\.id) == [5]) + } + + // MARK: - Registration token + + @Test("a registration-token response decodes") + func decodeRegistrationToken() throws { + // Verified: `shared.RegistrationToken` is `{Token string `json:"token"`}`. + let response = try decode(RegistrationTokenResponse.self, #"{"token": "AABBCCDDEEFF00112233"}"#) + #expect(response.token == "AABBCCDDEEFF00112233") + } + + // MARK: - Timestamps + + @Test("timestamps parse with and without fractional seconds") + func timestampParsing() throws { + // Go marshals time.Time as RFC 3339 Nano: the fractional part appears + // only when non-zero, so both spellings occur in one response. + let plain = try #require(GiteaClient.parseTimestamp("2026-08-07T09:15:04Z")) + let fractional = try #require(GiteaClient.parseTimestamp("2026-08-07T09:15:04.482913Z")) + #expect(fractional > plain) + #expect(fractional.timeIntervalSince(plain) < 1) + + // An offset rather than Z. + let offset = try #require(GiteaClient.parseTimestamp("2026-08-07T11:15:04+02:00")) + #expect(offset == plain) + + #expect(GiteaClient.parseTimestamp("not a date") == nil) + } + + @Test("a malformed timestamp fails the field, loudly") + func malformedTimestampThrows() { + let json = """ + {"total_count": 1, "jobs": [ + {"id": 1, "run_id": 1, "name": "j", "labels": [], "status": "queued", + "created_at": "yesterday"} + ]} + """ + #expect(throws: DecodingError.self) { + try decode(WorkflowJobsResponse.self, json) + } + } +} diff --git a/Tests/RunnerCoreTests/LabelsTests.swift b/Tests/RunnerCoreTests/LabelsTests.swift new file mode 100644 index 0000000..9710cf0 --- /dev/null +++ b/Tests/RunnerCoreTests/LabelsTests.swift @@ -0,0 +1,182 @@ +import Foundation +import Testing + +@testable import RunnerCore + +/// Tests for ``LabelSet`` matching against a job's bare `runs-on` labels. +/// +/// The semantics under test are exactly: a job matches iff its label array is +/// non-empty and every entry is one of ours. That is a subset test with an +/// explicit carve-out for the empty set, which subset semantics would otherwise +/// accept. +@Suite("LabelSet") +struct LabelsTests { + + // MARK: - Construction + + @Test("bare names are kept verbatim") + func bareNamesAreKept() { + let set = LabelSet(["macos-arm64", "macos"]) + #expect(set.names == ["macos-arm64", "macos"]) + } + + @Test("a ':schema' suffix is stripped at construction") + func schemaSuffixIsStripped() { + // Safe to hand this the same array the config file holds, even if an + // operator wrote the registration form by mistake. + let set = LabelSet(["macos-arm64:host", "macos:docker"]) + #expect(set.names == ["macos-arm64", "macos"]) + } + + @Test("empty and whitespace-only names are dropped") + func emptyNamesAreDropped() { + let set = LabelSet(["macos-arm64", "", " ", ":host"]) + #expect(set.names == ["macos-arm64"]) + } + + @Test("bareName strips at the first colon only") + func bareNameStripsAtFirstColon() { + #expect(LabelSet.bareName("macos-arm64") == "macos-arm64") + #expect(LabelSet.bareName("macos-arm64:host") == "macos-arm64") + #expect(LabelSet.bareName("a:b:c") == "a") + #expect(LabelSet.bareName(" macos:host") == "macos") + } + + // MARK: - Matching + + @Test("an exact single-label match succeeds") + func exactMatch() { + #expect(LabelSet(["macos-arm64"]).matches(jobLabels: ["macos-arm64"])) + } + + @Test("a job asking for a subset of our labels matches") + func subsetMatches() { + // The runner is a superset: it advertises more than the job needs. + let set = LabelSet(["macos-arm64", "macos", "self-hosted"]) + #expect(set.matches(jobLabels: ["macos-arm64"])) + #expect(set.matches(jobLabels: ["macos-arm64", "self-hosted"])) + #expect(set.matches(jobLabels: ["macos-arm64", "macos", "self-hosted"])) + } + + @Test("a job asking for anything we lack does not match") + func supersetDoesNotMatch() { + let set = LabelSet(["macos-arm64"]) + #expect(!set.matches(jobLabels: ["ubuntu-latest"])) + // One unknown label is enough to disqualify the whole job. + #expect(!set.matches(jobLabels: ["macos-arm64", "xcode-16"])) + } + + @Test("an empty job label array never matches") + func emptyJobLabelsDoNotMatch() { + // Subset semantics alone would say yes — the empty set is a subset of + // everything. A job that declares no requirement must not consume one of + // two scarce macOS VMs. + #expect(!LabelSet(["macos-arm64"]).matches(jobLabels: [])) + #expect(!LabelSet([]).matches(jobLabels: [])) + } + + @Test("an empty runner label set matches nothing") + func emptyRunnerLabelsMatchNothing() { + #expect(!LabelSet([]).matches(jobLabels: ["macos-arm64"])) + } + + @Test("matching is case-sensitive") + func matchingIsCaseSensitive() { + // Gitea compares labels case-sensitively, so `macOS-ARM64` in a workflow + // is a different label from `macos-arm64` and must not be claimed. + let set = LabelSet(["macos-arm64"]) + #expect(!set.matches(jobLabels: ["macOS-ARM64"])) + #expect(!set.matches(jobLabels: ["MACOS-ARM64"])) + #expect(set.matches(jobLabels: ["macos-arm64"])) + } + + @Test("a schema suffix on the job side is stripped before matching") + func jobLabelsAreNormalized() { + // The server stores bare names, so this is unusual — but a workflow that + // writes `runs-on: [macos-arm64:host]` would otherwise be skipped + // silently, with no log line to explain why. + let set = LabelSet(["macos-arm64", "macos"]) + #expect(set.matches(jobLabels: ["macos-arm64:host"])) + #expect(set.matches(jobLabels: ["macos-arm64:host", "macos"])) + #expect(!set.matches(jobLabels: ["ubuntu-latest:host"])) + // A label that is nothing but a schema separator is not a match. + #expect(!set.matches(jobLabels: [":host"])) + } + + @Test("duplicate job labels are harmless") + func duplicateJobLabels() { + #expect(LabelSet(["macos-arm64"]).matches(jobLabels: ["macos-arm64", "macos-arm64"])) + } + + // MARK: - Registration argument + + @Test("the registration argument appends the schema to every name") + func registrationArgumentAppendsSchema() { + #expect(LabelSet(["macos-arm64"]).registrationArgument() == "macos-arm64:host") + #expect( + LabelSet(["macos-arm64", "macos"]).registrationArgument() == "macos:host,macos-arm64:host") + } + + @Test("the registration argument is stable across calls") + func registrationArgumentIsStable() { + // `names` is a Set; sorting is what keeps the guest command line — and + // therefore its logs — reproducible. + let set = LabelSet(["zulu", "alpha", "mike"]) + #expect(set.registrationArgument() == set.registrationArgument()) + #expect(set.registrationArgument() == "alpha:host,mike:host,zulu:host") + } + + @Test("a non-default schema is honoured") + func registrationArgumentWithCustomSchema() { + #expect(LabelSet(["ubuntu"]).registrationArgument(schema: "docker") == "ubuntu:docker") + } + + @Test("an empty label set produces an empty registration argument") + func registrationArgumentOfEmptySet() { + #expect(LabelSet([]).registrationArgument() == "") + } +} + +/// Tests for ``RunnerNaming``. +@Suite("RunnerNaming") +struct RunnerNamingTests { + + @Test("a generated name carries the prefix and a lowercase UUID") + func generatedNameShape() throws { + let name = RunnerNaming.makeRunnerName(prefix: "macos-vm-") + #expect(name.hasPrefix("macos-vm-")) + + let suffix = String(name.dropFirst("macos-vm-".count)) + #expect(suffix == suffix.lowercased()) + #expect(UUID(uuidString: suffix) != nil) + } + + @Test("generated names are unique") + func generatedNamesAreUnique() { + // Uniqueness is what lets the reconcile loop decide a row is ours and + // unbacked; a collision would make that decision unsound. + let names = Set((0..<200).map { _ in RunnerNaming.makeRunnerName(prefix: "macos-vm-") }) + #expect(names.count == 200) + } + + @Test("prefix detection matches only our names") + func prefixDetection() { + #expect(RunnerNaming.hasPrefix("macos-vm-abc", prefix: "macos-vm-")) + #expect(!RunnerNaming.hasPrefix("linux-runner-1", prefix: "macos-vm-")) + #expect(!RunnerNaming.hasPrefix("MACOS-VM-abc", prefix: "macos-vm-")) + #expect(!RunnerNaming.hasPrefix("x-macos-vm-abc", prefix: "macos-vm-")) + } + + @Test("an empty prefix matches nothing") + func emptyPrefixMatchesNothing() { + // Otherwise the reconcile loop would consider every runner on the + // instance — including other hosts' — a deletion candidate. + #expect(!RunnerNaming.hasPrefix("anything", prefix: "")) + } + + @Test("a generated name is recognized by its own prefix") + func roundTrip() { + let name = RunnerNaming.makeRunnerName(prefix: "macos-vm-") + #expect(RunnerNaming.hasPrefix(name, prefix: "macos-vm-")) + } +} diff --git a/Tests/RunnerCoreTests/SchedulerCoreTests.swift b/Tests/RunnerCoreTests/SchedulerCoreTests.swift new file mode 100644 index 0000000..8c10e27 --- /dev/null +++ b/Tests/RunnerCoreTests/SchedulerCoreTests.swift @@ -0,0 +1,344 @@ +import Foundation +import Testing + +@testable import RunnerCore + +/// Tests for the pure scheduling state machine. +/// +/// Every case drives ``SchedulerCore/plan(state:queuedJobs:labels:maxVMs:now:jobTimeout:bootTimeout:)`` +/// with an injected `now`, so nothing here touches a clock, the network, or a VM. +@Suite("SchedulerCore") +struct SchedulerCoreTests { + // MARK: - Fixtures + + static let labels = LabelSet(["macos-arm64", "macos"]) + static let now = Date(timeIntervalSince1970: 1_700_000_000) + static let jobTimeout: TimeInterval = 3600 + static let bootTimeout: TimeInterval = 300 + + static func job(_ id: Int64, labels: [String] = ["macos-arm64"]) -> WorkflowJob { + WorkflowJob(id: id, runID: id * 10, name: "job-\(id)", status: "queued", labels: labels) + } + + /// Plans one tick with the suite's fixed labels and timeouts. + static func tick( + _ state: SchedulerState, + _ queued: [WorkflowJob], + maxVMs: Int = 2, + now: Date = SchedulerCoreTests.now + ) -> (SchedulerState, [SchedulerAction]) { + SchedulerCore.plan( + state: state, + queuedJobs: queued, + labels: labels, + maxVMs: maxVMs, + now: now, + jobTimeout: jobTimeout, + bootTimeout: bootTimeout + ) + } + + // MARK: - Baseline + + @Test("a fresh state has no VMs and no ledger") + func freshStateIsAllIdle() { + let state = SchedulerState(slotCount: 2) + #expect(state.slots.count == 2) + #expect(state.idleSlots.count == 2) + #expect(state.occupiedSlots.isEmpty) + #expect(state.dispatchedJobIDs.isEmpty) + } + + @Test("an empty queue is a no-op") + func emptyQueueDoesNothing() { + let state = SchedulerState(slotCount: 2) + let (next, actions) = Self.tick(state, []) + #expect(actions.isEmpty) + #expect(next == state) + } + + // MARK: - Booting + + @Test("one queued job boots one VM") + func oneJobBootsOneVM() { + let (next, actions) = Self.tick(SchedulerState(slotCount: 2), [Self.job(1)]) + + #expect(actions == [.bootVM(slot: 0, jobHint: 1)]) + #expect(next.slots[0].state == .provisioning(since: Self.now)) + #expect(next.slots[1].state == .idle) + #expect(next.dispatchedJobIDs == [1]) + } + + @Test("the same job across two ticks boots only one VM") + func dedupsAcrossTicks() { + let queue = [Self.job(1)] + let (afterFirst, firstActions) = Self.tick(SchedulerState(slotCount: 2), queue) + // The job is still queued a poll later: the VM has not registered yet. + let (afterSecond, secondActions) = Self.tick( + afterFirst, queue, now: Self.now.addingTimeInterval(10)) + + #expect(firstActions == [.bootVM(slot: 0, jobHint: 1)]) + #expect(secondActions.isEmpty) + #expect(afterSecond.occupiedSlots.count == 1) + #expect(afterSecond.dispatchedJobIDs == [1]) + } + + @Test("a job still queued while its VM is running does not boot a second VM") + func dedupSurvivesTheRunningTransition() { + let queue = [Self.job(1)] + let (booted, _) = Self.tick(SchedulerState(slotCount: 2), queue) + let running = SchedulerCore.markRunning(state: booted, slot: 0, jobHint: 1, now: Self.now) + + let (next, actions) = Self.tick(running, queue, now: Self.now.addingTimeInterval(30)) + #expect(actions.isEmpty) + #expect(next.occupiedSlots.count == 1) + } + + @Test("two jobs boot two VMs but a third waits for capacity") + func capacityIsCapped() { + let queue = [Self.job(1), Self.job(2), Self.job(3)] + let (next, actions) = Self.tick(SchedulerState(slotCount: 2), queue) + + #expect(actions == [.bootVM(slot: 0, jobHint: 1), .bootVM(slot: 1, jobHint: 2)]) + #expect(next.occupiedSlots.count == 2) + // Job 3 never entered the ledger, so it is eligible the moment a slot frees. + #expect(next.dispatchedJobIDs == [1, 2]) + } + + @Test("maxVMs above two is clamped to the kernel's concurrent-guest limit") + func maxVMsIsClampedToTwo() { + let queue = [Self.job(1), Self.job(2), Self.job(3), Self.job(4)] + let (next, actions) = Self.tick(SchedulerState(slotCount: 4), queue, maxVMs: 5) + + #expect(actions.count == 2) + #expect(next.occupiedSlots.count == 2) + } + + @Test("maxVMs of zero boots nothing") + func zeroCapacityBootsNothing() { + let (next, actions) = Self.tick(SchedulerState(slotCount: 2), [Self.job(1)], maxVMs: 0) + #expect(actions.isEmpty) + #expect(next.occupiedSlots.isEmpty) + } + + // MARK: - Label matching + + @Test("jobs whose labels do not match are ignored") + func nonMatchingLabelsAreIgnored() { + let queue = [ + Self.job(1, labels: ["ubuntu-latest"]), + Self.job(2, labels: ["windows-2022", "self-hosted"]), + ] + let (next, actions) = Self.tick(SchedulerState(slotCount: 2), queue) + + #expect(actions.isEmpty) + #expect(next.dispatchedJobIDs.isEmpty) + #expect(next.occupiedSlots.isEmpty) + } + + @Test("a matching job among non-matching ones still boots") + func matchingJobIsPickedOutOfAMixedQueue() { + let queue = [ + Self.job(1, labels: ["ubuntu-latest"]), + Self.job(2, labels: ["macos-arm64"]), + Self.job(3, labels: ["ubuntu-latest"]), + ] + let (_, actions) = Self.tick(SchedulerState(slotCount: 2), queue) + #expect(actions == [.bootVM(slot: 0, jobHint: 2)]) + } + + // MARK: - Ledger expiry + + @Test("a job that leaves the queue drops out of the dedup ledger") + func dequeuedJobClearsItsLedgerEntry() { + let (booted, _) = Self.tick(SchedulerState(slotCount: 2), [Self.job(1)]) + let running = SchedulerCore.markRunning(state: booted, slot: 0, jobHint: 1, now: Self.now) + #expect(running.dispatchedJobIDs == [1]) + + // The VM registered and claimed job 1, so Gitea no longer reports it queued. + let (next, actions) = Self.tick(running, [], now: Self.now.addingTimeInterval(60)) + #expect(actions.isEmpty) + #expect(next.dispatchedJobIDs.isEmpty) + #expect(next.occupiedSlots.count == 1) + } + + @Test("releasing a job lets a still-queued job boot again after a failure") + func releasedJobIsRedispatched() { + // A boot that failed (no disk, clone error, dead SSH) leaves its job + // queued, so the ledger's "no longer queued" expiry never fires for it. + // Without the explicit release the job is stranded for good. + let (booted, _) = Self.tick(SchedulerState(slotCount: 2), [Self.job(1)]) + #expect(booted.dispatchedJobIDs == [1]) + + let failed = SchedulerCore.releaseJob( + state: SchedulerCore.markIdle(state: booted, slot: 0), + jobID: 1 + ) + #expect(failed.dispatchedJobIDs.isEmpty) + + let (next, actions) = Self.tick(failed, [Self.job(1)], now: Self.now.addingTimeInterval(30)) + #expect(actions == [.bootVM(slot: 0, jobHint: 1)]) + #expect(next.dispatchedJobIDs == [1]) + } + + @Test("releasing an id that was never dispatched is a no-op") + func releasingAnUnknownJobIsHarmless() { + let (booted, _) = Self.tick(SchedulerState(slotCount: 2), [Self.job(1)]) + let after = SchedulerCore.releaseJob(state: booted, jobID: 99) + #expect(after.dispatchedJobIDs == [1]) + #expect(SchedulerCore.releaseJob(state: after, jobID: 1).dispatchedJobIDs.isEmpty) + } + + @Test("a freed slot is re-earned by a genuinely new job") + func freedCapacityServesTheNextJob() { + let (booted, _) = Self.tick(SchedulerState(slotCount: 2), [Self.job(1)]) + let done = SchedulerCore.markIdle(state: booted, slot: 0) + + let (next, actions) = Self.tick(done, [Self.job(2)], now: Self.now.addingTimeInterval(120)) + #expect(actions == [.bootVM(slot: 0, jobHint: 2)]) + #expect(next.dispatchedJobIDs == [2]) + } + + // MARK: - Timeouts + + @Test("a stuck boot is torn down and replaced in the same pass") + func bootTimeoutTearsDownAndAllowsAReplacement() { + let stuckSince = Self.now.addingTimeInterval(-(Self.bootTimeout + 60)) + let state = SchedulerState( + slots: [ + VMSlot(id: 0, state: .provisioning(since: stuckSince)), + VMSlot(id: 1, state: .idle), + ], + dispatchedJobIDs: [1] + ) + + let (next, actions) = Self.tick(state, [Self.job(1)]) + + // Teardown first so the orchestrator frees the slot before reusing it. + #expect(actions.count == 2) + if case .teardownVM(let slot, let reason) = actions[0] { + #expect(slot == 0) + #expect(reason.contains("boot timeout")) + } else { + Issue.record("expected a teardown first, got \(actions[0])") + } + #expect(actions[1] == .bootVM(slot: 0, jobHint: 1)) + #expect(next.slots[0].state == .provisioning(since: Self.now)) + #expect(next.dispatchedJobIDs == [1]) + } + + @Test("a boot inside its timeout is left alone") + func youngBootIsNotTornDown() { + let state = SchedulerState( + slots: [VMSlot(id: 0, state: .provisioning(since: Self.now.addingTimeInterval(-10)))], + dispatchedJobIDs: [1] + ) + let (next, actions) = Self.tick(state, [Self.job(1)]) + #expect(actions.isEmpty) + #expect(next == state) + } + + @Test("a run that overshoots the job timeout is torn down") + func jobTimeoutTearsDownARunningSlot() { + let startedAt = Self.now.addingTimeInterval(-(Self.jobTimeout + 300)) + let state = SchedulerState( + slots: [ + VMSlot(id: 0, state: .running(jobHint: 7, since: startedAt)), + VMSlot(id: 1, state: .idle), + ], + dispatchedJobIDs: [7] + ) + + let (next, actions) = Self.tick(state, []) + + #expect(actions.count == 1) + if case .teardownVM(let slot, let reason) = actions[0] { + #expect(slot == 0) + #expect(reason.contains("job timeout")) + } else { + Issue.record("expected a teardown, got \(actions[0])") + } + #expect(next.slots[0].state == .idle) + #expect(next.dispatchedJobIDs.isEmpty) + } + + @Test("a run inside its timeout is left alone") + func youngRunIsNotTornDown() { + let state = SchedulerState( + slots: [VMSlot(id: 0, state: .running(jobHint: 7, since: Self.now.addingTimeInterval(-60)))] + ) + let (next, actions) = Self.tick(state, []) + #expect(actions.isEmpty) + #expect(next == state) + } + + @Test("both slots can time out on the same tick") + func bothSlotsCanTimeOutTogether() { + let state = SchedulerState( + slots: [ + VMSlot(id: 0, state: .provisioning(since: Self.now.addingTimeInterval(-1000))), + VMSlot(id: 1, state: .running(jobHint: 9, since: Self.now.addingTimeInterval(-100_000))), + ], + dispatchedJobIDs: [9] + ) + let (next, actions) = Self.tick(state, []) + #expect(actions.count == 2) + #expect(next.occupiedSlots.isEmpty) + } + + // MARK: - Transitions + + @Test("the mark* helpers move a slot through its lifecycle") + func slotTransitions() { + var state = SchedulerState(slotCount: 2) + + state = SchedulerCore.markProvisioning(state: state, slot: 1, jobHint: 42, now: Self.now) + #expect(state.slots[1].state == .provisioning(since: Self.now)) + #expect(state.dispatchedJobIDs == [42]) + + let live = Self.now.addingTimeInterval(90) + state = SchedulerCore.markRunning(state: state, slot: 1, jobHint: 42, now: live) + #expect(state.slots[1].state == .running(jobHint: 42, since: live)) + + state = SchedulerCore.markIdle(state: state, slot: 1) + #expect(state.slots[1].state == .idle) + #expect(state.occupiedSlots.isEmpty) + } + + @Test("markRunning keeps an existing hint and tolerates an unknown slot") + func markRunningIsForgiving() { + var state = SchedulerState(slotCount: 1) + state = SchedulerCore.markRunning(state: state, slot: 0, jobHint: 5, now: Self.now) + + let later = Self.now.addingTimeInterval(30) + let carried = SchedulerCore.markRunning(state: state, slot: 0, now: later) + #expect(carried.slots[0].state == .running(jobHint: 5, since: later)) + + // A slot id we do not own is ignored rather than trapping. + #expect(SchedulerCore.markRunning(state: state, slot: 99, now: later) == state) + #expect(SchedulerCore.markIdle(state: state, slot: 99) == state) + } + + // MARK: - Purity + + @Test("planning is deterministic and leaves its input untouched") + func planningIsPure() { + let state = SchedulerState(slotCount: 2) + let queue = [Self.job(1), Self.job(2)] + + let (firstState, firstActions) = Self.tick(state, queue) + let (secondState, secondActions) = Self.tick(state, queue) + + #expect(firstState == secondState) + #expect(firstActions == secondActions) + // The value passed in is unchanged — `plan` returns a new state. + #expect(state.occupiedSlots.isEmpty) + #expect(state.dispatchedJobIDs.isEmpty) + } + + @Test("the plan never contains an explicit no-op action") + func noOpIsAnEmptyPlan() { + let (_, actions) = Self.tick(SchedulerState(slotCount: 2), [Self.job(1)]) + #expect(!actions.contains(.none)) + } +} diff --git a/docs/DESIGN.md b/docs/DESIGN.md new file mode 100644 index 0000000..957db27 --- /dev/null +++ b/docs/DESIGN.md @@ -0,0 +1,581 @@ +# gitea-macos-runner — Design + +## 1. Overview + +`gitea-macos-runner` is a single-host daemon for an Apple Silicon Mac. It watches +a Gitea instance for queued Actions jobs that require macOS, and for each one it +boots a **fresh, ephemeral macOS VM** on Apple's Virtualization.framework, +registers a single-use runner inside it, lets the job run, and then destroys the +VM. + +The design goal is that **no state survives a job**. Not a checkout, not a +keychain entry, not a `~/Library` mutation, not a leftover process. The guest +that runs job *N+1* is a byte-identical copy-on-write clone of the same base +image that job *N* started from. This is the property that a persistent +self-hosted Mac runner cannot offer, and it is the whole reason this tool exists. + +Three constraints shape everything below: + +1. **Apple's kernel allows at most two concurrent macOS guests per host.** Not a + policy, not a licence term we chose — a hard limit that surfaces as + `VZError.virtualMachineLimitExceeded` from `start()`. Concurrency is therefore + 2, permanently, and the config value is clamped rather than trusted. +2. **Virtualization needs a GUI session and a signed bundle.** The daemon runs as + a LaunchAgent in a logged-in user session, from inside an ad-hoc-signed `.app` + carrying `com.apple.security.virtualization`. +3. **Gitea decides which job a runner claims, not us.** We supply capacity; the + server matches. Trying to pin a specific job to a specific VM would mean + reimplementing Gitea's matching rules, and would be wrong the moment they + change. + +### Non-goals + +* Multi-host scheduling. One daemon, one Mac, two slots. +* Container-based execution. Gitea's `host` schema runs jobs directly on the + guest; that is the point of having a real macOS VM. +* Bridged networking. NAT only — see §6. +* Guest reuse or warm pools. See §9 for why save/restore is deferred rather than + rejected. + +--- + +## 2. Component diagram + +``` +┌──────────────────────────────── Host (Apple Silicon Mac, macOS 26+) ─────────────────────────────┐ +│ │ +│ LaunchAgent (user session, auto-login, login.keychain unlocked) │ +│ └── GiteaMacosRunner.app (ad-hoc signed, com.apple.security.virtualization, LSUIElement) │ +│ │ │ +│ │ NSApplication(.prohibited).run() ── main thread, required by Virtualization │ +│ │ │ +│ ┌─────▼──────────────────────────── Orchestrator (actor) ────────────────────────────────┐ │ +│ │ │ │ +│ │ poll loop ──► GiteaClient.listQueuedJobs() ──► [WorkflowJob] │ │ +│ │ │ │ │ +│ │ ├──────► SchedulerCore.plan(...) ── PURE, no I/O ──► [SchedulerAction] │ │ +│ │ │ │ │ +│ │ ├──► bootVM(slot,jobHint) ──► VMStore.cloneImage ──► VMInstance.start │ │ +│ │ │ │ │ │ │ +│ │ │ │ ▼ │ │ +│ │ │ │ DHCPLeaseParser(/var/db/…) │ │ +│ │ │ │ │ │ │ +│ │ │ │ ▼ │ │ +│ │ │ │ SSHExecutor ──► gitea-runner │ │ +│ │ │ │ register+daemon │ │ +│ │ └──► teardownVM(slot,reason) ──► VMInstance.requestStopThenForce ──► deleteClone │ │ +│ │ │ │ +│ │ reconcile loop ──► GiteaClient.listRunners / deleteRunner (sweep orphaned rows) │ │ +│ └────────────────────────────────────────────────────────────────────────────────────────┘ │ +│ │ +│ / │ +│ images/default/{disk.asif, nvram.bin, config.json} ← built once, provisioned, read-only │ +│ vms//{disk.asif, nvram.bin, config.json} ← APFS CoW clones, destroyed per job │ +│ ipsw/ ← downloaded restore images │ +│ state.json ← the two persistent per-slot MACs │ +│ │ +│ ┌──── VM slot 0 (MAC A) ────┐ ┌──── VM slot 1 (MAC B) ────┐ ← at most 2, kernel-enforced │ +│ │ macOS guest │ │ macOS guest │ │ +│ │ gitea-runner --ephemeral │ │ gitea-runner --ephemeral │ │ +│ │ node, git, bash │ │ node, git, bash │ │ +│ └───────────┬───────────────┘ └───────────┬───────────────┘ │ +└──────────────┼───────────────────────────────┼───────────────────────────────────────────────────┘ + │ NAT (vmenet, bootpd) │ + └───────────────┬───────────────┘ + ▼ + ┌─────────────────────┐ + │ Gitea 1.25+ │ + │ /api/v1/admin/… │ + └─────────────────────┘ +``` + +### Module boundaries + +| Target | Contains | Constraint | +|---|---|---| +| `RunnerCore` | Config, Gitea models + client, `LabelSet`, `DHCPLeaseParser`, `SSHExec`, `SchedulerCore` | **No `import Virtualization`.** Builds on Linux, so scheduling and parsing logic can be unit-tested anywhere. | +| `RunnerHost` | `VMBundle`, `VMStore`, `VZConfigFactory`, `VMInstance`, `IPSW`, `ImageBuilder`, `GuestProvisioner`, `Orchestrator`, `LaunchdService`, `Doctor` | macOS-only. Everything that touches the framework. | +| `gitea-macos-runner` | CLI + daemon entry point | Depends on both. | + +The split is not cosmetic: `SchedulerCore` being pure and portable is what makes +the scheduling policy — the part most likely to have subtle bugs — testable +without a Mac, a VM, or a Gitea instance. + +--- + +## 3. Job lifecycle + +``` +Gitea Orchestrator VMStore / VMInstance Guest + │ │ │ │ + │◄── listQueuedJobs ────────┤ (every pollIntervalSeconds) │ │ + ├─── [job 4711, labels ─────► │ │ + │ ["macos-arm64"]] │ │ │ + │ ├── LabelSet.matches? ─────────┤ │ + │ ├── SchedulerCore.plan ────────┤ │ + │ │ → .bootVM(slot: 0, │ │ + │ │ jobHint: 4711) │ │ + │ │ │ │ + │ ├── ensureFreeSpace(minGB) ───►│ │ + │ ├── cloneImage("default", ───►│ APFS CoW copy │ + │ │ slotMAC: MAC-A) │ + rewrite config.json │ + │ ├── VMInstance.start() ───────►│ ──── boot ───────────►│ + │ │ │ │ + │ ├── poll /var/db/dhcpd_leases ─┤◄─── DHCP request ──────┤ + │ │ until MAC-A has an IP │ │ + │ ├── waitForSSH(ip) ────────────┼───────────────────────►│ + │ │ │ │ + │ ├── uploadData(token, 0600) ───┼───────────────────────►│ + │ ├── ssh: gitea-runner register ┼───────────────────────►│ + │◄────────────────────── register (name=macos-vm-, --ephemeral) ──────────────┤ + │ ├── ssh: rm -f │ │ + │ ├── ssh: gitea-runner daemon ──┼───────────────────────►│ + │ │ │ │ + │◄────────────────────── poll for task ─────────────────────────────────────────────┤ + ├─── assign job 4711 ───────────────────────────────────────────────────────────────► + │ │ │ ...running... │ + │◄────────────────────── job result, logs ──────────────────────────────────────────┤ + ├─── auto-deregister runner (server-enforced --ephemeral) ──────────────────────────► + │ │ │ daemon exits │ + │ │◄─ SSH command returns ───────┼────────────────────────┤ + │ ├── teardownVM(slot: 0) ──────►│ │ + │ │ requestStopThenForce ──────┼───────────────────────►│ (halt) + │ │ deleteClone ──────────────►│ rm -rf vms/ │ + │ ├── markIdle(slot: 0) │ │ +``` + +Two details in that sequence carry more weight than their size suggests. + +**The token goes through a file, not an argument.** `gitea-runner register` is +invoked with `--token-file `, where `` was written by `uploadData` with +mode `0600` and is `rm -f`'d in the same shell command. Passing `--token` would +put a fleet-wide credential into the guest's process table, visible to any +process the job spawns — and the job is arbitrary code from a repository. + +**`--ephemeral`, not `--once`.** `--ephemeral` (Gitea 1.24+) is enforced *by the +server*: it hands this runner exactly one task and then deletes the registration. +`--once` is a runner-side convention only — the server still considers the runner +live, and a misbehaving or patched runner could claim more work. Since the whole +security story here rests on "one VM, one job", the enforcement has to live on +the side we don't hand to the job. + +--- + +## 4. Image build pipeline + +Base images are built once with `image build`, and every job clones one. The +build is slow (most of an hour, mostly a ~15 GB download); the clone is +milliseconds. + +``` + image build --name default [--ipsw PATH] + │ + ├─ 1. IPSWProvider.latestSupported() → CDN url + buildVersion + │ (VZMacOSRestoreImage.latestSupported returns a NETWORK url — + │ it cannot be handed to the installer) + ├─ 2. IPSWProvider.download() → /ipsw/*.ipsw + ├─ 3. IPSWProvider.load(localPath:) → VZMacOSRestoreImage + │ (resolveSymlinksInPath first; the framework rejects symlinks) + │ + ├─ 4. restoreImage.mostFeaturefulSupportedConfiguration + │ nil ⇒ this host cannot run this image. Fail loudly; do not guess. + │ + ├─ 5. createBundle() + │ hardwareModel.dataRepresentation → config.json + │ VZMacMachineIdentifier() (fresh) → config.json + │ VZMacAuxiliaryStorage(creatingStorageAt:hardwareModel:) → nvram.bin + │ disk: diskutil image create blank --fs none --format ASIF --size G + │ └─ fallback: sparse RAW file (format recorded in config.json) + │ + ├─ 6. VZMacOSInstaller(virtualMachine:restoringFromImageAt:) on a STOPPED vm + │ KVO on installer.progress → percentage + │ + ├─ 7. first boot with Setup Assistant automation + │ #available(macOS 27.0, *): + │ VZMacGuestProvisioningOptions(username/password/fullName, + │ logsInAutomatically: true, + │ enablesRemoteLogin: true) + │ → VZMacOSVirtualMachineStartOptions.setGuestProvisioning(_:) + │ ⚠ an OLDER GUEST SILENTLY IGNORES THIS — no error, no account, no SSH + │ + ├─ 8. wait for DHCP lease (by MAC) → wait for SSH → GuestProvisioner + │ provision.sh (sudoers, no-sleep, no-Spotlight, maxfiles, known_hosts) + │ Node.js (official arm64 .pkg → installer -pkg) ← REQUIRED + │ verify git / bash / node + │ gitea-runner (host downloads asset → upload → chmod +x) + │ [optional] Xcode from a .xip + │ + └─ 9. clean shutdown → config.provisioned = true ← only now is it clonable +``` + +### On step 7 and its failure mode + +`VZMacGuestProvisioningOptions` needs **macOS 27 or newer on both the host and +the guest**. The host side is a compile/availability check we control. The guest +side is not: an older guest accepts the boot and simply ignores the options. +There is no error to catch. The observable symptom is that the VM boots, sits at +Setup Assistant forever, never requests a DHCP lease with a usable hostname, and +never answers SSH — so the build fails at step 8 with a timeout that says nothing +useful. + +`firstBootAndProvision` therefore detects the timeout and reports it as an +explicit "guest is too old for unattended setup; supply a macOS 27+ IPSW" +failure. A `--manual-setup` flow that opens a window and lets a human click +through Setup Assistant once is **out of scope for v1** — deliberately, because a +GUI step in a tool whose whole purpose is unattended operation is a trap. It is +noted here so the omission is a decision rather than an oversight. + +### On step 5's disk format + +ASIF is preferred because it is sparse: a 64 GB nominal disk costs what the guest +actually writes, and it CoW-clones cleanly on APFS. `diskutil image create` is +shelled out to because there is no framework API for it. If that call fails for +any reason — older `diskutil`, unusual volume — a sparse RAW file is created +instead and the format is recorded in `config.json`, so `VZConfigFactory` attaches +the right file without re-probing. + +--- + +## 5. Scheduling semantics + +`SchedulerCore.plan` is a pure function: `(state, queuedJobs, labels, maxVMs, +now, jobTimeout, bootTimeout) → (state', [action])`. It performs no I/O, reads no +clock, and is fully deterministic — which is what allows the entire scheduling +policy to be tested with a fixed `now` and a synthetic job list. + +### Capacity, not assignment + +This is the central idea and the easiest thing to get wrong. + +A booted VM is **capacity**. It is not a promise to run a particular job. We see +job 4711 queued, we boot a VM, we register an ephemeral runner — and the *server* +then decides which queued job that runner claims. It may well claim job 4712 +instead. That is fine and in fact preferable: Gitea's matching rules (labels, +repo permissions, ordering, priority) are its business, and any attempt to +predict them here would be a reimplementation that drifts out of sync. + +The `jobHint` threaded through `SchedulerAction.bootVM` and +`SlotState.running` exists for exactly two purposes: log messages, and the dedup +ledger below. Nothing else may depend on it. + +### Dedup by job id + +`SchedulerState.dispatchedJobIDs` is a `Set` of jobs that have already +caused a boot. + +Without it, the loop is pathological. A VM takes tens of seconds to boot, +provision, and register. The poll interval is 5 seconds. So a single queued job +would still be queued on the next poll, and the next, and the next — triggering a +second boot, then exhausting the slot budget, all for one job. + +The ledger is expired against reality rather than against a timer: any id no +longer appearing in the queued set is dropped. That way a slot freed by a +completed job can be re-earned by a genuinely new job, but a job that is *still* +waiting does not double-book. + +### The cap + +`maxVMs` is clamped to 2 in `plan`, and again in `RunnerConfig.validated()`. Both +places, because the kernel limit is not something a config file gets to +negotiate: a third `start()` raises `VZError.virtualMachineLimitExceeded`, which +`VMInstance.mapVZError` translates into `CoreError.vmLimitExceeded` and the +scheduler treats as transient back-pressure rather than a failure. + +### Timeouts + +* A slot in `.provisioning(since:)` longer than `bootTimeoutSeconds` (default + 300) is torn down. Covers a guest that never gets a lease, never starts `sshd`, + or hangs in Setup Assistant. +* A slot in `.running(jobHint:since:)` longer than `jobTimeoutMinutes` (default + 120) is torn down. Covers a job that hangs. This is comfortably below Gitea's + own `ABANDONED_JOB_TIMEOUT` (24 h), so our teardown always happens first and + the server sees a clean deregistration rather than an abandonment. + +Teardown actions are emitted **before** boot actions in the returned list, so a +slot freed in one pass can be reused in that same pass. + +### Reconcile + +Every `reconcileIntervalSeconds` (default 300), the orchestrator lists runners and +deletes any that are: + +* `ephemeral == true`, **and** +* `busy == false`, **and** +* `name` starts with our configured `namePrefix`, **and** +* not backed by a live VM in this process. + +All four conditions, because deleting a live runner fails a running job. The loop +is deliberately conservative: a row we are unsure about is left alone, and will be +revisited in five minutes. + +This loop is not optional housekeeping — it is load-bearing. See Verified Fact 7. + +--- + +## 6. Security model + +### The threat + +A CI job is arbitrary code from a repository, running with the privileges of the +account it executes under. On a persistent self-hosted Mac runner, that code can +read every previous job's checkout, poison caches, install launch agents, and +harvest whatever credentials the machine has accumulated. Every subsequent job on +that host inherits the compromise. + +### The mitigation: genuinely ephemeral guests + +* **One job per VM, enforced server-side.** `--ephemeral` means Gitea hands the + runner exactly one task and then deletes the registration. A patched or + hijacked runner binary cannot ask for more work, because the server will not + give it any. +* **The VM is destroyed after that job.** Not reset, not cleaned — the clone + directory is `rm -rf`'d and the next job clones the base image afresh. There is + no path by which job *N* influences job *N+1* short of compromising the host. +* **The guest holds nothing worth stealing.** Its account credentials + (`admin`/`admin` by default) are meaningful only on a host-private NAT link to a + machine that is about to be deleted. + +### The shared registration token + +Registration tokens in Gitea are **reusable and scope-wide**, and minting a new +one for a scope **invalidates all prior tokens of that scope**. That makes +per-VM tokens actively harmful: generating one for each VM would break every +other runner registered against that scope, including ones on other hosts. + +So the fleet shares one token. The mitigations are: + +* It is written into the guest as a **file with mode `0600`**, never as a command + argument (arguments are world-readable via `ps`). +* It is **deleted immediately** after `gitea-runner register` consumes it, in the + same `&&` chain, before `gitea-runner daemon` starts and long before any job + code runs. +* It is a *registration* token, not an API token: it grants the ability to + register a runner, not to read repositories or act as a user. + +The residual risk is real but bounded — a job that wins a race against `rm -f` +could register additional runners for that scope. The recommended deployment +seeds a fixed token server-side via `GITEA_RUNNER_REGISTRATION_TOKEN` so that +rotating it is a deliberate, coordinated act rather than an API call side effect. + +### `:host` schema risk + +Jobs run in `host` schema: directly on the guest OS, not in a container. That is +the point — a macOS job needs real macOS. But it means the job has full user-level +access to the guest, including `sudo` (which `provision.sh` makes passwordless, +because Xcode and `installer` need it). Everything above rests on the guest being +disposable and isolated, not on the job being constrained inside it. + +### Host-side posture + +* The daemon runs as a **LaunchAgent in a user session**, not as root. The + entitlement it carries (`com.apple.security.virtualization`) grants VM creation + and nothing else. +* Networking is **NAT**, not bridged. Guests can reach the LAN and Gitea, but are + not first-class hosts on it. Bridged networking would require the restricted + `com.apple.vm.networking` entitlement, which ad-hoc signing cannot grant — a + constraint that happens to align with what we want anyway. +* **SSH host keys are not verified.** The peer is a VM this process booted + moments ago on a link no other machine shares; pinning would break on every + clone and add nothing. + +--- + +## 7. Failure modes and recovery + +| Failure | Detection | Recovery | +|---|---|---| +| Guest never gets a DHCP lease | `bootTimeout` in `waitForLease` | Teardown, slot recycled, retried next poll | +| Guest never answers SSH | `bootTimeout` in `waitForSSH` | Same | +| Job hangs | `jobTimeoutMinutes` | Teardown; Gitea reaps the task via its zombie sweep (~10–15 min) | +| VM dies uncleanly | Runner row left behind, task stuck Running | Reconcile loop deletes the row (§5); Gitea's zombie sweep handles the task | +| Daemon crashes with VMs live | Clones orphaned on disk | `purgeClones()` at startup, then one immediate reconcile pass | +| Third VM requested | `VZError.virtualMachineLimitExceeded` | Mapped to `CoreError.vmLimitExceeded`, treated as back-pressure | +| Disk fills | `ensureFreeSpace(minGB:)` before each clone | Boot refused, logged; jobs stay queued (safe — Gitea holds them ~24 h) | +| Gitea unreachable | Request error in the poll loop | Logged, retried next tick; no state change | + +--- + +## 8. Configuration and operations summary + +Config lives at `~/.config/gitea-macos-runner/config.json`; see +`Resources/config.example.json`. Order of operations for a new host: + +``` +gitea-macos-runner doctor # verify arch, macOS, entitlement, keychain, Gitea +gitea-macos-runner config init # write an annotated config +gitea-macos-runner image build # ~1 hour, mostly IPSW download +gitea-macos-runner vm boot # optional smoke test: boot a clone, print its IP +gitea-macos-runner service install # LaunchAgent, RunAtLoad + KeepAlive +gitea-macos-runner doctor # again, now that it runs from the signed .app +``` + +`doctor` exists because every one of its checks corresponds to a failure that +otherwise appears as an opaque error deep inside a VM boot. The most common by +far: running from `.build/` instead of the signed `.app`, so the entitlement is +absent. + +--- + +## 9. Future work + +* **Save/restore for warm boots.** `VZVirtualMachine.saveMachineStateTo` (macOS + 14+) could cut per-job boot from ~60 s to near-instant by restoring a snapshot + taken just after `gitea-runner` is ready. The blocker is that restore forbids + changing the MAC address or ECID, which collides with our per-slot MAC scheme + (§ Verified Fact 12) — a restored state would have to be captured per slot, and + the interaction with DHCP lease reuse needs care. Deferred, not rejected. +* **vsock control channel.** `VZVirtioSocketDeviceConfiguration` is already in the + VM configuration. Replacing SSH with a vsock agent would remove password auth, + the `waitForSSH` poll, and the Local Network privacy prompt entirely. +* **`--manual-setup`** for pre-macOS-27 guests (§4). +* **Image versioning / garbage collection** for multiple base images. + +--- + +## Appendix: Verified Facts + +Researched facts this design depends on, each with its consequence for the +implementation. Anyone changing the corresponding code should re-verify the fact +first. + +**1. Job discovery is `GET /api/v1/admin/actions/jobs?status=queued` (Gitea +1.25+).** The `labels` field on a returned job is the workflow's `runs-on:` +value. The external status string `queued` maps to Gitea's internal +`StatusWaiting`, meaning "ready, waiting for a matching runner". +→ *Consequence:* the external string `waiting` means something entirely +different — the job is **blocked** on a dependency — and must **never** be +treated as schedulable. `WorkflowJob.isQueued` checks `status == "queued"` and +nothing else. + +**2. A queued job waits for a matching runner up to `ABANDONED_JOB_TIMEOUT` +(default 24 h, swept every 6 h).** +→ *Consequence:* there is no urgency in the poll loop. A 5-second interval is for +responsiveness, not correctness; a daemon that is down for an hour loses nothing. +It also sets the ceiling that `jobTimeoutMinutes` (default 120) must stay well +under, so our teardown always precedes the server's abandonment. + +**3. The runner binary is `gitea-runner` v3.x**, renamed from `act_runner` and +published from `gitea.com/gitea/runner`. The register-time flag `--ephemeral` +(Gitea 1.24+) is **server-enforced**: single job, then automatic deregistration. +`--once` is weaker — runner-side only. +→ *Consequence:* `--ephemeral` is mandatory and `--once` is never used. The +"one VM, one job" guarantee in §6 rests on the server enforcing it, not on the +runner cooperating. Download URLs and the binary name must reference +`gitea-runner`, not `act_runner`. + +**4. Registration tokens are REUSABLE, and minting a new token for a scope +INVALIDATES prior tokens of that scope.** `POST +/api/v1/admin/actions/runners/registration-token` in practice returns the +existing active token. Seeding server-side via `GITEA_RUNNER_REGISTRATION_TOKEN` +is the recommended deployment. +→ *Consequence:* **never pre-generate a token per VM** — doing so would break +every other runner registered against that scope. One token is resolved once, +cached for the process lifetime, and shared by the fleet; +`fetchRegistrationTokenViaAPI` defaults to `false`. + +**5. Label syntax is `name:schema`, schema defaults to `host`, and only BARE +names are stored server-side.** If the guest's runner `config.yaml` sets +`runner.labels`, it **silently overrides** `--labels` passed at registration. +→ *Consequence:* `LabelSet` matches bare names (`macos-arm64`) and only appends +`:host` when building the `register --labels` argument. The guest must **never** +ship a `config.yaml` containing labels — noted in both `GuestProvisioner` and +`Resources/provision.sh`, because the failure is silent: the runner registers +successfully and is simply never matched. + +**6. Guest requirements: the `gitea-runner` binary, `node`, `git`, `bash`, and a +writable `$HOME`.** JavaScript actions such as `actions/checkout` spawn `node` +**directly**. +→ *Consequence:* Node.js installation is **not optional** and not a convenience — +without it essentially every real workflow fails at its first step. +`GuestProvisioner.verifyToolchain` fails the build rather than shipping an image +that will break at job time. + +**7. An unclean VM death leaves both a runner row and a Running task behind.** +The task is reaped by Gitea's zombie sweep in roughly 10–15 minutes. The runner +row is swept only at midnight — and **never at all** if that runner claimed no +task. Rows are removed with `DELETE /api/v1/admin/actions/runners/{id}`. +→ *Consequence:* the reconcile loop (§5) is load-bearing, not housekeeping. +Without it, every crashed boot leaves a permanent phantom runner. It is also why +every VM registers under a **globally unique** name (`namePrefix` + UUID): that +uniqueness is what lets us look at a row and decide with certainty that it is +ours and unbacked. + +**8. macOS guests are capped at 2 concurrent per host, enforced by the kernel.** +A third `start()` raises `VZError.virtualMachineLimitExceeded`. +→ *Consequence:* `maxConcurrentVMs` is hard-clamped to 2 in both +`RunnerConfig.validated()` and `SchedulerCore.plan`; the slot table is fixed-size; +and the error is mapped to `CoreError.vmLimitExceeded` and treated as transient +back-pressure rather than a failure. + +**9. `VZMacGuestProvisioningOptions` (macOS 27+ host AND guest) automates Setup +Assistant**, including `enablesRemoteLogin` (SSH). Older guests **silently +ignore** it. +→ *Consequence:* the API is gated at `#available(macOS 27.0, *)`, and because the +guest-side failure produces no error, `firstBootAndProvision` must translate its +lease/SSH timeout into an explicit "guest too old" message rather than a bare +timeout. The `--manual-setup` fallback is out of v1 scope (§4). + +**10. Headless Virtualization requires an `NSApplication` run loop with +`.prohibited` activation policy, inside a signed `.app` bundle** carrying +`com.apple.security.virtualization`. Ad-hoc signing (`codesign -s -`) suffices. +Bridged networking would additionally need a restricted entitlement; NAT does +not. +→ *Consequence:* `CommandDaemon` starts `NSApplication` and runs the orchestrator +in a detached `Task`. This applies to **every** command that starts a VM, not +just the daemon: `vm boot`, `image build`, and `image provision` all go through +the same `VZAppRuntime.run` host, since `VZMacOSInstaller` and the first-boot +provisioning pass need the run loop exactly as much as a job VM does. The +`Makefile` has `bundle` and `sign` targets and +`install` deliberately installs the bundle rather than the bare binary; +`Info.plist` sets `LSUIElement`; `Doctor` checks the entitlement on the running +binary because running from `.build/` is the most common setup failure. +Two packaging constraints follow from the same fact. The entitlements plist must +contain **no XML comments**: `plutil -lint` accepts them, but `codesign` hands +the file to AMFI's stricter parser, which fails with `AMFIUnserializeXML: syntax +error` and then signs the bundle with *zero* entitlements — a silent +downgrade that only surfaces as a failed VM start. And `bundle` must copy +`provision.sh`, `launchd.plist.template`, and `config.example.json` into +`Contents/Resources`, since `GuestProvisioner`, `LaunchdService`, and +`config init` look there before falling back to repo-relative paths; an installed +`.app` without them is a working binary with a broken `image build`, +`service install`, and `config init`. + +**11. macOS 15+ requires an unlocked `login.keychain` to start a VM.** +→ *Consequence:* the service **must** be a LaunchAgent in the auto-logged-in +user's session, never a LaunchDaemon (which has no session and no unlocked +keychain). `LaunchdService` only ever writes to `~/Library/LaunchAgents`, and +`Doctor` probes with `security show-keychain-info login.keychain`. + +**12. Guest IPs come from parsing `/var/db/dhcpd_leases`, keyed by MAC.** +`hw_address` lines carry a `1,` hardware-type prefix and octets that may lack +zero-padding (`aa:bb:c:dd:ee:ff`). Duplicate MACs occur; the newest lease wins. +macOS's DHCP lease time is 24 hours. +→ *Consequence:* `DHCPLeaseParser.normalizeMAC` must strip the prefix and +zero-pad, or lookups silently fail against `VZMACAddress.string`. And ephemeral +fleets must **not** randomize MACs per clone — a day of dead leases would +accumulate and exhaust the NAT subnet. Hence exactly two **persistent per-slot +MACs**, generated once with `VZMACAddress.randomLocallyAdministered()` and stored +in `state.json`. + +**13. APFS copy-on-write cloning via `FileManager.copyItem` requires source and +destination on the same volume.** Clones grow as the guest writes. +→ *Consequence:* base images and ephemeral clones both live under `storeDir`, and +cloning is done **per file** rather than by copying a directory wholesale. +`ensureFreeSpace(minGB:)` runs before every clone, with a floor (default 20 GB) +well above one clone's nominal cost, because the apparent size and the real cost +diverge over a job's lifetime. + +**14. The ASIF sparse disk format is created via `diskutil image create` (macOS +26+); RAW is the fallback.** +→ *Consequence:* `ImageBuilder.createDisk` shells out to +`/usr/sbin/diskutil image create blank --fs none --format ASIF --size G ` +and falls back to a sparse RAW file, recording which was used in +`VMBundleConfig.diskFormat` so `VZConfigFactory` attaches the right file without +re-probing. This is also the reason the package's minimum platform is macOS 26. + +**15. Save/restore (macOS 14+) could give near-instant warm boots, but forbids +changing the MAC address or ECID.** +→ *Consequence:* documented as future work (§9) rather than implemented. The +prohibition collides directly with the per-slot MAC scheme from Fact 12, so +adopting it would require per-slot saved states and a careful look at DHCP lease +reuse — not a drop-in optimization. diff --git a/docs/security.md b/docs/security.md new file mode 100644 index 0000000..16f80b6 --- /dev/null +++ b/docs/security.md @@ -0,0 +1,150 @@ +# Security model + +This document states what `gitea-macos-runner` protects, what it does not, and the operational +choices that follow. Read it before pointing the runner at an instance where untrusted code can +open a pull request. + +## Summary + +Workflow code runs **as the guest admin user, with passwordless sudo, and with no container +isolation**. The only isolation boundary is the virtual machine, and that VM is destroyed after a +single job. This is the same trust posture as GitHub's hosted macOS runners: the job owns the +machine, and the machine is thrown away. + +## Trust boundary + +``` +┌─ host Mac ─────────────────────────────────────────────┐ +│ daemon, admin PAT, registration token, base images │ ← trusted +│ ┌─ ephemeral VM ─────────────────────────────────┐ │ +│ │ gitea-runner + workflow code (root-capable) │ │ ← untrusted +│ └────────────────────────────────────────────────┘ │ +└────────────────────────────────────────────────────────┘ +``` + +**The VM is the boundary.** Everything inside it is treated as untrusted and disposable; +everything outside it is trusted. A guest that escapes Virtualization.framework and compromises the +host is **out of scope** — the design assumes Apple's hypervisor holds. If your threat model +includes hypervisor escape, this tool is not sufficient on its own; isolate the host at the network +level and treat it as a machine that may be compromised. + +## What runs where + +- **In the guest:** the `gitea-runner` binary and every step of the workflow. The runner registers + with `--labels "macos-arm64:host"` — the `:host` execution mode means steps run directly on the + guest OS rather than inside a container. There is no second layer of isolation, by design; + container isolation is not meaningfully available for macOS builds, and it would defeat the point + of a real macOS environment. +- **On the host:** the daemon, the Gitea admin PAT, the registration token, base images, and the + clone/boot/destroy lifecycle. **The admin PAT is never copied into a guest.** + +Because the guest admin has passwordless sudo, a job can install software, load kernel extensions +the guest permits, read every file in the image, and reconfigure the guest arbitrarily. None of +that persists: the clone is deleted when the job ends and the next job starts from the untouched +base image. Nothing a job writes is visible to any later job. + +## Ephemeral registration is server-enforced + +Each VM registers a fresh runner with `--ephemeral`. Ephemerality is enforced **by the Gitea +server**, not by the runner or by this daemon: + +- After the runner accepts one job, Gitea **refuses to dispatch a second job** to that + registration. +- Gitea **deletes the registration** once that job completes. + +The security consequence matters: a runner token exfiltrated from inside a running job cannot be +used to fetch additional jobs, because the server has already spent that registration. The worst an +attacker can do with it is nothing. This is why a compromised job does not become a persistent +foothold in your CI queue — the compromised VM is destroyed and its credential is already dead. + +## The shared registration token and its blast radius + +The registration token is a different matter, and it is the sharpest edge in this design. + +Gitea's registration tokens are **reusable by construction**, and creating a new token for a scope +**invalidates the previous one**. There is no API for minting a single-use, per-VM registration +token. A per-VM token is therefore impossible — not merely unimplemented. + +So every VM presents the same registration token, and that token is necessarily present inside an +untrusted guest for the duration of registration. + +**Blast radius if the token leaks:** an attacker can register arbitrary runners against the scope +the token covers (instance-wide, if you seeded `GITEA_RUNNER_REGISTRATION_TOKEN` on the server). +Such a runner can advertise your labels and thereby **receive and execute jobs**, which means it +can read whatever secrets those jobs are given and return forged results. It does not by itself +grant API access to Gitea, read repositories the runner is not assigned jobs from, or confer admin +rights. + +Mitigations, in order of effectiveness: + +1. **Constrain who can dispatch jobs to the label** (below) so a rogue runner has a small pool of + jobs to intercept. +2. **Register at org or repo scope** rather than instance-wide when only a few repos need macOS. + The token's reach is then limited to that scope. +3. **Rotate the token** by changing `GITEA_RUNNER_REGISTRATION_TOKEN` and restarting Gitea. Update + `gitea.registrationTokenFile` on the host at the same time; in-flight VMs that have already + registered are unaffected. +4. **Keep secrets out of macOS jobs where possible.** Prefer short-lived, narrowly scoped + credentials injected per job over long-lived org-level secrets. + +## Limiting who can use the runner + +By default an instance-wide runner will execute any job from any repo that writes +`runs-on: macos-arm64`. On an instance where untrusted users can push branches or open PRs that +trigger workflows, that is effectively arbitrary code execution on your CI Mac's VM. + +Restrict it: + +- **Register at repo or org scope** instead of instance-wide — the runner is then only offered jobs + from that repo or org. +- Use Gitea's per-repo and per-org **Actions runner settings** to control which repositories may + use a shared runner. +- Configure Gitea so that **workflows from forked-repository pull requests require approval** before + running. This is the single most important setting if your instance accepts outside + contributions. + +Decide this deliberately. Nothing in the daemon restricts job origin; that control lives entirely +in Gitea. + +## Secrets hygiene + +- **Prefer the file forms of every credential:** `gitea.adminTokenFile` and + `gitea.registrationTokenFile` rather than `adminToken`/`registrationToken` inline in + `config.json`. For the admin token this is not merely a preference in one direction: config + validation requires **exactly one** of `adminToken` and `adminTokenFile`, so a stale inline token + cannot sit unnoticed beside a live token file. +- **`chmod 600` every token file** and keep it owned by the runner user: + + ```sh + chmod 600 ~/.config/gitea-macos-runner/admin-token \ + ~/.config/gitea-macos-runner/registration-token + ``` + + `gitea-macos-runner doctor` checks this: its `token file permissions` check warns, and prints the + exact `chmod` to run, if any configured token file is group- or world-readable. + +- **Token *files* keep secrets off `ps`.** Any local user can read another process's argument + vector; a token passed as `--token …` is visible there for the life of the process, and often in + shell history and logs too. The `--token-file` form passes a path instead, so the secret never + enters the command line. +- **Do not commit `config.json`** or any token file. If your config lives in a dotfiles repo, keep + the token files outside it. +- **The `guest.password`** is a credential for a throwaway machine, but it appears in the host's + config file and grants SSH into running guests. Treat the config file itself as sensitive + (`chmod 600`) and do not reuse a password from anywhere else. +- **Rotate the admin PAT** on the normal schedule you use for admin credentials. Its scope + (`read:admin` + `write:admin`) is genuinely powerful — it can enumerate and delete runner + registrations instance-wide — so it is the highest-value secret on the host. It lives only on the + host and never enters a VM; keep it that way. + +## Host hardening notes + +- Use a **dedicated Mac** for CI. The auto-login recommendation in + [setup.md](setup.md#25-install-the-service) means the disk is unlocked at boot, which is + acceptable for a purpose-built CI machine and not for a workstation holding other data. +- Run the daemon as a **non-administrator user** where practical. It needs a GUI session and + virtualization entitlements, not host root. +- Place the Mac on a **network segment that cannot reach production**. Guests get NAT'd egress + through the host; anything the host can reach, a job can reach. +- Keep the base image current. It is rebuilt from an IPSW, so refreshing macOS and the toolchain is + an `image build` away — and every job automatically picks up the new image. diff --git a/docs/setup.md b/docs/setup.md new file mode 100644 index 0000000..f678462 --- /dev/null +++ b/docs/setup.md @@ -0,0 +1,479 @@ +# Setup + +This walkthrough covers everything needed to go from a bare Apple Silicon Mac and a self-hosted +Gitea instance to a working macOS CI runner: Gitea-side configuration, host build and signing, +base image creation, service installation, and verification. + +Work through it in order. The Gitea side can be done from any machine; the host side must be done +on the Mac that will run the VMs. + +--- + +## 1. Gitea-side configuration + +### 1.1 Version requirements + +| Component | Minimum | Notes | +| --- | --- | --- | +| Gitea server | **1.25** | 1.25 added `GET /api/v1/admin/actions/jobs`, including the `labels` field that carries the job's `runs-on`. The daemon cannot work without it. | +| Gitea server | 1.26+ recommended | Later fixes to Actions job dispatch and runner cleanup. | +| Runner binary in the guest | **`gitea-runner` v3.x** | This is Gitea's own runner. It is *not* the older `act_runner`; do not substitute it. | + +Actions must be enabled on the instance (`[actions] ENABLED = true` in `app.ini`) and on any repo +that will use the runner. + +### 1.2 Create the admin token (PAT) + +The daemon needs a personal access token belonging to a **site administrator** so it can read the +queued-jobs list and delete stale runner registrations. + +1. Sign in as a site-admin user. +2. **Settings → Applications → Manage Access Tokens → Generate New Token.** +3. Grant the token **`read:admin` and `write:admin`** scopes. Nothing else is required. +4. Copy the token immediately — Gitea shows it once. + +Store it on the host in a file readable only by the runner user rather than inline in the config: + +```sh +install -m 600 /dev/null ~/.config/gitea-macos-runner/admin-token +printf '%s' 'PASTE_TOKEN_HERE' > ~/.config/gitea-macos-runner/admin-token +``` + +Then set `gitea.adminTokenFile` to that path. See [security.md](security.md) for why the file form +is preferred. + +### 1.3 Choose a registration token strategy + +Every VM registers itself as a runner before it can accept a job, and registration requires a +registration token. Gitea's registration tokens are **reusable**, and creating a new token for a +given scope **invalidates the previous one** — so a distinct token per VM is impossible by design. +You have two options. + +**Option A (recommended): seed a stable instance-wide token on the Gitea server.** + +Set an environment variable on the Gitea server process before it starts: + +```sh +# Must be at least 32 characters. +GITEA_RUNNER_REGISTRATION_TOKEN=<32+ character random string> +``` + +For example, in a systemd unit: + +```ini +[Service] +Environment=GITEA_RUNNER_REGISTRATION_TOKEN=<32+ character random string> +``` + +or in `docker-compose.yml`: + +```yaml +services: + gitea: + environment: + - GITEA_RUNNER_REGISTRATION_TOKEN=<32+ character random string> +``` + +Restart Gitea, then put the same value on the host in `gitea.registrationTokenFile`. This token is +stable across restarts and is not invalidated by someone clicking "create new token" in the UI for +a different scope. + +**Option B: let the daemon fetch a token via the admin API.** + +Set `gitea.fetchRegistrationTokenViaAPI: true` and omit `gitea.registrationToken`/`…File`. The +daemon requests a token with the admin PAT when it needs one. This is simpler to set up, but any +out-of-band token creation for the same scope invalidates the token the daemon is holding, and the +daemon must re-fetch. Option A is more predictable for an unattended host. + +Generate a token value with: + +```sh +openssl rand -hex 24 +``` + +### 1.4 Target the runner from a workflow + +```yaml +# .gitea/workflows/macos.yml +name: macOS build +on: + push: + branches: [main] + +jobs: + build: + runs-on: macos-arm64 # bare label name — must match runner.labels + steps: + - uses: actions/checkout@v4 + - name: Show host + run: sw_vers && uname -m + - name: Build + run: swift build +``` + +Two things to know about labels: + +- Use **bare label names** in `runs-on`. The runner registers with `--labels "macos-arm64:host"`; + the `:host` suffix is runner-side only and declares that the label executes directly on the host + OS rather than in a container. It never appears in workflow YAML. +- Every label in `runs-on` must be present in the runner's `runner.labels`. A typo means the job + sits queued forever with no error. + +Because no runner exists until a job appears, jobs necessarily wait for a VM to boot. Gitea holds a +queued job for `ABANDONED_JOB_TIMEOUT` (default **24 hours**) before marking it abandoned, so +scale-from-zero is safe. If your queue can legitimately back up for more than a day — a long +maintenance window, for example — raise that setting: + +```ini +[actions] +ABANDONED_JOB_TIMEOUT = 72h +``` + +### 1.5 Optionally restrict which repos may use the runner + +The runner is registered instance-wide by default, meaning any repo whose workflow says +`runs-on: macos-arm64` can execute code on it. If that is broader than you want, register the +runner at the org or repo level instead, or restrict via Gitea's runner settings. See +[security.md](security.md#limiting-who-can-use-the-runner). + +--- + +## 2. Host-side setup + +### 2.1 Host requirements + +| Requirement | Value | +| --- | --- | +| Architecture | Apple Silicon (arm64). Intel Macs cannot run macOS guests. | +| Host macOS | 26 minimum; **27+ strongly recommended** (see below). | +| Guest macOS | 27+ if you want unattended image builds. | +| Free disk | ~60 GB for a vanilla image; **140 GB+** with Xcode installed. | +| RAM | 16 GB minimum; each guest defaults to 8 GB. | +| Concurrent VMs | **2 maximum**, enforced by the macOS kernel. `scheduler.maxConcurrentVMs` must be ≤ 2. | + +The macOS 27 recommendation is not cosmetic. The automated image builder uses +`VZMacGuestProvisioningOptions` — introduced in macOS 27 — to create the admin account and skip +Setup Assistant during first boot. Both host and guest must be 27+. An older guest **silently +ignores** the options: the install succeeds, the VM boots, and then sits at Setup Assistant with no +SSH server, so the build appears to hang. See +[troubleshooting.md](troubleshooting.md). + +### 2.2 Build, sign, and install + +Virtualization.framework refuses to run unless the calling binary carries the +`com.apple.security.virtualization` entitlement, and entitlements are only honoured on a signed +binary inside a proper `.app` bundle. Ad-hoc signing (`codesign -s -`) satisfies this — **no paid +Apple developer account is needed.** + +```sh +git clone && cd gitea-macos-runner +make install +``` + +`make install` runs the full chain and places the result: + +| Target | What it does | +| --- | --- | +| `make build` | `swift build -c release --arch arm64` | +| `make bundle` | Assemble `GiteaMacosRunner.app` around the binary: `Contents/MacOS/gitea-macos-runner`, `Contents/Info.plist`, and `Contents/Resources/` (`provision.sh`, `launchd.plist.template`, `config.example.json`) | +| `make sign` | `codesign --sign - --entitlements …` (ad-hoc) and print the resulting entitlements | +| `make all` | `build` + `bundle` + `sign`. The default target. | +| `make install` | Runs `all`, copies the app to `~/Applications`, and symlinks the CLI to `/usr/local/bin/gitea-macos-runner` | +| `make dev` | Debug build + bundle + sign, for fast iteration. Does **not** install. | +| `make test` | `swift test` | +| `make uninstall` | Remove the installed app and symlink | +| `make clean` | Remove `.build/` | +| `make help` | List the targets | + +The three files under `Contents/Resources/` are not decoration. `image build` +uploads `provision.sh` into the guest, `service install` renders +`launchd.plist.template`, and `config init` writes `config.example.json`. The +code looks in `Contents/Resources` first and only then falls back to +repo-relative paths, so an installed `.app` missing them is a working binary +with three broken commands. + +If `/usr/local/bin` isn't writable, `make install` says so and prints the `sudo ln -sf …` command to +run yourself. + +Never run the raw binary from `.build/release/` — it is outside the signed bundle, so every VM +operation fails with an entitlement error. Always invoke the symlink (or +`~/Applications/GiteaMacosRunner.app/Contents/MacOS/gitea-macos-runner`). + +### 2.3 Create the configuration file + +```sh +gitea-macos-runner config init # add --force to overwrite an existing file +$EDITOR "$(gitea-macos-runner config path)" +gitea-macos-runner config show # parse + validate, printing secrets redacted +``` + +`config init` writes the annotated example shipped in the app bundle, so the +file you land in has a `_comment` key explaining each section; those keys are +ignored when the config is read back. Pass `--instance-url https://…` to seed +`gitea.instanceURL` instead of the placeholder. Every subcommand takes +`--config PATH` (`-c`) if you keep the file somewhere else, and `--verbose`. + +A minimal working config: + +```json +{ + "gitea": { + "instanceURL": "https://gitea.example.com", + "adminTokenFile": "~/.config/gitea-macos-runner/admin-token", + "registrationTokenFile": "~/.config/gitea-macos-runner/registration-token" + }, + "runner": { + "labels": ["macos-arm64"], + "namePrefix": "macos-vm-" + }, + "scheduler": { + "maxConcurrentVMs": 2 + }, + "guest": { + "username": "admin", + "password": "CHANGE_ME", + "cpuCount": 4, + "memoryGB": 8, + "diskGB": 64 + } +} +``` + +#### Configuration reference + +**`gitea`** + +| Key | Default | Description | +| --- | --- | --- | +| `instanceURL` | — (required) | Base URL of the Gitea instance, e.g. `https://gitea.example.com`. Must be `http://` or `https://` with a host. A trailing slash is harmless; a subpath is preserved. | +| `adminToken` | — | Site-admin PAT with `read:admin` + `write:admin`, inline. | +| `adminTokenFile` | — | Path to a file containing the PAT (tilde-expanded). Should be `chmod 600`. | +| `registrationToken` | — | Runner registration token, inline. Prefer `registrationTokenFile`. | +| `registrationTokenFile` | — | Path to a file containing the registration token (tilde-expanded). Takes precedence over `registrationToken` if both are set. | +| `fetchRegistrationTokenViaAPI` | `false` | Fetch a registration token with the admin PAT when no static one is configured. A *fallback*, not an override: a configured token still wins. | + +Set **exactly one** of `adminToken` and `adminTokenFile`. Setting both is a +validation error — a stale inline token next to a live token file is precisely +the ambiguity that turns into a baffling 401 later — and setting neither is too. + +For the registration token the rule is looser: configure `registrationToken`, +`registrationTokenFile`, or `fetchRegistrationTokenViaAPI: true`. At least one +is required; the file form wins over the inline form when both are present. + +**`runner`** + +| Key | Default | Description | +| --- | --- | --- | +| `labels` | `["macos-arm64"]` | Labels this runner offers. A queued job runs here only if its `runs-on` labels are all in this set. **Bare names only** — a `:schema` suffix here is a validation error; the `:host` suffix is appended automatically at registration. | +| `namePrefix` | `"macos-vm-"` | Prefix for generated runner names in the Gitea UI; a unique suffix is appended per VM. | +| `runnerDownloadURL` | `https://gitea.com/gitea/runner/releases/download/v{version}/gitea-runner-{version}-darwin-arm64` | Release asset installed into the guest. `{version}` is substituted with `version`. | +| `version` | `"3.0.2"` | The `gitea-runner` version to install. | + +**`scheduler`** + +| Key | Default | Description | +| --- | --- | --- | +| `maxConcurrentVMs` | `2` | Simultaneous VMs. **Hard-capped at 2 by macOS.** A larger value is silently clamped to 2 at load rather than rejected; the kernel is not something a config file gets to negotiate. | +| `pollIntervalSeconds` | `5` | How often to poll the queued-jobs API. | +| `reconcileIntervalSeconds` | `300` | How often to sweep Gitea for orphaned runner registrations from uncleanly-killed VMs. | +| `jobTimeoutMinutes` | `120` | Wall-clock limit for one job; the VM is destroyed when exceeded. | +| `bootTimeoutSeconds` | `300` | Time allowed from VM start to a usable SSH connection. | + +**`guest`** + +| Key | Default | Description | +| --- | --- | --- | +| `username` | `"admin"` | Guest admin account created during image build. Has passwordless sudo. | +| `password` | — (required) | Password for that account. Also used for SSH if key auth is unavailable. | +| `cpuCount` | `4` | vCPUs per guest. | +| `memoryGB` | `8` | RAM per guest. Two concurrent guests at 8 GB need a 16 GB+ host with headroom. | +| `diskGB` | `64` | Guest disk size. Sparse, so this is a ceiling, not immediate consumption. Raise for Xcode. | + +**`storage`** + +| Key | Default | Description | +| --- | --- | --- | +| `storeDir` | `~/Library/Application Support/gitea-macos-runner` | Base images (`images/`) and running clones (`vms/`) live here. | +| `minFreeDiskGB` | `20` | The scheduler refuses to start a VM below this free-space threshold rather than filling the disk mid-job. | + +### 2.4 Build the base image + +Download a macOS **27 or newer** IPSW for Apple Silicon (Apple's restore images; the URL for the +current release is also discoverable from Virtualization.framework's latest-supported-restore-image +endpoint), then: + +```sh +gitea-macos-runner image build \ + --ipsw ~/Downloads/UniversalMac_27.0_XXXXX_Restore.ipsw \ + --disk-gb 64 +``` + +`--name` defaults to `default`, which is also what `daemon` and `vm boot` look +for. If you name the image something else, pass the same name to +`daemon --image NAME` (and to `vm boot --image NAME`) or the daemon will not +find it. `--ipsw` is optional: omit it and the latest supported restore image is +downloaded into `storeDir/ipsw/` first, which is most of the build's wall-clock +time. `--disk-gb` overrides `guest.diskGB` for this image only. + +This takes a long time — macOS installs from the IPSW, boots, and is then provisioned over SSH. +**Do not interrupt it during the install phase.** Stopping a VM mid-install leaves the disk image +in an undefined state; delete the image and start over rather than trying to resume. + +The finished image contains: + +- macOS installed from the IPSW. +- The `guest.username` admin account, auto-created via provisioning options, with SSH (Remote + Login) enabled and **passwordless sudo**. +- Sleep, screensaver, and Spotlight indexing disabled — a sleeping guest stalls a job, and Spotlight + wastes I/O on a throwaway machine. +- Raised file-descriptor limits, which Node- and Xcode-based builds routinely exhaust at the + default. +- **Node.js — required, not optional.** Gitea Actions' JavaScript actions (`actions/checkout` and + most of the ecosystem) are executed by spawning `node` in the guest. Without it, every workflow + fails on its first step. +- `git`, verified present. +- The `gitea-runner` v3.x binary. + +Verify: + +```sh +gitea-macos-runner image list +``` + +#### Adding Xcode (optional) + +Xcode is not installed automatically. Apple requires an authenticated Apple ID for the download and +its download endpoints are impractical to drive unattended, so you fetch the `.xip` yourself from +[developer.apple.com/download](https://developer.apple.com/download/) and hand it to the +provisioner: + +```sh +gitea-macos-runner image provision default --xcode-xip ~/Downloads/Xcode_XX.xip +``` + +Budget disk accordingly: **~60 GB free is enough for a vanilla image, but plan on 140 GB+ with +Xcode**, and remember each running VM is a copy-on-write clone whose divergence from the base +consumes additional space while it runs. Raise `guest.diskGB` (e.g. to 120) before building if the +image will carry Xcode. + +### 2.5 Install the service + +```sh +gitea-macos-runner service install +gitea-macos-runner service status +``` + +This installs a **LaunchAgent in the logged-in user's GUI session — not a LaunchDaemon.** Two +reasons this is not negotiable: + +1. Virtualization.framework requires a GUI session; a system-context daemon cannot start VMs. +2. macOS 15 and later require an **unlocked `login.keychain`** for key operations the VM lifecycle + performs. In a locked or headless session these fail with `SecKeyCreateRandomKey` / + "Interaction is not allowed" errors. + +**Recommended: enable auto-login for the runner user on a dedicated CI Mac.** +**System Settings → Users & Groups → Automatically log in as → \.** Combine with +disabling sleep (`sudo pmset -a sleep 0 disablesleep 1`) so the Mac comes back into a live session +after a power event without a human present. This does mean the disk is effectively unlocked at +boot — appropriate for a dedicated CI machine, not for a shared workstation. + +`service uninstall` removes the LaunchAgent; it does not delete images or config. + +### 2.6 macOS 15+ Local Network privacy prompt + +Starting with macOS 15, a process that contacts other hosts on the local network triggers a +one-time Local Network permission prompt. A LaunchAgent that is denied (or that never gets a human +to click Allow) cannot reach the guest's NAT address, so VMs boot but SSH never connects. + +Grant it interactively the first time — run `gitea-macos-runner vm boot` from a +Terminal in the GUI session and click **Allow** — or pre-authorize the VM subnet: + +```sh +sudo defaults write com.apple.network.local-network \ + AllowedEthernetLocalNetworkAddresses -array "192.168.0.0/16" +``` + +Adjust the range to match the subnet Virtualization.framework's NAT hands out on your host (check +`/var/db/dhcpd_leases` after a VM boots). Reboot, or restart the service, for the change to take +effect. Also confirm the runner is enabled under **System Settings → Privacy & Security → Local +Network**. + +--- + +## 3. Verification + +### 3.1 Preflight + +```sh +gitea-macos-runner doctor +``` + +It reports one line per check, each `✓` pass, `✗` fail, `!` warn, or `·` note, +with a remediation hint under anything that is not a pass, and exits non-zero if +anything failed. `--json` emits the same results machine-readably; `--no-fail` +exits zero regardless, which is what you want when running it from a script that +handles the results itself. + +The checks, in order: + +| Check | What it looks at | +| --- | --- | +| `host capability` | Apple Silicon, and host macOS ≥ 26 | +| `Virtualization.framework` | `VZVirtualMachine.isSupported` | +| `virtualization entitlement` | `com.apple.security.virtualization` on the *running* executable — this is the check that catches running from `.build/` instead of the signed `.app` | +| `configuration` | The config file loads, parses, and passes validation | +| `free disk space` | Free space on the `storeDir` volume against `storage.minFreeDiskGB` | +| `login.keychain unlocked` | `security show-keychain-info login.keychain` | +| `gitea admin token` / `gitea admin api` | The admin token resolves, and an admin-only endpoint answers with it (so a non-admin PAT fails here rather than at 3am) | +| `registration token` | A static token resolves, or one can be fetched when `fetchRegistrationTokenViaAPI` is on | +| `runner download url` | The `gitea-runner` release asset is reachable | +| `token file permissions` | Warns — not fails — when a token file is group- or world-readable | +| `local network access` | An informational note about the macOS 15+ Local Network prompt | + +If the config file is missing or invalid, the host checks still run and the rest +are skipped — which is exactly the state a first-time operator is in. Resolve +everything it reports before going further. + +There is deliberately **no** "a base image exists" check; use `image list`. + +### 3.2 Boot a VM by hand + +```sh +gitea-macos-runner vm boot +``` + +This clones the base image and boots it without registering a runner — the fastest way to confirm +that virtualization, networking, and SSH all work. Once it is up, check that the guest got a lease: + +```sh +cat /var/db/dhcpd_leases +``` + +and that you can reach it: + +```sh +ssh admin@ 'sw_vers; node --version; git --version; which gitea-runner' +``` + +All four should answer. A guest at Setup Assistant instead of a login window means the image was +built from a pre-27 IPSW; rebuild. + +### 3.3 Watch a real job + +Start the service, push the example workflow from §1.4, and watch both sides: + +```sh +# Host: daemon logs +log stream --predicate 'process == "gitea-macos-runner"' --info + +# Host: VM lifecycle +gitea-macos-runner service status +``` + +In the Gitea UI, the job should move from queued to running within roughly one poll interval plus +boot time; a runner named `macos-vm-` appears under **Site Administration → Actions → +Runners** while the job runs and disappears when it finishes. That disappearance is Gitea deleting +the ephemeral registration itself, and it is the signal that the whole loop worked. + +If the job stays queued, or the runner appears but the job fails immediately, go to +[troubleshooting.md](troubleshooting.md). diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md new file mode 100644 index 0000000..4f80745 --- /dev/null +++ b/docs/troubleshooting.md @@ -0,0 +1,314 @@ +# Troubleshooting + +Start with `gitea-macos-runner doctor` — it catches most misconfiguration before you go +symptom-hunting. Then find your symptom below. + +Useful log commands throughout: + +```sh +# Daemon logs, live +log stream --predicate 'process == "gitea-macos-runner"' --info + +# Daemon logs, last hour +log show --predicate 'process == "gitea-macos-runner"' --info --last 1h + +gitea-macos-runner service status +``` + +--- + +## Quick reference + +| Symptom | Cause | Fix | +| --- | --- | --- | +| VM won't start; entitlement / `com.apple.security.virtualization` error | Running an unsigned binary, or one outside the signed `.app` bundle | `make sign` (or re-run `make install`); invoke the installed bundle, never `.build/release/…` | +| `virtualMachineLimitExceeded` at boot | macOS allows at most **2** concurrent macOS VMs | Set `scheduler.maxConcurrentVMs` ≤ 2; kill stray VMs from earlier runs | +| VM boots but never gets an IP | DHCP lease not yet written, or Local Network privacy denial (macOS 15+) | Check `/var/db/dhcpd_leases`; grant Local Network permission or pre-authorize the subnet | +| SSH times out on a freshly built image | Guest macOS < 27, so provisioning options were ignored and Setup Assistant is waiting | Rebuild the image from a macOS **27+** IPSW | +| `SecKeyCreateRandomKey` / "Interaction is not allowed" | `login.keychain` is locked — no GUI session | Run as a LaunchAgent in an unlocked GUI session; enable auto-login | +| Job stays queued forever | Label mismatch, or the daemon isn't running/reaching Gitea | Use bare label names in `runs-on`; match `runner.labels`; check daemon logs | +| `actions/checkout` fails instantly | Node.js missing from the guest image | `gitea-macos-runner image provision ` | +| Runner rows piling up in the Gitea UI | VMs killed uncleanly; registrations orphaned | Reconcile loop cleans them; force it by restarting the daemon; delete manually if needed | +| Disk filling up | Copy-on-write clones grow as jobs write | Raise `storage.minFreeDiskGB`; delete stale clones in `storeDir/vms` | +| `image build` appears to hang during install | Normal — macOS install is slow | Wait. **Do not stop the VM mid-install**; if you did, delete the image and rebuild | + +--- + +## VM won't start — entitlement error + +**Symptom.** Any VM operation fails immediately with an error naming +`com.apple.security.virtualization`, or a generic "operation not permitted" from +Virtualization.framework. + +**Cause.** Virtualization.framework checks the entitlement on the calling binary, and entitlements +are only honoured on a signed binary inside a proper `.app` bundle. The raw product of +`swift build` has neither. + +**Fix.** + +```sh +make install # build → bundle → sign → install (make sign alone re-signs in place) +codesign -d --entitlements - ~/Applications/GiteaMacosRunner.app # verify +``` + +The output must list `com.apple.security.virtualization`. Then confirm the command you're running +resolves to the installed bundle's binary — `which -a gitea-macos-runner` should point at +`~/Applications/GiteaMacosRunner.app/Contents/MacOS/gitea-macos-runner`, not at +`.build/release/gitea-macos-runner`. Ad-hoc signing is sufficient; you do not need a paid developer +account. + +--- + +## `virtualMachineLimitExceeded` + +**Symptom.** The first VM boots fine; a second or third fails with `virtualMachineLimitExceeded`. + +**Cause.** macOS permits **two** concurrent macOS guests per host. This is an Apple kernel and +licensing limit, not a resource constraint — more RAM will not raise it. + +**Fix.** Set `scheduler.maxConcurrentVMs` to 2 or less. If you're already at 2 and still hitting +the limit, a VM from a previous run is still alive — check for stray processes and for leftover +directories under `storeDir/vms`, then restart the daemon so it starts from a clean state. + +To handle more macOS jobs in parallel, add another Mac. + +--- + +## VM starts but never gets an IP + +**Symptom.** The VM boots (you can see it progress if you use `vm boot`), but the daemon reports +that it could not resolve the guest address, or gives up at `scheduler.bootTimeoutSeconds`. + +**Cause.** The daemon resolves the guest's NAT address from the host's DHCP lease file, which is +only written once the guest requests a lease — several seconds after the boot screen appears. If the +address never appears at all, the usual culprit on macOS 15+ is the **Local Network privacy +prompt**: a LaunchAgent that was never granted permission (or was denied) cannot talk to the guest. + +**Fix.** + +1. Check the lease file after the guest has been up for ~30 seconds: + + ```sh + cat /var/db/dhcpd_leases + ``` + + An entry with a recent `lease` timestamp and the guest's MAC means networking is fine and the + problem is timing — raise `scheduler.bootTimeoutSeconds`. + +2. No entry at all: check **System Settings → Privacy & Security → Local Network** and enable the + runner. If it isn't listed, trigger the prompt interactively from a Terminal in the GUI session: + + ```sh + gitea-macos-runner vm boot + ``` + + and click **Allow**. + +3. Or pre-authorize the VM subnet, then reboot: + + ```sh + sudo defaults write com.apple.network.local-network \ + AllowedEthernetLocalNetworkAddresses -array "192.168.0.0/16" + ``` + + Match the range to what your host's NAT actually hands out. + +--- + +## SSH times out on a freshly built image + +**Symptom.** The VM boots and gets an IP, but SSH never connects. Attaching a display to the guest +shows **Setup Assistant** — the region/Apple ID welcome flow — rather than a login window. + +**Cause.** The image builder uses `VZMacGuestProvisioningOptions` to create the admin account, enable +Remote Login, and skip Setup Assistant. That API requires **macOS 27 or newer in the guest as well +as the host**. An older guest **silently ignores** the options: no error, no account, no SSH server +— it just sits at first-run setup forever. + +**Fix.** Rebuild the base image from a macOS 27+ IPSW: + +```sh +gitea-macos-runner image delete default +gitea-macos-runner image build --ipsw ~/Downloads/UniversalMac_27.0_XXXXX_Restore.ipsw +``` + +Verify the host is also 27+ (`sw_vers`). There is no way to make a pre-27 guest work unattended +with this builder. + +--- + +## `SecKeyCreateRandomKey` / "Interaction is not allowed" + +**Symptom.** The daemon starts but fails during VM setup with a Security-framework error mentioning +`SecKeyCreateRandomKey`, `errSecInteractionNotAllowed`, or "Interaction is not allowed". Often it +works when you run the daemon by hand in Terminal and fails under launchd. + +**Cause.** The **`login.keychain` is locked.** macOS 15+ requires it unlocked for key operations the +VM lifecycle performs, and it is only unlocked inside a live, logged-in GUI session. A LaunchDaemon, +an SSH-only session, or a Mac sitting at the login window all fail this. + +**Fix.** + +1. Confirm the service is installed as a **LaunchAgent**, not a LaunchDaemon: + `gitea-macos-runner service install` does the right thing; a hand-written plist in + `/Library/LaunchDaemons` does not. +2. Ensure the runner user is actually logged in with the desktop loaded. Enable auto-login: + **System Settings → Users & Groups → Automatically log in as**. +3. Prevent the machine from returning to a locked state: + + ```sh + sudo pmset -a sleep 0 disablesleep 1 + ``` + + and disable "Require password after screen saver begins" for the runner user. + +Connecting over Screen Sharing to a Mac at the login window does not unlock `login.keychain` for +launchd's session — auto-login is the reliable answer on a dedicated CI Mac. + +--- + +## Job stays queued and no VM boots + +**Symptom.** The workflow shows as queued in Gitea indefinitely. Nothing appears in the daemon logs +about it. + +**Causes and fixes, in the order worth checking:** + +1. **Label mismatch.** `runs-on` must use **bare label names** (`macos-arm64`), and every label + listed must appear in the host config's `runner.labels`. The `:host` suffix used at registration + is runner-side only and must never appear in workflow YAML. A single typo produces exactly this + symptom with no error anywhere. + +2. **Daemon not running or not polling.** + + ```sh + gitea-macos-runner service status + log show --predicate 'process == "gitea-macos-runner"' --info --last 15m + ``` + + You should see a poll every `scheduler.pollIntervalSeconds`. + +3. **Gitea too old.** The queued-jobs API with the `labels` field requires **Gitea ≥ 1.25**. On an + older instance the daemon can never see jobs. `doctor` reports the server version. + +4. **Admin PAT wrong or under-scoped.** The token must belong to a **site admin** and carry + `read:admin` + `write:admin`. Test it: + + ```sh + curl -H "Authorization: token $TOKEN" \ + "https://gitea.example.com/api/v1/admin/actions/jobs?status=queued" + ``` + + A 403 means scope or admin status; a 404 usually means the Gitea version predates the endpoint. + +5. **A VM booted but its runner never came online.** Then the job is queued *and* you see VM + activity in the logs. Look at registration failures — most often an invalid or invalidated + registration token (see below). + +6. **Job expired.** Gitea abandons a job after `ABANDONED_JOB_TIMEOUT` (default 24h). If the daemon + was down longer than that, the job is gone; re-run it. + +### Registration fails with an invalid token + +Registration tokens are reusable, but **creating a new token for a scope invalidates the previous +one**. If someone clicked "create new registration token" in the Gitea UI, the token in your +`registrationTokenFile` is now dead. Re-seed +`GITEA_RUNNER_REGISTRATION_TOKEN` on the server (and restart Gitea), or switch to +`fetchRegistrationTokenViaAPI: true`. See +[setup.md §1.3](setup.md#13-choose-a-registration-token-strategy). + +--- + +## `actions/checkout` fails instantly + +**Symptom.** The job starts, the runner connects, and the very first step fails immediately — +typically a spawn error naming `node`, or an unhelpful non-zero exit before any output. + +**Cause.** Gitea Actions' JavaScript actions (`actions/checkout` and most of the ecosystem) run by +spawning `node` inside the guest. **Node.js is required in the image**, and if provisioning was +interrupted it may be absent. + +**Fix.** Confirm, then reprovision: + +```sh +gitea-macos-runner vm boot +ssh admin@ 'node --version && git --version' + +gitea-macos-runner image provision default +``` + +If `git` is also missing, the provisioning step failed early — check the build log and rerun +provisioning. + +--- + +## Runner rows piling up in the Gitea UI + +**Symptom.** **Site Administration → Actions → Runners** accumulates offline `macos-vm-…` entries. + +**Cause.** Gitea deletes an ephemeral registration when its job completes normally. A VM that is +killed uncleanly — daemon crash, host power loss, `jobTimeoutMinutes` kill — never reaches that +point, so the row is orphaned. + +**Fix.** Usually nothing: the daemon's reconcile loop sweeps orphaned registrations every +`scheduler.reconcileIntervalSeconds` (default 300) via +`DELETE /api/v1/admin/actions/runners/{id}`, plus a daily midnight sweep. Orphans should clear +within a few minutes. + +If they persist, the daemon's admin PAT probably lacks `write:admin` — check the logs for delete +failures. To clear them by hand, delete the rows in the Gitea UI; they are inert (offline +registrations that have already been spent cannot receive jobs). + +--- + +## Disk filling up + +**Symptom.** Free space falls steadily; the daemon starts refusing to launch VMs, citing +`storage.minFreeDiskGB`. + +**Cause.** Each VM is an APFS copy-on-write clone of the base image. The clone is free at creation +but **grows as the job writes** — dependency caches, build outputs, Xcode's derived data. Clones +from uncleanly-killed VMs are not reclaimed automatically. + +**Fix.** + +```sh +# See what's there. +du -sh ~/Library/Application\ Support/gitea-macos-runner/* +ls -la ~/Library/Application\ Support/gitea-macos-runner/vms + +# With the daemon stopped, remove stale clones. +gitea-macos-runner service uninstall # or stop the daemon +rm -rf ~/Library/Application\ Support/gitea-macos-runner/vms/ +gitea-macos-runner service install +``` + +Only delete entries under `vms/` — `images/` holds the base images you'd otherwise have to rebuild. +Longer term: raise `storage.minFreeDiskGB` so the guard trips earlier, delete unused base images +with `image delete`, and remember an Xcode image needs 140 GB+ of headroom, more with two +concurrent clones diverging. + +--- + +## `image build` hangs at install + +**Symptom.** `image build` sits for a long time at the macOS install phase with little visible +progress. + +**Cause.** Usually none — installing macOS from an IPSW genuinely takes a long time (tens of +minutes, longer on slower storage). The install phase is largely silent. + +**Fix.** **Wait, and do not stop the VM mid-install.** Interrupting the installer leaves the disk +image in an undefined state; the resulting image may boot and then fail in confusing ways later. +There is no resume. + +If you did interrupt it, or the build genuinely failed: + +```sh +gitea-macos-runner image delete +gitea-macos-runner image build --ipsw --name +``` + +Before rebuilding, verify the IPSW is complete and matches your host architecture (Apple Silicon) +and version requirement (macOS 27+ for unattended provisioning), and that you have enough free disk +for the IPSW plus the target disk size.