Files
nucleic/Sources/NucleicCore/DelegatedIntelligence.swift
T
abkslmandClaude Opus 4.8 de53ee9d01 Unwrap AFM tool-calling scaffolding; retune two prompts for macOS 27 beta 4
The Foundation Models seed in macOS 27 beta 4 (26A5388g) answers a plain
`respond(to:)` as if it were driving a tool-calling harness, so free-form
replies now arrive wrapped: a bare JSON object, a fenced ```json block, a
`tool_call: {…}` literal, `[start_x]`/`[No tools needed]` markers, or a
persona preamble ahead of the answer. Every consumer takes the reply's
first non-empty line, so the wrapper landed verbatim in chat titles and
collapsed summaries — dev even carries a commit named
"Nucleic: {task:Update Website Hero Text}". Guided generation
(@Generable) still binds its schema and is untouched.

Add ModelOutput to NucleicProtocol — the one layer the Mac app, core and
the iOS client all link — and run it at each provider's text choke point,
before the existing line-oriented parsing. Plain replies pass through
byte-identical; an all-scaffolding reply yields nil, which is the "no
result" every caller already handles by falling back to its heuristic, so
no new failure path. DelegatedIntelligence unwraps again on receipt: the
mesh is mixed-version and the agent-CLI backend never unwraps.

Also retune the two prompts the seed broke, measured against the live
on-device model rather than by inspection:

- classifyTurn: the old wording let the agentically-tuned seed reason
  about what the agent should do NEXT, so it answered AWAITING for
  plainly finished turns — 12/17 on a labeled set (3 samples, majority
  vote) with 4 false AWAITING, which silently stops autoship. Reframing
  it as "a classifier, NOT an assistant" scores 17/17 with none. Guided
  generation was tried here and is worse (81% struct, 68% enum). Applied
  to both hand-duplicated twins.
- sessionKeywords: 9 of 9 runs returned a JSON/tool-call envelope, one
  inventing a fake Package.swift to "search". Anchoring extraction to
  terms already in the input returns 9 of 9 clean comma lists.

Tests pin the captured beta 4 shapes verbatim and the classifier clauses
that earned the accuracy, since they read like boilerplate.

Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
2026-07-22 09:14:35 +00:00

363 lines
18 KiB
Swift
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import Foundation
import NucleicProtocol
// The runner's replacement for Apple Foundation Models (docs/COVALENCE_RUNNER.md §5, item 6):
// one `IntelligenceProviding` implementation that renders each call into a
// `WireIntelligenceRequest`, runs it through a text-generation backend — the mesh-wide AFM
// queue (`nucleic.runner.intelligenceMode = mesh`) or a one-shot agent CLI (`agent`) — and
// validates the answer with the same heuristics-as-guardrails discipline the AFM provider
// uses, falling back to `HeuristicIntelligence` whenever the backend can't serve.
// MARK: - Backends
/// Where a delegated intelligence request gets its text generated. `nil` means "couldn't
/// serve" (no eligible device, deadline, CLI failure) — the provider falls back to heuristics.
public protocol IntelligenceGenerationBackend: Sendable {
func run(_ request: WireIntelligenceRequest) async -> WireIntelligenceResult?
}
/// Mode `mesh`: hand the request to the mesh-wide AFM queue, which places it on a connected
/// Apple-Intelligence-capable device per the priority/device-class rules.
public struct MeshIntelligenceBackend: IntelligenceGenerationBackend {
public let queue: MeshIntelligenceQueue
public init(queue: MeshIntelligenceQueue) { self.queue = queue }
public func run(_ request: WireIntelligenceRequest) async -> WireIntelligenceResult? {
await queue.submit(request)
}
}
/// Mode `agent`: render the shared template and run it as ONE non-interactive agent turn
/// (`claude -p … --model <small SKU> --output-format text`) — the credential mesh already put
/// the agent's login on the runner. Work is serialized through a private `AFMRequestQueue` so
/// a burst of soft-AI jobs can't fork a pile of CLI processes, and each invocation is killed
/// at its deadline so a hung CLI can't wedge the queue.
public struct AgentCLIIntelligenceBackend: IntelligenceGenerationBackend {
public let executable: String
public let model: String
/// Serializes CLI invocations (and orders a backlog by the wire priority).
private let queue: AFMRequestQueue
public init(executable: String = "claude", model: String = "haiku") {
self.executable = executable
self.model = model
self.queue = AFMRequestQueue(maxConcurrent: 1)
}
public func run(_ request: WireIntelligenceRequest) async -> WireIntelligenceResult? {
guard let job = IntelligenceDelegate.job(for: request) else {
return WireIntelligenceResult(
id: request.id, error: "unsupported kind: \(request.kind.rawValue)")
}
let deadline = request.deadlineSeconds ?? 60
let executable = self.executable
let model = self.model
let text = await queue.run(
priority: AFMRequestQueue.Priority(wire: request.priority),
label: "Agent \(request.kind.rawValue)", detail: job.prompt,
resultText: { $0?.trimmingCharacters(in: .whitespacesAndNewlines) }
) {
await Self.oneShot(executable: executable, model: model, job: job, deadline: deadline)
}
guard let text else { return nil }
return WireIntelligenceResult(id: request.id, outputs: [text])
}
/// One print-mode turn: instructions + prompt as the single prompt argument, plain-text
/// output, SIGKILL at the deadline. Any failure (CLI missing, non-zero exit, empty reply)
/// is nil — the provider's heuristic fallback covers it.
private static func oneShot(
executable: String, model: String, job: IntelligenceDelegate.Job, deadline: Double
) async -> String? {
let spec = ProcessSpec(
executable: executable,
args: [
"-p", "\(job.instructions)\n\n\(job.prompt)",
"--model", model, "--output-format", "text",
],
cwd: NSTemporaryDirectory(), stdinMode: .closed)
guard let handle = try? await ProcessHost().launch(spec) else { return nil }
let killer = Task {
try? await Task.sleep(for: .seconds(deadline))
guard !Task.isCancelled else { return }
handle.sendSignal(SIGKILL)
}
defer { killer.cancel() }
var lines: [String] = []
do {
for try await line in handle.stdoutLines {
lines.append(String(decoding: line, as: UTF8.self))
}
} catch {}
guard await handle.wait() == 0 else { return nil }
let text = lines.joined(separator: "\n").trimmingCharacters(in: .whitespacesAndNewlines)
return text.isEmpty ? nil : text
}
}
extension AFMRequestQueue.Priority {
/// The local-queue slot for a wire priority tier; unknown/omitted tiers run as background.
public init(wire: IntelligencePriority?) {
switch wire ?? .background {
case .interactive: self = .interactive
case .bashSummary: self = .bashSummary
case .completion: self = .completion
default: self = .background
}
}
}
// MARK: - Device-side executor
/// Runs one delegated request on THIS device's provider (the executor half of §5): render the
/// shared template, generate through `IntelligenceProviding.generateText` at the wire-carried
/// priority, and answer — with `error` set whenever no model could serve, so the delegating
/// host falls back to heuristics immediately instead of waiting out its deadline.
public enum IntelligenceDelegateExecutor {
public static func execute(
_ request: WireIntelligenceRequest, provider: any IntelligenceProviding
) async -> WireIntelligenceResult {
guard let job = IntelligenceDelegate.job(for: request) else {
return WireIntelligenceResult(
id: request.id, error: "unsupported kind: \(request.kind.rawValue)")
}
let text = await provider.generateText(
instructions: job.instructions, prompt: job.prompt,
priority: AFMRequestQueue.Priority(wire: request.priority))
guard let text, !text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
return WireIntelligenceResult(id: request.id, error: "no local model available")
}
return WireIntelligenceResult(id: request.id, outputs: [text])
}
}
// MARK: - Provider
/// `IntelligenceProviding` over a delegated text backend — the runner's `agent` and `mesh`
/// intelligence modes (§5 modes 12). Every method mirrors `AppleIntelligenceProvider`'s
/// shape: encode → generate → validate with the deterministic heuristics → fall back to the
/// model-free answer. `auditHostExecReason` stays fail-closed: no backend answer is
/// `.notChecked`, and a garbled verdict token is `.unclear` — never a fabricated "consistent".
public struct DelegatedIntelligenceProvider: IntelligenceProviding {
private let backend: any IntelligenceGenerationBackend
public init(backend: any IntelligenceGenerationBackend) {
self.backend = backend
}
/// Mode `mesh`: delegate to Apple-Intelligence-capable mesh devices via the queue.
public static func mesh(queue: MeshIntelligenceQueue) -> DelegatedIntelligenceProvider {
DelegatedIntelligenceProvider(backend: MeshIntelligenceBackend(queue: queue))
}
/// Mode `agent` (the doc's `AgentIntelligenceProvider`): a small/cheap agent SKU behind
/// strict prompt templates.
public static func agent(
executable: String = "claude", model: String = "haiku"
) -> DelegatedIntelligenceProvider {
DelegatedIntelligenceProvider(
backend: AgentCLIIntelligenceBackend(executable: executable, model: model))
}
/// How long each tier is worth waiting on before the heuristic renders instead. Interactive
/// work is watched live, so it gives up fastest; background work can afford a long queue.
static func deadline(for priority: IntelligencePriority) -> Double {
switch priority {
case .interactive: 20
case .bashSummary: 30
case .completion: 45
default: 90
}
}
/// Encode one call, run it, and hand back the raw output text — nil whenever the backend
/// couldn't serve (every caller falls back to its deterministic answer).
private func generate(
_ kind: IntelligenceRequestKind, inputs: [String], context: String? = nil,
priority: IntelligencePriority
) async -> String? {
let request = WireIntelligenceRequest(
id: UUID().uuidString, kind: kind, inputs: inputs, context: context,
deadlineSeconds: Self.deadline(for: priority), priority: priority)
guard let result = await backend.run(request), result.error == nil else { return nil }
// Unwrap again on receipt even though a current phone worker already did: the mesh is
// mixed-version (a peer on an older build still sends the raw tool-calling envelope), and
// the agent-CLI backend never unwraps at all. `unwrap` leaves plain text alone, so running
// it twice costs nothing and closes both gaps.
return ModelOutput.usableText(result.outputs.joined(separator: "\n"))
}
/// The first non-empty line of a reply, stripped of leading list/bullet markers — the
/// single glanceable line most kinds want (shared with the AFM provider).
private func firstLine(of raw: String) -> String {
HeuristicTitle.firstModelLine(of: raw)
}
// MARK: IntelligenceProviding
public func summarize(session: Session, events: [AgentEvent]) async -> String {
guard events.contains(where: {
if case .userText = $0.kind { return true } else { return false }
}) else {
return HeuristicSummary.text(session: session, events: events)
}
let digest = HeuristicSummary.sessionDigest(events)
if let text = await generate(.summarizeSession, inputs: [digest], priority: .completion) {
return text
}
return HeuristicSummary.text(session: session, events: events)
}
public func sessionName(fromFirstMessage message: String) async -> String? {
if let raw = await generate(.sessionName, inputs: [message], priority: .interactive),
let name = HeuristicTitle.sanitizeModelTitle(raw), HeuristicTitle.looksLikeTitle(name) {
return name
}
return HeuristicTitle.fromMessage(message)
}
public func sessionName(fromFirstMessage message: String, context digest: String) async -> String? {
if let raw = await generate(
.sessionName, inputs: [message], context: digest, priority: .background),
let name = HeuristicTitle.sanitizeModelTitle(raw), HeuristicTitle.looksLikeTitle(name) {
return name
}
return nil
}
public func summarizeTodos(_ items: [String], group: String) async -> String {
let cleaned = items
.map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }
.filter { !$0.isEmpty }
guard !cleaned.isEmpty else { return "" }
guard cleaned.count > 1 else { return HeuristicSummary.todoGist(cleaned) }
// `group` drives only per-group coalescing on the AFM path; the template doesn't read
// it, so it stays off the wire.
if let text = await generate(.summarizeTodos, inputs: cleaned, priority: .background) {
let line = firstLine(of: text)
if !line.isEmpty { return line }
}
return HeuristicSummary.todoGist(cleaned)
}
public func summarizeTodo(_ text: String) async -> String {
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return "" }
guard trimmed.split(whereSeparator: { $0 == " " || $0.isNewline }).count > 8 else {
return HeuristicSummary.todoLine(trimmed)
}
if let out = await generate(.summarizeTodo, inputs: [trimmed], priority: .background) {
let line = firstLine(of: out)
if !line.isEmpty { return line }
}
return HeuristicSummary.todoLine(trimmed)
}
public func classifyTurn(lastReply: String) async -> TurnDisposition {
let trimmed = lastReply.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return .completed }
let tail = String(trimmed.suffix(600))
if let raw = await generate(.classifyTurn, inputs: [tail], priority: .completion) {
let upper = raw.uppercased()
if upper.contains("AWAIT") { return .awaitingInput }
if upper.contains("DONE") { return .completed }
}
return HeuristicTurnClassifier.classify(lastReply)
}
public func triageTodos(_ items: [TriageInput], group: String) async -> [TriageItem] {
guard !items.isEmpty else { return [] }
guard items.count > 1 else { return HeuristicTriage.triage(items) }
// Like `summarizeTodos`, `group` only keys AFM-side coalescing — not sent.
guard let raw = await generate(
.triageTodos, inputs: items.map(\.text), priority: .background)
else {
return HeuristicTriage.triage(items)
}
// Parse the strict "N|LEVEL|reason" lines the template demands; anything malformed is
// skipped, and `fromModelRanking` itself falls back to the heuristic ranking when
// nothing valid survives.
let verdicts = raw.split(whereSeparator: \.isNewline).compactMap { line -> HeuristicTriage.ModelVerdict? in
let parts = line.split(separator: "|", maxSplits: 2, omittingEmptySubsequences: false)
.map { $0.trimmingCharacters(in: .whitespaces) }
guard parts.count >= 2,
let number = Int(parts[0].trimmingCharacters(in: CharacterSet(charactersIn: ".) ")))
else { return nil }
return HeuristicTriage.ModelVerdict(
number: number, level: parts[1], reason: parts.count > 2 ? parts[2] : "")
}
guard !verdicts.isEmpty else { return HeuristicTriage.triage(items) }
return HeuristicTriage.fromModelRanking(verdicts, items: items)
}
public func summarizeBashCommand(_ command: String) async -> String {
guard let raw = await generate(
.summarizeBashCommand, inputs: [command], priority: .bashSummary)
else { return command }
let line = firstLine(of: raw)
// Same guardrail as the AFM provider: a parroted command isn't a summary.
if line.isEmpty || HeuristicSummary.echoesCommands(line, commands: [command]) {
return command
}
return line
}
public func mergeBashSummaries(_ summaries: [String]) async -> String {
let fallback = HeuristicSummary.joinBashSummaries(summaries)
guard summaries.count > 1 else { return summaries.first.map(firstLine) ?? fallback }
guard let raw = await generate(
.mergeBashSummaries, inputs: summaries, priority: .bashSummary)
else { return fallback }
// Reject an ungrounded line (a parroted instruction example) for the deterministic merge.
let line = firstLine(of: raw)
if line.isEmpty || !HeuristicSummary.isGrounded(line, in: summaries) { return fallback }
return line
}
public func summarizeToolFamily(_ calls: [ToolCall], worktreeRoot: String?) async -> [String] {
let families = HeuristicSummary.families(calls)
let fallback = families.map { HeuristicSummary.line(for: $0, relativeTo: worktreeRoot) }
let actions = calls.compactMap { HeuristicSummary.actionDescription($0) }
guard !actions.isEmpty else { return fallback }
// The template routes on the verb: an all-Grep family takes the search-tailored prompt.
let isSearch = calls.allSatisfy { $0.name == "Grep" }
let verb = isSearch ? IntelligenceDelegate.searchFamilyVerb : (families.first?.verb ?? "")
guard let raw = await generate(
.summarizeToolFamily, inputs: actions, context: verb, priority: .background)
else { return fallback }
let lines = raw.split(whereSeparator: \.isNewline)
.map { $0.trimmingCharacters(in: CharacterSet(charactersIn: " -•*\t")) }
.filter { !$0.isEmpty }
return lines.isEmpty ? fallback : lines
}
public func updateTrunkSummary(previous: String, with digest: String) async -> String {
let prev = previous.trimmingCharacters(in: .whitespacesAndNewlines)
let digestTrimmed = digest.trimmingCharacters(in: .whitespacesAndNewlines)
guard !digestTrimmed.isEmpty else { return prev }
if let raw = await generate(
.updateTrunkSummary, inputs: [prev, digestTrimmed], priority: .background) {
let line = firstLine(of: raw)
if !line.isEmpty { return line }
}
return HeuristicSummary.foldTrunkSummary(previous: previous, digest: digest)
}
public func auditHostExecReason(command: String, reason: String) async -> HostExecReasonAudit {
let cmd = command.trimmingCharacters(in: .whitespacesAndNewlines)
let why = reason.trimmingCharacters(in: .whitespacesAndNewlines)
guard !cmd.isEmpty, !why.isEmpty else { return .notChecked }
// Fail-closed both ways: no answer → `.notChecked` (never a spoofable clean bill), and
// an unrecognized verdict token → `.unclear` via `trust(fromModelToken:)`.
guard let raw = await generate(
.auditHostExecReason, inputs: [cmd, why], priority: .interactive)
else { return .notChecked }
let line = firstLine(of: raw)
guard !line.isEmpty else { return .notChecked }
let parts = line.split(separator: "|", maxSplits: 1, omittingEmptySubsequences: false)
.map { $0.trimmingCharacters(in: .whitespaces) }
return HostExecReasonAudit(
trust: HostExecReasonAudit.trust(fromModelToken: parts[0]),
rationale: parts.count > 1 ? parts[1] : "")
}
}