Merge nucleic/sleek-ember-seal-uady into dev

purpose_data.py

@@ -364,6 +364,73 @@ def curate_records(
     return CurationResult(records=kept, duplicates=duplicates)
 
 
+def exclude_reviewed_duplicates(
+    records: Sequence[SourceRecord],
+    decisions: Sequence[dict[str, Any]],
+) -> CurationResult:
+    """Apply human-reviewed semantic exclusions, failing closed on corpus drift."""
+
+    by_hash = {prompt_hash(record.value["prompt"]): record for record in records}
+    if len(by_hash) != len(records):
+        raise DataError("reviewed exclusions require an exact-deduplicated population")
+
+    drops: dict[str, tuple[str, float]] = {}
+    for index, decision in enumerate(decisions, 1):
+        try:
+            dropped_hash = decision["droppedPromptHash"]
+            matched_hash = decision["matchedPromptHash"]
+            similarity = float(decision["similarity"])
+        except (KeyError, TypeError, ValueError) as exc:
+            raise DataError(
+                f"semantic exclusion {index}: invalid decision fields"
+            ) from exc
+        if (
+            not isinstance(dropped_hash, str)
+            or not isinstance(matched_hash, str)
+            or len(dropped_hash) != 64
+            or len(matched_hash) != 64
+            or not 0.0 <= similarity <= 1.0
+        ):
+            raise DataError(f"semantic exclusion {index}: invalid hashes/similarity")
+        if dropped_hash == matched_hash:
+            raise DataError(f"semantic exclusion {index}: cannot match itself")
+        if dropped_hash in drops:
+            raise DataError(f"semantic exclusion {index}: duplicate dropped hash")
+        drops[dropped_hash] = (matched_hash, similarity)
+
+    missing = sorted((set(drops) | {item[0] for item in drops.values()}) - set(by_hash))
+    if missing:
+        raise DataError(
+            "reviewed semantic exclusion no longer matches the curated corpus: "
+            + ", ".join(missing)
+        )
+
+    kept = []
+    duplicates = []
+    for record in records:
+        dropped_hash = prompt_hash(record.value["prompt"])
+        decision = drops.get(dropped_hash)
+        if decision is None:
+            kept.append(record)
+            continue
+        matched_hash, similarity = decision
+        matched = by_hash[matched_hash]
+        if record.value["purpose"] != matched.value["purpose"]:
+            raise DataError(
+                "reviewed semantic duplicate labels no longer agree: "
+                f"{dropped_hash} vs {matched_hash}"
+            )
+        duplicates.append(
+            Duplicate(
+                dropped=record,
+                matched_prompt_hash=matched_hash,
+                kind="semantic-reviewed",
+                similarity=similarity,
+            )
+        )
+    return CurationResult(records=kept, duplicates=duplicates)
+
+
 def _stable_digest(seed: int, prompt: str) -> str:
     material = f"{seed}\0{normalized_key(prompt)}".encode("utf-8")
     return hashlib.sha256(material).hexdigest()