Merge nucleic/sleek-ember-seal-uady into dev
This commit is contained in:
@@ -0,0 +1,493 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fail-closed contract for purpose-classifier human review artifacts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from collections import Counter, defaultdict
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Sequence
|
||||
|
||||
from purpose_data import (
|
||||
LABEL_SET,
|
||||
SLICES,
|
||||
DataError,
|
||||
SourceRecord,
|
||||
file_sha256,
|
||||
normalize_prompt,
|
||||
prompt_hash,
|
||||
validate_source_record,
|
||||
write_json,
|
||||
)
|
||||
|
||||
|
||||
REVIEW_CSV_FIELDS = (
|
||||
"promptHash",
|
||||
"source",
|
||||
"line",
|
||||
"prompt",
|
||||
"generatedPurpose",
|
||||
"generatedSecondary",
|
||||
"generatedMixed",
|
||||
"generatedDifficulty",
|
||||
"generatedSlice",
|
||||
"generatedLanguage",
|
||||
"reviewedPurpose",
|
||||
"reviewedSecondary",
|
||||
"reviewedDifficulty",
|
||||
"reviewedSlice",
|
||||
"reviewStatus",
|
||||
"reviewNotes",
|
||||
)
|
||||
REVIEW_STATUSES = frozenset({"accept", "relabel", "reject"})
|
||||
NONE_SENTINELS = frozenset({"<none>", "none", "null"})
|
||||
CHANGE_FIELDS = frozenset({"purpose", "secondary", "difficulty", "slice"})
|
||||
DEFAULT_SAMPLE_SEED = 0xA11D17
|
||||
DEFAULT_SAMPLE_FRACTION = 0.10
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReviewProgress:
|
||||
records: int
|
||||
accepted: int
|
||||
relabeled: int
|
||||
rejected: int
|
||||
incomplete: int
|
||||
decisions: list[dict[str, Any]]
|
||||
|
||||
@property
|
||||
def completed(self) -> int:
|
||||
return self.records - self.incomplete
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HumanReviewResult:
|
||||
records: list[SourceRecord]
|
||||
summary: dict[str, Any]
|
||||
|
||||
|
||||
def _stable_rank(seed: int, record: SourceRecord) -> str:
|
||||
material = f"{seed}\0{prompt_hash(record.value['prompt'])}".encode("utf-8")
|
||||
return hashlib.sha256(material).hexdigest()
|
||||
|
||||
|
||||
def _review_stratum(record: SourceRecord) -> tuple[str, str, str]:
|
||||
language = record.value["lang"].split("-", 1)[0].casefold()
|
||||
return record.value["purpose"], record.value["slice"], language
|
||||
|
||||
|
||||
def stratified_review_sample(
|
||||
records: Sequence[SourceRecord],
|
||||
*,
|
||||
fraction: float,
|
||||
seed: int,
|
||||
) -> list[SourceRecord]:
|
||||
"""Choose exactly round(N*fraction), apportioned by purpose/slice/language."""
|
||||
|
||||
if not records:
|
||||
raise DataError("cannot sample an empty review population")
|
||||
if not 0.0 < fraction <= 1.0:
|
||||
raise DataError("review fraction must be in (0, 1]")
|
||||
|
||||
target = round(len(records) * fraction)
|
||||
groups: dict[tuple[str, str, str], list[SourceRecord]] = defaultdict(list)
|
||||
for record in records:
|
||||
groups[_review_stratum(record)].append(record)
|
||||
|
||||
allocations: dict[tuple[str, str, str], int] = {}
|
||||
remainders: list[tuple[float, str, tuple[str, str, str]]] = []
|
||||
allocated = 0
|
||||
for key in sorted(groups):
|
||||
quota = len(groups[key]) * target / len(records)
|
||||
base = math.floor(quota)
|
||||
allocations[key] = base
|
||||
allocated += base
|
||||
tie_break = hashlib.sha256(f"{seed}\0{key}".encode("utf-8")).hexdigest()
|
||||
remainders.append((quota - base, tie_break, key))
|
||||
for _, _, key in sorted(remainders, reverse=True)[: target - allocated]:
|
||||
allocations[key] += 1
|
||||
|
||||
selected: list[SourceRecord] = []
|
||||
for key in sorted(groups):
|
||||
ordered = sorted(groups[key], key=lambda record: _stable_rank(seed, record))
|
||||
selected.extend(ordered[: allocations[key]])
|
||||
return sorted(selected, key=lambda record: _stable_rank(seed + 1, record))
|
||||
|
||||
|
||||
def _review_row(
|
||||
record: SourceRecord,
|
||||
source_formatter: Callable[[Path], str],
|
||||
) -> dict[str, str]:
|
||||
value = record.value
|
||||
return {
|
||||
"promptHash": prompt_hash(value["prompt"]),
|
||||
"source": source_formatter(record.source),
|
||||
"line": str(record.line),
|
||||
"prompt": normalize_prompt(value["prompt"]),
|
||||
"generatedPurpose": value["purpose"],
|
||||
"generatedSecondary": value["secondary"] or "",
|
||||
"generatedMixed": str(value["mixed"]).lower(),
|
||||
"generatedDifficulty": str(value["difficulty"]),
|
||||
"generatedSlice": value["slice"],
|
||||
"generatedLanguage": value["lang"],
|
||||
"reviewedPurpose": "",
|
||||
"reviewedSecondary": "",
|
||||
"reviewedDifficulty": "",
|
||||
"reviewedSlice": "",
|
||||
"reviewStatus": "",
|
||||
"reviewNotes": "",
|
||||
}
|
||||
|
||||
|
||||
def write_review_csv(
|
||||
path: Path,
|
||||
records: Sequence[SourceRecord],
|
||||
*,
|
||||
source_formatter: Callable[[Path], str] = str,
|
||||
) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8", newline="") as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=REVIEW_CSV_FIELDS)
|
||||
writer.writeheader()
|
||||
for record in records:
|
||||
writer.writerow(_review_row(record, source_formatter))
|
||||
|
||||
|
||||
def _parse_review_changes(
|
||||
row: dict[str, str],
|
||||
record: SourceRecord,
|
||||
*,
|
||||
row_number: int,
|
||||
) -> dict[str, Any]:
|
||||
value = record.value
|
||||
changes: dict[str, Any] = {}
|
||||
|
||||
purpose = row["reviewedPurpose"].strip()
|
||||
if purpose:
|
||||
if purpose not in LABEL_SET:
|
||||
raise DataError(f"review CSV row {row_number}: invalid reviewed purpose")
|
||||
if purpose != value["purpose"]:
|
||||
changes["purpose"] = purpose
|
||||
|
||||
secondary_text = row["reviewedSecondary"].strip()
|
||||
secondary = value["secondary"]
|
||||
if secondary_text:
|
||||
if secondary_text.casefold() in NONE_SENTINELS:
|
||||
secondary = None
|
||||
elif secondary_text in LABEL_SET:
|
||||
secondary = secondary_text
|
||||
else:
|
||||
raise DataError(f"review CSV row {row_number}: invalid reviewed secondary")
|
||||
if secondary != value["secondary"]:
|
||||
changes["secondary"] = secondary
|
||||
|
||||
difficulty_text = row["reviewedDifficulty"].strip()
|
||||
if difficulty_text:
|
||||
try:
|
||||
difficulty = float(difficulty_text)
|
||||
except ValueError as exc:
|
||||
raise DataError(
|
||||
f"review CSV row {row_number}: invalid reviewed difficulty"
|
||||
) from exc
|
||||
if not math.isfinite(difficulty) or not 0.0 <= difficulty <= 1.0:
|
||||
raise DataError(
|
||||
f"review CSV row {row_number}: reviewed difficulty must be in [0, 1]"
|
||||
)
|
||||
if difficulty != float(value["difficulty"]):
|
||||
changes["difficulty"] = difficulty
|
||||
|
||||
slice_name = row["reviewedSlice"].strip()
|
||||
if slice_name:
|
||||
if slice_name not in SLICES:
|
||||
raise DataError(f"review CSV row {row_number}: invalid reviewed slice")
|
||||
if slice_name != value["slice"]:
|
||||
changes["slice"] = slice_name
|
||||
|
||||
candidate = dict(value)
|
||||
candidate.update(changes)
|
||||
candidate["mixed"] = candidate["secondary"] is not None
|
||||
validate_source_record(candidate, f"review CSV row {row_number}")
|
||||
return changes
|
||||
|
||||
|
||||
def inspect_review_csv(
|
||||
path: Path,
|
||||
expected_sample: Sequence[SourceRecord],
|
||||
*,
|
||||
source_formatter: Callable[[Path], str] = str,
|
||||
) -> ReviewProgress:
|
||||
try:
|
||||
with path.open("r", encoding="utf-8", newline="") as handle:
|
||||
reader = csv.DictReader(handle)
|
||||
if tuple(reader.fieldnames or ()) != REVIEW_CSV_FIELDS:
|
||||
raise DataError(
|
||||
f"{path}: review CSV fields do not match the current schema; "
|
||||
"regenerate the blank artifact before reviewing"
|
||||
)
|
||||
rows = list(reader)
|
||||
except (OSError, UnicodeError, csv.Error) as exc:
|
||||
raise DataError(f"{path}: cannot read review CSV: {exc}") from exc
|
||||
|
||||
expected_by_hash = {
|
||||
prompt_hash(record.value["prompt"]): record for record in expected_sample
|
||||
}
|
||||
if len(expected_by_hash) != len(expected_sample):
|
||||
raise DataError("human-review sample contains duplicate prompt hashes")
|
||||
if len(rows) != len(expected_sample):
|
||||
raise DataError(
|
||||
f"{path}: expected {len(expected_sample)} review rows, found {len(rows)}"
|
||||
)
|
||||
|
||||
seen: set[str] = set()
|
||||
decisions: list[dict[str, Any]] = []
|
||||
counts: Counter[str] = Counter()
|
||||
incomplete = 0
|
||||
for row_number, row in enumerate(rows, 2):
|
||||
if None in row or any(row.get(field) is None for field in REVIEW_CSV_FIELDS):
|
||||
raise DataError(f"{path}:{row_number}: malformed review CSV column count")
|
||||
row_hash = row["promptHash"].strip()
|
||||
if row_hash in seen:
|
||||
raise DataError(f"{path}:{row_number}: duplicate promptHash {row_hash}")
|
||||
seen.add(row_hash)
|
||||
record = expected_by_hash.get(row_hash)
|
||||
if record is None:
|
||||
raise DataError(
|
||||
f"{path}:{row_number}: promptHash is not in the deterministic sample"
|
||||
)
|
||||
|
||||
expected = _review_row(record, source_formatter)
|
||||
for field in REVIEW_CSV_FIELDS[:10]:
|
||||
actual = row[field]
|
||||
if field == "generatedDifficulty":
|
||||
try:
|
||||
matches = float(actual) == float(expected[field])
|
||||
except ValueError:
|
||||
matches = False
|
||||
else:
|
||||
matches = actual == expected[field]
|
||||
if not matches:
|
||||
raise DataError(
|
||||
f"{path}:{row_number}: generated field {field} no longer "
|
||||
"matches the curated corpus"
|
||||
)
|
||||
|
||||
status = row["reviewStatus"].strip().casefold()
|
||||
reviewer_values = [row[field].strip() for field in REVIEW_CSV_FIELDS[10:14]]
|
||||
notes = row["reviewNotes"].strip()
|
||||
if not status:
|
||||
if any(reviewer_values) or notes:
|
||||
raise DataError(
|
||||
f"{path}:{row_number}: reviewer fields require a reviewStatus"
|
||||
)
|
||||
incomplete += 1
|
||||
continue
|
||||
if status not in REVIEW_STATUSES:
|
||||
raise DataError(f"{path}:{row_number}: invalid reviewStatus {status!r}")
|
||||
|
||||
changes = _parse_review_changes(row, record, row_number=row_number)
|
||||
if status == "accept":
|
||||
if any(reviewer_values):
|
||||
raise DataError(
|
||||
f"{path}:{row_number}: accept must leave reviewed fields blank"
|
||||
)
|
||||
elif status == "reject":
|
||||
if any(reviewer_values):
|
||||
raise DataError(
|
||||
f"{path}:{row_number}: reject must leave reviewed fields blank"
|
||||
)
|
||||
if not notes:
|
||||
raise DataError(f"{path}:{row_number}: reject requires reviewNotes")
|
||||
else:
|
||||
if not changes:
|
||||
raise DataError(
|
||||
f"{path}:{row_number}: relabel must change at least one field"
|
||||
)
|
||||
if not notes:
|
||||
raise DataError(f"{path}:{row_number}: relabel requires reviewNotes")
|
||||
|
||||
decision: dict[str, Any] = {
|
||||
"promptHash": row_hash,
|
||||
"status": status,
|
||||
}
|
||||
if changes:
|
||||
decision["changes"] = changes
|
||||
if notes:
|
||||
decision["notes"] = notes
|
||||
decisions.append(decision)
|
||||
counts[status] += 1
|
||||
|
||||
missing = sorted(set(expected_by_hash) - seen)
|
||||
if missing:
|
||||
raise DataError(f"{path}: deterministic sample rows are missing")
|
||||
return ReviewProgress(
|
||||
records=len(rows),
|
||||
accepted=counts["accept"],
|
||||
relabeled=counts["relabel"],
|
||||
rejected=counts["reject"],
|
||||
incomplete=incomplete,
|
||||
decisions=decisions,
|
||||
)
|
||||
|
||||
|
||||
def finalize_human_review(
|
||||
csv_path: Path,
|
||||
output_path: Path,
|
||||
population: Sequence[SourceRecord],
|
||||
*,
|
||||
dataset_version: str,
|
||||
fraction: float,
|
||||
seed: int,
|
||||
source_formatter: Callable[[Path], str] = str,
|
||||
) -> dict[str, Any]:
|
||||
sample = stratified_review_sample(population, fraction=fraction, seed=seed)
|
||||
progress = inspect_review_csv(
|
||||
csv_path,
|
||||
sample,
|
||||
source_formatter=source_formatter,
|
||||
)
|
||||
if progress.incomplete:
|
||||
raise DataError(
|
||||
f"{csv_path}: human review is incomplete: {progress.completed}/"
|
||||
f"{progress.records} rows completed"
|
||||
)
|
||||
|
||||
summary = {
|
||||
"accepted": progress.accepted,
|
||||
"relabeled": progress.relabeled,
|
||||
"rejected": progress.rejected,
|
||||
}
|
||||
artifact = {
|
||||
"schemaVersion": 1,
|
||||
"datasetVersion": dataset_version,
|
||||
"status": "complete",
|
||||
"populationRecords": len(population),
|
||||
"sampleFraction": fraction,
|
||||
"sampleRecords": len(sample),
|
||||
"seed": seed,
|
||||
"sourceCSVSha256": file_sha256(csv_path),
|
||||
"summary": summary,
|
||||
"decisions": progress.decisions,
|
||||
}
|
||||
if output_path.exists():
|
||||
try:
|
||||
existing = json.loads(output_path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
||||
raise DataError(
|
||||
f"{output_path}: cannot verify existing human-review ledger: {exc}"
|
||||
) from exc
|
||||
if existing != artifact:
|
||||
raise DataError(
|
||||
f"{output_path}: refusing to replace a different human-review ledger"
|
||||
)
|
||||
return artifact
|
||||
write_json(output_path, artifact)
|
||||
return artifact
|
||||
|
||||
|
||||
def apply_completed_human_review(
|
||||
path: Path,
|
||||
population: Sequence[SourceRecord],
|
||||
*,
|
||||
dataset_version: str,
|
||||
) -> HumanReviewResult:
|
||||
try:
|
||||
artifact = json.loads(path.read_text(encoding="utf-8"))
|
||||
if artifact["schemaVersion"] != 1:
|
||||
raise DataError(f"{path}: unsupported human-review schema")
|
||||
if artifact["datasetVersion"] != dataset_version:
|
||||
raise DataError(f"{path}: human-review dataset version does not match")
|
||||
if artifact["status"] != "complete":
|
||||
raise DataError(f"{path}: human review is not complete")
|
||||
fraction = float(artifact["sampleFraction"])
|
||||
seed = int(artifact["seed"])
|
||||
decisions = artifact["decisions"]
|
||||
except DataError:
|
||||
raise
|
||||
except (OSError, UnicodeError, json.JSONDecodeError, KeyError, TypeError, ValueError) as exc:
|
||||
raise DataError(f"{path}: cannot read completed human review: {exc}") from exc
|
||||
if not isinstance(decisions, list):
|
||||
raise DataError(f"{path}: human-review decisions must be an array")
|
||||
if artifact.get("populationRecords") != len(population):
|
||||
raise DataError(f"{path}: reviewed population no longer matches the corpus")
|
||||
|
||||
sample = stratified_review_sample(population, fraction=fraction, seed=seed)
|
||||
expected_hashes = {prompt_hash(record.value["prompt"]) for record in sample}
|
||||
if artifact.get("sampleRecords") != len(sample):
|
||||
raise DataError(f"{path}: reviewed sample size no longer matches the corpus")
|
||||
|
||||
by_hash: dict[str, dict[str, Any]] = {}
|
||||
counts: Counter[str] = Counter()
|
||||
for index, decision in enumerate(decisions, 1):
|
||||
if not isinstance(decision, dict):
|
||||
raise DataError(f"{path}: human-review decision {index} must be an object")
|
||||
row_hash = decision.get("promptHash")
|
||||
status = decision.get("status")
|
||||
changes = decision.get("changes", {})
|
||||
if not isinstance(row_hash, str) or not isinstance(status, str):
|
||||
raise DataError(f"{path}: invalid human-review decision {index}")
|
||||
if row_hash in by_hash:
|
||||
raise DataError(f"{path}: duplicate human-review promptHash")
|
||||
if row_hash not in expected_hashes or status not in REVIEW_STATUSES:
|
||||
raise DataError(f"{path}: invalid human-review decision {index}")
|
||||
if not isinstance(changes, dict) or not set(changes) <= CHANGE_FIELDS:
|
||||
raise DataError(f"{path}: invalid changes in human-review decision {index}")
|
||||
if status != "relabel" and changes:
|
||||
raise DataError(f"{path}: only relabel decisions may contain changes")
|
||||
if status == "relabel" and not changes:
|
||||
raise DataError(f"{path}: relabel decision {index} has no changes")
|
||||
if "purpose" in changes and (
|
||||
not isinstance(changes["purpose"], str)
|
||||
or changes["purpose"] not in LABEL_SET
|
||||
):
|
||||
raise DataError(f"{path}: invalid purpose in human-review decision {index}")
|
||||
if "secondary" in changes and (
|
||||
changes["secondary"] is not None
|
||||
and (
|
||||
not isinstance(changes["secondary"], str)
|
||||
or changes["secondary"] not in LABEL_SET
|
||||
)
|
||||
):
|
||||
raise DataError(f"{path}: invalid secondary in human-review decision {index}")
|
||||
if "slice" in changes and (
|
||||
not isinstance(changes["slice"], str) or changes["slice"] not in SLICES
|
||||
):
|
||||
raise DataError(f"{path}: invalid slice in human-review decision {index}")
|
||||
by_hash[row_hash] = decision
|
||||
counts[status] += 1
|
||||
if set(by_hash) != expected_hashes:
|
||||
raise DataError(f"{path}: decisions do not exactly cover the deterministic sample")
|
||||
|
||||
expected_summary = {
|
||||
"accepted": counts["accept"],
|
||||
"relabeled": counts["relabel"],
|
||||
"rejected": counts["reject"],
|
||||
}
|
||||
if artifact.get("summary") != expected_summary:
|
||||
raise DataError(f"{path}: human-review summary does not match its decisions")
|
||||
|
||||
result: list[SourceRecord] = []
|
||||
for record in population:
|
||||
decision = by_hash.get(prompt_hash(record.value["prompt"]))
|
||||
if decision is None or decision["status"] == "accept":
|
||||
result.append(record)
|
||||
continue
|
||||
if decision["status"] == "reject":
|
||||
continue
|
||||
value = dict(record.value)
|
||||
value.update(decision["changes"])
|
||||
value["mixed"] = value["secondary"] is not None
|
||||
validate_source_record(value, f"{path}: {decision['promptHash']}")
|
||||
result.append(SourceRecord(value=value, source=record.source, line=record.line))
|
||||
|
||||
return HumanReviewResult(
|
||||
records=result,
|
||||
summary={
|
||||
**expected_summary,
|
||||
"sampleRecords": len(sample),
|
||||
"retainedRecords": len(result),
|
||||
},
|
||||
)
|
||||
Reference in New Issue
Block a user