Merge nucleic/fuzzy-dewy-urchin-hpvd into dev

rebuild_sol_high.py

@@ -34,6 +34,7 @@ from purpose_data import (
     file_sha256,
     jsonl_bytes,
     load_jsonl,
+    prompt_hash,
     validate_source_record,
     write_json,
     write_jsonl,
@@ -463,7 +464,41 @@ def _replace_transaction(candidates: Sequence[tuple[Path, Path]], backup_root: P
         raise
 
 
-def promote(stage: Path, confirmation: str | None) -> dict[str, Any]:
+def _exclude_reviewed_real_lines(
+    records: Sequence[dict[str, Any]],
+    source_lines: Sequence[int],
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    unique_lines = sorted(set(source_lines))
+    if len(unique_lines) != len(source_lines):
+        raise DataError("--exclude-real-line contains a duplicate line")
+    for line in unique_lines:
+        if not 1 <= line <= len(records):
+            raise DataError(
+                f"--exclude-real-line {line} is outside the combined real population "
+                f"(1-{len(records)})"
+            )
+
+    excluded = set(unique_lines)
+    review = [
+        {
+            "sourceLine": line,
+            "promptHash": prompt_hash(records[line - 1]["prompt"]),
+            "purpose": records[line - 1]["purpose"],
+            "reason": "reviewed-near-duplicate-label-conflict",
+        }
+        for line in unique_lines
+    ]
+    return (
+        [record for line, record in enumerate(records, 1) if line not in excluded],
+        review,
+    )
+
+
+def promote(
+    stage: Path,
+    confirmation: str | None,
+    excluded_real_lines: Sequence[int] = (),
+) -> dict[str, Any]:
     if confirmation != CONFIRMATION:
         raise DataError(f"promotion requires --confirm {CONFIRMATION}")
     load_config(stage)
@@ -514,9 +549,12 @@ def promote(stage: Path, confirmation: str | None) -> dict[str, Any]:
         state_by_name["fixtures"],
     )
     combined_real = promotion / "combined-real.labeled.jsonl"
-    real_records = _records_from_states(
+    all_real_records = _records_from_states(
         state_by_name["history"]
     ) + _records_from_states(state_by_name["swe"])
+    real_records, real_review = _exclude_reviewed_real_lines(
+        all_real_records, excluded_real_lines
+    )
     write_jsonl(combined_real, real_records)
 
     public_dataset = promotion / "dataset-public"
@@ -558,6 +596,10 @@ def promote(stage: Path, confirmation: str | None) -> dict[str, Any]:
     combined_manifest["policy"]["historyUsage"] = (
         "Nucleic history and SWE-chat are training-only"
     )
+    combined_manifest["policy"]["reviewedRealLabelConflicts"] = (
+        "explicitly excluded before near-duplicate curation"
+    )
+    combined_manifest["reviewedRealExclusions"] = real_review
     combined_manifest["sources"]["baseDataset"]["path"] = _relative(
         PUBLIC_DATASET_DESTINATION
     )
@@ -608,6 +650,7 @@ def promote(stage: Path, confirmation: str | None) -> dict[str, Any]:
         "teacher": {"model": MODEL, "reasoningEffort": REASONING_EFFORT},
         "publicLabeled": sum(len(value) for value in public_records.values()),
         "realLabeled": len(real_records),
+        "realReviewExclusions": real_review,
         "combinedTrain": combined_manifest["outputs"]["train"]["records"],
         "validation": combined_manifest["outputs"]["validation"]["records"],
         "test": combined_manifest["outputs"]["test"]["records"],
@@ -663,6 +706,16 @@ def build_parser() -> argparse.ArgumentParser:
         "promote", help="curate and replace canonical labels"
     )
     promote_parser.add_argument("--confirm")
+    promote_parser.add_argument(
+        "--exclude-real-line",
+        action="append",
+        default=[],
+        type=int,
+        help=(
+            "exclude a reviewed 1-based line from the combined history+SWE training "
+            "augmentation; repeat for each adjudicated conflict"
+        ),
+    )
     commands_parser = subparsers.add_parser(
         "train-commands", help="print clean from-base training commands"
     )
@@ -701,7 +754,13 @@ def main(argv: Sequence[str] | None = None) -> int:
         elif args.command == "status":
             print(json.dumps(status(stage), indent=2, sort_keys=True))
         elif args.command == "promote":
-            print(json.dumps(promote(stage, args.confirm), indent=2, sort_keys=True))
+            print(
+                json.dumps(
+                    promote(stage, args.confirm, args.exclude_real_line),
+                    indent=2,
+                    sort_keys=True,
+                )
+            )
         else:
             print(train_commands(args.python))
     except (