Merge nucleic/sleek-ember-seal-uady into dev

2026-07-30 18:53:22 -07:00
parent ddbff97191
commit bb6d53a520
10 changed files with 962 additions and 110 deletions
@@ -0,0 +1,154 @@
+import csv
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+
+MODULE_DIR = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(MODULE_DIR))
+
+import purpose_data
+import review_contract
+
+
+def record(index: int) -> purpose_data.SourceRecord:
+    return purpose_data.SourceRecord(
+        value={
+            "prompt": f"Purpose review prompt {index}",
+            "purpose": purpose_data.LABELS[index % len(purpose_data.LABELS)],
+            "secondary": None,
+            "mixed": False,
+            "difficulty": 0.4,
+            "slice": "core",
+            "lang": "en",
+        },
+        source=Path("source.jsonl"),
+        line=index + 1,
+    )
+
+
+def update_csv(path: Path, update):
+    with path.open("r", encoding="utf-8", newline="") as handle:
+        rows = list(csv.DictReader(handle))
+    update(rows)
+    with path.open("w", encoding="utf-8", newline="") as handle:
+        writer = csv.DictWriter(
+            handle,
+            fieldnames=review_contract.REVIEW_CSV_FIELDS,
+        )
+        writer.writeheader()
+        writer.writerows(rows)
+
+
+class HumanReviewContractTests(unittest.TestCase):
+    def test_blank_review_reports_progress_without_claiming_completion(self):
+        population = [record(index) for index in range(10)]
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "review.csv"
+            review_contract.write_review_csv(path, population)
+            progress = review_contract.inspect_review_csv(path, population)
+
+        self.assertEqual(0, progress.completed)
+        self.assertEqual(10, progress.incomplete)
+        self.assertEqual([], progress.decisions)
+
+    def test_finalize_and_apply_accept_relabel_and_reject(self):
+        population = [record(index) for index in range(10)]
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            csv_path = root / "review.csv"
+            artifact_path = root / "human-review.json"
+            review_contract.write_review_csv(csv_path, population)
+
+            def complete(rows):
+                for row in rows:
+                    row["reviewStatus"] = "accept"
+                rows[0]["reviewStatus"] = "relabel"
+                rows[0]["reviewedPurpose"] = "writing"
+                rows[0]["reviewNotes"] = "Primary intent is prose."
+                rows[1]["reviewStatus"] = "reject"
+                rows[1]["reviewNotes"] = "Prompt is not classifiable."
+                rows[2]["reviewStatus"] = "relabel"
+                rows[2]["reviewedSecondary"] = "review"
+                rows[2]["reviewedSlice"] = "mixed"
+                rows[2]["reviewNotes"] = "Two explicit intents."
+
+            update_csv(csv_path, complete)
+            artifact = review_contract.finalize_human_review(
+                csv_path,
+                artifact_path,
+                population,
+                dataset_version="test-v1",
+                fraction=1.0,
+                seed=7,
+            )
+            self.assertEqual(
+                artifact,
+                review_contract.finalize_human_review(
+                    csv_path,
+                    artifact_path,
+                    population,
+                    dataset_version="test-v1",
+                    fraction=1.0,
+                    seed=7,
+                ),
+            )
+            result = review_contract.apply_completed_human_review(
+                artifact_path,
+                population,
+                dataset_version="test-v1",
+            )
+
+        self.assertEqual(
+            {"accepted": 7, "relabeled": 2, "rejected": 1},
+            artifact["summary"],
+        )
+        self.assertEqual(9, len(result.records))
+        by_prompt = {item.value["prompt"]: item.value for item in result.records}
+        self.assertEqual("writing", by_prompt["Purpose review prompt 0"]["purpose"])
+        self.assertNotIn("Purpose review prompt 1", by_prompt)
+        self.assertEqual("review", by_prompt["Purpose review prompt 2"]["secondary"])
+        self.assertTrue(by_prompt["Purpose review prompt 2"]["mixed"])
+        self.assertEqual("mixed", by_prompt["Purpose review prompt 2"]["slice"])
+
+    def test_finalize_fails_closed_when_rows_are_incomplete(self):
+        population = [record(index) for index in range(5)]
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            csv_path = root / "review.csv"
+            review_contract.write_review_csv(csv_path, population)
+            with self.assertRaisesRegex(
+                purpose_data.DataError,
+                "human review is incomplete: 0/5",
+            ):
+                review_contract.finalize_human_review(
+                    csv_path,
+                    root / "human-review.json",
+                    population,
+                    dataset_version="test-v1",
+                    fraction=1.0,
+                    seed=7,
+                )
+
+    def test_relabel_must_preserve_the_source_contract(self):
+        population = [record(0)]
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "review.csv"
+            review_contract.write_review_csv(path, population)
+
+            def invalidate(rows):
+                rows[0]["reviewStatus"] = "relabel"
+                rows[0]["reviewedSecondary"] = "review"
+                rows[0]["reviewNotes"] = "Two intents."
+
+            update_csv(path, invalidate)
+            with self.assertRaisesRegex(
+                purpose_data.DataError,
+                "mixed slice and mixed field disagree",
+            ):
+                review_contract.inspect_review_csv(path, population)
+
+
+if __name__ == "__main__":
+    unittest.main()