import csv import sys import tempfile import unittest from pathlib import Path MODULE_DIR = Path(__file__).resolve().parents[1] sys.path.insert(0, str(MODULE_DIR)) import purpose_data import review_contract def record(index: int) -> purpose_data.SourceRecord: return purpose_data.SourceRecord( value={ "prompt": f"Purpose review prompt {index}", "purpose": purpose_data.LABELS[index % len(purpose_data.LABELS)], "secondary": None, "mixed": False, "difficulty": 0.4, "slice": "core", "lang": "en", }, source=Path("source.jsonl"), line=index + 1, ) def update_csv(path: Path, update): with path.open("r", encoding="utf-8", newline="") as handle: rows = list(csv.DictReader(handle)) update(rows) with path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter( handle, fieldnames=review_contract.REVIEW_CSV_FIELDS, ) writer.writeheader() writer.writerows(rows) class HumanReviewContractTests(unittest.TestCase): def test_blank_review_reports_progress_without_claiming_completion(self): population = [record(index) for index in range(10)] with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "review.csv" review_contract.write_review_csv(path, population) progress = review_contract.inspect_review_csv(path, population) self.assertEqual(0, progress.completed) self.assertEqual(10, progress.incomplete) self.assertEqual([], progress.decisions) def test_finalize_and_apply_accept_relabel_and_reject(self): population = [record(index) for index in range(10)] with tempfile.TemporaryDirectory() as directory: root = Path(directory) csv_path = root / "review.csv" artifact_path = root / "human-review.json" review_contract.write_review_csv(csv_path, population) def complete(rows): for row in rows: row["reviewStatus"] = "accept" rows[0]["reviewStatus"] = "relabel" rows[0]["reviewedPurpose"] = "writing" rows[0]["reviewNotes"] = "Primary intent is prose." rows[1]["reviewStatus"] = "reject" rows[1]["reviewNotes"] = "Prompt is not classifiable." rows[2]["reviewStatus"] = "relabel" rows[2]["reviewedSecondary"] = "review" rows[2]["reviewedSlice"] = "mixed" rows[2]["reviewNotes"] = "Two explicit intents." update_csv(csv_path, complete) artifact = review_contract.finalize_human_review( csv_path, artifact_path, population, dataset_version="test-v1", fraction=1.0, seed=7, ) self.assertEqual( artifact, review_contract.finalize_human_review( csv_path, artifact_path, population, dataset_version="test-v1", fraction=1.0, seed=7, ), ) result = review_contract.apply_completed_human_review( artifact_path, population, dataset_version="test-v1", ) self.assertEqual( {"accepted": 7, "relabeled": 2, "rejected": 1}, artifact["summary"], ) self.assertEqual(9, len(result.records)) by_prompt = {item.value["prompt"]: item.value for item in result.records} self.assertEqual("writing", by_prompt["Purpose review prompt 0"]["purpose"]) self.assertNotIn("Purpose review prompt 1", by_prompt) self.assertEqual("review", by_prompt["Purpose review prompt 2"]["secondary"]) self.assertTrue(by_prompt["Purpose review prompt 2"]["mixed"]) self.assertEqual("mixed", by_prompt["Purpose review prompt 2"]["slice"]) def test_finalize_fails_closed_when_rows_are_incomplete(self): population = [record(index) for index in range(5)] with tempfile.TemporaryDirectory() as directory: root = Path(directory) csv_path = root / "review.csv" review_contract.write_review_csv(csv_path, population) with self.assertRaisesRegex( purpose_data.DataError, "human review is incomplete: 0/5", ): review_contract.finalize_human_review( csv_path, root / "human-review.json", population, dataset_version="test-v1", fraction=1.0, seed=7, ) def test_relabel_must_preserve_the_source_contract(self): population = [record(0)] with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "review.csv" review_contract.write_review_csv(path, population) def invalidate(rows): rows[0]["reviewStatus"] = "relabel" rows[0]["reviewedSecondary"] = "review" rows[0]["reviewNotes"] = "Two intents." update_csv(path, invalidate) with self.assertRaisesRegex( purpose_data.DataError, "mixed slice and mixed field disagree", ): review_contract.inspect_review_csv(path, population) if __name__ == "__main__": unittest.main()