Merge nucleic/sleek-ember-seal-uady into dev
This commit is contained in:
@@ -1,3 +1,4 @@
|
||||
import csv
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
@@ -10,6 +11,7 @@ sys.path.insert(0, str(MODULE_DIR))
|
||||
|
||||
import prepare_data
|
||||
import purpose_data
|
||||
import review_contract
|
||||
|
||||
|
||||
def example(index: int):
|
||||
@@ -72,6 +74,66 @@ class PrepareIntegrationTests(unittest.TestCase):
|
||||
train = purpose_data.load_jsonl(output / "train.jsonl")
|
||||
self.assertFalse(any(row["slice"] == "vague-eval" for row in train))
|
||||
|
||||
def test_completed_human_review_is_applied_and_recorded(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
source = root / "source.jsonl"
|
||||
purpose_data.write_jsonl(source, (example(index) for index in range(80)))
|
||||
fixtures = root / "fixtures.json"
|
||||
fixtures.write_text(
|
||||
json.dumps(
|
||||
[{"prompt": "Plan the cache migration", "purpose": "planning"}]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
source_records = purpose_data.load_sources([source])
|
||||
review_population = purpose_data.curate_records(
|
||||
source_records,
|
||||
purpose_data.load_classifiable_fixtures(fixtures),
|
||||
).records
|
||||
review_csv = root / "review.csv"
|
||||
review_contract.write_review_csv(review_csv, review_population)
|
||||
with review_csv.open("r", encoding="utf-8", newline="") as handle:
|
||||
rows = list(csv.DictReader(handle))
|
||||
for row in rows:
|
||||
row["reviewStatus"] = "accept"
|
||||
rows[0]["reviewStatus"] = "reject"
|
||||
rows[0]["reviewNotes"] = "Not classifiable after human review."
|
||||
with review_csv.open("w", encoding="utf-8", newline="") as handle:
|
||||
writer = csv.DictWriter(
|
||||
handle,
|
||||
fieldnames=review_contract.REVIEW_CSV_FIELDS,
|
||||
)
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
human_review = root / "human-review.json"
|
||||
review_contract.finalize_human_review(
|
||||
review_csv,
|
||||
human_review,
|
||||
review_population,
|
||||
dataset_version=prepare_data.DATASET_VERSION,
|
||||
fraction=1.0,
|
||||
seed=41,
|
||||
)
|
||||
|
||||
manifest = prepare_data.prepare(
|
||||
sources=[source],
|
||||
fixtures_path=fixtures,
|
||||
output_dir=root / "output",
|
||||
frozen_test_path=root / "frozen.jsonl",
|
||||
manifest_path=root / "manifest.json",
|
||||
refresh_frozen_test=True,
|
||||
seed=23,
|
||||
near_duplicate_threshold=0.92,
|
||||
human_review_path=human_review,
|
||||
)
|
||||
|
||||
self.assertEqual(79, manifest["curation"]["retainedRecords"])
|
||||
self.assertEqual(
|
||||
1,
|
||||
manifest["curation"]["humanReview"]["summary"]["rejected"],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
import csv
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
MODULE_DIR = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(MODULE_DIR))
|
||||
|
||||
import purpose_data
|
||||
import review_contract
|
||||
|
||||
|
||||
def record(index: int) -> purpose_data.SourceRecord:
|
||||
return purpose_data.SourceRecord(
|
||||
value={
|
||||
"prompt": f"Purpose review prompt {index}",
|
||||
"purpose": purpose_data.LABELS[index % len(purpose_data.LABELS)],
|
||||
"secondary": None,
|
||||
"mixed": False,
|
||||
"difficulty": 0.4,
|
||||
"slice": "core",
|
||||
"lang": "en",
|
||||
},
|
||||
source=Path("source.jsonl"),
|
||||
line=index + 1,
|
||||
)
|
||||
|
||||
|
||||
def update_csv(path: Path, update):
|
||||
with path.open("r", encoding="utf-8", newline="") as handle:
|
||||
rows = list(csv.DictReader(handle))
|
||||
update(rows)
|
||||
with path.open("w", encoding="utf-8", newline="") as handle:
|
||||
writer = csv.DictWriter(
|
||||
handle,
|
||||
fieldnames=review_contract.REVIEW_CSV_FIELDS,
|
||||
)
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
|
||||
class HumanReviewContractTests(unittest.TestCase):
|
||||
def test_blank_review_reports_progress_without_claiming_completion(self):
|
||||
population = [record(index) for index in range(10)]
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "review.csv"
|
||||
review_contract.write_review_csv(path, population)
|
||||
progress = review_contract.inspect_review_csv(path, population)
|
||||
|
||||
self.assertEqual(0, progress.completed)
|
||||
self.assertEqual(10, progress.incomplete)
|
||||
self.assertEqual([], progress.decisions)
|
||||
|
||||
def test_finalize_and_apply_accept_relabel_and_reject(self):
|
||||
population = [record(index) for index in range(10)]
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
csv_path = root / "review.csv"
|
||||
artifact_path = root / "human-review.json"
|
||||
review_contract.write_review_csv(csv_path, population)
|
||||
|
||||
def complete(rows):
|
||||
for row in rows:
|
||||
row["reviewStatus"] = "accept"
|
||||
rows[0]["reviewStatus"] = "relabel"
|
||||
rows[0]["reviewedPurpose"] = "writing"
|
||||
rows[0]["reviewNotes"] = "Primary intent is prose."
|
||||
rows[1]["reviewStatus"] = "reject"
|
||||
rows[1]["reviewNotes"] = "Prompt is not classifiable."
|
||||
rows[2]["reviewStatus"] = "relabel"
|
||||
rows[2]["reviewedSecondary"] = "review"
|
||||
rows[2]["reviewedSlice"] = "mixed"
|
||||
rows[2]["reviewNotes"] = "Two explicit intents."
|
||||
|
||||
update_csv(csv_path, complete)
|
||||
artifact = review_contract.finalize_human_review(
|
||||
csv_path,
|
||||
artifact_path,
|
||||
population,
|
||||
dataset_version="test-v1",
|
||||
fraction=1.0,
|
||||
seed=7,
|
||||
)
|
||||
self.assertEqual(
|
||||
artifact,
|
||||
review_contract.finalize_human_review(
|
||||
csv_path,
|
||||
artifact_path,
|
||||
population,
|
||||
dataset_version="test-v1",
|
||||
fraction=1.0,
|
||||
seed=7,
|
||||
),
|
||||
)
|
||||
result = review_contract.apply_completed_human_review(
|
||||
artifact_path,
|
||||
population,
|
||||
dataset_version="test-v1",
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
{"accepted": 7, "relabeled": 2, "rejected": 1},
|
||||
artifact["summary"],
|
||||
)
|
||||
self.assertEqual(9, len(result.records))
|
||||
by_prompt = {item.value["prompt"]: item.value for item in result.records}
|
||||
self.assertEqual("writing", by_prompt["Purpose review prompt 0"]["purpose"])
|
||||
self.assertNotIn("Purpose review prompt 1", by_prompt)
|
||||
self.assertEqual("review", by_prompt["Purpose review prompt 2"]["secondary"])
|
||||
self.assertTrue(by_prompt["Purpose review prompt 2"]["mixed"])
|
||||
self.assertEqual("mixed", by_prompt["Purpose review prompt 2"]["slice"])
|
||||
|
||||
def test_finalize_fails_closed_when_rows_are_incomplete(self):
|
||||
population = [record(index) for index in range(5)]
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
csv_path = root / "review.csv"
|
||||
review_contract.write_review_csv(csv_path, population)
|
||||
with self.assertRaisesRegex(
|
||||
purpose_data.DataError,
|
||||
"human review is incomplete: 0/5",
|
||||
):
|
||||
review_contract.finalize_human_review(
|
||||
csv_path,
|
||||
root / "human-review.json",
|
||||
population,
|
||||
dataset_version="test-v1",
|
||||
fraction=1.0,
|
||||
seed=7,
|
||||
)
|
||||
|
||||
def test_relabel_must_preserve_the_source_contract(self):
|
||||
population = [record(0)]
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "review.csv"
|
||||
review_contract.write_review_csv(path, population)
|
||||
|
||||
def invalidate(rows):
|
||||
rows[0]["reviewStatus"] = "relabel"
|
||||
rows[0]["reviewedSecondary"] = "review"
|
||||
rows[0]["reviewNotes"] = "Two intents."
|
||||
|
||||
update_csv(path, invalidate)
|
||||
with self.assertRaisesRegex(
|
||||
purpose_data.DataError,
|
||||
"mixed slice and mixed field disagree",
|
||||
):
|
||||
review_contract.inspect_review_csv(path, population)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user