Merge nucleic/lucid-north-quail-rnvt into dev

validate-data.py

@@ -7,7 +7,7 @@ process violations are errors.  Approximate targets (the requirements written as
 
 Usage:
     python3 ml/purpose-classifier/validate-data.py
-    python3 ml/purpose-classifier/validate-data.py path/to/batch.jsonl
+    python3 ml/purpose-classifier/validate-data.py path/to/batch.jsonl --batch-size 200
     python3 ml/purpose-classifier/validate-data.py --strict
 """
 
@@ -28,6 +28,10 @@ from typing import Any, Iterable
 
 SCRIPT_DIR = Path(__file__).resolve().parent
 DEFAULT_DATA_DIR = SCRIPT_DIR / "data"
+DEFAULT_CANONICAL_FILES = (
+    DEFAULT_DATA_DIR / "purpose-prompts.jsonl",
+    DEFAULT_DATA_DIR / "purpose-prompts-round2.jsonl",
+)
 DEFAULT_FIXTURES = (
     SCRIPT_DIR.parent.parent
     / "Tests"
@@ -177,8 +181,8 @@ class Validator:
     def __init__(
         self,
         *,
-        batch_size: int = 200,
-        expected_total: int = 8_000,
+        batch_size: int = 0,
+        expected_total: int = 12_214,
         fixture_path: Path | None = DEFAULT_FIXTURES,
         process_checks: bool = True,
     ) -> None:
@@ -520,7 +524,7 @@ class Validator:
     def _check_manifests(self, roots: list[Path]) -> None:
         directories = sorted({root if root.is_dir() else root.parent for root in roots})
         for directory in directories:
-            candidates = sorted(directory.glob("*manifest*.json"))
+            candidates = sorted(directory.glob("*generation-manifest*.json"))
             if not candidates:
                 self.warning(
                     directory,
@@ -687,14 +691,14 @@ def build_parser() -> argparse.ArgumentParser:
     parser.add_argument(
         "--batch-size",
         type=int,
-        default=200,
-        help="required generation batch size; use 0 to disable (default: 200)",
+        default=0,
+        help="required generation batch size; use 200 for raw batches (default: disabled)",
     )
     parser.add_argument(
         "--expected-total",
         type=int,
-        default=8_000,
-        help="expected total record count; use 0 to disable (default: 8000)",
+        default=12_214,
+        help="expected canonical record count; use 0 to disable (default: 12214)",
     )
     parser.add_argument(
         "--fixtures",
@@ -727,7 +731,7 @@ def main(argv: list[str] | None = None) -> int:
         print("error: numeric options must be non-negative", file=sys.stderr)
         return 2
 
-    targets = args.paths or [DEFAULT_DATA_DIR]
+    targets = args.paths or list(DEFAULT_CANONICAL_FILES)
     paths, roots, discovery_errors = discover_paths(targets)
     if discovery_errors:
         for message in discovery_errors: