Merge nucleic/sleek-ember-seal-uady into dev

tests/test_eval.py

@@ -23,5 +23,33 @@ class TierDriftTests(unittest.TestCase):
         self.assertLessEqual(report["maximumTierDrift"], 1)
 
 
+class PredictionAgreementTests(unittest.TestCase):
+    def test_reports_accuracy_transitions_and_scorable_agreement(self):
+        records = [
+            {"prompt": "one", "slice": "core"},
+            {"prompt": "two", "slice": "boundary"},
+            {"prompt": "three", "slice": "vague-eval"},
+            {"prompt": "four", "slice": "core"},
+        ]
+        report = purpose_eval.prediction_agreement(
+            records,
+            actual=[0, 1, 2, 3],
+            reference=[0, 0, 3, 4],
+            candidate=[1, 1, 4, 5],
+        )
+        self.assertEqual(0.0, report["labelAgreement"])
+        self.assertEqual(0.0, report["scoredLabelAgreement"])
+        self.assertEqual(
+            {
+                "correctToIncorrect": 1,
+                "differentIncorrectLabel": 2,
+                "incorrectToCorrect": 1,
+            },
+            report["transitionCounts"],
+        )
+        self.assertEqual(2, report["bySlice"]["core"]["records"])
+        self.assertEqual(4, len(report["disagreements"]))
+
+
 if __name__ == "__main__":
     unittest.main()