56 lines
1.8 KiB
Python
56 lines
1.8 KiB
Python
import sys
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
|
|
MODULE_DIR = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(MODULE_DIR))
|
|
|
|
import eval as purpose_eval
|
|
|
|
|
|
class TierDriftTests(unittest.TestCase):
|
|
def test_current_routing_matrix_bounds_every_label_pair(self):
|
|
records = [{"prompt": f"prompt {index}"} for index in range(8 * 8)]
|
|
actual = []
|
|
predicted = []
|
|
for expected in range(8):
|
|
for got in range(8):
|
|
actual.append(expected)
|
|
predicted.append(got)
|
|
report = purpose_eval.routing_tier_drift(records, actual, predicted)
|
|
self.assertTrue(report["passed"])
|
|
self.assertLessEqual(report["maximumTierDrift"], 1)
|
|
|
|
|
|
class PredictionAgreementTests(unittest.TestCase):
|
|
def test_reports_accuracy_transitions_and_scorable_agreement(self):
|
|
records = [
|
|
{"prompt": "one", "slice": "core"},
|
|
{"prompt": "two", "slice": "boundary"},
|
|
{"prompt": "three", "slice": "vague-eval"},
|
|
{"prompt": "four", "slice": "core"},
|
|
]
|
|
report = purpose_eval.prediction_agreement(
|
|
records,
|
|
actual=[0, 1, 2, 3],
|
|
reference=[0, 0, 3, 4],
|
|
candidate=[1, 1, 4, 5],
|
|
)
|
|
self.assertEqual(0.0, report["labelAgreement"])
|
|
self.assertEqual(0.0, report["scoredLabelAgreement"])
|
|
self.assertEqual(
|
|
{
|
|
"correctToIncorrect": 1,
|
|
"differentIncorrectLabel": 2,
|
|
"incorrectToCorrect": 1,
|
|
},
|
|
report["transitionCounts"],
|
|
)
|
|
self.assertEqual(2, report["bySlice"]["core"]["records"])
|
|
self.assertEqual(4, len(report["disagreements"]))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|