Merge nucleic/plucky-north-vole-sdna into dev
This commit is contained in:
+6
-1
@@ -217,7 +217,12 @@ def attach_first_responses(
|
||||
continue
|
||||
previous = responses.get(turn.session_id)
|
||||
if previous is not None:
|
||||
raise DataError(f"ambiguous assistant response after first prompt in session {turn.session_id!r}")
|
||||
# The dataset can contain multiple transcript rows at one conversational
|
||||
# ordinal. `turn_number` is the documented checked tie-breaker; retain the
|
||||
# earliest actual response and make the condition visible in the funnel.
|
||||
funnel["multipleAssistantResponsesAtFirstResponseOrdinal"] += 1
|
||||
if (turn.turn_number, turn.turn_id) >= (previous.turn_number, previous.turn_id):
|
||||
continue
|
||||
responses[turn.session_id] = turn
|
||||
|
||||
preliminary: list[Candidate] = []
|
||||
|
||||
@@ -68,6 +68,18 @@ class ExportSWEChatTests(unittest.TestCase):
|
||||
self.assertEqual("answer", value["teacherResponse"])
|
||||
self.assertNotIn("answer", value["teacherResponseHash"])
|
||||
|
||||
def test_duplicate_response_ordinal_uses_turn_number_tie_breaker(self):
|
||||
rows = [
|
||||
row("s", "t1", "first", conversation_turn_number=0, turn_number=10),
|
||||
row("s", "t3", "later response", conversation_turn_number=1, turn_number=12, turn_type="assistant_response", role="assistant"),
|
||||
row("s", "t2", "first response", conversation_turn_number=1, turn_number=11, turn_type="assistant_response", role="assistant"),
|
||||
]
|
||||
candidates, funnel = export_swe_chat.select_candidates(
|
||||
rows, sessions={}, max_per_repo=10, max_per_user=10
|
||||
)
|
||||
self.assertEqual("first response", candidates[0].response_turn.prompt)
|
||||
self.assertEqual(1, funnel["multipleAssistantResponsesAtFirstResponseOrdinal"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user