86 lines
4.1 KiB
JSON
86 lines
4.1 KiB
JSON
{
|
|
"schemaVersion": 1,
|
|
"datasetVersion": "purpose-dataset-v1",
|
|
"wordTrigramExclusionReview": {
|
|
"status": "complete",
|
|
"reviewedRecords": 18,
|
|
"confirmedTemplateDuplicates": 18,
|
|
"rejectedExclusions": 0,
|
|
"decision": "All 18 pairs preserve the same task, requested outcome, and purpose label; differences are generated identifiers, ticket numbers, cosmetic context prefixes, or difficulty/slice metadata. Keep the earliest member and exclude the later template copy.",
|
|
"reviewedDroppedPromptHashes": [
|
|
"f29de1d646055017f80346055bb3ac1468c49af7ab0c6249f45f19cf386f9fa7",
|
|
"dabbcd0a19d2918199b5d0d35fac49fd5413cb17a5164252b789114fc68ec43b",
|
|
"ac3a166e07a1db67475c773ef3d996d12fe627d11b7bb91a7c524c9c67c841b0",
|
|
"d5f9d735490874ded856105e10df99ed3c2c703c09ec7909fd07aa1e9e7a0c91",
|
|
"6376e5f6f0d7c3d6faae9e50975034113f43a7f959bcbbea67531d3b4f0cfd16",
|
|
"10933534254a1f73163e0ba7883225e2d0c896b565590753698c1052743af664",
|
|
"cf4deb437c380216601b176e238964bfce79bf2645d54e1eb659f8479b980186",
|
|
"fe6b2e05da51a12ea07d6fa015933196931fb1023b62c0eb267b80318f083ba0",
|
|
"c0fddb025279caf6438453b53c99b266ccfb61c6c19609b43c6f731d88812813",
|
|
"15c28d9f67ba103378db711c82193aba7330acbe602d9df54995a5db20e1bbb1",
|
|
"b3583a130811ba7d2f425042c41e1f69244c622764a0c952ed42bb5114e18735",
|
|
"bba760782054c4ed405611cf8b55bc99d710424bb3f9bab0682dc2c1cf4e2a52",
|
|
"bd4d2d8ea68a4e56eaed5c6465815abefa6c8ae78fb533eb81547dedac9c71ed",
|
|
"8c1ab1a866cadd294c3d2a6c1d6eefbede62549220c81e1dd28a886717aa285d",
|
|
"c074ce20e7cd81c28d390aed919bab30cabe9211817103e52c9b229e6a3c94ff",
|
|
"fdc04376376dbffcaf9928e9edcef29376203640b49c200c2c117f78891125a1",
|
|
"432e67ead52d6cd57e4a5034cd8bcde14bd2da13f88cb5532c42fd8fbd8daf05",
|
|
"24f3759ff4a3fb9fc77bed4c8525f30f9af2479a6cffbb0c4842043f99a7d8f1"
|
|
]
|
|
},
|
|
"semanticDuplicateReview": {
|
|
"status": "complete",
|
|
"auditVersion": "purpose-semantic-audit-v1",
|
|
"reviewedCandidates": 4,
|
|
"excludedCandidates": 3,
|
|
"retainedCandidates": 1,
|
|
"decision": "Exclude three same-purpose generated template copies. One pair ('make the screen nicer' / 'Make this screen nicer') remains because it is a deliberate vague-language variant, lives entirely in validation, and does not contaminate the frozen evaluation boundary.",
|
|
"excluded": [
|
|
{
|
|
"droppedPromptHash": "7348d764d602cfff72dd2b1d369ad0375a96ea52b38174aa0bd6168a548bcc2f",
|
|
"matchedPromptHash": "3ad9cfbb664504f0e7ae84b905520937e7812d4e879fa706415782341f2cd400",
|
|
"similarity": 0.984897
|
|
},
|
|
{
|
|
"droppedPromptHash": "a0c6e7984e92adc11ea6855c39688d733bf7665f5f965939325b15f3ba39e937",
|
|
"matchedPromptHash": "17f79a890a323c08770193a94b100dadaa00116405bdb7d14c1b89abc3c3066a",
|
|
"similarity": 0.980998
|
|
},
|
|
{
|
|
"droppedPromptHash": "bfd7791069fd04c13797f39ba99224ad68ca2deebbc912f08b10815954a7d223",
|
|
"matchedPromptHash": "dc48031efb18f25a56a3beddcdd814f26473b23cd496482ce1884ae6f697417a",
|
|
"similarity": 0.973329
|
|
}
|
|
],
|
|
"retained": [
|
|
{
|
|
"leftPromptHash": "70944e70e0062e947476f73b704ec01ecee19420d06fd174c58a41c04e8112f6",
|
|
"rightPromptHash": "cb3e63992cbf5830a222933dc7b44be39933baedcb46e335be02e3c379685517",
|
|
"similarity": 0.970376
|
|
}
|
|
]
|
|
},
|
|
"humanLabelAndDifficultyReview": {
|
|
"status": "planned",
|
|
"populationRecords": 12193,
|
|
"sampleFraction": 0.1,
|
|
"sampleRecords": 1219,
|
|
"seed": 10558743,
|
|
"strata": [
|
|
"purpose",
|
|
"slice",
|
|
"primary language"
|
|
],
|
|
"reviewFields": [
|
|
"purpose",
|
|
"secondary",
|
|
"difficulty",
|
|
"slice",
|
|
"review status",
|
|
"notes"
|
|
],
|
|
"generatedArtifact": "ml/purpose-classifier/.artifacts/human-review-v1.csv",
|
|
"completionRule": "Every sampled row must be marked accept, relabel, or reject by a human reviewer. review_data.py validates the exact sample and writes a versioned complete ledger; prepare_data.py applies relabel/reject decisions before splitting. The revised split and frozen evaluation must then be intentionally reviewed and versioned before the dataset can be called fully curated."
|
|
}
|
|
}
|