Merge nucleic/sleek-ember-seal-uady into dev
This commit is contained in:
@@ -0,0 +1,84 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"datasetVersion": "purpose-dataset-v1",
|
||||
"wordTrigramExclusionReview": {
|
||||
"status": "complete",
|
||||
"reviewedRecords": 18,
|
||||
"confirmedTemplateDuplicates": 18,
|
||||
"rejectedExclusions": 0,
|
||||
"decision": "All 18 pairs preserve the same task, requested outcome, and purpose label; differences are generated identifiers, ticket numbers, cosmetic context prefixes, or difficulty/slice metadata. Keep the earliest member and exclude the later template copy.",
|
||||
"reviewedDroppedPromptHashes": [
|
||||
"f29de1d646055017f80346055bb3ac1468c49af7ab0c6249f45f19cf386f9fa7",
|
||||
"dabbcd0a19d2918199b5d0d35fac49fd5413cb17a5164252b789114fc68ec43b",
|
||||
"ac3a166e07a1db67475c773ef3d996d12fe627d11b7bb91a7c524c9c67c841b0",
|
||||
"d5f9d735490874ded856105e10df99ed3c2c703c09ec7909fd07aa1e9e7a0c91",
|
||||
"6376e5f6f0d7c3d6faae9e50975034113f43a7f959bcbbea67531d3b4f0cfd16",
|
||||
"10933534254a1f73163e0ba7883225e2d0c896b565590753698c1052743af664",
|
||||
"cf4deb437c380216601b176e238964bfce79bf2645d54e1eb659f8479b980186",
|
||||
"fe6b2e05da51a12ea07d6fa015933196931fb1023b62c0eb267b80318f083ba0",
|
||||
"c0fddb025279caf6438453b53c99b266ccfb61c6c19609b43c6f731d88812813",
|
||||
"15c28d9f67ba103378db711c82193aba7330acbe602d9df54995a5db20e1bbb1",
|
||||
"b3583a130811ba7d2f425042c41e1f69244c622764a0c952ed42bb5114e18735",
|
||||
"bba760782054c4ed405611cf8b55bc99d710424bb3f9bab0682dc2c1cf4e2a52",
|
||||
"bd4d2d8ea68a4e56eaed5c6465815abefa6c8ae78fb533eb81547dedac9c71ed",
|
||||
"8c1ab1a866cadd294c3d2a6c1d6eefbede62549220c81e1dd28a886717aa285d",
|
||||
"c074ce20e7cd81c28d390aed919bab30cabe9211817103e52c9b229e6a3c94ff",
|
||||
"fdc04376376dbffcaf9928e9edcef29376203640b49c200c2c117f78891125a1",
|
||||
"432e67ead52d6cd57e4a5034cd8bcde14bd2da13f88cb5532c42fd8fbd8daf05",
|
||||
"24f3759ff4a3fb9fc77bed4c8525f30f9af2479a6cffbb0c4842043f99a7d8f1"
|
||||
]
|
||||
},
|
||||
"semanticDuplicateReview": {
|
||||
"status": "complete",
|
||||
"auditVersion": "purpose-semantic-audit-v1",
|
||||
"reviewedCandidates": 4,
|
||||
"excludedCandidates": 3,
|
||||
"retainedCandidates": 1,
|
||||
"decision": "Exclude three same-purpose generated template copies. One pair ('make the screen nicer' / 'Make this screen nicer') remains because it is a deliberate vague-language variant, lives entirely in validation, and does not contaminate the frozen evaluation boundary.",
|
||||
"excluded": [
|
||||
{
|
||||
"droppedPromptHash": "7348d764d602cfff72dd2b1d369ad0375a96ea52b38174aa0bd6168a548bcc2f",
|
||||
"matchedPromptHash": "3ad9cfbb664504f0e7ae84b905520937e7812d4e879fa706415782341f2cd400",
|
||||
"similarity": 0.984897
|
||||
},
|
||||
{
|
||||
"droppedPromptHash": "a0c6e7984e92adc11ea6855c39688d733bf7665f5f965939325b15f3ba39e937",
|
||||
"matchedPromptHash": "17f79a890a323c08770193a94b100dadaa00116405bdb7d14c1b89abc3c3066a",
|
||||
"similarity": 0.980998
|
||||
},
|
||||
{
|
||||
"droppedPromptHash": "bfd7791069fd04c13797f39ba99224ad68ca2deebbc912f08b10815954a7d223",
|
||||
"matchedPromptHash": "dc48031efb18f25a56a3beddcdd814f26473b23cd496482ce1884ae6f697417a",
|
||||
"similarity": 0.973329
|
||||
}
|
||||
],
|
||||
"retained": [
|
||||
{
|
||||
"leftPromptHash": "70944e70e0062e947476f73b704ec01ecee19420d06fd174c58a41c04e8112f6",
|
||||
"rightPromptHash": "cb3e63992cbf5830a222933dc7b44be39933baedcb46e335be02e3c379685517",
|
||||
"similarity": 0.970376
|
||||
}
|
||||
]
|
||||
},
|
||||
"humanLabelAndDifficultyReview": {
|
||||
"status": "planned",
|
||||
"populationRecords": 12193,
|
||||
"sampleFraction": 0.1,
|
||||
"sampleRecords": 1219,
|
||||
"seed": 10558743,
|
||||
"strata": [
|
||||
"purpose",
|
||||
"slice",
|
||||
"primary language"
|
||||
],
|
||||
"reviewFields": [
|
||||
"purpose",
|
||||
"secondary",
|
||||
"difficulty",
|
||||
"review status",
|
||||
"notes"
|
||||
],
|
||||
"generatedArtifact": "ml/purpose-classifier/.artifacts/human-review-v1.csv",
|
||||
"completionRule": "Every sampled row must be marked accept, relabel, or reject by a human reviewer; relabel/reject decisions are applied to canonical source data before dataset-v1 can be called fully curated."
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,15 @@
|
||||
{
|
||||
"curation": {
|
||||
"excludedDuplicates": {
|
||||
"near": 18
|
||||
"near": 18,
|
||||
"semantic-reviewed": 3
|
||||
},
|
||||
"inputRecords": 12214,
|
||||
"nearDuplicateMethod": "word-trigram Jaccard after SimHash LSH candidate search",
|
||||
"nearDuplicateThreshold": 0.92,
|
||||
"retainedRecords": 12196,
|
||||
"retainedRecords": 12193,
|
||||
"reviewPath": "ml/purpose-classifier/data/curation-review-v1.json",
|
||||
"reviewSha256": "2db763df027b545b4fd654108dcadb0717f9c9c0a63555bf8cea26b791cda370",
|
||||
"vagueEvalPolicy": "validation/test only"
|
||||
},
|
||||
"datasetVersion": "purpose-dataset-v1",
|
||||
@@ -23,7 +26,7 @@
|
||||
"shippedFixturesPath": "Tests/NucleicCoreTests/Fixtures/purpose-prompts.json",
|
||||
"shippedFixturesSha256": "6c8f0eedce35f9da7a407934009e87737b5e85c7371f6a118982d9fae2ea7aae",
|
||||
"syntheticPath": "ml/purpose-classifier/data/frozen-test-v1.jsonl",
|
||||
"syntheticSha256": "e5a6b501a43ce256092932ab05b0af01ff9137aee42d85b7d58bb4ce65f2943f"
|
||||
"syntheticSha256": "5d34c8ac3d39c5b4e2e739c4c9058093beb6f790a42a1397cedefae6271654ab"
|
||||
},
|
||||
"ratios": {
|
||||
"test": 0.1,
|
||||
@@ -50,7 +53,7 @@
|
||||
"distribution": {
|
||||
"language": {
|
||||
"de": 7,
|
||||
"en": 1111,
|
||||
"en": 1110,
|
||||
"es": 9,
|
||||
"fr": 5,
|
||||
"ja": 3,
|
||||
@@ -63,7 +66,7 @@
|
||||
"frontendImpl": 157,
|
||||
"planning": 132,
|
||||
"quickFix": 140,
|
||||
"refactor": 137,
|
||||
"refactor": 136,
|
||||
"review": 127,
|
||||
"writing": 129
|
||||
},
|
||||
@@ -72,19 +75,19 @@
|
||||
"core": 507,
|
||||
"mixed": 80,
|
||||
"pasted-context": 84,
|
||||
"vague-eval": 292
|
||||
"vague-eval": 291
|
||||
}
|
||||
},
|
||||
"hardSyntheticRecords": 635,
|
||||
"logicalRecords": 1229,
|
||||
"sha256": "e5a6b501a43ce256092932ab05b0af01ff9137aee42d85b7d58bb4ce65f2943f",
|
||||
"syntheticRecords": 1142
|
||||
"hardSyntheticRecords": 634,
|
||||
"logicalRecords": 1228,
|
||||
"sha256": "5d34c8ac3d39c5b4e2e739c4c9058093beb6f790a42a1397cedefae6271654ab",
|
||||
"syntheticRecords": 1141
|
||||
},
|
||||
"train": {
|
||||
"distribution": {
|
||||
"language": {
|
||||
"de": 100,
|
||||
"en": 9326,
|
||||
"en": 9324,
|
||||
"es": 100,
|
||||
"fr": 81,
|
||||
"ja": 77,
|
||||
@@ -92,24 +95,24 @@
|
||||
"zh": 72
|
||||
},
|
||||
"purpose": {
|
||||
"backendImpl": 1225,
|
||||
"backendImpl": 1226,
|
||||
"debugging": 1245,
|
||||
"frontendImpl": 1235,
|
||||
"frontendImpl": 1234,
|
||||
"planning": 1223,
|
||||
"quickFix": 1230,
|
||||
"refactor": 1225,
|
||||
"refactor": 1223,
|
||||
"review": 1230,
|
||||
"writing": 1213
|
||||
},
|
||||
"slice": {
|
||||
"boundary": 2065,
|
||||
"core": 5820,
|
||||
"boundary": 2066,
|
||||
"core": 5818,
|
||||
"mixed": 906,
|
||||
"pasted-context": 1035
|
||||
"pasted-context": 1034
|
||||
}
|
||||
},
|
||||
"records": 9826,
|
||||
"sha256": "ca0ec9fe6fef6bc4013bf09bdfed5877327f69c98f8e5552c082cbb4e76d9b7d"
|
||||
"records": 9824,
|
||||
"sha256": "40a5064c2a95d4394d71b3edbad7d03dce7d67b2e5ba9ab9320476e35e3a4953"
|
||||
},
|
||||
"validation": {
|
||||
"distribution": {
|
||||
@@ -123,25 +126,25 @@
|
||||
"zh": 7
|
||||
},
|
||||
"purpose": {
|
||||
"backendImpl": 184,
|
||||
"backendImpl": 183,
|
||||
"debugging": 157,
|
||||
"frontendImpl": 168,
|
||||
"planning": 141,
|
||||
"quickFix": 151,
|
||||
"refactor": 148,
|
||||
"refactor": 149,
|
||||
"review": 140,
|
||||
"writing": 139
|
||||
},
|
||||
"slice": {
|
||||
"boundary": 196,
|
||||
"boundary": 195,
|
||||
"core": 541,
|
||||
"mixed": 83,
|
||||
"pasted-context": 95,
|
||||
"vague-eval": 313
|
||||
"vague-eval": 314
|
||||
}
|
||||
},
|
||||
"records": 1228,
|
||||
"sha256": "7eca32f98105eec1c1a34c324851d52004d8dc720ca5853a1e91fa0a0dd620f4"
|
||||
"sha256": "3d0a3730931db360dd8346cf12d03901ac139acf5be1d9e84975ac591186c588"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -747,7 +747,6 @@
|
||||
{"prompt":"add a compact density option to the data grid — user setting, persists, rows go 44 -> 32 and the font drops a step","purpose":"frontendImpl","secondary":null,"mixed":false,"difficulty":0.5,"slice":"core","lang":"en"}
|
||||
{"prompt":"Ticket OPS-52152: retire the legacy replay path for LongbowFernSnapshotCoordinator\n\nBackground\nThe mobile clients now send monotonic cursors, but the worker still supports the pre-2024 offset token. That compatibility branch performs an extra database lookup on every page and owns a separate retry counter. Support confirmed that the oldest active client is version 8.14, which uses cursors.\n\nAcceptance notes\n- existing cursor tokens must remain valid during a two-release overlap\n- operators need one metric showing legacy traffic by client version\n- a rollback must not require restoring deleted columns\n- tenant isolation remains enforced in the storage query\n- SDK examples should show how to persist the continuation token\n- the public route and current 429 envelope cannot change\n\nOpen questions\nDo we delete the offset decoder immediately or quarantine it behind a flag? Who owns the mobile version gate? Can the database index be removed in the same deploy? Security wants evidence that forged cursors cannot select another tenant.\n\nSlack excerpt\nMina 09:42: “I can provide client adoption numbers, but not a guarantee about sideloaded builds.”\nRavi 09:47: “Please make the rollback trigger explicit; queue lag alone is too noisy.”\n\nAssess LongbowFernSnapshotCoordinator for durability, tenant isolation, races, and misleading observability. Separate blockers from questions and do not produce a patch.","purpose":"review","secondary":null,"mixed":false,"difficulty":0.79,"slice":"pasted-context","lang":"en"}
|
||||
{"prompt":"Propose an architecture for claim-event analytics with privacy, late corrections, financial snapshots, model features, retention, and reproducibility.","purpose":"planning","secondary":null,"mixed":false,"difficulty":0.91,"slice":"core","lang":"en"}
|
||||
{"prompt":"CalderaQuartzPlayerCoordinator: could this be clearer","purpose":"refactor","secondary":null,"mixed":false,"difficulty":0.58,"slice":"vague-eval","lang":"en"}
|
||||
{"prompt":"Create an operator runbook for draining a NATS JetStream consumer, replaying a dead-letter subject, and confirming that lag has recovered.","purpose":"writing","secondary":null,"mixed":false,"difficulty":0.4,"slice":"core","lang":"en"}
|
||||
{"prompt":"Investigate why expired offers remain visible in saved-search results.","purpose":"debugging","secondary":null,"mixed":false,"difficulty":0.61,"slice":"boundary","lang":"en"}
|
||||
{"prompt":"Zephyr: Incident timeline — INC-49147\n\n08:02 deploy IsotopeOspreyJobFlow 6.18.0 begins in eu-west\n08:06 write latency rises from 22 ms to 410 ms; error rate remains below 0.2%\n08:08 autoscaler adds three workers; queue lag continues rising\n08:11 on-call disables feature flag `parallel_commit_v2`\n08:13 latency returns to 31 ms, but duplicate-event warnings increase\n08:19 traffic moved to the previous worker pool\n08:27 queue lag clears; no customer data loss observed\n\nWhat changed\nThe release moved metrics emission into the same Promise.all as the durable write and added NFC normalization to event payloads. Database CPU peaked at 64%, Redis stayed normal, and the downstream consumer reported 183 duplicate keys that its uniqueness constraint ignored.\n\nConstraints from incident command\n- no emergency schema change\n- preserve tenant ordering\n- canary must include a forced consumer rebalance\n- rollback decision must use two independent signals\n- ownership between storage and ingestion teams must be explicit\n\nThe next regular release window is in six days.\n\nMap a safe route from the current IsotopeOspreyJobFlow behavior to the desired one, comparing two approaches and naming telemetry, failure drills, and rollback responsibility.","purpose":"planning","secondary":null,"mixed":false,"difficulty":0.91,"slice":"pasted-context","lang":"en"}
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
{
|
||||
"auditVersion": "purpose-semantic-audit-v1",
|
||||
"candidatePolicy": {
|
||||
"automaticExclusion": false,
|
||||
"crossLabelCosineThreshold": 0.985,
|
||||
"nearestNeighborsPerRecord": 8,
|
||||
"note": "Embedding similarity proposes review candidates only. It cannot safely distinguish deliberate boundary pairs from duplicated labels.",
|
||||
"sameLabelCosineThreshold": 0.97
|
||||
},
|
||||
"candidates": [
|
||||
{
|
||||
"crossesFrozenBoundary": false,
|
||||
"left": {
|
||||
"line": 4639,
|
||||
"origin": "synthetic",
|
||||
"promptHash": "70944e70e0062e947476f73b704ec01ecee19420d06fd174c58a41c04e8112f6",
|
||||
"purpose": "frontendImpl",
|
||||
"source": "ml/purpose-classifier/data/purpose-prompts.jsonl",
|
||||
"split": "validation"
|
||||
},
|
||||
"right": {
|
||||
"line": 8435,
|
||||
"origin": "synthetic",
|
||||
"promptHash": "cb3e63992cbf5830a222933dc7b44be39933baedcb46e335be02e3c379685517",
|
||||
"purpose": "frontendImpl",
|
||||
"source": "ml/purpose-classifier/data/purpose-prompts.jsonl",
|
||||
"split": "validation"
|
||||
},
|
||||
"sameLabel": true,
|
||||
"similarity": 0.970376
|
||||
}
|
||||
],
|
||||
"embedding": {
|
||||
"device": "cpu",
|
||||
"fixedInputShape": [
|
||||
1,
|
||||
128
|
||||
],
|
||||
"model": "sentence-transformers/all-MiniLM-L6-v2",
|
||||
"pooling": "attention-mask mean pooling followed by L2 normalization",
|
||||
"revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41",
|
||||
"truncation": {
|
||||
"headTokens": 63,
|
||||
"strategy": "head-tail-pair",
|
||||
"tailTokens": 62
|
||||
}
|
||||
},
|
||||
"humanReviewSample": {
|
||||
"fraction": 0.1,
|
||||
"generatedCSV": "ml/purpose-classifier/.artifacts/human-review-v1.csv",
|
||||
"populationRecords": 12193,
|
||||
"requiredFields": [
|
||||
"reviewedPurpose",
|
||||
"reviewedSecondary",
|
||||
"reviewedDifficulty",
|
||||
"reviewStatus"
|
||||
],
|
||||
"samplePurposeCounts": {
|
||||
"backendImpl": 158,
|
||||
"debugging": 154,
|
||||
"frontendImpl": 155,
|
||||
"planning": 152,
|
||||
"quickFix": 150,
|
||||
"refactor": 149,
|
||||
"review": 150,
|
||||
"writing": 151
|
||||
},
|
||||
"sampleRecords": 1219,
|
||||
"sampleSliceCounts": {
|
||||
"boundary": 245,
|
||||
"core": 689,
|
||||
"mixed": 108,
|
||||
"pasted-context": 117,
|
||||
"vague-eval": 60
|
||||
},
|
||||
"seed": 10558743,
|
||||
"status": "planned",
|
||||
"strata": [
|
||||
"purpose",
|
||||
"slice",
|
||||
"primary language"
|
||||
]
|
||||
},
|
||||
"population": {
|
||||
"auditedRecords": 12280,
|
||||
"canonicalSourceRecords": 12214,
|
||||
"classifiableShippedFixtures": 87,
|
||||
"curatedSyntheticRecords": 12193
|
||||
},
|
||||
"schemaVersion": 1,
|
||||
"summary": {
|
||||
"candidates": 1,
|
||||
"crossLabelCandidates": 0,
|
||||
"frozenBoundaryCandidates": 0,
|
||||
"sameLabelCandidates": 1
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user