blakeslee-research/nucleic-purpose-classifier
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge nucleic/lucid-north-quail-rnvt into dev

Browse Source
This commit is contained in:
abkslm
2026-07-30 03:48:00 -07:00
parent e2a5314f1b
commit 9c43be6df1
17 changed files with 3325 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
data/dataset-v1-manifest.json
+147
View File
@@ -0,0 +1,147 @@
{
"curation": {
"excludedDuplicates": {
"near": 18
},
"inputRecords": 12214,
"nearDuplicateMethod": "word-trigram Jaccard after SimHash LSH candidate search",
"nearDuplicateThreshold": 0.92,
"retainedRecords": 12196,
"vagueEvalPolicy": "validation/test only"
},
"datasetVersion": "purpose-dataset-v1",
"frozenEval": {
"classifiableShippedFixtureRecords": 87,
"excludedGeneralFixtureRecords": 5,
"hardSliceDefinition": [
"boundary",
"mixed",
"pasted-context",
"vague-eval"
],
"shippedFixtureRecords": 92,
"shippedFixturesPath": "Tests/NucleicCoreTests/Fixtures/purpose-prompts.json",
"shippedFixturesSha256": "6c8f0eedce35f9da7a407934009e87737b5e85c7371f6a118982d9fae2ea7aae",
"syntheticPath": "ml/purpose-classifier/data/frozen-test-v1.jsonl",
"syntheticSha256": "e5a6b501a43ce256092932ab05b0af01ff9137aee42d85b7d58bb4ce65f2943f"
},
"ratios": {
"test": 0.1,
"train": 0.8,
"validation": 0.1
},
"schemaVersion": 1,
"seed": 3248837105,
"sources": [
{
"path": "ml/purpose-classifier/data/purpose-prompts.jsonl",
"records": 9214,
"sha256": "3c7f8b496ef2014794a50433a78e3125c0dd4a03dc2a7773b52ee25587ac48e7"
},
{
"path": "ml/purpose-classifier/data/purpose-prompts-round2.jsonl",
"records": 3000,
"sha256": "0a35eac2549e95b31518cd0eee99b83b03761c26a0fb258f427fe04a632699ed"
}
],
"splits": {
"test": {
"classifiableFixtureRecords": 87,
"distribution": {
"language": {
"de": 7,
"en": 1111,
"es": 9,
"fr": 5,
"ja": 3,
"pt": 4,
"zh": 3
},
"purpose": {
"backendImpl": 171,
"debugging": 149,
"frontendImpl": 157,
"planning": 132,
"quickFix": 140,
"refactor": 137,
"review": 127,
"writing": 129
},
"slice": {
"boundary": 179,
"core": 507,
"mixed": 80,
"pasted-context": 84,
"vague-eval": 292
}
},
"hardSyntheticRecords": 635,
"logicalRecords": 1229,
"sha256": "e5a6b501a43ce256092932ab05b0af01ff9137aee42d85b7d58bb4ce65f2943f",
"syntheticRecords": 1142
},
"train": {
"distribution": {
"language": {
"de": 100,
"en": 9326,
"es": 100,
"fr": 81,
"ja": 77,
"pt": 70,
"zh": 72
},
"purpose": {
"backendImpl": 1225,
"debugging": 1245,
"frontendImpl": 1235,
"planning": 1223,
"quickFix": 1230,
"refactor": 1225,
"review": 1230,
"writing": 1213
},
"slice": {
"boundary": 2065,
"core": 5820,
"mixed": 906,
"pasted-context": 1035
}
},
"records": 9826,
"sha256": "ca0ec9fe6fef6bc4013bf09bdfed5877327f69c98f8e5552c082cbb4e76d9b7d"
},
"validation": {
"distribution": {
"language": {
"de": 8,
"en": 1178,
"es": 12,
"fr": 7,
"ja": 8,
"pt": 8,
"zh": 7
},
"purpose": {
"backendImpl": 184,
"debugging": 157,
"frontendImpl": 168,
"planning": 141,
"quickFix": 151,
"refactor": 148,
"review": 140,
"writing": 139
},
"slice": {
"boundary": 196,
"core": 541,
"mixed": 83,
"pasted-context": 95,
"vague-eval": 313
}
},
"records": 1228,
"sha256": "7eca32f98105eec1c1a34c324851d52004d8dc720ca5853a1e91fa0a0dd620f4"
}
}
}
data/frozen-test-v1.jsonl
+1142
View File
File diff suppressed because it is too large Load Diff
data/generation-manifest.json
+57
View File
@@ -0,0 +1,57 @@
{
"schemaVersion": 1,
"dataset": "purpose-classifier-source-v1",
"canonicalFiles": [
"purpose-prompts.jsonl",
"purpose-prompts-round2.jsonl"
],
"derivedBatchFiles": [
"round2-01.jsonl",
"round2-02.jsonl",
"round2-03.jsonl",
"round2-04.jsonl",
"round2-05.jsonl",
"round2-06.jsonl",
"round2-07.jsonl",
"round2-08.jsonl",
"round2-09.jsonl",
"round2-10.jsonl",
"round2-11.jsonl",
"round2-12.jsonl",
"round2-13.jsonl",
"round2-14.jsonl",
"round2-15.jsonl"
],
"generations": [
{
"file": "purpose-prompts.jsonl",
"model": "mixed frontier-model runs (legacy sol/opus aliases; exact model IDs were not retained)",
"date": "2026-07-29",
"prompt": "../datagen-prompt.md",
"topics": [
"web and mobile",
"backend and data",
"infrastructure and systems",
"developer tooling"
],
"notes": "The canonical aggregate was curated from the original per-model batches. Three exact overlaps with the shipped eval fixture were removed on 2026-07-30."
},
{
"file": "purpose-prompts-round2.jsonl",
"model": "Nucleic frontier-model generator (exact underlying model ID was not retained)",
"date": "2026-07-30",
"prompt": "../datagen-prompt-2.md",
"topics": [
"boundary confusion pairs",
"pasted context",
"mixed intent",
"non-English developer prompts"
],
"notes": "Corrective generation that counterbalances round-one label, slice, length, opener, and language drift."
}
],
"limitations": [
"The exact generator model IDs and sampling parameters were not recorded when the source corpora were created.",
"The round2-NN files are retained generation batches and duplicate the round-two canonical aggregate; dataset tooling must read canonicalFiles only."
]
}
data/purpose-prompts.jsonl
-3
View File
@@ -22,7 +22,6 @@
{"prompt":"feature flag `new_billing_summary` is still gated to staff only, open it to everyone","purpose":"quickFix","secondary":null,"mixed":false,"difficulty":0.15,"slice":"core","lang":"en"}
{"prompt":"dedupe the three copies of retryWithBackoff","purpose":"refactor","secondary":null,"mixed":false,"difficulty":0.35,"slice":"core","lang":"en"}
{"prompt":"the /search endpoint got 4x slower after we shipped last thursday and nothing in that diff touches search. p99 went 180ms -> 750ms. where do i even start","purpose":"debugging","secondary":null,"mixed":false,"difficulty":0.75,"slice":"boundary","lang":"en"}
{"prompt":"make it pop","purpose":"frontendImpl","secondary":null,"mixed":false,"difficulty":0.3,"slice":"vague-eval","lang":"en"}
{"prompt":"is our jwt refresh flow safe against replay if someone grabs a refresh token off a stolen device? read through auth/refresh.go and tell me what you think, don't change anything yet","purpose":"review","secondary":null,"mixed":false,"difficulty":0.65,"slice":"core","lang":"en"}
{"prompt":"compare our current celery setup against just using postgres SKIP LOCKED for the job queue. we do maybe 200 jobs/min, mostly short","purpose":"review","secondary":null,"mixed":false,"difficulty":0.6,"slice":"core","lang":"en"}
{"prompt":"explain what pkg/ledger/reconcile.go actually does, line by line if you have to. inherited it and nobody left knows","purpose":"review","secondary":null,"mixed":false,"difficulty":0.55,"slice":"boundary","lang":"en"}
@@ -72,7 +71,6 @@
{"prompt":"review the diff on my branch before i open the PR, specifically the transaction boundaries in the transfer path","purpose":"review","secondary":null,"mixed":false,"difficulty":0.6,"slice":"core","lang":"en"}
{"prompt":"what's the difference between our useSyncedQuery hook and just using react-query's useQuery with our fetcher? feels like we reimplemented it","purpose":"review","secondary":null,"mixed":false,"difficulty":0.45,"slice":"core","lang":"en"}
{"prompt":"document the event payload schemas for the pubsub topics in docs/events.md AND add the missing JSON schema files under schemas/ so we can validate in CI","purpose":"writing","secondary":"backendImpl","mixed":true,"difficulty":0.55,"slice":"mixed","lang":"en"}
{"prompt":"continue","purpose":"backendImpl","secondary":null,"mixed":false,"difficulty":0.4,"slice":"vague-eval","lang":"en"}
{"prompt":"figure out why the checkout total is off by a cent for some carts and then write up the postmortem, we told the customer we'd have both by friday","purpose":"debugging","secondary":"writing","mixed":true,"difficulty":0.7,"slice":"mixed","lang":"en"}
{"prompt":"picking the payment-flow bug back up from yesterday — the 3DS redirect lands on a blank page maybe a third of the time in safari. no console error, network tab shows the POST to /confirm returning 302 and then nothing","purpose":"debugging","secondary":null,"mixed":false,"difficulty":0.75,"slice":"core","lang":"en"}
{"prompt":"come up with the migration plan for moving our 4TB mysql instance to aurora with under 15 min of write downtime, and list the go/no-go checks","purpose":"planning","secondary":null,"mixed":false,"difficulty":0.9,"slice":"core","lang":"en"}
@@ -99,7 +97,6 @@
{"prompt":"internal blog post about the latency work we did last quarter. audience is other engineers here, ~800 words, i can give you the numbers","purpose":"writing","secondary":null,"mixed":false,"difficulty":0.45,"slice":"core","lang":"en"}
{"prompt":"runbook for the on-call rotation covering the four alerts that actually page us","purpose":"writing","secondary":null,"mixed":false,"difficulty":0.5,"slice":"core","lang":"en"}
{"prompt":"the CONTRIBUTING.md is 3 lines. write a real one — branch naming, how to run the test suite, what we expect in a PR, and the codegen step people always forget","purpose":"writing","secondary":null,"mixed":false,"difficulty":0.4,"slice":"core","lang":"en"}
{"prompt":"do the thing we discussed","purpose":"backendImpl","secondary":null,"mixed":false,"difficulty":0.4,"slice":"vague-eval","lang":"en"}
{"prompt":"spec out the multiplayer lobby then get the netcode skeleton in — matchmaking rules first as a doc, then the actual go server with the room state machine","purpose":"planning","secondary":"backendImpl","mixed":true,"difficulty":0.85,"slice":"mixed","lang":"en"}
{"prompt":"evaluate whether we should adopt bazel. monorepo, ~60 services, mixed go/ts/python, current builds are makefiles and 14 min of CI. i want a recommendation with a phased rollout, not a yes/no","purpose":"planning","secondary":null,"mixed":false,"difficulty":0.85,"slice":"core","lang":"en"}
{"prompt":"what's a reasonable retention + partitioning strategy for the raw telemetry table? we ingest ~90GB/day and only ever query the last 14 days interactively","purpose":"planning","secondary":null,"mixed":false,"difficulty":0.7,"slice":"core","lang":"en"}