{
 "description": "Paired closed-loop comparisons (candidate vs unedited base on the same slots) from the Claude/Opus weightsmith runs on the reference GPU host (Nov 4-6, 2026) and, for reference, the Astra study's reported comparisons. Counts are successes; fixed = base failed and candidate succeeded; broke = base succeeded and candidate failed.",
 "runtime_notes": {
  "claude": "weightsmith main runtime, RoboLab on Isaac Sim (performance preset), pi05_droid_jointpos, RTX 3090 GPUs (one eval lane for the Nov 4-5 runs; two lanes for the Nov 5-6 binding-edit runs, no lane effect; see EVIDENCE.md). Rows of type 'heldout' are operator-run weightsmith/heldout.py scores on unseen slots >= 90000 (files in heldout/); all other Claude rows use practice slots.",
  "astra": "Values copied from study/RESULTS.md (original and fixed16 runtimes, separate host). Not pooled with Claude rows."
 },
 "rows": [
  {
   "level": "Perception: wrong target object",
   "task": "Stack yellow on red",
   "label": "v1a1: attention gain on identity axis",
   "editor": "Claude Opus",
   "type": "weight",
   "values_changed": 30400000,
   "n": 64,
   "base": 28,
   "cand": 21,
   "fixed": 9,
   "broke": 16
  },
  {
   "level": "Perception: wrong target object",
   "task": "Stack yellow on red",
   "label": "dkv2: VLM key/value gain",
   "editor": "Claude Opus",
   "type": "weight",
   "values_changed": 12000000,
   "n": 64,
   "base": 28,
   "cand": 21,
   "fixed": 11,
   "broke": 18
  },
  {
   "level": "Perception: wrong target object",
   "task": "Stack yellow on red",
   "label": "combo1: dkv2 + head rewire (run 1)",
   "editor": "Claude Opus",
   "type": "weight",
   "values_changed": 12200000,
   "n": 64,
   "base": 28,
   "cand": 21,
   "fixed": 9,
   "broke": 16
  },
  {
   "level": "Perception: wrong target object",
   "task": "Stack yellow on red",
   "label": "combo1 rerun, same weights (run 2)",
   "editor": "Claude Opus",
   "type": "weight",
   "values_changed": 12200000,
   "n": 64,
   "base": 28,
   "cand": 21,
   "fixed": 8,
   "broke": 15
  },
  {
   "level": "Perception: wrong target object",
   "task": "Stack yellow on red",
   "label": "Astra A1 p1 (4,412 values)",
   "editor": "GPT-6 Astra",
   "type": "weight",
   "values_changed": 4412,
   "n": 32,
   "base": 11,
   "cand": 12,
   "fixed": 7,
   "broke": 6
  },
  {
   "level": "Perception: wrong target object",
   "task": "Stack yellow on red",
   "label": "Astra A2 p1 (1,543 values)",
   "editor": "GPT-6 Astra",
   "type": "weight",
   "values_changed": 1543,
   "n": 32,
   "base": 11,
   "cand": 13,
   "fixed": 9,
   "broke": 7
  },
  {
   "level": "Perception: wrong target object",
   "task": "Stack yellow on red",
   "label": "Astra A2 p3 (791 values)",
   "editor": "GPT-6 Astra",
   "type": "weight",
   "values_changed": 791,
   "n": 32,
   "base": 11,
   "cand": 13,
   "fixed": 7,
   "broke": 5
  },
  {
   "level": "Perception: wrong target object",
   "task": "Stack yellow on red",
   "label": "Astra fixed16 p1 (6,225 values)",
   "editor": "GPT-6 Astra",
   "type": "weight",
   "values_changed": 6225,
   "n": 32,
   "base": 17,
   "cand": 11,
   "fixed": 5,
   "broke": 11
  },
  {
   "level": "Perception: wrong target object",
   "task": "Stack yellow on red",
   "label": "Claude, Astra protocol p1 (64 rows, L17)",
   "editor": "Claude Opus",
   "type": "weight",
   "values_changed": null,
   "n": 32,
   "base": 12,
   "cand": 16,
   "fixed": 8,
   "broke": 4,
   "source": "projects/claude-rq123; gate: not eligible (broke 4 > 2)"
  },
  {
   "level": "Perception: wrong target object",
   "task": "Stack yellow on red",
   "label": "Claude, Astra protocol p2 (35,210 values)",
   "editor": "Claude Opus",
   "type": "weight",
   "values_changed": 35210,
   "n": 32,
   "base": 12,
   "cand": 11,
   "fixed": 6,
   "broke": 7,
   "source": "projects/claude-rq123; gate: not eligible"
  },
  {
   "level": "Language binding",
   "task": "Cube then banana",
   "label": "Explicit two-step wording (no weight change)",
   "editor": "Claude Opus",
   "type": "prompt",
   "values_changed": 0,
   "n": 32,
   "base": 3,
   "cand": 22,
   "fixed": 20,
   "broke": 1
  },
  {
   "level": "Language binding",
   "task": "Cube then banana",
   "label": "v2mi-f: binding edit, practice seeds",
   "editor": "Claude Opus",
   "type": "weight",
   "values_changed": 450468837,
   "n": 32,
   "base": 3,
   "cand": 25,
   "fixed": 22,
   "broke": 0
  },
  {
   "level": "Language binding",
   "task": "Cube then banana",
   "label": "v2mi-f: binding edit, HELD-OUT seeds",
   "editor": "Claude Opus",
   "type": "heldout",
   "values_changed": 450468837,
   "n": 128,
   "base": 27,
   "cand": 106,
   "fixed": 81,
   "broke": 2,
   "source": "heldout/v2mi-f-RubiksCubeThenBananaTask-90000-128/results.json; McNemar p=7.2e-22"
  },
  {
   "level": "Language binding",
   "task": "Cube then banana",
   "label": "binding edit + preservation keys (v2p-f)",
   "editor": "Claude Opus",
   "type": "weight",
   "values_changed": null,
   "n": 32,
   "base": 3,
   "cand": 24,
   "fixed": 21,
   "broke": 0
  },
  {
   "level": "Language binding",
   "task": "Cube then banana",
   "label": "minimal edit (0.16%), HELD-OUT seeds",
   "editor": "Claude Opus",
   "type": "heldout",
   "values_changed": 248057499,
   "n": 128,
   "base": 19,
   "cand": 102,
   "fixed": 86,
   "broke": 3
  },
  {
   "level": "Language binding",
   "task": "Cube then banana",
   "label": "random directions, same size as v2mi-f, HELD-OUT seeds",
   "editor": "Claude Opus",
   "type": "weight",
   "values_changed": 502291925,
   "n": 64,
   "base": 13,
   "cand": 9,
   "fixed": 6,
   "broke": 10,
   "source": "controls/ctl1-90000-64.results.json (random-direction control: v2mi-f's layers, rank and singular values, random orthonormal directions, seed 0) vs the unedited run of heldout/v2mi-f-RubiksCubeThenBananaTask-90000-128 on slots 90000-90063; McNemar p=0.45"
  },
  {
   "level": "Language binding",
   "task": "Cube then banana",
   "label": "Explicit two-step wording, HELD-OUT seeds (no weight change)",
   "editor": "Claude Opus",
   "type": "prompt",
   "values_changed": 0,
   "n": 64,
   "base": 13,
   "cand": 57,
   "fixed": 45,
   "broke": 1,
   "source": "controls/ctl1-90000-64 (explicit instruction, unedited weights) vs the unedited run of heldout/v2mi-f-RubiksCubeThenBananaTask-90000-128 on slots 90000-90063; McNemar p=1.3e-12; vs distilled edit 57 vs 50 (11/4, p=0.12), vs 0.16% variant 57 vs 54 (10/7, p=0.63)"
  },
  {
   "level": "Binding edit on other tasks",
   "task": "Cube left of bowl",
   "label": "binding edit, HELD-OUT seeds",
   "editor": "Claude Opus",
   "type": "heldout",
   "values_changed": 450468837,
   "n": 64,
   "base": 8,
   "cand": 36,
   "fixed": 33,
   "broke": 5
  },
  {
   "level": "Binding edit on other tasks",
   "task": "Cube left of bowl",
   "label": "binding edit",
   "editor": "Claude Opus",
   "type": "weight",
   "values_changed": 450468837,
   "n": 32,
   "base": 3,
   "cand": 22,
   "fixed": 20,
   "broke": 1
  },
  {
   "level": "Binding edit on other tasks",
   "task": "Cube right of bowl",
   "label": "binding edit",
   "editor": "Claude Opus",
   "type": "weight",
   "values_changed": 450468837,
   "n": 32,
   "base": 0,
   "cand": 0,
   "fixed": 0,
   "broke": 0
  },
  {
   "level": "Binding edit on other tasks",
   "task": "Stack three cubes",
   "label": "binding edit",
   "editor": "Claude Opus",
   "type": "weight",
   "values_changed": 450468837,
   "n": 32,
   "base": 14,
   "cand": 19,
   "fixed": 9,
   "broke": 4
  },
  {
   "level": "Binding edit on other tasks",
   "task": "Block tower",
   "label": "binding edit",
   "editor": "Claude Opus",
   "type": "weight",
   "values_changed": 450468837,
   "n": 32,
   "base": 15,
   "cand": 14,
   "fixed": 7,
   "broke": 8
  },
  {
   "level": "Binding edit on other tasks",
   "task": "Yogurt into bowl",
   "label": "binding edit",
   "editor": "Claude Opus",
   "type": "weight",
   "values_changed": 450468837,
   "n": 32,
   "base": 3,
   "cand": 2,
   "fixed": 2,
   "broke": 3
  },
  {
   "level": "Binding edit on other tasks",
   "task": "Grab a bagel",
   "label": "binding edit",
   "editor": "Claude Opus",
   "type": "weight",
   "values_changed": 450468837,
   "n": 32,
   "base": 0,
   "cand": 1,
   "fixed": 1,
   "broke": 0
  },
  {
   "level": "Binding edit on other tasks",
   "task": "Marker in mug",
   "label": "binding edit",
   "editor": "Claude Opus",
   "type": "weight",
   "values_changed": 450468837,
   "n": 32,
   "base": 0,
   "cand": 1,
   "fixed": 1,
   "broke": 0
  },
  {
   "level": "Binding edit on other tasks",
   "task": "Pick up eyeglasses",
   "label": "binding edit, HELD-OUT seeds",
   "editor": "Claude Opus",
   "type": "heldout",
   "values_changed": 450468837,
   "n": 64,
   "base": 22,
   "cand": 44,
   "fixed": 31,
   "broke": 9
  },
  {
   "level": "Binding edit on other tasks",
   "task": "Pick up eyeglasses",
   "label": "binding edit, practice (2 runs)",
   "editor": "Claude Opus",
   "type": "weight",
   "values_changed": 450468837,
   "n": 64,
   "base": 26,
   "cand": 46,
   "fixed": 28,
   "broke": 8
  },
  {
   "level": "Binding edit on other tasks",
   "task": "Mustard into bin",
   "label": "binding edit",
   "editor": "Claude Opus",
   "type": "weight",
   "values_changed": 450468837,
   "n": 32,
   "base": 7,
   "cand": 12,
   "fixed": 11,
   "broke": 6
  },
  {
   "level": "Binding edit on other tasks",
   "task": "Stack yellow on red",
   "label": "binding edit",
   "editor": "Claude Opus",
   "type": "weight",
   "values_changed": 450468837,
   "n": 32,
   "base": 14,
   "cand": 17,
   "fixed": 9,
   "broke": 6
  },
  {
   "level": "Binding edit on other tasks",
   "task": "Stack bowls",
   "label": "binding edit",
   "editor": "Claude Opus",
   "type": "weight",
   "values_changed": 450468837,
   "n": 32,
   "base": 19,
   "cand": 21,
   "fixed": 8,
   "broke": 6
  },
  {
   "level": "Binding edit on other tasks",
   "task": "Butter box onto raisin box",
   "label": "binding edit (vs first base run)",
   "editor": "Claude Opus",
   "type": "weight",
   "values_changed": 450468837,
   "n": 16,
   "base": 14,
   "cand": 10,
   "fixed": 0,
   "broke": 4
  },
  {
   "level": "Binding edit on other tasks",
   "task": "Butter box onto raisin box",
   "label": "binding edit + preservation keys (vs first base run)",
   "editor": "Claude Opus",
   "type": "weight",
   "values_changed": 450468837,
   "n": 16,
   "base": 14,
   "cand": 10,
   "fixed": 0,
   "broke": 4
  },
  {
   "level": "Binding edit on other tasks",
   "task": "Cube into bowl",
   "label": "binding edit",
   "editor": "Claude Opus",
   "type": "weight",
   "values_changed": 450468837,
   "n": 16,
   "base": 16,
   "cand": 16,
   "fixed": 0,
   "broke": 0
  },
  {
   "level": "Binding edit on other tasks",
   "task": "Banana onto plate",
   "label": "binding edit",
   "editor": "Claude Opus",
   "type": "weight",
   "values_changed": 450468837,
   "n": 16,
   "base": 16,
   "cand": 16,
   "fixed": 0,
   "broke": 0
  },
  {
   "level": "Motor: missing competence",
   "task": "Pick up flat green box",
   "label": "10 edits (readout, MLP rows, neuron flip, biases)",
   "editor": "Claude Opus",
   "type": "weight",
   "values_changed": null,
   "n": 16,
   "base": 0,
   "cand": 0,
   "fixed": 0,
   "broke": 0,
   "note": "gb4, wr1gb4, wr2gb4, wr2gb6, wr2e1, ro8, and3, and3rot, rw1 each 0/16; e2-contact6 0/32. Base 0/64.",
   "display": "0/16 each vs 0/64"
  },
  {
   "level": "Motor: grasp physics",
   "task": "Mustard into left bin",
   "label": "gthr04: gripper bias (1 value)",
   "editor": "Claude Opus",
   "type": "weight",
   "values_changed": 1,
   "n": 64,
   "base": 13,
   "cand": 13,
   "fixed": 10,
   "broke": 10
  },
  {
   "level": "Gripper commitment (all tasks)",
   "task": "Stack yellow on red",
   "label": "gthr04: gripper bias (1 value)",
   "editor": "Claude Opus",
   "type": "weight",
   "values_changed": 1,
   "n": 64,
   "base": 27,
   "cand": 27,
   "fixed": 12,
   "broke": 12
  },
  {
   "level": "Gripper commitment (all tasks)",
   "task": "Pick up eyeglasses",
   "label": "gthr04: gripper bias (1 value)",
   "editor": "Claude Opus",
   "type": "weight",
   "values_changed": 1,
   "n": 32,
   "base": 12,
   "cand": 13,
   "fixed": 7,
   "broke": 6
  },
  {
   "level": "Gripper commitment (all tasks)",
   "task": "Stack left bowl on right bowl",
   "label": "gthr04: gripper bias (1 value)",
   "editor": "Claude Opus",
   "type": "weight",
   "values_changed": 1,
   "n": 32,
   "base": 19,
   "cand": 18,
   "fixed": 6,
   "broke": 7
  },
  {
   "level": "Gripper commitment (all tasks)",
   "task": "Butter box onto raisin box",
   "label": "gthr04: gripper bias (1 value)",
   "editor": "Claude Opus",
   "type": "weight",
   "values_changed": 1,
   "n": 16,
   "base": 14,
   "cand": 14,
   "fixed": 2,
   "broke": 2
  },
  {
   "level": "Gripper commitment (all tasks)",
   "task": "Cube into bowl",
   "label": "gthr04: gripper bias (1 value)",
   "editor": "Claude Opus",
   "type": "weight",
   "values_changed": 1,
   "n": 16,
   "base": 16,
   "cand": 16,
   "fixed": 0,
   "broke": 0
  },
  {
   "level": "Gripper commitment (all tasks)",
   "task": "Banana onto plate",
   "label": "gthr04: gripper bias (1 value)",
   "editor": "Claude Opus",
   "type": "weight",
   "values_changed": 1,
   "n": 16,
   "base": 16,
   "cand": 16,
   "fixed": 0,
   "broke": 0
  },
  {
   "level": "Gripper commitment (all tasks)",
   "task": "Stack yellow on red",
   "label": "Astra study: runtime threshold 0.4",
   "editor": "operator",
   "type": "runtime",
   "values_changed": 0,
   "n": 64,
   "base": 23,
   "cand": 36,
   "fixed": 21,
   "broke": 8
  },
  {
   "level": "Unchanged weights (noise controls)",
   "task": "Stack yellow on red",
   "label": "Astra study: original identity",
   "editor": "operator",
   "type": "identity",
   "values_changed": 0,
   "n": 64,
   "base": 26,
   "cand": 23,
   "fixed": 17,
   "broke": 20
  },
  {
   "level": "Unchanged weights (noise controls)",
   "task": "Stack yellow on red",
   "label": "Astra study: extra unchanged repeat",
   "editor": "operator",
   "type": "identity",
   "values_changed": 0,
   "n": 64,
   "base": 26,
   "cand": 35,
   "fixed": 21,
   "broke": 12
  },
  {
   "level": "Unchanged weights (noise controls)",
   "task": "Stack yellow on red",
   "label": "Astra study: fixed16 identity",
   "editor": "operator",
   "type": "identity",
   "values_changed": 0,
   "n": 64,
   "base": 24,
   "cand": 27,
   "fixed": 16,
   "broke": 13
  },
  {
   "level": "Unchanged weights (noise controls)",
   "task": "Butter box onto raisin box",
   "label": "this work: unedited rerun, same seeds",
   "editor": "operator",
   "type": "identity",
   "values_changed": 0,
   "n": 16,
   "base": 14,
   "cand": 8,
   "fixed": 0,
   "broke": 6
  }
 ]
}