{ "title": "Weight editing across robot models", "question": "A world-action model and a reasoning-backbone VLA: where does weight-space programming fix failures?", "status": "draft", "shared_slots": "practice 30000–30031, held-out 96000–96063", "models": [ { "id": "cosmos", "name": "Cosmos3 Edge Policy", "short": "world-action model", "kind": "world-action model", "status": "final", "checkpoint": "`nvidia/Cosmos3-Edge-Policy-DROID` at `a7c7288`, 3.37B parameters by the track’s count (the model card lists 4B)", "architecture": "One transformer denoises an action chunk together with future video latents, conditioned on the instruction: a mixture of transformers with 28 layers of width 2048, each holding a text pathway and a generation pathway for video and action tokens. It sees the wrist and both shoulder cameras as one composite image and returns 32 joint-position steps per request, run open loop at 15 Hz.", "entry": "Text tokens attend only to text; video and action tokens attend to the text keys and values in every layer, so the instruction reaches the actions only through those keys and values (read from the code that served Cosmos3 Nano).", "where": "text pathway / video-action (generation) pathway", "evaluation": "Closed loop on the shared RoboLab DROID tasks and slots; settings of the Same Goal, Different Words study (4 sampling steps, guidance 3).", "source": { "kind": "branch", "branch": "cross-cosmos/v1", "path": "study/results/cross/metrics_cosmos.json", "url": "https://github.com/adeeb10abbas/weightsmith/blob/cross-cosmos/v1/study/results/cross/metrics_cosmos.json", "commit": "2444083b32c2" } }, { "id": "groot", "name": "GR00T N1.7", "short": "reasoning-backbone VLA", "kind": "VLA on a reasoning backbone", "status": "final", "checkpoint": "`nvidia/GR00T-N1.7-DROID` at `05e7cc9`, 3.14B parameters", "architecture": "A Cosmos-Reason2-2B vision-language backbone (Qwen3-VL), cut to its first 16 language layers, reads an exterior and a wrist camera (180 × 320 each) and the instruction. The action head, 4 self-attention layers and 32 diffusion-transformer blocks, turns the backbone's tokens and the robot state into 40-step chunks by flow matching; the first 8 steps run at 15 Hz.", "entry": "As text tokens in the Cosmos-Reason2 backbone, next to the image tokens (lowercased, punctuation removed). The action head reads the instruction only through the backbone's tokens.", "where": "Cosmos-Reason2 VLM backbone / flow-matching DiT action head", "evaluation": "Closed loop on the shared RoboLab DROID tasks and slots, NVIDIA's validated client and embodiment setting.", "source": { "kind": "branch", "branch": "cross-groot/v1", "path": "study/results/cross/metrics_groot.json", "url": "https://github.com/adeeb10abbas/weightsmith/blob/cross-groot/v1/study/results/cross/metrics_groot.json", "commit": "f1dfb229b095" } }, { "id": "pi05", "name": "π0.5", "short": "OpenPI VLA, control", "kind": "VLA with an action expert (OpenPI)", "status": "final", "checkpoint": "`pi05_droid_jointpos` (OpenPI), about 3.35 billion parameters", "architecture": "An 18-layer PaliGemma vision-language model (VLM) and an 18-layer Gemma action expert. The action expert attends to the VLM's keys and values and produces 15-step action chunks, including a gripper command, by flow matching.", "entry": "As text tokens in the VLM, together with image tokens from two cameras and robot-state tokens, under bidirectional attention, so the image tokens read the instruction. The distilled edit goes into the VLM's MLP weights.", "where": "VLM backbone / action expert / action output projection", "evaluation": "Closed loop on RoboLab in Isaac Sim: on the shared starts with the default render preset, and, for the comparisons not rerun there, the paper’s evaluation (RTX 3090, performance render preset), each with its own slots and number of starts. Trained on DROID data and none of these tasks.", "source": { "kind": "branch", "branch": "cross-pi05/v1", "path": "study/results/cross/metrics_pi05.json", "url": "https://github.com/adeeb10abbas/weightsmith/blob/cross-pi05/v1/study/results/cross/metrics_pi05.json", "commit": "05b5a28fd7b5" } } ], "tasks": [ { "id": "T1", "name": "Cube then banana", "robolab": "RubiksCubeThenBananaTask", "pi05_failure": "lets go of the cube early: binds “in the bowl” to the banana" }, { "id": "T2", "name": "Pick up the eyeglasses", "robolab": "PickGlassesTask", "pi05_failure": "grasps one temple arm and lifts too little" }, { "id": "T3", "name": "Cube left of the bowl", "robolab": "RubiksCubeLeftOfBowlTask", "pi05_failure": "carries along a diagonal and leaves the cube behind the bowl" }, { "id": "T4", "name": "Stack yellow on red", "robolab": "StackYellowOnRedTask", "pi05_failure": "goes first for a lookalike yellow object" }, { "id": "T5", "name": "Mustard into the left bin", "robolab": "MustardInLeftBinTask", "pi05_failure": "the bottle slips from a cap grasp mid-carry" }, { "id": "T6", "name": "Stack three cubes", "robolab": "Stack3RubiksCubeTask", "pi05_failure": "builds two cubes, then closes on air above the stack" }, { "id": "T7", "name": "Pick up the flat box", "robolab": "PickUpGreenObjectTask", "pi05_failure": "never lifts it: the box is wider than the open gripper" }, { "id": "T8", "name": "Mustard right of the raisin box, reference-first", "robolab": "MustardRightOfRaisinRFTask", "pi05_failure": "takes the wrong object or carries it the wrong way" }, { "id": "R1", "name": "Retention task", "robolab": "RubiksCubeTask", "pi05_failure": "already solved" } ], "tldr": { "items": [ { "id": "fix", "lead": "The fix.", "text": "A closed-form weight edit makes a world-action model, Cosmos3 Edge, succeed from 56 of 64 new starts, as often as rewording does.", "kind": null, "tag": null }, { "id": "where", "lead": "When it works.", "text": "Only in the VLM or text pathway. Outside the action-expert VLA, only when the clear wording swaps words in place, as predicted.", "kind": null, "tag": null }, { "id": "cost", "lead": "The cost.", "text": "The edit overgeneralises the spatial relation it fixes. Other instructions using that relation break, and preservation constraints did not prevent it.", "kind": null, "tag": null } ], "note": "" }, "tldr_models": { "items": [ { "id": "pi05", "lead": "π0.5", "kind": "OpenPI VLA, control", "tag": null, "prov": true, "text": "Edits lifted success on Cube then banana from 2 to 40 of 64 unseen starts (p = 7.3 × 10⁻¹²) and on Mustard right of raisin box from 0 to 28 of 64 (p = 7.5 × 10⁻⁹). A random edit of the same size scored 2 of 64 on Cube then banana, against 2 unedited. No detectable gain on Yellow on red, Mustard in left bin, Three-cube tower, Flat box, failures of perception, contact or missing skill." }, { "id": "cosmos", "lead": "Cosmos3 Edge Policy", "kind": "world-action model", "tag": null, "prov": null, "text": "Edits lifted success on Mustard right of raisin box from 0 to 56 of 64 unseen starts (p = 2.8 × 10⁻¹⁷). A random edit of the same size scored 0 of 64 on Mustard right of raisin box, against 0 unedited." }, { "id": "groot", "lead": "GR00T N1.7", "kind": "reasoning-backbone VLA", "tag": null, "prov": null, "text": "Unedited, all 9 tasks: Cube then banana 9 of 32, Cube left of bowl 1 of 32 and Cube in bowl 28 of 32; no success on Eyeglasses and Yellow on red, Mustard in left bin, Three-cube tower, Flat box, Mustard right of raisin box. Clearer wording, same weights: Cube then banana 9 → 16 of 32 (p = 0.12); no gain on Cube left of bowl and Mustard right of raisin box. Cube then banana edits in the Cosmos-Reason2 VLM backbone and in the flow-matching DiT action head each lowered success on the 32 practice starts they were built from, 9 → 0 (p = 0.0039 each); random edits of the same sizes, 11 and 10 of 32; no held-out runs, since no edit passed practice. Render check: banana onto plate succeeded on 5 of 16 starts under RoboLab’s performance preset and 16 of 16 under the default preset, which all three models now use." }, { "id": "side-effects", "lead": "Side effects", "kind": null, "tag": null, "text": "π0.5’s Cube then banana edit, run unchanged on Eyeglasses, Cube left of bowl, Yellow on red, Mustard in left bin, Three-cube tower, Flat box, Mustard right of raisin box and Cube in bowl, lifted Eyeglasses (12 → 21 of 32), made none detectably worse, and kept the retention task Cube in bowl at 31 of 32. Cosmos3 Edge Policy’s Mustard right of raisin box edit, run unchanged on Cube then banana, Eyeglasses, Cube left of bowl, Yellow on red, Mustard in left bin, Three-cube tower, Flat box and Cube in bowl, lifted no task, lowered Cube left of bowl (13 → 0 of 32) and Mustard in left bin (9 → 0 of 32), and moved the retention task Cube in bowl from 24 to 28 of 32. Rebuilt with preservation keys, the edit kept Mustard right of raisin box (0 → 12 and 0 → 13 of 16 practice starts) but did not prevent the break on practice starts: with keys from 65 unrelated instructions, Mustard in left bin fell from 8 to 0 of 16; with keys from 24 instructions from other tasks that use the same spatial relations, Mustard in left bin fell from 8 to 2 of 16 and Cube left of bowl from 4 to 1 of 16 (the original edit: 0). GR00T N1.7: not run, because its Cube then banana edits lowered success on Cube then banana itself." } ], "note": "Lifted and worse mean an exact McNemar p < 0.05 on the same starts, edited against unedited. Some of π0.5’s numbers are from the paper’s evaluation, each on its own slots and number of starts." }, "sections": [ { "id": "glance", "title": "At a glance", "level": 3 }, { "id": "per-model", "title": "Per model", "level": 3 }, { "id": "model-cards", "title": "The three models in detail", "level": 3 }, { "id": "edit", "title": "The closed-form edit", "level": 3 }, { "id": "protocol", "title": "Protocol", "level": 3 }, { "id": "failure-map", "title": "Failure map", "level": 3 }, { "id": "wording", "title": "Wording sensitivity", "level": 3 }, { "id": "edit-gain", "title": "Edits on unseen starts", "level": 3 }, { "id": "where-numbers", "title": "Where to edit", "level": 3 }, { "id": "side-effect-numbers", "title": "Side effects", "level": 3 }, { "id": "caveats", "title": "Caveats in full", "level": 3 }, { "id": "noise", "title": "Noise floor", "level": 3 }, { "id": "records", "title": "Records and reproduction", "level": 3 }, { "id": "clips", "title": "Clips", "level": 3 }, { "id": "references", "title": "References", "level": 2 }, { "id": "citation", "title": "Citation", "level": 2 } ], "summary": { "id": "summary", "title": "Where weight-space programming fixed failures", "rows": [ { "key": "T1", "label": "Cube then banana", "cells": { "cosmos": { "v": "Mustard right of raisin box edit: 18 → 18 of 32", "s": "ok", "d": "9 / 9; p = 1" }, "groot": { "v": "9 → 0 of 32", "s": "ok", "d": "0 / 9, p = 0.0039", "prac": true }, "pi05": { "v": "2 → 40 of 64", "s": "ok", "d": "38 / 0, p = 7.3 × 10⁻¹²; paper’s evaluation: fixed: 27 → 106 of 128" } } }, { "key": "T2", "label": "Eyeglasses", "cells": { "cosmos": { "v": "Mustard right of raisin box edit: 4 → 1 of 16", "s": "ok", "d": "0 / 3; p = 0.25" }, "groot": { "v": "unedited 0 of 32", "s": "ok", "d": "not edited" }, "pi05": { "v": "Cube then banana edit: 12 → 21 of 32", "s": "ok", "d": "12 / 3; p = 0.035; paper’s evaluation: Cube then banana edit: 22 → 44 of 64" } } }, { "key": "T3", "label": "Cube left of bowl", "cells": { "cosmos": { "v": "Mustard right of raisin box edit: 13 → 0 of 32", "s": "ok", "d": "0 / 13; p = 2.4 × 10⁻⁴" }, "groot": { "v": "unedited 1 of 32", "s": "ok", "d": "not edited" }, "pi05": { "v": "Cube then banana edit: 1 → 6 of 32", "s": "ok", "d": "6 / 1; p = 0.12; paper’s evaluation: Cube then banana edit: 8 → 36 of 64" } } }, { "key": "T4", "label": "Yellow on red", "cells": { "cosmos": { "v": "Mustard right of raisin box edit: 24 → 22 of 32", "s": "ok", "d": "4 / 6; p = 0.75" }, "groot": { "v": "unedited 0 of 16", "s": "ok", "d": "not edited" }, "pi05": { "v": "no gain", "s": "ok", "d": "lookalike perception; no clear wording; shared slots, unedited: 9 of 16; shared slots, Cube then banana edit: 21 → 27 of 32", "prov": true } } }, { "key": "T5", "label": "Mustard in left bin", "cells": { "cosmos": { "v": "Mustard right of raisin box edit: 9 → 0 of 32", "s": "ok", "d": "0 / 9; p = 0.0039" }, "groot": { "v": "unedited 0 of 32", "s": "ok", "d": "not edited" }, "pi05": { "v": "no gain", "s": "ok", "d": "own edit 8 → 11 of 64; the grasp slips; shared slots, unedited: 3 of 16; shared slots, Cube then banana edit: 7 → 6 of 32", "prov": true } } }, { "key": "T6", "label": "Three-cube tower", "cells": { "cosmos": { "v": "Mustard right of raisin box edit: 1 → 1 of 32", "s": "ok", "d": "1 / 1; p = 1" }, "groot": { "v": "unedited 0 of 16", "s": "ok", "d": "not edited" }, "pi05": { "v": "no gain", "s": "ok", "d": "own edit 35 → 39 of 64; shared slots, unedited: 6 of 16; shared slots, Cube then banana edit: 11 → 11 of 32", "prov": true } } }, { "key": "T7", "label": "Flat box", "cells": { "cosmos": { "v": "Mustard right of raisin box edit: 0 → 0 of 16", "s": "ok", "d": "0 / 0; p = 1" }, "groot": { "v": "unedited 0 of 32", "s": "ok", "d": "not edited" }, "pi05": { "v": "no gain", "s": "ok", "d": "a missing skill; 0 of 64 unedited; shared slots, unedited: 0 of 16; shared slots, Cube then banana edit: 0 → 0 of 32", "prov": true } } }, { "key": "T8", "label": "Mustard right of raisin box", "cells": { "cosmos": { "v": "0 → 56 of 64", "s": "ok", "d": "56 / 0, p = 2.8 × 10⁻¹⁷" }, "groot": { "v": "unedited 0 of 32", "s": "ok", "d": "not edited" }, "pi05": { "v": "0 → 28 of 64", "s": "ok", "d": "28 / 0, p = 7.5 × 10⁻⁹; paper’s evaluation: fixed: 0 → 27 of 64" } } }, { "key": "R1", "label": "Cube in bowl", "cells": { "cosmos": { "v": "24 → 28 of 32", "s": "ok" }, "groot": { "v": "unedited 28 of 32", "s": "ok", "d": "not edited" }, "pi05": { "v": "31 → 31 of 32", "s": "ok", "d": "paper’s evaluation: kept: 32 → 32 of 32" } } }, { "key": "render", "label": "Rendering: performance → default preset", "sub": true, "cells": { "cosmos": { "v": "—", "s": "na" }, "groot": { "v": "5 → 16 of 16", "s": "ok", "d": "Banana onto plate" }, "pi05": { "v": "—", "s": "na" } } }, { "key": "control", "label": "Random control, same size", "cells": { "cosmos": { "v": "control 0, unedited 0, edit 56 of 64", "s": "ok", "d": "Mustard right of raisin box" }, "groot": { "v": "control 11, unedited 9, edit 0 of 32", "s": "ok", "d": "Cube then banana", "prac": true }, "pi05": { "v": "control 2, unedited 2, edit 40 of 64", "s": "ok", "d": "Cube then banana" } }, "sub": true }, { "key": "where", "label": "Where to edit", "cells": { "cosmos": { "v": "text pathway 13, video-action (generation) pathway 0 of 16", "s": "ok", "d": "Mustard right of raisin box, 16 practice starts; unedited 0" }, "groot": { "v": "Cosmos-Reason2 VLM backbone 0, flow-matching DiT action head 0 of 32", "s": "ok", "d": "Cube then banana, 32 practice starts; unedited 9" }, "pi05": { "v": "VLM 55, action expert 20, output projection 0 of 64", "s": "ok", "d": "Cube then banana; unedited 5; only the VLM edit transfers", "prov": true } }, "sub": true } ] }, "prevention": { "model": "cosmos", "base": "t8und", "target": "T8", "left": [ "T5", "T3" ], "rows": [ { "name": "t8und", "label": "the edit for Mustard right of raisin box, as above", "plain": "The edit", "cells": { "T8": { "v": "0 → 56 of 64", "s": "ok", "d": "p = 2.8 × 10⁻¹⁷; practice 0 → 13 of 16; random control 0 of 64", "a": 0, "b": 56, "n": 64, "p": 2.7755575615628914e-17, "delta": 56, "held": true, "runs": { "practice": [ 0, 13, 16, "30000; 30015" ], "unseen": [ 0, 56, 64, "96000; 96063" ] }, "slots": "96000; 96063" }, "T5": { "v": "9 → 0 of 32", "s": "ok", "d": "p = 0.0039; practice 11 → 0 of 32", "a": 9, "b": 0, "n": 32, "p": 0.00390625, "delta": -9, "held": true, "runs": { "practice": [ 11, 0, 32, "30000; 30031" ], "unseen": [ 9, 0, 32, "96000; 96031" ] }, "slots": "96000; 96031" }, "T3": { "v": "13 → 0 of 32", "s": "ok", "d": "p = 2.4 × 10⁻⁴; practice 13 → 0 of 32", "a": 13, "b": 0, "n": 32, "p": 0.000244140625, "delta": -13, "held": true, "runs": { "practice": [ 13, 0, 32, "30000; 30031" ], "unseen": [ 13, 0, 32, "96000; 96031" ] }, "slots": "96000; 96031" }, "gap": { "v": "91%", "s": "ok", "d": "30 recorded moments; random edit: −1.8%", "x": 91.2, "ctl": -1.8 }, "dev": { "v": "13.7 cm", "s": "ok", "d": "38 recorded moments; Cube left of bowl: 29.6 cm on 20", "x": 13.73, "x3": 29.64 } }, "gate_failed": false, "measured": true }, { "name": "t8undp", "label": "+ preservation keys from 65 generic RoboLab instructions", "plain": "The edit + 65 unrelated instructions kept unchanged", "cells": { "T8": { "v": "0 → 12 of 16", "s": "ok", "d": "p = 4.9 × 10⁻⁴", "a": 0, "b": 12, "n": 16, "p": 0.00048828125, "delta": 12, "prac": true, "runs": { "practice": [ 0, 12, 16, "30000; 30015" ] }, "slots": "30000; 30015" }, "T5": { "v": "8 → 0 of 16", "s": "ok", "d": "p = 0.0078", "a": 8, "b": 0, "n": 16, "p": 0.0078125, "delta": -8, "prac": true, "runs": { "practice": [ 8, 0, 16, "30000; 30015" ] }, "slots": "30000; 30015" }, "T3": { "v": "—", "s": "na", "d": "not run: we went on only if a rebuild saved Mustard in left bin" }, "gap": { "v": "93%", "s": "ok", "d": "30 recorded moments", "x": 93.1 }, "dev": { "v": "13.9 cm", "s": "ok", "d": "38 recorded moments", "x": 13.89 } }, "gate_failed": false, "measured": true }, { "name": "t8undr", "label": "+ preservation keys from 24 left/right instructions of other RoboLab tasks", "plain": "The edit + 24 left/right instructions from other tasks kept unchanged", "cells": { "T8": { "v": "0 → 13 of 16", "s": "ok", "d": "p = 2.4 × 10⁻⁴", "a": 0, "b": 13, "n": 16, "p": 0.000244140625, "delta": 13, "prac": true, "runs": { "practice": [ 0, 13, 16, "30000; 30015" ] }, "slots": "30000; 30015" }, "T5": { "v": "8 → 2 of 16", "s": "ok", "d": "p = 0.031", "a": 8, "b": 2, "n": 16, "p": 0.03125, "delta": -6, "prac": true, "runs": { "practice": [ 8, 2, 16, "30000; 30015" ] }, "slots": "30000; 30015" }, "T3": { "v": "4 → 1 of 16", "s": "ok", "d": "p = 0.38", "a": 4, "b": 1, "n": 16, "p": 0.375, "delta": -3, "prac": true, "runs": { "practice": [ 4, 1, 16, "30000; 30015" ] }, "slots": "30000; 30015" }, "gap": { "v": "94%", "s": "ok", "d": "30 recorded moments; random edit: −10.5%", "x": 93.6, "ctl": -10.5 }, "dev": { "v": "5.9 cm", "s": "ok", "d": "38 recorded moments; Cube left of bowl: 6.5 cm on 20", "x": 5.88, "x3": 6.46 } }, "gate_failed": false, "measured": true }, { "name": "t8unds", "label": "only the instruction template aligned, + the 65 generic preservation keys", "plain": "Copying only the words both wordings share (+ the 65 unrelated instructions)", "cells": { "T8": { "v": "—", "s": "na", "d": "not run: in replays it reproduced too little of the clear wording’s effect" }, "T5": { "v": "—", "s": "na", "d": "not run: in replays it reproduced too little of the clear wording’s effect" }, "T3": { "v": "—", "s": "na", "d": "not run: in replays it reproduced too little of the clear wording’s effect" }, "gap": { "v": "1.9%", "s": "ok", "d": "30 recorded moments; random edit: 2.6%", "x": 1.9, "ctl": 2.6 }, "dev": { "v": "2.0 cm", "s": "ok", "d": "38 recorded moments; Cube left of bowl: 0.8 cm on 20", "x": 2.04, "x3": 0.78 } }, "gate_failed": true, "measured": true } ], "sources": [ "cross-cosmos/v1:study/results/cross/metrics_cosmos.json", "cross-cosmos/v1:study/results/cross/cosmos/README.md" ], "text": "`t8undp` (preservation keys from 65 generic instructions) keeps the Mustard right of raisin box fix (0 → 12 of 16 practice starts) but does not prevent the break: Mustard in left bin still falls from 8 to 0 of 16 practice starts (p = 0.0078); offline it closes 93% of the Mustard right of raisin box gap and moves Mustard in left bin’s actions 13.9 cm (`t8und`: 13.7 cm) from the unedited policy’s. `t8undr` (preservation keys from 24 left/right instructions of other tasks) keeps the Mustard right of raisin box fix (0 → 13 of 16 practice starts) but does not prevent the break: Mustard in left bin still falls from 8 to 2 of 16 practice starts (p = 0.031) and Cube left of bowl goes from 4 to 1 of 16 practice starts (p = 0.38; `t8und`: 0); offline it closes 94% of the Mustard right of raisin box gap and moves Mustard in left bin’s actions 5.9 cm (`t8und`: 13.7 cm) and Cube left of bowl’s 6.5 cm (`t8und`: 29.6 cm) from the unedited policy’s, so the smaller offline deviation on Mustard in left bin and Cube left of bowl does not carry over to the closed loop. `t8unds` (only the instruction template aligned and the 65 generic preservation keys): offline it closes 1.9% of the Mustard right of raisin box gap and moves Mustard in left bin’s actions 2.0 cm (`t8und`: 13.7 cm) and Cube left of bowl’s 0.8 cm (`t8und`: 29.6 cm) from the unedited policy’s; it failed the offline check, so it did not run on the robot." }, "second_edit": { "model": "cosmos", "cols": [ "T8", "T3", "T5" ], "rows": [ { "name": "t8und", "plain": "The mustard edit", "built": "built for Mustard right of raisin box", "cells": { "T8": { "v": "0 → 56 of 64", "s": "ok", "d": "p = 2.8 × 10⁻¹⁷; practice 0 → 13 of 16; random control 0 of 64", "a": 0, "b": 56, "n": 64, "p": 2.7755575615628914e-17, "delta": 56, "held": true, "runs": { "practice": [ 0, 13, 16, "30000; 30015" ], "unseen": [ 0, 56, 64, "96000; 96063" ] }, "slots": "96000; 96063" }, "T3": { "v": "13 → 0 of 32", "s": "ok", "d": "p = 2.4 × 10⁻⁴; practice 13 → 0 of 32", "a": 13, "b": 0, "n": 32, "p": 0.000244140625, "delta": -13, "held": true, "runs": { "practice": [ 13, 0, 32, "30000; 30031" ], "unseen": [ 13, 0, 32, "96000; 96031" ] }, "slots": "96000; 96031" }, "T5": { "v": "9 → 0 of 32", "s": "ok", "d": "p = 0.0039; practice 11 → 0 of 32", "a": 9, "b": 0, "n": 32, "p": 0.00390625, "delta": -9, "held": true, "runs": { "practice": [ 11, 0, 32, "30000; 30031" ], "unseen": [ 9, 0, 32, "96000; 96031" ] }, "slots": "96000; 96031" } }, "measured": true, "gate_failed": null, "build": null }, { "name": "t3und", "plain": "The cube-left edit", "built": "built for Cube left of bowl", "cells": { "T8": { "v": "—", "s": "na", "d": "didn’t pass the offline check, not run on the robot" }, "T3": { "v": "—", "s": "na", "d": "didn’t pass the offline check, not run on the robot" }, "T5": { "v": "—", "s": "na", "d": "didn’t pass the offline check, not run on the robot" } }, "measured": false, "gate_failed": { "passed": false, "gaps": [ [ "T3", -85.62569667613064 ] ], "refs": {}, "ctl": -2.5011124555874398, "n": 30, "rule": "Cube left of bowl gap closed >= 85% and |control gap closed| <= 15%", "path": "results/offline/ali-ms/t3und_gate.json" }, "build": { "orig": 159, "good": 167, "matched": 151, "align": "ps" } }, { "name": "t8t3und", "plain": "Both edits together", "built": "the cube-left edit built on top of the mustard edit", "cells": { "T8": { "v": "—", "s": "na", "d": "didn’t pass the offline check, not run on the robot" }, "T3": { "v": "—", "s": "na", "d": "didn’t pass the offline check, not run on the robot" }, "T5": { "v": "—", "s": "na", "d": "didn’t pass the offline check, not run on the robot" } }, "measured": false, "gate_failed": { "passed": false, "gaps": [ [ "T3", 20.863413277283595 ], [ "T8", 60.83571808150297 ] ], "refs": { "T8": 91.16889868770983 }, "ctl": null, "n": 30, "rule": "Cube left of bowl gap (T8-edited default -> unedited specific) closed >= 85% and Mustard right of raisin box raisin-box-first -> mustard-first gap closed >= 85%", "path": "results/offline/ali-ms/t8t3und_gate.json" }, "build": { "orig": 159, "good": 167, "matched": 151, "align": "ps" } } ], "wording": { "v": "13 → 26 of 32", "s": "ok", "d": "15 / 2, p = 0.0023; +40.6 points [18.8, 59.4]; slots 30000–30031; “Put the rubiks cube to the left of the bowl” → “Pick up the rubiks cube and place it on the table to the left side of the bowl”", "a": 13, "b": 26, "n": 32, "pv": 0.002349853515625, "fl": "Cube left of bowl (32 practice)" }, "wording_pair": [ "Put the rubiks cube to the left of the bowl", "Pick up the rubiks cube and place it on the table to the left side of the bowl" ], "text": "The cube-left edit didn’t pass the offline check, so it never ran on the robot: in replays of 30 held-aside moments, it moved the robot’s planned motion further from the clearer wording’s on Cube left of bowl (−86% of the gap); a random edit of the same size, −2.5%. The two wordings are 159 and 167 tokens long, so the edit could only copy the 151 template positions they share, as in the rebuild above that copies only the words both wordings share. Both edits together didn’t pass the offline check either: in replays of 30 held-aside moments, they moved the robot’s planned motion 21% of the way toward the clearer wording’s on Cube left of bowl and 61% of the way toward the clearer wording’s on Mustard right of raisin box, down from 91% with the mustard edit alone.", "sources": [ "cross-cosmos/v1:study/results/cross/metrics_cosmos.json", "cross-cosmos/v1:study/results/cross/cosmos/README.md" ] }, "when_text": "The edit copies the clearer wording’s effect onto the matching tokens of the original, position by position. Each token’s input is a key, as in model editing. On the mustard task the two wordings have the same length and swap words in place, so every token has a match, and the edit works. When the clearer wording adds words, the new tokens have nothing to land on. That is why the edit failed on the world-action model’s Cube left of bowl and the reasoning-backbone VLA’s Cube then banana. Copying only the words both wordings share does nothing at all. The action-expert VLA is the exception, for a reason. π0.5’s VLM backbone attends over the image tokens and the whole instruction together, so a clearer wording’s effect also reaches the image tokens, which both wordings share, and the edit can copy it from there. π0.5 is known to hold spatial information at its image-token positions. We wrote this rule down before testing a new pair of wordings. On a pair of equal length, Cube left of bowl worded bowl-first vs cube-first, the edit lifted the task from 1 to 12 of 32 unseen starts, far above a random edit. On a pair of unequal length, it failed its offline check, as predicted. Case The two wordings Result Mustard right of raisin box, world-action model Same length, swapped in place Fixed Cube left of bowl, world-action model Words added Failed the offline check Cube then banana, reasoning-backbone VLA Words added Broke the task Mustard right of raisin box, only the shared words copied Only the shared words No better than a random edit, in replays Cube left of bowl, worded bowl-first vs cube-first, predicted Same length, swapped in place Fixed, as predicted Cube in front of bowl, worded bowl-first vs cube-first, predicted Words added Failed the offline check, as predicted Even where it works, though, the edit is not free.", "diagrams": { "where": [ { "id": "pi05", "name": "π0.5", "task": "Cube then banana", "n": 64, "base": 5, "kind": "unseen", "cams": "A", "blocks": [ { "let": "A", "detail": "vision-language model", "sites": [ { "variant": "whole part", "row": "VLM, all layers", "oc": "fixed", "k": 55, "n": 64, "a": 5, "p": 1.7763568394002505e-15, "ctl": null }, { "variant": "image part only", "row": "VLM, image tokens only (0.16% of parameters)", "oc": "partial", "k": 41, "n": 64, "a": 5, "p": 1.4933903003111482e-09, "ctl": null } ] }, { "let": "B", "detail": "action expert", "sites": [ { "variant": "this part alone", "row": "Action expert", "oc": "partial", "k": 20, "n": 64, "a": 5, "p": 0.004077315330505371, "ctl": null }, { "variant": "A and B together", "row": "VLM + action expert", "oc": "partial", "k": 43, "n": 64, "a": 5, "p": 1.6183321349672042e-09, "ctl": null } ] }, { "let": "C", "detail": "action output projection", "sites": [ { "variant": "always on", "row": "Action output projection, unconditional", "oc": "none", "k": 0, "n": 64, "a": 5, "p": 0.0625, "ctl": null }, { "variant": "with an on/off switch", "row": "Action output projection, gated*", "oc": "partial", "k": 30, "n": 64, "a": 5, "p": 4.649162292480469e-06, "ctl": null } ] } ] }, { "id": "cosmos", "name": "Cosmos3 Edge", "task": "Mustard right of raisin box", "n": 16, "base": 0, "kind": "practice", "cams": "B", "blocks": [ { "let": "A", "detail": "text pathway", "sites": [ { "variant": "", "row": "Text pathway", "oc": "fixed", "k": 13, "n": 16, "a": 0, "p": 0.000244140625, "ctl": 1 } ] }, { "let": "B", "detail": "video and action pathway", "sites": [ { "variant": "", "row": "Video/action pathway", "oc": "none", "k": 0, "n": 16, "a": 0, "p": 1.0, "ctl": null } ] }, { "let": "C", "detail": "", "sites": [] } ] }, { "id": "groot", "name": "GR00T N1.7", "task": "Cube then banana", "n": 32, "base": 9, "kind": "practice", "cams": "A", "blocks": [ { "let": "A", "detail": "Cosmos-Reason2 VLM backbone", "sites": [ { "variant": "", "row": "VLM backbone", "oc": "broke", "k": 0, "n": 32, "a": 9, "p": 0.00390625, "ctl": 11 } ] }, { "let": "B", "detail": "flow-matching DiT action head", "sites": [ { "variant": "", "row": "Flow-matching DiT action head", "oc": "broke", "k": 0, "n": 32, "a": 9, "p": 0.00390625, "ctl": 10 } ] }, { "let": "C", "detail": "", "sites": [] } ] } ] }, "tables": [ { "id": "failure_map", "title": "Failure map", "definition": "Success of the unedited model on each task, and how it fails where diagnosed.", "rows": [ { "key": "T1", "label": "Cube then banana", "cells": { "cosmos": { "v": "21 of 32", "s": "ok", "d": "66%; slots 30000–30031", "k": 21, "n": 32 }, "groot": { "v": "9 of 32", "s": "ok", "d": "28%; slots 30000–30031", "k": 9, "n": 32 }, "pi05": { "v": "3 of 16", "s": "ok", "d": "19%; slots 30000–30015; paper’s evaluation: 27 of 128", "k": 3, "n": 16 } } }, { "key": "T2", "label": "Eyeglasses", "cells": { "cosmos": { "v": "2 of 32", "s": "ok", "d": "6%; slots 30000–30031", "k": 2, "n": 32 }, "groot": { "v": "0 of 32", "s": "ok", "d": "0%; slots 30000–30031", "k": 0, "n": 32 }, "pi05": { "v": "7 of 16", "s": "ok", "d": "44%; slots 30000–30015; paper’s evaluation: 18 of 64", "k": 7, "n": 16 } } }, { "key": "T3", "label": "Cube left of bowl", "cells": { "cosmos": { "v": "13 of 32", "s": "ok", "d": "41%; slots 30000–30031", "k": 13, "n": 32 }, "groot": { "v": "1 of 32", "s": "ok", "d": "3%; slots 30000–30031", "k": 1, "n": 32 }, "pi05": { "v": "0 of 16", "s": "ok", "d": "0%; slots 30000–30015; paper’s evaluation: 8 of 64", "k": 0, "n": 16 } } }, { "key": "T4", "label": "Yellow on red", "cells": { "cosmos": { "v": "22 of 32", "s": "ok", "d": "69%; slots 30000–30031", "k": 22, "n": 32 }, "groot": { "v": "0 of 16", "s": "ok", "d": "0%; slots 30000–30015", "k": 0, "n": 16 }, "pi05": { "v": "9 of 16", "s": "ok", "d": "56%; slots 30000–30015; paper’s evaluation: 13 of 32", "k": 9, "n": 16 } } }, { "key": "T5", "label": "Mustard in left bin", "cells": { "cosmos": { "v": "11 of 32", "s": "ok", "d": "34%; slots 30000–30031", "k": 11, "n": 32 }, "groot": { "v": "0 of 32", "s": "ok", "d": "0%; slots 30000–30031", "k": 0, "n": 32 }, "pi05": { "v": "3 of 16", "s": "ok", "d": "19%; slots 30000–30015; paper’s evaluation: 8 of 64", "k": 3, "n": 16 } } }, { "key": "T6", "label": "Three-cube tower", "cells": { "cosmos": { "v": "2 of 32", "s": "ok", "d": "6%; slots 30000–30031", "k": 2, "n": 32 }, "groot": { "v": "0 of 16", "s": "ok", "d": "0%; slots 30000–30015", "k": 0, "n": 16 }, "pi05": { "v": "6 of 16", "s": "ok", "d": "38%; slots 30000–30015; paper’s evaluation: 35 of 64", "k": 6, "n": 16 } } }, { "key": "T7", "label": "Flat box", "cells": { "cosmos": { "v": "0 of 32", "s": "ok", "d": "0%; slots 30000–30031", "k": 0, "n": 32 }, "groot": { "v": "0 of 32", "s": "ok", "d": "0%; slots 30000–30031", "k": 0, "n": 32 }, "pi05": { "v": "0 of 16", "s": "ok", "d": "0%; slots 30000–30015; paper’s evaluation: 0 of 32", "k": 0, "n": 16 } } }, { "key": "T8", "label": "Mustard right of raisin box", "cells": { "cosmos": { "v": "0 of 32", "s": "ok", "d": "0%; slots 30000–30031", "k": 0, "n": 32 }, "groot": { "v": "0 of 32", "s": "ok", "d": "0%; slots 30000–30031", "k": 0, "n": 32 }, "pi05": { "v": "0 of 16", "s": "ok", "d": "0%; slots 30000–30015; paper’s evaluation: 0 of 64", "k": 0, "n": 16 } } }, { "key": "R1", "label": "Cube in bowl", "cells": { "cosmos": { "v": "21 of 32", "s": "ok", "d": "66%; slots 30000–30031", "k": 21, "n": 32 }, "groot": { "v": "28 of 32", "s": "ok", "d": "88%; slots 30000–30031", "k": 28, "n": 32 }, "pi05": { "v": "16 of 16", "s": "ok", "d": "100%; slots 30000–30015; paper’s evaluation: 32 of 32", "k": 16, "n": 16 } } } ], "note": "π0.5's cells are from the shared slots where it was run there, else from the paper’s evaluation (the largest run, on its own slots). Its unedited rate on Cube then banana ranged from 8% to 21% across six start sets, so compare models on the shared slots.", "sources": [] }, { "id": "render", "title": "Rendering sensitivity", "definition": "The unedited model on the same starts under RoboLab's performance render preset and its default preset.", "rows": [ { "key": "render", "label": "Same starts: performance → default preset", "cells": { "cosmos": { "v": "—", "s": "na" }, "groot": { "v": "5 → 16 of 16", "s": "ok", "d": "performance → default; Banana onto plate, the same 16 starts (slots 0–15); 11 / 0, p = 9.8 × 10⁻⁴; NVIDIA's own serving path, default preset: 16 of 16", "src": [ "cross-groot/v1:study/results/cross/groot/results/validation/official-bop16.episode_results.jsonl", "cross-groot/v1:study/results/cross/groot/results/validation/smoke-bop-s0-default.json", "cross-groot/v1:study/results/cross/groot/results/validation/smoke-bop-s0.json" ], "a": 5, "b": 16, "n": 16, "task": "Banana onto plate" }, "pi05": { "v": "—", "s": "na" } } } ], "note": "The performance preset renders without the RTX denoiser, which leaves speckle noise on every surface; the default preset is NVIDIA's validated configuration. All three models are compared under the default preset. The paper’s evaluation of π0.5 used the performance preset.", "sources": [ "cross-groot/v1:study/results/cross/groot/results/validation/official-bop16.episode_results.jsonl", "cross-groot/v1:study/results/cross/groot/results/validation/smoke-bop-s0-default.json", "cross-groot/v1:study/results/cross/groot/results/validation/smoke-bop-s0.json" ] }, { "id": "wording", "title": "Wording sensitivity", "definition": "The unedited model under the original and the clear wording of the same goal, on the same starts.", "rows": [ { "key": "T1", "label": "Cube then banana", "cells": { "cosmos": { "v": "21 → 15 of 32", "s": "ok", "d": "3 / 9, p = 0.15; −18.8 points [−37.5, 0.0]; slots 30000–30031; “Put the cube then the banana in the bowl” → “Put the cube in the bowl then put the banana in the bowl”", "a": 21, "b": 15, "n": 32, "pv": 0.14599609375, "fl": "Cube then banana (32 practice)" }, "groot": { "v": "9 → 16 of 32", "s": "ok", "d": "11 / 4, p = 0.12; +21.9 points [0.0, 43.8]; slots 30000–30031; “Put the cube then the banana in the bowl” → “Put the cube in the bowl then put the banana in the bowl”", "a": 9, "b": 16, "n": 32, "pv": 0.11846923828125, "fl": "Cube then banana (32 practice)" }, "pi05": { "v": "2 → 36 of 64", "s": "ok", "d": "+53.1 points; slots 96000–96063; Put the cube then the banana in the bowl; paper’s evaluation: 13 → 57 of 64", "a": 2, "b": 36, "n": 64, "fl": "Cube then banana (64 unseen)" } } }, { "key": "T3", "label": "Cube left of bowl", "cells": { "cosmos": { "v": "13 → 26 of 32", "s": "ok", "d": "15 / 2, p = 0.0023; +40.6 points [18.8, 59.4]; slots 30000–30031; “Put the rubiks cube to the left of the bowl” → “Pick up the rubiks cube and place it on the table to the left side of the bowl”", "a": 13, "b": 26, "n": 32, "pv": 0.002349853515625, "fl": "Cube left of bowl (32 practice)" }, "groot": { "v": "1 → 0 of 32", "s": "ok", "d": "0 / 1, p = 1; −3.1 points [−9.4, 0.0]; slots 30000–30031; “Put the rubiks cube to the left of the bowl” → “Pick up the rubiks cube and place it on the table to the left of the bowl”", "a": 1, "b": 0, "n": 32, "pv": 1.0, "fl": "Cube left of bowl (32 practice)" }, "pi05": { "v": "—", "s": "na" } } }, { "key": "T4", "label": "Yellow on red", "cells": { "cosmos": { "v": "—", "s": "na" }, "groot": { "v": "—", "s": "na" }, "pi05": { "v": "no clear wording found", "s": "ok", "d": "no rewording moved the arm in the blind editor's replays, so there was nothing to distil", "prov": true } } }, { "key": "T8", "label": "Mustard right of raisin box", "cells": { "cosmos": { "v": "0 → 31 of 32", "s": "ok", "d": "31 / 0, p = 9.3 × 10⁻¹⁰; +96.9 points [90.6, 100.0]; slots 30000–30031; “Move the mustard bottle on the table so that the raisin box is to the left of the mustard bottle.” → “Move the mustard bottle on the table so that the mustard bottle is to the right of the raisin box.”", "a": 0, "b": 31, "n": 32, "pv": 9.313225746154785e-10, "fl": "Mustard right of raisin box (32 practice)" }, "groot": { "v": "0 → 0 of 32", "s": "ok", "d": "0 / 0, p = 1; 0.0 points [0.0, 0.0]; slots 30000–30031; “Move the mustard bottle on the table so that the raisin box is to the left of the mustard bottle.” → “Move the mustard bottle on the table so that the mustard bottle is to the right of the raisin box.”", "a": 0, "b": 0, "n": 32, "pv": 1.0, "fl": "Mustard right of raisin box (32 practice)" }, "pi05": { "v": "0 → 52 of 64", "s": "ok", "d": "+81.2 points; slots 96000–96063; Move the mustard bottle on the table so that the raisin box is to the left of the mustard bottle.; paper’s evaluation: 0 → 27 of 32", "a": 0, "b": 52, "n": 64, "fl": "Mustard right of raisin box (64 unseen)" } } } ], "note": "Reference-first describes the placement from the reference object's side, mustard-first from the moved object's side. In the Same Goal, Different Words study (8 starts per scene, the same starts for every wording, not this page's slots), raisin-box-first wording lowered stable placement by 8.1 points for Cosmos3 Edge across 12 goals; on this goal Edge placed 4 of 8 under mustard-first and 0 of 8 under raisin-box-first.", "sources": [] }, { "id": "edit_gain", "title": "Edits on unseen starts", "definition": "Unedited → edited on held-out starts never used to build the edit: fixed / broke pairs, exact McNemar p, paired bootstrap 95% CI of the difference.", "rows": [ { "key": "T1", "label": "Cube then banana", "cells": { "cosmos": { "v": "—", "s": "na" }, "groot": { "v": "9 → 0 of 32", "s": "ok", "d": "scored on the practice starts the edit was built from, not on unseen starts; Cosmos-Reason2 LLM layers 0-15, mlp.down_proj (VLM backbone); 0 / 9, p = 0.0039; −28.1 points [−43.8, −12.5]; slots 30000–30031; 1 more edit(s) on this task in the where-to-edit table", "a": 9, "b": 0, "n": 32, "fx": 0, "br": 9, "pv": 0.00390625, "ci": [ -28.1, -43.8, -12.5 ], "color": "teal", "prac": true, "loc": "Cosmos-Reason2 LLM layers 0-15, mlp.down_proj (VLM backbone)", "alts": [ { "loc": "GR00T action head: 32 AlternateVLDiT blocks, ff.net.2 (diffusion transformer)", "a": 9, "b": 0, "n": 32, "p": 0.00390625, "prac": true, "ctl": 10 } ], "fl": "Cube then banana, practice starts", "fb": "Cube then banana practice" }, "pi05": { "v": "2 → 40 of 64", "s": "ok", "d": "VLM (PaliGemma) MLPs, layers 0-16: prompt + state + image keys; 38 / 0, p = 7.3 × 10⁻¹²; +59.4 points [46.9, 71.9]; slots 96000–96063; paper’s evaluation: 27 → 106 of 128", "a": 2, "b": 40, "n": 64, "fx": 38, "br": 0, "pv": 7.276e-12, "ci": [ 59.4, 46.9, 71.9 ], "color": "teal", "loc": "VLM (PaliGemma) MLPs, layers 0-16: prompt + state + image keys", "fl": "Cube then banana, VLM (PaliGemma) MLPs, laye…", "fb": "Cube then banana VLM (PaliGemma) M" } } }, { "key": "T1b", "label": "Cube then banana, built end to end by the packaged tool (`weightsmith distill`)", "cells": { "cosmos": { "v": "—", "s": "na" }, "groot": { "v": "—", "s": "na" }, "pi05": { "v": "8 → 36 of 64", "s": "ok", "d": "decisions chosen by rule; 33 / 5, p = 4.3 × 10⁻⁶; +43.8 points [28.1, 59.4]; held-out slots 95008–95071", "src": [ "study/results/tutorial/tutorial-ctb__ctb-first-95008-64.results.json" ], "a": 8, "b": 36, "n": 64, "ci": [ 43.8, 28.1, 59.4 ], "color": "teal", "fl": "Cube then banana, packaged tool, end to end", "fb": "Cube then banana packaged tool", "fx": 33, "br": 5, "pv": 4.255962267052382e-06, "prov": true } } }, { "key": "T1c", "label": "Cube then banana, a blind autonomous editor's own edit", "cells": { "cosmos": { "v": "—", "s": "na" }, "groot": { "v": "—", "s": "na" }, "pi05": { "v": "11 → 27 of 64", "s": "ok", "d": "gated action-head change, 1,024 values; 22 / 6, p = 0.0037; +25.0 points [9.4, 40.6]; held-out slots 91008–91071", "src": [ "study/results/ctrl/heldout/ctrl-ctb__p1-RubiksCubeThenBananaTask-91008-64.json" ], "a": 11, "b": 27, "n": 64, "ci": [ 25.0, 9.4, 40.6 ], "color": "dark", "fl": "Cube then banana, blind editor's own edit", "fb": "Cube then banana blind editor", "fx": 22, "br": 6, "pv": 0.0037191659212112427, "prov": true } } }, { "key": "T2", "label": "Eyeglasses", "cells": { "cosmos": { "v": "—", "s": "na" }, "groot": { "v": "—", "s": "na" }, "pi05": { "v": "18 → 27 of 64", "s": "ok", "d": "blind editor's edit; not significant; 18 / 9, p = 0.12; +14.1 points [−1.6, 29.7]; held-out slots 91008–91071", "src": [ "study/results/ctrl/heldout/ctrl-glasses__p1-PickGlassesTask-91008-64.json" ], "a": 18, "b": 27, "n": 64, "ci": [ 14.1, -1.6, 29.7 ], "color": "dark", "fl": "Eyeglasses, blind editor", "fb": "Eyeglasses blind editor", "fx": 18, "br": 9, "pv": 0.12207812070846558, "prov": true } } }, { "key": "T3", "label": "Cube left of bowl", "cells": { "cosmos": { "v": "—", "s": "na" }, "groot": { "v": "—", "s": "na" }, "pi05": { "v": "no edit submitted", "s": "ok", "d": "the blind editor's candidate scored 5 → 3 of 32 on practice starts and broke grasping", "src": [ "study/results/ctrl/summary.json" ], "prov": true } } }, { "key": "T4", "label": "Yellow on red", "cells": { "cosmos": { "v": "—", "s": "na" }, "groot": { "v": "—", "s": "na" }, "pi05": { "v": "no gain", "s": "ok", "d": "three edits, 28 → 21 of 64 each on practice starts; no clear wording to distil", "src": [ "study/results/opus/comparisons.json" ], "prov": true } } }, { "key": "T5", "label": "Mustard in left bin", "cells": { "cosmos": { "v": "—", "s": "na" }, "groot": { "v": "—", "s": "na" }, "pi05": { "v": "8 → 11 of 64", "s": "ok", "d": "blind editor's edit; not significant; 9 / 6, p = 0.61; +4.7 points [−6.2, 17.2]; held-out slots 91008–91071", "src": [ "study/results/ctrl/heldout/ctrl-mustard__p1-MustardInLeftBinTask-91008-64.json" ], "a": 8, "b": 11, "n": 64, "ci": [ 4.7, -6.2, 17.2 ], "color": "dark", "fl": "Mustard in left bin, blind editor", "fb": "Mustard in left bin blind editor", "fx": 9, "br": 6, "pv": 0.60723876953125, "prov": true } } }, { "key": "T6", "label": "Three-cube tower", "cells": { "cosmos": { "v": "—", "s": "na" }, "groot": { "v": "—", "s": "na" }, "pi05": { "v": "35 → 39 of 64", "s": "ok", "d": "blind editor's edit; not significant; 17 / 13, p = 0.58; +6.2 points [−10.9, 23.4]; held-out slots 91008–91071", "src": [ "study/results/ctrl/heldout/ctrl-stack3__p1-Stack3RubiksCubeTask-91008-64.json" ], "a": 35, "b": 39, "n": 64, "ci": [ 6.2, -10.9, 23.4 ], "color": "dark", "fl": "Three-cube tower, blind editor", "fb": "Three-cube tower blind editor", "fx": 17, "br": 13, "pv": 0.584664711728692, "prov": true } } }, { "key": "T7", "label": "Flat box", "cells": { "cosmos": { "v": "—", "s": "na" }, "groot": { "v": "—", "s": "na" }, "pi05": { "v": "no gain", "s": "ok", "d": "unedited 0 of 64; 10 edits, none succeeded (a skill the policy lacks)", "src": [ "study/results/opus/comparisons.json" ], "prov": true } } }, { "key": "T8", "label": "Mustard right of raisin box", "cells": { "cosmos": { "v": "0 → 56 of 64", "s": "ok", "d": "text pathway (und mlp.down_proj, layers 0-26); 56 / 0, p = 2.8 × 10⁻¹⁷; +87.5 points [78.1, 95.3]; slots 96000–96063; on the practice starts it was built from: 0 → 13 of 16; 1 more edit(s) on this task in the where-to-edit table", "a": 0, "b": 56, "n": 64, "fx": 56, "br": 0, "pv": 2.7755575615628914e-17, "ci": [ 87.5, 78.1, 95.3 ], "color": "teal", "loc": "text pathway (und mlp.down_proj, layers 0-26)", "alts": [ { "loc": "video/action pathway (gen mlp_moe_gen.down_proj, layers 8-20)", "a": 0, "b": 0, "n": 16, "p": 1.0, "prac": true, "ctl": null } ], "fl": "Mustard right of raisin box, text pathway (und mlp.down…", "fb": "Mustard right of raisin box text pathway (und" }, "groot": { "v": "—", "s": "na" }, "pi05": { "v": "0 → 28 of 64", "s": "ok", "d": "VLM MLP down-projections language.blocks.08-16.mlp.linear (9 tensors), 3 key groups (approach, grasp, transport), rank <= 96 per layer; 28 / 0, p = 7.5 × 10⁻⁹; +43.8 points [31.2, 56.2]; slots 96000–96063; paper’s evaluation: 0 → 27 of 64", "a": 0, "b": 28, "n": 64, "fx": 28, "br": 0, "pv": 7.451e-09, "ci": [ 43.8, 31.2, 56.2 ], "color": "teal", "loc": "VLM MLP down-projections language.blocks.08-16.mlp.linear (9 tensors), 3 key groups (approach, grasp, transport), rank <= 96 per layer", "fl": "Mustard right of raisin box, VLM MLP down-projections l…", "fb": "Mustard right of raisin box VLM MLP down-proj" } } } ], "note": "Distilled edits copy the clear wording's internal effect (teal in the figures); the blind editors designed their own edits under the same rules (dark). The packaged tool's 2 × 2 table happens to equal the Cube then banana edit's transfer to Cube left of bowl (8 → 36 of 64, 33 / 5): different tasks, different starts.", "sources": [ "study/results/ctrl/heldout/ctrl-ctb__p1-RubiksCubeThenBananaTask-91008-64.json", "study/results/ctrl/heldout/ctrl-glasses__p1-PickGlassesTask-91008-64.json", "study/results/ctrl/heldout/ctrl-mustard__p1-MustardInLeftBinTask-91008-64.json", "study/results/ctrl/heldout/ctrl-stack3__p1-Stack3RubiksCubeTask-91008-64.json", "study/results/ctrl/summary.json", "study/results/opus/comparisons.json", "study/results/tutorial/tutorial-ctb__ctb-first-95008-64.results.json" ] }, { "id": "random_control", "title": "Random-direction control", "definition": "The edit's layers, rank and singular values in random directions, the same size, scored on the same starts.", "rows": [ { "key": "T1", "label": "Cube then banana", "cells": { "cosmos": { "v": "—", "s": "na" }, "groot": { "v": "control 11, unedited 9, edit 0 of 32", "s": "ok", "d": "practice starts the edit was built from; slots 30000–30031", "u": 9, "r": 11, "e": 0, "n": 32, "t": 16, "tn": 32, "prac": true, "loc": "Cosmos-Reason2 LLM layers 0-15, mlp.down_proj (VLM backbone)", "fl": "Cube then banana, VLM backbone edit (32 practice)" }, "pi05": { "v": "control 2, unedited 2, edit 40 of 64", "s": "ok", "d": "slots 96000–96063; paper’s evaluation: control 9, unedited 13, edit 50 of 64", "u": 2, "r": 2, "e": 40, "n": 64, "t": 36, "tn": 64, "loc": "VLM (PaliGemma) MLPs, layers 0-16: prompt + state + image keys", "fl": "Cube then banana (64 unseen)" } } }, { "key": "T8", "label": "Mustard right of raisin box", "cells": { "cosmos": { "v": "control 0, unedited 0, edit 56 of 64", "s": "ok", "d": "slots 96000–96063", "u": 0, "r": 0, "e": 56, "n": 64, "t": 56, "tn": 64, "loc": "text pathway (und mlp.down_proj, layers 0-26)", "fl": "Mustard right of raisin box, text pathway edit (64 unseen)" }, "groot": { "v": "—", "s": "na" }, "pi05": { "v": "control 0, unedited 0, edit 28 of 64", "s": "ok", "d": "slots 96000–96063; paper’s evaluation: control 0, unedited 0, edit 27 of 64", "u": 0, "r": 0, "e": 28, "n": 64, "t": 52, "tn": 64, "loc": "VLM MLP down-projections language.blocks.08-16.mlp.linear (9 tensors), 3 key groups (approach, grasp, transport), rank <= 96 per layer", "fl": "Mustard right of raisin box (64 unseen)" } } } ], "note": "The control keeps every layer's singular values and replaces the singular vectors with random orthonormal bases (seed 0), so its Frobenius norm matches the edit's in every layer.", "sources": [] }, { "id": "recovery", "title": "Teacher recovery", "definition": "(edited − unedited) / (teacher − unedited), where the teacher is the unedited model given the clear wording.", "rows": [ { "key": "T1", "label": "Cube then banana", "cells": { "cosmos": { "v": "—", "s": "na" }, "groot": { "v": "The edit did clearly less well than the clearer wording (0 vs 16 of 32)", "s": "ok", "d": "no recovery %: the clearer wording’s own gain (9 → 16 of 32) is not a clear difference (p = 0.12); practice starts", "prac": true, "a": 9, "b": 0, "t": 16, "n": 32, "words": true }, "pi05": { "v": "The edit did at least as well as the clearer wording (40 vs 36 of 64; not a clear difference)", "s": "ok", "d": "paper’s evaluation: 84%", "a": 2, "b": 40, "t": 36, "n": 64, "words": true } } }, { "key": "T8", "label": "Mustard right of raisin box", "cells": { "cosmos": { "v": "The edit did as well as the clearer wording (56 vs 56 of 64)", "s": "ok", "d": null, "a": 0, "b": 56, "t": 56, "n": 64, "words": true }, "groot": { "v": "—", "s": "na" }, "pi05": { "v": "54%", "s": "ok", "d": "as reported; (28 − 0) / (52 − 0) of 64; paper’s evaluation: 59%", "a": 0, "b": 28, "t": 52, "n": 64 } } } ], "note": "100% would mean the edit does everything the clear wording does while the original wording stays.", "sources": [] }, { "id": "where", "title": "Where to edit", "definition": "The same distillation written into the backbone / text pathway or into the action pathway, on the same starts.", "rows": [ { "key": "starts", "label": "Task, starts", "cells": { "cosmos": { "v": "Mustard right of raisin box, 16 practice starts", "s": "ok", "d": "slots 30000–30015" }, "groot": { "v": "Cube then banana, 32 practice starts", "s": "ok", "d": "slots 30000–30031" }, "pi05": { "v": "Cube then banana, 64 unseen starts", "s": "ok", "d": "held-out slots 94016–94079", "src": [ "study/results/opus/where/results/heldout_analysis.json" ], "prov": true } } }, { "key": "unedited", "label": "Unedited", "cells": { "cosmos": { "v": "0 of 16", "s": "ok" }, "groot": { "v": "9 of 32", "s": "ok" }, "pi05": { "v": "5 of 64", "s": "ok", "src": [ "study/results/opus/where/results/heldout_analysis.json" ], "prov": true } } }, { "key": "backbone", "label": "Backbone / text pathway", "cells": { "cosmos": { "v": "text pathway (und mlp.down_proj, layers 0-26): 13 of 16", "s": "ok" }, "groot": { "v": "Cosmos-Reason2 LLM layers 0-15, mlp.down_proj (VLM backbone): 0 of 32", "s": "ok" }, "pi05": { "v": "VLM MLPs: 55 of 64", "s": "ok", "d": "50 / 0, p = 1.8 × 10⁻¹⁵; the VLM's weights also process the image tokens, where this edit is keyed", "src": [ "study/results/opus/where/results/heldout_analysis.json" ], "prov": true } } }, { "key": "action", "label": "Action pathway", "cells": { "cosmos": { "v": "video/action pathway (gen mlp_moe_gen.down_proj, layers 8-20): 0 of 16", "s": "ok" }, "groot": { "v": "GR00T action head: 32 AlternateVLDiT blocks, ff.net.2 (diffusion transformer): 0 of 32", "s": "ok" }, "pi05": { "v": "action expert 20, action output projection 0 of 64", "s": "ok", "d": "expert 20 / 5, p = 0.0041; output projection 0 / 5; an output-projection edit with a hand-built gate (blind editor): 30", "src": [ "study/results/opus/where/results/heldout_analysis.json" ], "prov": true } } }, { "key": "both", "label": "Both", "cells": { "cosmos": { "v": "—", "s": "na" }, "groot": { "v": "—", "s": "na" }, "pi05": { "v": "VLM + action expert: 43 of 64", "s": "ok", "d": "below the VLM edit alone, p = 0.017", "src": [ "study/results/opus/where/results/heldout_analysis.json" ], "prov": true } } }, { "key": "offline_where", "label": "Offline check: share of the wording’s action change reproduced", "cells": { "cosmos": { "v": "text pathway (und mlp.down_proj, layers 0-26): 91% (random, same size: −2%); video/action pathway (gen mlp_moe_gen.down_proj, layers 8-20): 23% (random, same size: 1%)", "s": "ok", "d": "Mustard right of raisin box, 30 recorded decisions held aside, never used to build either edit; share of the clearer wording’s action change reproduced, by end-effector distance at the end of the action chunk; offline, no rollouts", "parts": [ [ "text pathway (und mlp.down_proj, layers 0-26)", 91.2, -1.8 ], [ "video/action pathway (gen mlp_moe_gen.down_proj, layers 8-20)", 23.4, 0.7 ] ] }, "groot": { "v": "backbone: −15% (random, same size: 0%); action head: 22%", "s": "ok", "d": "Cube then banana, 11 failed practice decisions that no build used; share of the clearer wording’s action change reproduced, as the projection of the edit’s change onto the wording’s over the executed actions (joints and gripper); offline, no rollouts", "parts": [ [ "backbone", -15.310076334760012, -0.26354435264788945 ], [ "action head", 21.875615897501223, null ] ] }, "pi05": { "v": "—", "s": "na" } } }, { "key": "transfers", "label": "Which one transfers", "cells": { "cosmos": { "v": "—", "s": "na" }, "groot": { "v": "—", "s": "na" }, "pi05": { "v": "only the backbone edit", "s": "ok", "d": "eyeglasses 14 → 25, cube left of the bowl 4 → 20 of 32; action expert 11 and 2, gated head 12 and 1 (all p ≥ 0.38); slots 94016–94047", "src": [ "study/results/opus/where/results/transfer_glasses_analysis.json", "study/results/opus/where/results/transfer_left_analysis.json" ], "prov": true } } }, { "key": "size", "label": "Size of each", "cells": { "cosmos": { "v": "—", "s": "na" }, "groot": { "v": "—", "s": "na" }, "pi05": { "v": "VLM 13.4%, expert 1.8% of values; head 8,056", "s": "ok", "d": "median change per layer: VLM 1.7% of the weight norm, action expert 99%", "src": [ "study/results/opus/comparisons.json", "study/results/opus/where/results/size_summary.json" ], "prov": true } } } ], "note": "In π0.5 the VLM's weights process image and text tokens alike, and the edit is keyed at the image tokens. In Cosmos3 the video and action tokens have their own weights (the generation pathway), so its closest counterpart is the video-action pathway; the text pathway is the other candidate. In GR00T the backbone is Cosmos-Reason2-2B and the action pathway is the flow-matching DiT action head.", "sources": [ "study/results/opus/comparisons.json", "study/results/opus/where/results/heldout_analysis.json", "study/results/opus/where/results/size_summary.json", "study/results/opus/where/results/transfer_glasses_analysis.json", "study/results/opus/where/results/transfer_left_analysis.json" ] }, { "id": "size", "title": "Edit size", "definition": "Values changed, rank per layer, and share of the parameters, also as low-rank factors.", "rows": [ { "key": "T1", "label": "Cube then banana", "cells": { "cosmos": { "v": "—", "s": "na" }, "groot": { "v": "167,481,274 values", "s": "ok", "d": "Cosmos-Reason2 LLM layers 0-15, mlp.down_proj (VLM backbone)" }, "pi05": { "v": "450,468,837 values", "s": "ok", "d": "VLM (PaliGemma) MLPs, layers 0-16: prompt + state + image keys; paper’s evaluation: 450,468,837 values (13.4%)" } } }, { "key": "T1v", "label": "Cube then banana, smaller variant (image cells, layers 8–16)", "cells": { "cosmos": { "v": "—", "s": "na" }, "groot": { "v": "—", "s": "na" }, "pi05": { "v": "5.3M as factors (0.16%)", "s": "ok", "d": "image cells, layers 8–16, rank ≤ 32 per layer; 19 → 102 of 128 unseen starts; stored values changed 248,057,499; the packaged tool's edit uses this recipe", "src": [ "study/results/opus/heldout/cube-then-banana__abl-imgl8-RubiksCubeThenBananaTask-90000-128.json" ], "prov": true } } }, { "key": "T1c", "label": "Cube then banana, smallest edit that passed unseen starts", "cells": { "cosmos": { "v": "—", "s": "na" }, "groot": { "v": "—", "s": "na" }, "pi05": { "v": "1,024 values", "s": "ok", "d": "the blind editor's gated action-head change", "src": [ "study/results/ctrl/summary.json" ], "prov": true } } }, { "key": "T8", "label": "Mustard right of raisin box", "cells": { "cosmos": { "v": "179,010,015 values", "s": "ok", "d": "text pathway (und mlp.down_proj, layers 0-26)" }, "groot": { "v": "—", "s": "na" }, "pi05": { "v": "275,482,328 values", "s": "ok", "d": "VLM MLP down-projections language.blocks.08-16.mlp.linear (9 tensors), 3 key groups (approach, grasp, transport), rank <= 96 per layer; paper’s evaluation: 15.9M as factors (0.47%)" } } } ], "note": "A low-rank update is dense in its weight matrix: the 0.16% variant changes 7.4% of stored values; as factors it is 0.16% of the parameters.", "sources": [ "study/results/ctrl/summary.json", "study/results/opus/heldout/cube-then-banana__abl-imgl8-RubiksCubeThenBananaTask-90000-128.json" ] }, { "id": "side_effects", "title": "Side-effect map", "definition": "Change in successes, edited minus unedited, on every suite task, paired on the same slots; significant at exact p < 0.05.", "rows": [ { "key": "T1", "label": "Cube then banana", "cells": { "cosmos": { "v": "±0: 18 → 18 of 32", "s": "ok", "d": "9 / 9; p = 1; slots 96000–96031; practice starts: 21 → 18 of 32, p = 0.63 (slots 30000–30031)", "delta": 0, "a": 18, "b": 18, "n": 32, "p": 1.0 }, "groot": { "v": "−9: 9 → 0 of 32", "s": "ok", "d": "0 / 9; p = 0.0039; the target task; practice starts; slots 30000–30031", "delta": -9, "a": 9, "b": 0, "n": 32, "p": 0.00390625, "target": true, "prac": true }, "pi05": { "v": "+38: 2 → 40 of 64", "s": "ok", "d": "38 / 0; p = 7.3 × 10⁻¹²; the target task; slots 96000–96063; paper’s evaluation: +79: 27 → 106 of 128", "delta": 38, "a": 2, "b": 40, "n": 64, "p": 7.276e-12, "target": true } } }, { "key": "T2", "label": "Eyeglasses", "cells": { "cosmos": { "v": "−3: 4 → 1 of 16", "s": "ok", "d": "0 / 3; p = 0.25; slots 96000–96015; practice starts: 2 → 0 of 32, p = 0.5 (slots 30000–30031)", "delta": -3, "a": 4, "b": 1, "n": 16, "p": 0.25 }, "groot": { "v": "not run", "s": "na", "d": "dropped: both Cube then banana edits lowered success on Cube then banana itself, so no side-effect map was run for them (cross-groot/v1 639a29b)" }, "pi05": { "v": "+9: 12 → 21 of 32", "s": "ok", "d": "12 / 3; p = 0.035; slots 96000–96031; paper’s evaluation: +22: 22 → 44 of 64", "delta": 9, "a": 12, "b": 21, "n": 32, "p": 0.03516 } } }, { "key": "T3", "label": "Cube left of bowl", "cells": { "cosmos": { "v": "−13: 13 → 0 of 32", "s": "ok", "d": "0 / 13; p = 2.4 × 10⁻⁴; slots 96000–96031; practice starts: 13 → 0 of 32, p = 2.4 × 10⁻⁴ (slots 30000–30031)", "delta": -13, "a": 13, "b": 0, "n": 32, "p": 0.000244140625 }, "groot": { "v": "not run", "s": "na", "d": "dropped: both Cube then banana edits lowered success on Cube then banana itself, so no side-effect map was run for them (cross-groot/v1 639a29b)" }, "pi05": { "v": "+5: 1 → 6 of 32", "s": "ok", "d": "6 / 1; p = 0.12; slots 96000–96031; paper’s evaluation: +28: 8 → 36 of 64", "delta": 5, "a": 1, "b": 6, "n": 32, "p": 0.125 } } }, { "key": "T4", "label": "Yellow on red", "cells": { "cosmos": { "v": "−2: 24 → 22 of 32", "s": "ok", "d": "4 / 6; p = 0.75; slots 96000–96031; practice starts: 22 → 26 of 32, p = 0.39 (slots 30000–30031)", "delta": -2, "a": 24, "b": 22, "n": 32, "p": 0.75390625 }, "groot": { "v": "not run", "s": "na", "d": "dropped: both Cube then banana edits lowered success on Cube then banana itself, so no side-effect map was run for them (cross-groot/v1 639a29b)" }, "pi05": { "v": "+6: 21 → 27 of 32", "s": "ok", "d": "9 / 3; p = 0.15; slots 96000–96031; paper’s evaluation: +3: 14 → 17 of 32", "delta": 6, "a": 21, "b": 27, "n": 32, "p": 0.146 } } }, { "key": "T5", "label": "Mustard in left bin", "cells": { "cosmos": { "v": "−9: 9 → 0 of 32", "s": "ok", "d": "0 / 9; p = 0.0039; slots 96000–96031; practice starts: 11 → 0 of 32, p = 9.8 × 10⁻⁴ (slots 30000–30031)", "delta": -9, "a": 9, "b": 0, "n": 32, "p": 0.00390625 }, "groot": { "v": "not run", "s": "na", "d": "dropped: both Cube then banana edits lowered success on Cube then banana itself, so no side-effect map was run for them (cross-groot/v1 639a29b)" }, "pi05": { "v": "−1: 7 → 6 of 32", "s": "ok", "d": "6 / 7; p = 1; slots 96000–96031; paper’s evaluation: +5: 7 → 12 of 32", "delta": -1, "a": 7, "b": 6, "n": 32, "p": 1.0 } } }, { "key": "T6", "label": "Three-cube tower", "cells": { "cosmos": { "v": "±0: 1 → 1 of 32", "s": "ok", "d": "1 / 1; p = 1; slots 96000–96031; practice starts: 2 → 3 of 32, p = 1 (slots 30000–30031)", "delta": 0, "a": 1, "b": 1, "n": 32, "p": 1.0 }, "groot": { "v": "not run", "s": "na", "d": "dropped: both Cube then banana edits lowered success on Cube then banana itself, so no side-effect map was run for them (cross-groot/v1 639a29b)" }, "pi05": { "v": "±0: 11 → 11 of 32", "s": "ok", "d": "7 / 7; p = 1; slots 96000–96031; paper’s evaluation: +5: 14 → 19 of 32", "delta": 0, "a": 11, "b": 11, "n": 32, "p": 1.0 } } }, { "key": "T7", "label": "Flat box", "cells": { "cosmos": { "v": "±0: 0 → 0 of 16", "s": "ok", "d": "0 / 0; p = 1; slots 96000–96015; practice starts: 0 → 0 of 32, p = 1 (slots 30000–30031)", "delta": 0, "a": 0, "b": 0, "n": 16, "p": 1.0 }, "groot": { "v": "not run", "s": "na", "d": "dropped: both Cube then banana edits lowered success on Cube then banana itself, so no side-effect map was run for them (cross-groot/v1 639a29b)" }, "pi05": { "v": "±0: 0 → 0 of 32", "s": "ok", "d": "0 / 0; p = 1; slots 96000–96031; paper’s evaluation: ±0: 0 → 0 of 32", "delta": 0, "a": 0, "b": 0, "n": 32, "p": 1.0 } } }, { "key": "T8", "label": "Mustard right of raisin box", "cells": { "cosmos": { "v": "+56: 0 → 56 of 64", "s": "ok", "d": "56 / 0; p = 2.8 × 10⁻¹⁷; the target task; slots 96000–96063", "delta": 56, "a": 0, "b": 56, "n": 64, "p": 2.7755575615628914e-17, "target": true }, "groot": { "v": "not run", "s": "na", "d": "dropped: both Cube then banana edits lowered success on Cube then banana itself, so no side-effect map was run for them (cross-groot/v1 639a29b)" }, "pi05": { "v": "±0: 0 → 0 of 32", "s": "ok", "d": "0 / 0; p = 1; slots 96000–96031; paper’s evaluation: not run", "delta": 0, "a": 0, "b": 0, "n": 32, "p": 1.0 } } }, { "key": "R1", "label": "Cube in bowl", "cells": { "cosmos": { "v": "+4: 24 → 28 of 32", "s": "ok", "d": "7 / 3; p = 0.34; slots 96000–96031; practice starts: 21 → 27 of 32, p = 0.15 (slots 30000–30031)", "delta": 4, "a": 24, "b": 28, "n": 32, "p": 0.34375 }, "groot": { "v": "not run", "s": "na", "d": "dropped: both Cube then banana edits lowered success on Cube then banana itself, so no side-effect map was run for them (cross-groot/v1 639a29b)" }, "pi05": { "v": "±0: 31 → 31 of 32", "s": "ok", "d": "0 / 0; p = 1; slots 96000–96031; paper’s evaluation: ±0: 32 → 32 of 32", "delta": 0, "a": 31, "b": 31, "n": 32, "p": 1.0 } } } ], "note": "Each model's main edit, unchanged, on every suite task; π0.5's is the Cube then banana edit. In the paper’s evaluation it harmed none of fifteen other surveyed tasks detectably.", "sources": [], "edit": { "cosmos": "the Mustard right of raisin box edit (text pathway (und mlp.down_proj, layers 0-26))", "groot": "the Cube then banana edit on practice starts (Cosmos-Reason2 LLM layers 0-15, mlp.down_proj (VLM backbone))", "pi05": "the Cube then banana edit (VLM (PaliGemma) MLPs, layers 0-16: prompt + state + image keys)" } }, { "id": "retention", "title": "Retention", "definition": "After the edit: a task the model already solves, and the clear wording itself.", "rows": [ { "key": "R1", "label": "Cube in bowl", "cells": { "cosmos": { "v": "24 → 28 of 32", "s": "ok", "d": "the Mustard right of raisin box edit, unchanged; 7 / 3; p = 0.34; slots 96000–96031; practice starts: 21 → 27 of 32, p = 0.15 (slots 30000–30031)" }, "groot": { "v": "—", "s": "na" }, "pi05": { "v": "31 → 31 of 32", "s": "ok", "d": "the Cube then banana edit, unchanged; 0 / 0; p = 1; slots 96000–96031; paper’s evaluation: 32 → 32 of 32" } } }, { "key": "clear", "label": "The clear wording, after the edit", "cells": { "cosmos": { "v": "—", "s": "na" }, "groot": { "v": "—", "s": "na" }, "pi05": { "v": "27 → 23 of 32", "s": "ok", "d": "Mustard right of raisin box under the mustard-first wording, with the Mustard right of raisin box edit; 5 / 9, p = 0.42: not detectable at 32 pairs, not excluded; practice slots 20000–20031", "src": [ "study/results/sgdw/results/practice_rf_grp3_vs_base.json", "study/results/sgdw/results/practice_tf_grp3_vs_base.json" ], "prov": true } } }, { "key": "offline", "label": "Recorded decisions it must not change", "cells": { "cosmos": { "v": "—", "s": "na" }, "groot": { "v": "—", "s": "na" }, "pi05": { "v": "unchanged", "s": "ok", "d": "all 9 recorded correct carries and all 19 banana-phase decisions unchanged (Cube then banana edit, offline replay)", "prov": true } } } ], "note": "At these sample sizes a retention check rules out only large losses (see the noise floor).", "sources": [ "study/results/sgdw/results/practice_rf_grp3_vs_base.json", "study/results/sgdw/results/practice_tf_grp3_vs_base.json" ] }, { "id": "noise", "title": "Noise floor", "definition": "Paired outcomes that differ when identical weights are rerun on the same starts; it sets the smallest detectable change.", "rows": [ { "key": "unedited2", "label": "Unedited, run twice on the same starts", "cells": { "cosmos": { "v": "—", "s": "na" }, "groot": { "v": "—", "s": "na" }, "pi05": { "v": "13 vs 9 of 64", "s": "ok", "d": "12 of 64 pairs differ (19%); Cube then banana, held-out slots 90000–90063", "src": [ "study/results/opus/controls/ctl1-90000-64.analysis.json" ], "prov": true } } }, { "key": "edited2", "label": "Identical edited weights, run twice", "cells": { "cosmos": { "v": "—", "s": "na" }, "groot": { "v": "—", "s": "na" }, "pi05": { "v": "agreed on 46 of 64", "s": "ok", "d": "28% of pairs differ (paper, Appendix B)", "prov": true } } }, { "key": "butter", "label": "Unedited rerun, butter on the raisin box", "cells": { "cosmos": { "v": "—", "s": "na" }, "groot": { "v": "—", "s": "na" }, "pi05": { "v": "14 vs 8 of 16", "s": "ok", "d": "38% of pairs differ; exact p = 0.031 with nothing changed", "src": [ "study/results/opus/comparisons.json" ], "prov": true } } }, { "key": "typical", "label": "Typical share of pairs that differ", "cells": { "cosmos": { "v": "—", "s": "na" }, "groot": { "v": "—", "s": "na" }, "pi05": { "v": "about 30%", "s": "ok", "d": "a near-neutral one-weight edit: 70 of 192", "src": [ "study/results/opus/comparisons.json" ], "prov": true } } }, { "key": "mde", "label": "Smallest detectable change", "cells": { "cosmos": { "v": "19 points at 64 pairs", "s": "ok", "d": "if about 30% of pairs differ" }, "groot": { "v": "27 points at 32 pairs", "s": "ok", "d": "if about 30% of pairs differ" }, "pi05": { "v": "27 / 19 / 14 / 10 points", "s": "ok", "d": "at 32 / 64 / 128 / 256 pairs, 80% power, α = 0.05, 30% of pairs differing", "prov": true } } } ], "note": "Simulated physics on GPUs is not bit-repeatable, and a policy that executes many actions between looks amplifies small differences. Every comparison pairs starts and uses exact tests for that reason.", "sources": [ "study/results/opus/comparisons.json", "study/results/opus/controls/ctl1-90000-64.analysis.json" ] } ], "cross_rows": { "cosmos": [ { "experiment": "failure_map", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "RubiksCubeThenBananaTask", "code": "T1", "wording_pair": [ "Put the cube then the banana in the bowl" ], "n": 32, "slots": [ 30000, 30031 ], "unedited_k": 21, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "wilson95_pct": [ 48.3, 79.6 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "unedited policy, task's default instruction, practice slots, default render", "_ptask": null, "_task": "T1", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "failure_map", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "PickGlassesTask", "code": "T2", "wording_pair": [ "Pick up the eye glasses" ], "n": 32, "slots": [ 30000, 30031 ], "unedited_k": 2, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "wilson95_pct": [ 1.7, 20.1 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "unedited policy, task's default instruction, practice slots, default render", "_ptask": null, "_task": "T2", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "failure_map", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "RubiksCubeLeftOfBowlTask", "code": "T3", "wording_pair": [ "Put the rubiks cube to the left of the bowl" ], "n": 32, "slots": [ 30000, 30031 ], "unedited_k": 13, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "wilson95_pct": [ 25.5, 57.7 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "unedited policy, task's default instruction, practice slots, default render", "_ptask": null, "_task": "T3", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "failure_map", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "StackYellowOnRedTask", "code": "T4", "wording_pair": [ "Stack the yellow block on the red block" ], "n": 32, "slots": [ 30000, 30031 ], "unedited_k": 22, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "wilson95_pct": [ 51.4, 82.0 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "unedited policy, task's default instruction, practice slots, default render", "_ptask": null, "_task": "T4", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "failure_map", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "MustardInLeftBinTask", "code": "T5", "wording_pair": [ "Put the mustard in the left bin" ], "n": 32, "slots": [ 30000, 30031 ], "unedited_k": 11, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "wilson95_pct": [ 20.4, 51.7 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "unedited policy, task's default instruction, practice slots, default render", "_ptask": null, "_task": "T5", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "failure_map", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "Stack3RubiksCubeTask", "code": "T6", "wording_pair": [ "Stack the rubiks cubes in a tower" ], "n": 32, "slots": [ 30000, 30031 ], "unedited_k": 2, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "wilson95_pct": [ 1.7, 20.1 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "unedited policy, task's default instruction, practice slots, default render", "_ptask": null, "_task": "T6", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "failure_map", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "PickUpGreenObjectTask", "code": "T7", "wording_pair": [ "Pick up the green vegetable block" ], "n": 32, "slots": [ 30000, 30031 ], "unedited_k": 0, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "wilson95_pct": [ 0.0, 10.7 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "unedited policy, task's default instruction, practice slots, default render", "_ptask": null, "_task": "T7", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "failure_map", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "MustardRightOfRaisinRFTask", "code": "T8", "wording_pair": [ "Move the mustard bottle on the table so that the raisin box is to the left of the mustard bottle." ], "n": 32, "slots": [ 30000, 30031 ], "unedited_k": 0, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "wilson95_pct": [ 0.0, 10.7 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "unedited policy, task's default instruction, practice slots, default render", "_ptask": null, "_task": "T8", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "failure_map", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "RubiksCubeTask", "code": "R1", "wording_pair": [ "Put the cube in the bowl" ], "n": 32, "slots": [ 30000, 30031 ], "unedited_k": 21, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "wilson95_pct": [ 48.3, 79.6 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "unedited policy, task's default instruction, practice slots, default render", "_ptask": null, "_task": "R1", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "wording", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "RubiksCubeThenBananaTask", "code": "T1", "wording_pair": [ "Put the cube then the banana in the bowl", "Put the cube in the bowl then put the banana in the bowl" ], "n": 32, "slots": [ 30000, 30031 ], "unedited_k": 21, "edited_k": null, "fixed": 3, "broke": 9, "p": 0.14599609375, "ci": { "diff_pts": -18.75, "bootstrap95_pts": [ -37.5, 0.0 ], "wilson95_orig_pct": [ 48.3, 79.6 ], "wilson95_alt_pct": [ 30.9, 63.6 ] }, "control_k": null, "teacher_k": 15, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "same weights, wording via prompt override; fixed = only the alternative wording succeeds. pi0.5's known-good explicit wording (paper, Section 2)", "_ptask": null, "_task": "T1", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "wording", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "RubiksCubeLeftOfBowlTask", "code": "T3", "wording_pair": [ "Put the rubiks cube to the left of the bowl", "Pick up the rubiks cube and place it on the table to the left side of the bowl" ], "n": 32, "slots": [ 30000, 30031 ], "unedited_k": 13, "edited_k": null, "fixed": 15, "broke": 2, "p": 0.002349853515625, "ci": { "diff_pts": 40.62, "bootstrap95_pts": [ 18.75, 59.38 ], "wilson95_orig_pct": [ 25.5, 57.7 ], "wilson95_alt_pct": [ 64.7, 91.1 ] }, "control_k": null, "teacher_k": 26, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "same weights, wording via prompt override; fixed = only the alternative wording succeeds. the task's own 'specific' instruction in RoboLab", "_ptask": null, "_task": "T3", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "wording", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "MustardRightOfRaisinRFTask", "code": "T8", "wording_pair": [ "Move the mustard bottle on the table so that the raisin box is to the left of the mustard bottle.", "Move the mustard bottle on the table so that the mustard bottle is to the right of the raisin box." ], "n": 32, "slots": [ 30000, 30031 ], "unedited_k": 0, "edited_k": null, "fixed": 31, "broke": 0, "p": 9.313225746154785e-10, "ci": { "diff_pts": 96.88, "bootstrap95_pts": [ 90.62, 100.0 ], "wilson95_orig_pct": [ 0.0, 10.7 ], "wilson95_alt_pct": [ 84.3, 99.4 ] }, "control_k": null, "teacher_k": 31, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "same weights, wording via prompt override; fixed = only the alternative wording succeeds. target-first wording of the same goal (Same Goal, Different Words; known-good for pi0.5)", "_ptask": null, "_task": "T8", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "wording_prospective", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "RubiksCubeLeftOfBowlTask", "code": "S1L", "wording_pair": [ "Move the rubiks cube on the table so that the bowl is to the right of the rubiks cube.", "Move the rubiks cube on the table so that the rubiks cube is to the left of the bowl." ], "n": 16, "slots": [ 30000, 30015 ], "unedited_k": 1, "edited_k": null, "fixed": 7, "broke": 1, "p": 0.0703125, "ci": { "diff_pts": 37.5, "bootstrap95_pts": [ 6.25, 68.75 ], "wilson95_orig_pct": [ 1.1, 28.3 ], "wilson95_alt_pct": [ 23.1, 66.8 ] }, "control_k": null, "teacher_k": 7, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "prospective token-count test: unedited, reference-first vs target-first wording of the same goal, both via prompt override", "_ptask": "S1L", "_task": null, "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "wording_prospective", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "RubiksCubeRightOfBowlTask", "code": "S1R", "wording_pair": [ "Move the rubiks cube on the table so that the bowl is to the left of the rubiks cube.", "Move the rubiks cube on the table so that the rubiks cube is to the right of the bowl." ], "n": 16, "slots": [ 30000, 30015 ], "unedited_k": 0, "edited_k": null, "fixed": 4, "broke": 0, "p": 0.125, "ci": { "diff_pts": 25.0, "bootstrap95_pts": [ 6.25, 50.0 ], "wilson95_orig_pct": [ 0.0, 19.4 ], "wilson95_alt_pct": [ 10.2, 49.5 ] }, "control_k": null, "teacher_k": 4, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "prospective token-count test: unedited, reference-first vs target-first wording of the same goal, both via prompt override", "_ptask": "S1R", "_task": null, "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "edit_practice", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "MustardRightOfRaisinRFTask", "code": "T8", "wording_pair": [ "Move the mustard bottle on the table so that the raisin box is to the left of the mustard bottle.", "Move the mustard bottle on the table so that the mustard bottle is to the right of the raisin box." ], "n": 16, "slots": [ 30000, 30015 ], "unedited_k": 0, "edited_k": 13, "fixed": 13, "broke": 0, "p": 0.000244140625, "ci": { "diff_pts": 81.25, "bootstrap95_pts": [ 62.5, 100.0 ], "wilson95_unedited_pct": [ 0.0, 19.4 ], "wilson95_edited_pct": [ 57.0, 93.4 ] }, "control_k": 1, "teacher_k": 15, "recovery_pct": 86.7, "transfer": null, "retention": null, "edit_location": "text pathway (und mlp.down_proj, layers 0-26)", "values_changed": 179010015, "notes": "edit t8und (sequential closed-form distillation of the TF wording's text states into the RF wording, all 171 positions matched one to one, plus 19 unconditional-pass preservation keys; lam_rel 1e-3); control: same rank and singular values, random directions; control vs unedited fixed 1 broke 0 p = 1; edit vs control p = 0.000488", "_ptask": null, "_task": "T8", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms", "control_n": 16, "teacher_n": 16, "edit_vs_control": { "n": 16, "ref_k": 1, "cand_k": 13, "fixed": 12, "broke": 0, "p": 0.00048828125, "slots": [ 30000, 30015 ] }, "edit_vs_teacher": { "n": 16, "ref_k": 15, "cand_k": 13, "fixed": 0, "broke": 2, "p": 0.5, "slots": [ 30000, 30015 ] }, "control_slots": [ 30000, 30015 ], "teacher_slots": [ 30000, 30015 ] } }, { "experiment": "edit_heldout", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "MustardRightOfRaisinRFTask", "code": "T8", "wording_pair": [ "Move the mustard bottle on the table so that the raisin box is to the left of the mustard bottle.", "Move the mustard bottle on the table so that the mustard bottle is to the right of the raisin box." ], "n": 64, "slots": [ 96000, 96063 ], "unedited_k": 0, "edited_k": 56, "fixed": 56, "broke": 0, "p": 2.7755575615628914e-17, "ci": { "diff_pts": 87.5, "bootstrap95_pts": [ 78.12, 95.31 ], "wilson95_unedited_pct": [ 0.0, 5.7 ], "wilson95_edited_pct": [ 77.2, 93.5 ] }, "control_k": 0, "teacher_k": 56, "recovery_pct": 100.0, "transfer": null, "retention": null, "edit_location": "text pathway (und mlp.down_proj, layers 0-26)", "values_changed": 179010015, "notes": "edit t8und (sequential closed-form distillation of the TF wording's text states into the RF wording, all 171 positions matched one to one, plus 19 unconditional-pass preservation keys; lam_rel 1e-3); control: same rank and singular values, random directions; control vs unedited fixed 0 broke 0 p = 1; edit vs control p = 2.78e-17", "_ptask": null, "_task": "T8", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms", "control_n": 64, "teacher_n": 64, "edit_vs_control": { "n": 64, "ref_k": 0, "cand_k": 56, "fixed": 56, "broke": 0, "p": 2.7755575615628914e-17, "slots": [ 96000, 96063 ] }, "edit_vs_teacher": { "n": 64, "ref_k": 56, "cand_k": 56, "fixed": 5, "broke": 5, "p": 1.0, "slots": [ 96000, 96063 ] }, "control_slots": [ 96000, 96063 ], "teacher_slots": [ 96000, 96063 ] } }, { "experiment": "side_effect", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "RubiksCubeThenBananaTask", "code": "T1", "wording_pair": [ "Put the cube then the banana in the bowl" ], "n": 32, "slots": [ 30000, 30031 ], "unedited_k": 21, "edited_k": 18, "fixed": 7, "broke": 10, "p": 0.629058837890625, "ci": { "diff_pts": -9.38, "bootstrap95_pts": [ -34.38, 15.62 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": -9.38, "retention": null, "edit_location": "text pathway (und mlp.down_proj, layers 0-26)", "values_changed": 179010015, "notes": "edit t8und built for T8, run on T1 with T1's default instruction, practice slots", "_ptask": null, "_task": "T1", "_edit_task": "T8", "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "side_effect", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "PickGlassesTask", "code": "T2", "wording_pair": [ "Pick up the eye glasses" ], "n": 32, "slots": [ 30000, 30031 ], "unedited_k": 2, "edited_k": 0, "fixed": 0, "broke": 2, "p": 0.5, "ci": { "diff_pts": -6.25, "bootstrap95_pts": [ -15.62, 0.0 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": -6.25, "retention": null, "edit_location": "text pathway (und mlp.down_proj, layers 0-26)", "values_changed": 179010015, "notes": "edit t8und built for T8, run on T2 with T2's default instruction, practice slots", "_ptask": null, "_task": "T2", "_edit_task": "T8", "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "side_effect", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "RubiksCubeLeftOfBowlTask", "code": "T3", "wording_pair": [ "Put the rubiks cube to the left of the bowl" ], "n": 32, "slots": [ 30000, 30031 ], "unedited_k": 13, "edited_k": 0, "fixed": 0, "broke": 13, "p": 0.000244140625, "ci": { "diff_pts": -40.62, "bootstrap95_pts": [ -59.38, -25.0 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": -40.62, "retention": null, "edit_location": "text pathway (und mlp.down_proj, layers 0-26)", "values_changed": 179010015, "notes": "edit t8und built for T8, run on T3 with T3's default instruction, practice slots", "_ptask": null, "_task": "T3", "_edit_task": "T8", "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "side_effect", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "StackYellowOnRedTask", "code": "T4", "wording_pair": [ "Stack the yellow block on the red block" ], "n": 32, "slots": [ 30000, 30031 ], "unedited_k": 22, "edited_k": 26, "fixed": 8, "broke": 4, "p": 0.3876953125, "ci": { "diff_pts": 12.5, "bootstrap95_pts": [ -9.38, 34.38 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": 12.5, "retention": null, "edit_location": "text pathway (und mlp.down_proj, layers 0-26)", "values_changed": 179010015, "notes": "edit t8und built for T8, run on T4 with T4's default instruction, practice slots", "_ptask": null, "_task": "T4", "_edit_task": "T8", "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "side_effect", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "MustardInLeftBinTask", "code": "T5", "wording_pair": [ "Put the mustard in the left bin" ], "n": 32, "slots": [ 30000, 30031 ], "unedited_k": 11, "edited_k": 0, "fixed": 0, "broke": 11, "p": 0.0009765625, "ci": { "diff_pts": -34.38, "bootstrap95_pts": [ -50.0, -18.75 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": -34.38, "retention": null, "edit_location": "text pathway (und mlp.down_proj, layers 0-26)", "values_changed": 179010015, "notes": "edit t8und built for T8, run on T5 with T5's default instruction, practice slots", "_ptask": null, "_task": "T5", "_edit_task": "T8", "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "side_effect", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "Stack3RubiksCubeTask", "code": "T6", "wording_pair": [ "Stack the rubiks cubes in a tower" ], "n": 32, "slots": [ 30000, 30031 ], "unedited_k": 2, "edited_k": 3, "fixed": 3, "broke": 2, "p": 1.0, "ci": { "diff_pts": 3.12, "bootstrap95_pts": [ -9.38, 15.62 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": 3.12, "retention": null, "edit_location": "text pathway (und mlp.down_proj, layers 0-26)", "values_changed": 179010015, "notes": "edit t8und built for T8, run on T6 with T6's default instruction, practice slots", "_ptask": null, "_task": "T6", "_edit_task": "T8", "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "side_effect", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "PickUpGreenObjectTask", "code": "T7", "wording_pair": [ "Pick up the green vegetable block" ], "n": 32, "slots": [ 30000, 30031 ], "unedited_k": 0, "edited_k": 0, "fixed": 0, "broke": 0, "p": 1.0, "ci": { "diff_pts": 0.0, "bootstrap95_pts": [ 0.0, 0.0 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": 0.0, "retention": null, "edit_location": "text pathway (und mlp.down_proj, layers 0-26)", "values_changed": 179010015, "notes": "edit t8und built for T8, run on T7 with T7's default instruction, practice slots", "_ptask": null, "_task": "T7", "_edit_task": "T8", "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "side_effect", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "RubiksCubeTask", "code": "R1", "wording_pair": [ "Put the cube in the bowl" ], "n": 32, "slots": [ 30000, 30031 ], "unedited_k": 21, "edited_k": 27, "fixed": 9, "broke": 3, "p": 0.14599609375, "ci": { "diff_pts": 18.75, "bootstrap95_pts": [ 0.0, 40.62 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": 18.75, "edit_location": "text pathway (und mlp.down_proj, layers 0-26)", "values_changed": 179010015, "notes": "edit t8und built for T8, run on R1 with R1's default instruction, practice slots", "_ptask": null, "_task": "R1", "_edit_task": "T8", "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "side_effect_heldout", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "RubiksCubeThenBananaTask", "code": "T1", "wording_pair": [ "Put the cube then the banana in the bowl" ], "n": 32, "slots": [ 96000, 96031 ], "unedited_k": 18, "edited_k": 18, "fixed": 9, "broke": 9, "p": 1.0, "ci": { "diff_pts": 0.0, "bootstrap95_pts": [ -25.0, 25.0 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": 0.0, "retention": null, "edit_location": "text pathway (und mlp.down_proj, layers 0-26)", "values_changed": 179010015, "notes": "edit t8und built for T8, run on T1 with T1's default instruction, heldout slots (never used for any choice on this task)", "_ptask": null, "_task": "T1", "_edit_task": "T8", "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "side_effect_heldout", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "PickGlassesTask", "code": "T2", "wording_pair": [ "Pick up the eye glasses" ], "n": 16, "slots": [ 96000, 96015 ], "unedited_k": 4, "edited_k": 1, "fixed": 0, "broke": 3, "p": 0.25, "ci": { "diff_pts": -18.75, "bootstrap95_pts": [ -37.5, 0.0 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": -18.75, "retention": null, "edit_location": "text pathway (und mlp.down_proj, layers 0-26)", "values_changed": 179010015, "notes": "edit t8und built for T8, run on T2 with T2's default instruction, heldout slots (never used for any choice on this task)", "_ptask": null, "_task": "T2", "_edit_task": "T8", "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "side_effect_heldout", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "RubiksCubeLeftOfBowlTask", "code": "T3", "wording_pair": [ "Put the rubiks cube to the left of the bowl" ], "n": 32, "slots": [ 96000, 96031 ], "unedited_k": 13, "edited_k": 0, "fixed": 0, "broke": 13, "p": 0.000244140625, "ci": { "diff_pts": -40.62, "bootstrap95_pts": [ -59.38, -25.0 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": -40.62, "retention": null, "edit_location": "text pathway (und mlp.down_proj, layers 0-26)", "values_changed": 179010015, "notes": "edit t8und built for T8, run on T3 with T3's default instruction, heldout slots (never used for any choice on this task)", "_ptask": null, "_task": "T3", "_edit_task": "T8", "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "side_effect_heldout", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "StackYellowOnRedTask", "code": "T4", "wording_pair": [ "Stack the yellow block on the red block" ], "n": 32, "slots": [ 96000, 96031 ], "unedited_k": 24, "edited_k": 22, "fixed": 4, "broke": 6, "p": 0.75390625, "ci": { "diff_pts": -6.25, "bootstrap95_pts": [ -25.0, 12.5 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": -6.25, "retention": null, "edit_location": "text pathway (und mlp.down_proj, layers 0-26)", "values_changed": 179010015, "notes": "edit t8und built for T8, run on T4 with T4's default instruction, heldout slots (never used for any choice on this task)", "_ptask": null, "_task": "T4", "_edit_task": "T8", "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "side_effect_heldout", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "MustardInLeftBinTask", "code": "T5", "wording_pair": [ "Put the mustard in the left bin" ], "n": 32, "slots": [ 96000, 96031 ], "unedited_k": 9, "edited_k": 0, "fixed": 0, "broke": 9, "p": 0.00390625, "ci": { "diff_pts": -28.12, "bootstrap95_pts": [ -43.75, -12.5 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": -28.12, "retention": null, "edit_location": "text pathway (und mlp.down_proj, layers 0-26)", "values_changed": 179010015, "notes": "edit t8und built for T8, run on T5 with T5's default instruction, heldout slots (never used for any choice on this task)", "_ptask": null, "_task": "T5", "_edit_task": "T8", "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "side_effect_heldout", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "Stack3RubiksCubeTask", "code": "T6", "wording_pair": [ "Stack the rubiks cubes in a tower" ], "n": 32, "slots": [ 96000, 96031 ], "unedited_k": 1, "edited_k": 1, "fixed": 1, "broke": 1, "p": 1.0, "ci": { "diff_pts": 0.0, "bootstrap95_pts": [ -9.38, 9.38 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": 0.0, "retention": null, "edit_location": "text pathway (und mlp.down_proj, layers 0-26)", "values_changed": 179010015, "notes": "edit t8und built for T8, run on T6 with T6's default instruction, heldout slots (never used for any choice on this task)", "_ptask": null, "_task": "T6", "_edit_task": "T8", "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "side_effect_heldout", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "PickUpGreenObjectTask", "code": "T7", "wording_pair": [ "Pick up the green vegetable block" ], "n": 16, "slots": [ 96000, 96015 ], "unedited_k": 0, "edited_k": 0, "fixed": 0, "broke": 0, "p": 1.0, "ci": { "diff_pts": 0.0, "bootstrap95_pts": [ 0.0, 0.0 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": 0.0, "retention": null, "edit_location": "text pathway (und mlp.down_proj, layers 0-26)", "values_changed": 179010015, "notes": "edit t8und built for T8, run on T7 with T7's default instruction, heldout slots (never used for any choice on this task)", "_ptask": null, "_task": "T7", "_edit_task": "T8", "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "side_effect_heldout", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "RubiksCubeTask", "code": "R1", "wording_pair": [ "Put the cube in the bowl" ], "n": 32, "slots": [ 96000, 96031 ], "unedited_k": 24, "edited_k": 28, "fixed": 7, "broke": 3, "p": 0.34375, "ci": { "diff_pts": 12.5, "bootstrap95_pts": [ -6.25, 31.25 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": 12.5, "edit_location": "text pathway (und mlp.down_proj, layers 0-26)", "values_changed": 179010015, "notes": "edit t8und built for T8, run on R1 with R1's default instruction, heldout slots (never used for any choice on this task)", "_ptask": null, "_task": "R1", "_edit_task": "T8", "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "edit_practice", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "MustardRightOfRaisinRFTask", "code": "T8", "wording_pair": [ "Move the mustard bottle on the table so that the raisin box is to the left of the mustard bottle.", "Move the mustard bottle on the table so that the mustard bottle is to the right of the raisin box." ], "n": 16, "slots": [ 30000, 30015 ], "unedited_k": 0, "edited_k": 0, "fixed": 0, "broke": 0, "p": 1.0, "ci": { "diff_pts": 0.0, "bootstrap95_pts": [ 0.0, 0.0 ], "wilson95_unedited_pct": [ 0.0, 19.4 ], "wilson95_edited_pct": [ 0.0, 19.4 ] }, "control_k": null, "teacher_k": 15, "recovery_pct": 0.0, "transfer": null, "retention": null, "edit_location": "video/action pathway (gen mlp_moe_gen.down_proj, layers 8-20)", "values_changed": null, "notes": "edit t8gen (closed-form distillation of the TF wording's effect on the action tokens and pooled video cells (per denoising pass, averaged over 30 RF-failure decisions of slots 30000-30005, replans 1-5), lam_rel 1e-3); control: same rank and singular values, random directions", "_ptask": null, "_task": "T8", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms", "control_n": null, "teacher_n": 16, "edit_vs_control": null, "edit_vs_teacher": { "n": 16, "ref_k": 15, "cand_k": 0, "fixed": 0, "broke": 15, "p": 6.103515625e-05, "slots": [ 30000, 30015 ] }, "control_slots": null, "teacher_slots": [ 30000, 30015 ] } }, { "experiment": "edit_practice", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "MustardRightOfRaisinRFTask", "code": "T8", "wording_pair": [ "Move the mustard bottle on the table so that the raisin box is to the left of the mustard bottle.", "Move the mustard bottle on the table so that the mustard bottle is to the right of the raisin box." ], "n": 16, "slots": [ 30000, 30015 ], "unedited_k": 0, "edited_k": 12, "fixed": 12, "broke": 0, "p": 0.00048828125, "ci": { "diff_pts": 75.0, "bootstrap95_pts": [ 50.0, 93.75 ], "wilson95_unedited_pct": [ 0.0, 19.4 ], "wilson95_edited_pct": [ 50.5, 89.8 ] }, "control_k": null, "teacher_k": 15, "recovery_pct": 80.0, "transfer": null, "retention": null, "edit_location": "text pathway (und mlp.down_proj, layers 0-26), with generic preservation keys", "values_changed": 449299249, "notes": "edit t8undp (t8und's sequential closed-form distillation plus zero-change preservation keys from 65 generic RoboLab instructions (tasks/preserve_generic.json: no shared-set task, no mustard/raisin/bin/left/right; every 3rd position after the prompt template's common prefix, 1562 keys); lam_rel 1e-3; built after the T5/T3 side effect of t8und was seen); control: same rank and singular values, random directions", "_ptask": null, "_task": "T8", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms", "control_n": null, "teacher_n": 16, "edit_vs_control": null, "edit_vs_teacher": { "n": 16, "ref_k": 15, "cand_k": 12, "fixed": 1, "broke": 4, "p": 0.375, "slots": [ 30000, 30015 ] }, "control_slots": null, "teacher_slots": [ 30000, 30015 ] } }, { "experiment": "side_effect", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "MustardInLeftBinTask", "code": "T5", "wording_pair": [ "Put the mustard in the left bin" ], "n": 16, "slots": [ 30000, 30015 ], "unedited_k": 8, "edited_k": 0, "fixed": 0, "broke": 8, "p": 0.0078125, "ci": { "diff_pts": -50.0, "bootstrap95_pts": [ -75.0, -25.0 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": -50.0, "retention": null, "edit_location": "text pathway (und mlp.down_proj, layers 0-26), with generic preservation keys", "values_changed": 449299249, "notes": "edit t8undp built for T8, run on T5 with T5's default instruction, practice slots", "_ptask": null, "_task": "T5", "_edit_task": "T8", "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "edit_practice", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "MustardRightOfRaisinRFTask", "code": "T8", "wording_pair": [ "Move the mustard bottle on the table so that the raisin box is to the left of the mustard bottle.", "Move the mustard bottle on the table so that the mustard bottle is to the right of the raisin box." ], "n": 16, "slots": [ 30000, 30015 ], "unedited_k": 0, "edited_k": 13, "fixed": 13, "broke": 0, "p": 0.000244140625, "ci": { "diff_pts": 81.25, "bootstrap95_pts": [ 62.5, 100.0 ], "wilson95_unedited_pct": [ 0.0, 19.4 ], "wilson95_edited_pct": [ 57.0, 93.4 ] }, "control_k": null, "teacher_k": 15, "recovery_pct": 86.7, "transfer": null, "retention": null, "edit_location": "text pathway (und mlp.down_proj, layers 0-26), with relation-aware preservation keys", "values_changed": 443281016, "notes": "edit t8undr (t8und's sequential closed-form distillation plus zero-change preservation keys from 24 left/right instructions of other RoboLab tasks (tasks/preserve_relation.json; T3/T5/T8 object pairs and all shared-set tasks excluded; 1816 keys at every position after the template's common prefix); lam_rel 1e-3; built after the T5/T3 side effect of t8und was seen); control: same rank and singular values, random directions", "_ptask": null, "_task": "T8", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms", "control_n": null, "teacher_n": 16, "edit_vs_control": null, "edit_vs_teacher": { "n": 16, "ref_k": 15, "cand_k": 13, "fixed": 0, "broke": 2, "p": 0.5, "slots": [ 30000, 30015 ] }, "control_slots": null, "teacher_slots": [ 30000, 30015 ] } }, { "experiment": "side_effect", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "RubiksCubeLeftOfBowlTask", "code": "T3", "wording_pair": [ "Put the rubiks cube to the left of the bowl" ], "n": 16, "slots": [ 30000, 30015 ], "unedited_k": 4, "edited_k": 1, "fixed": 1, "broke": 4, "p": 0.375, "ci": { "diff_pts": -18.75, "bootstrap95_pts": [ -43.75, 6.25 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": -18.75, "retention": null, "edit_location": "text pathway (und mlp.down_proj, layers 0-26), with relation-aware preservation keys", "values_changed": 443281016, "notes": "edit t8undr built for T8, run on T3 with T3's default instruction, practice slots", "_ptask": null, "_task": "T3", "_edit_task": "T8", "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "side_effect", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "MustardInLeftBinTask", "code": "T5", "wording_pair": [ "Put the mustard in the left bin" ], "n": 16, "slots": [ 30000, 30015 ], "unedited_k": 8, "edited_k": 2, "fixed": 0, "broke": 6, "p": 0.03125, "ci": { "diff_pts": -37.5, "bootstrap95_pts": [ -62.5, -12.5 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": -37.5, "retention": null, "edit_location": "text pathway (und mlp.down_proj, layers 0-26), with relation-aware preservation keys", "values_changed": 443281016, "notes": "edit t8undr built for T8, run on T5 with T5's default instruction, practice slots", "_ptask": null, "_task": "T5", "_edit_task": "T8", "_prov": false, "extra": { "machine": "ali-ms" } }, { "experiment": "edit_practice", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "RubiksCubeLeftOfBowlTask", "code": "S1L", "wording_pair": [ "Move the rubiks cube on the table so that the bowl is to the right of the rubiks cube.", "Move the rubiks cube on the table so that the rubiks cube is to the left of the bowl." ], "n": 16, "slots": [ 30000, 30015 ], "unedited_k": 1, "edited_k": 4, "fixed": 4, "broke": 1, "p": 0.375, "ci": { "diff_pts": 18.75, "bootstrap95_pts": [ -6.25, 43.75 ], "wilson95_unedited_pct": [ 1.1, 28.3 ], "wilson95_edited_pct": [ 10.2, 49.5 ] }, "control_k": 1, "teacher_k": 7, "recovery_pct": 50.0, "transfer": null, "retention": null, "edit_location": "text pathway (und mlp.down_proj, layers 0-26), --align all (169 = 169 tokens, every position matched)", "values_changed": 207159927, "notes": "edit s1lund (prospective test of the token-count rule: t8und's recipe unchanged on S1L's reference-first -> target-first pair (equal length), one prompt pair, unconditional-pass keys, lam_rel 1e-3); control: same rank and singular values, random directions; control vs unedited fixed 1 broke 1 p = 1; edit vs control p = 0.375", "_ptask": "S1L", "_task": null, "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms", "control_n": 16, "teacher_n": 16, "edit_vs_control": { "n": 16, "ref_k": 1, "cand_k": 4, "fixed": 4, "broke": 1, "p": 0.375, "slots": [ 30000, 30015 ] }, "edit_vs_teacher": { "n": 16, "ref_k": 7, "cand_k": 4, "fixed": 2, "broke": 5, "p": 0.453125, "slots": [ 30000, 30015 ] }, "control_slots": [ 30000, 30015 ], "teacher_slots": [ 30000, 30015 ] } }, { "experiment": "edit_heldout", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "RubiksCubeLeftOfBowlTask", "code": "S1L", "wording_pair": [ "Move the rubiks cube on the table so that the bowl is to the right of the rubiks cube.", "Move the rubiks cube on the table so that the rubiks cube is to the left of the bowl." ], "n": 32, "slots": [ 96032, 96063 ], "unedited_k": 1, "edited_k": 12, "fixed": 11, "broke": 0, "p": 0.0009765625, "ci": { "diff_pts": 34.38, "bootstrap95_pts": [ 18.75, 50.0 ], "wilson95_unedited_pct": [ 0.6, 15.7 ], "wilson95_edited_pct": [ 22.9, 54.7 ] }, "control_k": 3, "teacher_k": 8, "recovery_pct": 157.1, "transfer": null, "retention": null, "edit_location": "text pathway (und mlp.down_proj, layers 0-26), --align all (169 = 169 tokens, every position matched)", "values_changed": 207159927, "notes": "edit s1lund (prospective test of the token-count rule: t8und's recipe unchanged on S1L's reference-first -> target-first pair (equal length), one prompt pair, unconditional-pass keys, lam_rel 1e-3); control: same rank and singular values, random directions; control vs unedited fixed 3 broke 1 p = 0.625; edit vs control p = 0.0225", "_ptask": "S1L", "_task": null, "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms", "control_n": 32, "teacher_n": 32, "edit_vs_control": { "n": 32, "ref_k": 3, "cand_k": 12, "fixed": 11, "broke": 2, "p": 0.0224609375, "slots": [ 96032, 96063 ] }, "edit_vs_teacher": { "n": 32, "ref_k": 8, "cand_k": 12, "fixed": 8, "broke": 4, "p": 0.3876953125, "slots": [ 96032, 96063 ] }, "control_slots": [ 96032, 96063 ], "teacher_slots": [ 96032, 96063 ] } }, { "experiment": "where_to_edit_offline", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "MustardRightOfRaisinRFTask", "code": "T8", "wording_pair": [ "Move the mustard bottle on the table so that the raisin box is to the left of the mustard bottle.", "Move the mustard bottle on the table so that the mustard bottle is to the right of the raisin box." ], "n": 30, "slots": null, "unedited_k": null, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": null, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "text pathway (und mlp.down_proj, layers 0-26)", "values_changed": 179010015, "notes": "offline replay of 30 RF-failure decisions (build set: decisions the gen edit was built from, slots 30000-30005, replans 1-5): chunk-end end-effector distance to the TF-wording action chunk, same seed; gap closed = 1 - edited/unedited distance; control = same-size random directions", "_ptask": null, "_task": "T8", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms", "offline": { "machine": "ali-ms", "edit": "t8und", "group": "build", "n": 30, "unedited_ee_cm": 5.42, "edited_ee_cm": 0.5, "control_ee_cm": 5.21, "gap_closed_pct": 90.8, "control_gap_closed_pct": 3.8, "closer_to_teacher": 30, "control_closer_to_teacher": 18 } } }, { "experiment": "where_to_edit_offline", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "MustardRightOfRaisinRFTask", "code": "T8", "wording_pair": [ "Move the mustard bottle on the table so that the raisin box is to the left of the mustard bottle.", "Move the mustard bottle on the table so that the mustard bottle is to the right of the raisin box." ], "n": 30, "slots": null, "unedited_k": null, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": null, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "video/action pathway (gen mlp_moe_gen.down_proj, layers 8-20)", "values_changed": null, "notes": "offline replay of 30 RF-failure decisions (build set: decisions the gen edit was built from, slots 30000-30005, replans 1-5): chunk-end end-effector distance to the TF-wording action chunk, same seed; gap closed = 1 - edited/unedited distance; control = same-size random directions", "_ptask": null, "_task": "T8", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms", "offline": { "machine": "ali-ms", "edit": "t8gen", "group": "build", "n": 30, "unedited_ee_cm": 5.42, "edited_ee_cm": 4.55, "control_ee_cm": 5.07, "gap_closed_pct": 16.1, "control_gap_closed_pct": 6.5, "closer_to_teacher": 13, "control_closer_to_teacher": 20 } } }, { "experiment": "where_to_edit_offline", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "MustardRightOfRaisinRFTask", "code": "T8", "wording_pair": [ "Move the mustard bottle on the table so that the raisin box is to the left of the mustard bottle.", "Move the mustard bottle on the table so that the mustard bottle is to the right of the raisin box." ], "n": 30, "slots": null, "unedited_k": null, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": null, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "text pathway (und mlp.down_proj, layers 0-26)", "values_changed": 179010015, "notes": "offline replay of 30 RF-failure decisions (check set: held-aside decisions, slots 30006-30011, never used to build either edit, replans 1-5): chunk-end end-effector distance to the TF-wording action chunk, same seed; gap closed = 1 - edited/unedited distance; control = same-size random directions", "_ptask": null, "_task": "T8", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms", "offline": { "machine": "ali-ms", "edit": "t8und", "group": "check", "n": 30, "unedited_ee_cm": 8.75, "edited_ee_cm": 0.77, "control_ee_cm": 8.9, "gap_closed_pct": 91.2, "control_gap_closed_pct": -1.8, "closer_to_teacher": 25, "control_closer_to_teacher": 11 } } }, { "experiment": "where_to_edit_offline", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "MustardRightOfRaisinRFTask", "code": "T8", "wording_pair": [ "Move the mustard bottle on the table so that the raisin box is to the left of the mustard bottle.", "Move the mustard bottle on the table so that the mustard bottle is to the right of the raisin box." ], "n": 30, "slots": null, "unedited_k": null, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": null, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "video/action pathway (gen mlp_moe_gen.down_proj, layers 8-20)", "values_changed": null, "notes": "offline replay of 30 RF-failure decisions (check set: held-aside decisions, slots 30006-30011, never used to build either edit, replans 1-5): chunk-end end-effector distance to the TF-wording action chunk, same seed; gap closed = 1 - edited/unedited distance; control = same-size random directions", "_ptask": null, "_task": "T8", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms", "offline": { "machine": "ali-ms", "edit": "t8gen", "group": "check", "n": 30, "unedited_ee_cm": 8.75, "edited_ee_cm": 6.7, "control_ee_cm": 8.68, "gap_closed_pct": 23.4, "control_gap_closed_pct": 0.7, "closer_to_teacher": 16, "control_closer_to_teacher": 19 } } }, { "experiment": "preservation_offline", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "MustardInLeftBinTask", "code": "T5", "wording_pair": null, "n": null, "slots": null, "unedited_k": null, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": null, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "text pathway (und mlp.down_proj, layers 0-26), with generic preservation keys", "values_changed": 449299249, "notes": "t8undp vs t8und, offline replays at fixed seeds: T8 = chunk-end end-effector distance to the TF wording's chunk on 30 held-aside RF-failure decisions (slots 30006-30011, replans 1-5); T5 = distance of the edited chunk from the unedited chunk on 38 decisions of T5 practice episodes the unedited policy solved (replans 1-5); dev removed = 1 - t8undp/t8und", "_ptask": null, "_task": "T5", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms", "offline": { "machine": "ali-ms", "edit": "t8undp", "t8_n": 30, "t8_control_gap_closed_pct": null, "t3_n": null, "t3_t8und_dev_cm": null, "t3_variant_dev_cm": null, "t8_unedited_ee_cm": 8.75, "t8_edited_ee_cm": 0.61, "t8_gap_closed_pct": 93.1, "t8und_gap_closed_pct": 91.2, "t5_n": 38, "t5_t8und_dev_cm": 13.73, "t5_variant_dev_cm": 13.89, "t5_dev_removed_pct": -1.2 } } }, { "experiment": "preservation_offline", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "MustardInLeftBinTask", "code": "T5", "wording_pair": null, "n": null, "slots": null, "unedited_k": null, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": null, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "text pathway (und mlp.down_proj, layers 0-26), with relation-aware preservation keys", "values_changed": 443281016, "notes": "t8undr vs t8und, offline replays at fixed seeds: T8 = chunk-end end-effector distance to the TF wording's chunk on 30 held-aside RF-failure decisions (slots 30006-30011, replans 1-5); T5 = distance of the edited chunk from the unedited chunk on 38 decisions of T5 practice episodes the unedited policy solved (replans 1-5); dev removed = 1 - t8undr/t8und", "_ptask": null, "_task": "T5", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms", "offline": { "machine": "ali-ms", "edit": "t8undr", "t8_n": 30, "t8_control_gap_closed_pct": -10.5, "t3_n": 20, "t3_t8und_dev_cm": 29.64, "t3_variant_dev_cm": 6.46, "t8_unedited_ee_cm": 8.75, "t8_edited_ee_cm": 0.56, "t8_gap_closed_pct": 93.6, "t8und_gap_closed_pct": null, "t5_n": 38, "t5_t8und_dev_cm": 13.73, "t5_variant_dev_cm": 5.88, "t5_dev_removed_pct": 57.2 } } }, { "experiment": "preservation_offline", "model": "nvidia/Cosmos3-Edge-Policy-DROID@a7c7288", "task": "MustardInLeftBinTask", "code": "T5", "wording_pair": null, "n": null, "slots": null, "unedited_k": null, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": null, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "text pathway (und mlp.down_proj, layers 0-26), template positions only (--align ps), with generic preservation keys", "values_changed": 435316027, "notes": "t8unds vs t8und, offline replays at fixed seeds: T8 = chunk-end end-effector distance to the TF wording's chunk on 30 held-aside RF-failure decisions (slots 30006-30011, replans 1-5); T5 = distance of the edited chunk from the unedited chunk on 38 decisions of T5 practice episodes the unedited policy solved (replans 1-5); dev removed = 1 - t8unds/t8und", "_ptask": null, "_task": "T5", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms", "offline": { "machine": "ali-ms", "edit": "t8unds", "t8_n": 30, "t8_control_gap_closed_pct": 2.6, "t3_n": 20, "t3_t8und_dev_cm": 29.64, "t3_variant_dev_cm": 0.78, "t8_unedited_ee_cm": 8.75, "t8_edited_ee_cm": 8.58, "t8_gap_closed_pct": 1.9, "t8und_gap_closed_pct": null, "t5_n": 38, "t5_t8und_dev_cm": 13.73, "t5_variant_dev_cm": 2.04, "t5_dev_removed_pct": 85.2 } } } ], "groot": [ { "experiment": "failure_map", "model": "GR00T N1.7 DROID (nvidia/GR00T-N1.7-DROID@05e7cc97)", "task": "T1 RubiksCubeThenBananaTask", "wording_pair": [ "Put the cube then the banana in the bowl", null ], "n": 32, "unedited_k": 9, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "unedited_wilson95": [ 0.1556458049774277, 0.4537450942591657 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "slots": [ 30000, 30031 ], "notes": "tag fm-T1; envs 16; env seed 0; one fresh simulator process per 16-slot group", "_ptask": null, "_task": "T1", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms-rtx3090" } }, { "experiment": "failure_map", "model": "GR00T N1.7 DROID (nvidia/GR00T-N1.7-DROID@05e7cc97)", "task": "T2 PickGlassesTask", "wording_pair": [ "Pick up the eye glasses", null ], "n": 32, "unedited_k": 0, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "unedited_wilson95": [ 0.0, 0.10717919825507057 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "slots": [ 30000, 30031 ], "notes": "tag fm-T2; envs 16; env seed 0; one fresh simulator process per 16-slot group", "_ptask": null, "_task": "T2", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms-rtx3090" } }, { "experiment": "failure_map", "model": "GR00T N1.7 DROID (nvidia/GR00T-N1.7-DROID@05e7cc97)", "task": "T3 RubiksCubeLeftOfBowlTask", "wording_pair": [ "Put the rubiks cube to the left of the bowl", null ], "n": 32, "unedited_k": 1, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "unedited_wilson95": [ 0.005537860164003136, 0.1574426382001255 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "slots": [ 30000, 30031 ], "notes": "tag fm-T3; envs 16; env seed 0; one fresh simulator process per 16-slot group", "_ptask": null, "_task": "T3", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms-rtx3090" } }, { "experiment": "failure_map", "model": "GR00T N1.7 DROID (nvidia/GR00T-N1.7-DROID@05e7cc97)", "task": "T4 StackYellowOnRedTask", "wording_pair": [ "Stack the yellow block on the red block", null ], "n": 16, "unedited_k": 0, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "unedited_wilson95": [ 0.0, 0.1936076805344364 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "slots": [ 30000, 30015 ], "notes": "tag fm-T4; envs 16; env seed 0; one fresh simulator process per 16-slot group", "_ptask": null, "_task": "T4", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms-rtx3090" } }, { "experiment": "failure_map", "model": "GR00T N1.7 DROID (nvidia/GR00T-N1.7-DROID@05e7cc97)", "task": "T5 MustardInLeftBinTask", "wording_pair": [ "Put the mustard in the left bin", null ], "n": 32, "unedited_k": 0, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "unedited_wilson95": [ 0.0, 0.10717919825507057 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "slots": [ 30000, 30031 ], "notes": "tag fm-T5; envs 16; env seed 0; one fresh simulator process per 16-slot group", "_ptask": null, "_task": "T5", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms-rtx3090" } }, { "experiment": "failure_map", "model": "GR00T N1.7 DROID (nvidia/GR00T-N1.7-DROID@05e7cc97)", "task": "T6 Stack3RubiksCubeTask", "wording_pair": [ "Stack the rubiks cubes in a tower", null ], "n": 16, "unedited_k": 0, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "unedited_wilson95": [ 0.0, 0.1936076805344364 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "slots": [ 30000, 30015 ], "notes": "tag fm-T6; envs 16; env seed 0; one fresh simulator process per 16-slot group", "_ptask": null, "_task": "T6", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms-rtx3090" } }, { "experiment": "failure_map", "model": "GR00T N1.7 DROID (nvidia/GR00T-N1.7-DROID@05e7cc97)", "task": "T7 PickUpGreenObjectTask", "wording_pair": [ "Pick up the green vegetable block", null ], "n": 32, "unedited_k": 0, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "unedited_wilson95": [ 0.0, 0.10717919825507057 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "slots": [ 30000, 30031 ], "notes": "tag fm-T7; envs 16; env seed 0; one fresh simulator process per 16-slot group", "_ptask": null, "_task": "T7", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms-rtx3090" } }, { "experiment": "failure_map", "model": "GR00T N1.7 DROID (nvidia/GR00T-N1.7-DROID@05e7cc97)", "task": "T8 MustardRightOfRaisinRFTask", "wording_pair": [ "Move the mustard bottle on the table so that the raisin box is to the left of the mustard bottle.", null ], "n": 32, "unedited_k": 0, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "unedited_wilson95": [ 0.0, 0.10717919825507057 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "slots": [ 30000, 30031 ], "notes": "tag fm-T8; envs 16; env seed 0; one fresh simulator process per 16-slot group", "_ptask": null, "_task": "T8", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms-rtx3090" } }, { "experiment": "failure_map", "model": "GR00T N1.7 DROID (nvidia/GR00T-N1.7-DROID@05e7cc97)", "task": "R1 RubiksCubeTask", "wording_pair": [ "Put the cube in the bowl", null ], "n": 32, "unedited_k": 28, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "unedited_wilson95": [ 0.7193169451834979, 0.9502986561251993 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "slots": [ 30000, 30031 ], "notes": "tag fm-R1; envs 16; env seed 0; one fresh simulator process per 16-slot group", "_ptask": null, "_task": "R1", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms-rtx3090" } }, { "experiment": "wording_sensitivity", "model": "GR00T N1.7 DROID (nvidia/GR00T-N1.7-DROID@05e7cc97)", "task": "T1 RubiksCubeThenBananaTask", "wording_pair": [ "Put the cube then the banana in the bowl", "Put the cube in the bowl then put the banana in the bowl" ], "n": 32, "unedited_k": 9, "edited_k": null, "fixed": 11, "broke": 4, "p": 0.11846923828125, "ci": { "diff_bootstrap95": [ 0.0, 0.4375 ] }, "control_k": null, "teacher_k": 16, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "slots": [ 30000, 30031 ], "notes": "alternative wording given to the unedited model (teacher); fixed = succeeds only under the alternative wording; tags fm-T1 vs wd-T1-expl; GR00T's only wording-sensitive task among those screened (T1, T3, T8)", "_ptask": null, "_task": "T1", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms-rtx3090" } }, { "experiment": "wording_sensitivity", "model": "GR00T N1.7 DROID (nvidia/GR00T-N1.7-DROID@05e7cc97)", "task": "T3 RubiksCubeLeftOfBowlTask", "wording_pair": [ "Put the rubiks cube to the left of the bowl", "Pick up the rubiks cube and place it on the table to the left of the bowl" ], "n": 32, "unedited_k": 1, "edited_k": null, "fixed": 0, "broke": 1, "p": 1.0, "ci": { "diff_bootstrap95": [ -0.09375, 0.0 ] }, "control_k": null, "teacher_k": 0, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "slots": [ 30000, 30031 ], "notes": "alternative wording given to the unedited model (teacher); fixed = succeeds only under the alternative wording; tags fm-T3 vs wd-T3-alt; not wording-sensitive: under both wordings GR00T grasps the cube and holds it near the bowl until the episode ends (final frames of slots 30000-30005)", "_ptask": null, "_task": "T3", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms-rtx3090" } }, { "experiment": "wording_sensitivity", "model": "GR00T N1.7 DROID (nvidia/GR00T-N1.7-DROID@05e7cc97)", "task": "T8 MustardRightOfRaisinRFTask", "wording_pair": [ "Move the mustard bottle on the table so that the raisin box is to the left of the mustard bottle.", "Move the mustard bottle on the table so that the mustard bottle is to the right of the raisin box." ], "n": 32, "unedited_k": 0, "edited_k": null, "fixed": 0, "broke": 0, "p": 1.0, "ci": { "diff_bootstrap95": [ 0.0, 0.0 ] }, "control_k": null, "teacher_k": 0, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "slots": [ 30000, 30031 ], "notes": "alternative wording given to the unedited model (teacher); fixed = succeeds only under the alternative wording; tags fm-T8 vs wd-T8-tf; the target-first wording does not rescue GR00T (0/32 under both; Cosmos3 Edge went 0 -> 15/16 under TF in Track A); under TF every episode grasps the mustard (RF: 10/32 never grasp it) but none ends with it right of the raisin box", "_ptask": null, "_task": "T8", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms-rtx3090" } }, { "experiment": "adapter_validation_official_path", "model": "GR00T N1.7 DROID (nvidia/GR00T-N1.7-DROID@05e7cc97)", "task": "- BananaOnPlateTask", "wording_pair": null, "n": 16, "unedited_k": 16, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "wilson95": [ 0.8063923194655636, 1.0 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "slots": null, "notes": "unmodified official server + RoboLab client, default preset, 16 envs; this adapter: 16/16; file official-bop16.episode_results.jsonl", "_ptask": null, "_task": null, "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms-rtx3090" } }, { "experiment": "adapter_validation_official_path", "model": "GR00T N1.7 DROID (nvidia/GR00T-N1.7-DROID@05e7cc97)", "task": "T7 PickUpGreenObjectTask", "wording_pair": null, "n": 16, "unedited_k": 0, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "wilson95": [ 0.0, 0.1936076805344364 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "slots": null, "notes": "unmodified official server + RoboLab client, default preset, 16 envs; this adapter: 0/32; file official-t7-16.episode_results.jsonl", "_ptask": null, "_task": "T7", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms-rtx3090" } }, { "experiment": "wsp_edit_practice", "model": "GR00T N1.7 DROID (nvidia/GR00T-N1.7-DROID@05e7cc97)", "task": "T1 RubiksCubeThenBananaTask", "wording_pair": [ "Put the cube then the banana in the bowl", "Put the cube in the bowl then put the banana in the bowl" ], "n": 32, "unedited_k": 9, "edited_k": 0, "fixed": 0, "broke": 9, "p": 0.00390625, "ci": { "diff_bootstrap95": [ -0.4375, -0.125 ] }, "control_k": 11, "teacher_k": 16, "recovery_pct": -128.57142857142858, "transfer": null, "retention": null, "edit_location": "Cosmos-Reason2 LLM layers 0-15, mlp.down_proj (VLM backbone)", "values_changed": 167481274, "slots": [ 30000, 30031 ], "notes": "what was edited: Cosmos-Reason2 LLM layers 0-15, mlp.down_proj (16 tensors, rank 8 each: the 8 orig positions after the first difference carry the values; image cells and the shared prefix are pinned to zero); closed-form dW = K^T (K K^T + lam I)^-1 V, lam = 1e-3 mean diag(KK^T), keys/values averaged over 6 build decisions (the most divergent replan of each of the first 6 failed practice episodes). Offline (recorded decisions, noise seed fixed): teacher projection 0.28 on 17 check decisions from other failed episodes (matched random control 0.00); on the 11 of those used by neither this nor a 12-decision build it is -0.15. Closed loop on practice: 9/32 -> 0/32; every edited episode fails at object_grabbed(rubiks_cube): the edited GR00T reaches for the banana first and never grabs the cube (videos/RubiksCubeThenBananaTask_30004_*). Likely cause: token alignment only covers the 8 orig positions after the difference, while the explicit wording's inserted clause ('in the bowl then put') has no position in the original prompt; the transferred change at 'the banana in the bowl' arrives without the cube clause. Grasp-phase offline check (16 decisions, k = 4/8/12/16 of slots 30016-30019, none used to build): the teacher changes the executed chunk's end-effector target by 0.88 cm on average, the edit by 1.86 cm, away from the teacher (projection -0.07); pinning every image token to zero (bb_tok) gives 1.84 cm, so the break is not the edit leaking into the visual tokens.", "_ptask": null, "_task": "T1", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms-rtx3090", "control_fixed_broke": [ 9, 7 ], "control_p": 0.803619384765625, "edit_vs_control": { "n": 32, "ref_k": 11, "cand_k": 0, "fixed": 0, "broke": 11, "p": 0.0009765625 }, "values_total": 201326592, "control_values_changed": 179039148, "edit_fingerprint": "b5523fe55b6d94a2", "control_fingerprint": "1d2b485f3c0d1321" } }, { "experiment": "wsp_edit_practice", "model": "GR00T N1.7 DROID (nvidia/GR00T-N1.7-DROID@05e7cc97)", "task": "T1 RubiksCubeThenBananaTask", "wording_pair": [ "Put the cube then the banana in the bowl", "Put the cube in the bowl then put the banana in the bowl" ], "n": 32, "unedited_k": 9, "edited_k": 0, "fixed": 0, "broke": 9, "p": 0.00390625, "ci": { "diff_bootstrap95": [ -0.4375, -0.125 ] }, "control_k": 10, "teacher_k": 16, "recovery_pct": -128.57142857142858, "transfer": null, "retention": null, "edit_location": "GR00T action head: 32 AlternateVLDiT blocks, ff.net.2 (diffusion transformer)", "values_changed": 298523070, "slots": [ 30000, 30031 ], "notes": "same 6 build decisions as bb; keys = ff.net.2 inputs at every (denoising step, state/action token), 4 x 41 = 164 per block; values = explicit-minus-original block increments; lam = 1e-3 mean diag(KK^T); offline teacher projection +0.22 on the 11 failed episodes used for neither build (bb: -0.15); control = same singular values along random orthonormal directions (default_rng(0)). Closed loop on practice: 9/32 -> 0/32 (0 fixed / 9 broke); 31/32 edited episodes fail at object_grabbed(rubiks_cube), 1 at the banana.", "_ptask": null, "_task": "T1", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms-rtx3090", "control_fixed_broke": [ 9, 8 ], "control_p": 1.0, "edit_vs_control": { "n": 32, "ref_k": 10, "cand_k": 0, "fixed": 0, "broke": 10, "p": 0.001953125 }, "values_total": 301989888, "control_values_changed": 300072426, "edit_fingerprint": "a998bdc75628fbe6", "control_fingerprint": "afc8a78428b9ec07" } }, { "experiment": "wsp_offline_check", "model": "GR00T N1.7 DROID (nvidia/GR00T-N1.7-DROID@05e7cc97)", "task": "T1 RubiksCubeThenBananaTask", "wording_pair": [ "Put the cube then the banana in the bowl", "Put the cube in the bowl then put the banana in the bowl" ], "edit_location": "backbone", "notes": "teacher_projection = sum / sum |x-o|^2 over executed rows 0-7 (joints + gripper); shift = end-effector target of executed row 7. Only bb and dit were run closed loop.", "_ptask": null, "_task": "T1", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms-rtx3090", "edit": "bb", "grasp_phase": { "n_decisions": 16, "decisions": "k = 4, 8, 12, 16 of slots 30016-30019 (none used to build)", "mean_shift_vs_unedited_cm": 1.862347967835376, "teacher_mean_shift_vs_unedited_cm": 0.8770811306021642, "teacher_projection": -0.0716186033087405, "gripper_rows_matching_teacher": "120/128", "unedited_gripper_rows_matching_teacher": "121/128" }, "unseen_failures": { "n_decisions": 11, "decisions": "most-divergent replan of the 11 failed practice episodes used by neither the 6- nor a 12-decision build", "teacher_projection": -0.15310076334760012, "mean_shift_vs_unedited_cm": 2.3911862358958884, "teacher_mean_shift_vs_unedited_cm": 2.034298603592271 } } }, { "experiment": "wsp_offline_check", "model": "GR00T N1.7 DROID (nvidia/GR00T-N1.7-DROID@05e7cc97)", "task": "T1 RubiksCubeThenBananaTask", "wording_pair": [ "Put the cube then the banana in the bowl", "Put the cube in the bowl then put the banana in the bowl" ], "edit_location": "backbone", "notes": "teacher_projection = sum / sum |x-o|^2 over executed rows 0-7 (joints + gripper); shift = end-effector target of executed row 7. Only bb and dit were run closed loop.", "_ptask": null, "_task": "T1", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms-rtx3090", "edit": "bb_tok", "grasp_phase": { "n_decisions": 16, "decisions": "k = 4, 8, 12, 16 of slots 30016-30019 (none used to build)", "mean_shift_vs_unedited_cm": 1.8376651962171309, "teacher_mean_shift_vs_unedited_cm": 0.8770811306021642, "teacher_projection": -0.08338396692116268, "gripper_rows_matching_teacher": "120/128", "unedited_gripper_rows_matching_teacher": "121/128" }, "unseen_failures": { "n_decisions": 11, "decisions": "most-divergent replan of the 11 failed practice episodes used by neither the 6- nor a 12-decision build", "teacher_projection": -0.1660978186142906, "mean_shift_vs_unedited_cm": 2.396676090376621, "teacher_mean_shift_vs_unedited_cm": 2.032215103761039 } } }, { "experiment": "wsp_offline_check", "model": "GR00T N1.7 DROID (nvidia/GR00T-N1.7-DROID@05e7cc97)", "task": "T1 RubiksCubeThenBananaTask", "wording_pair": [ "Put the cube then the banana in the bowl", "Put the cube in the bowl then put the banana in the bowl" ], "edit_location": "backbone", "notes": "teacher_projection = sum / sum |x-o|^2 over executed rows 0-7 (joints + gripper); shift = end-effector target of executed row 7. Only bb and dit were run closed loop.", "_ptask": null, "_task": "T1", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms-rtx3090", "edit": "bb_span", "grasp_phase": { "n_decisions": 16, "decisions": "k = 4, 8, 12, 16 of slots 30016-30019 (none used to build)", "mean_shift_vs_unedited_cm": 3.495357137762767, "teacher_mean_shift_vs_unedited_cm": 0.8770811306021642, "teacher_projection": -0.13042248521510413, "gripper_rows_matching_teacher": "119/128", "unedited_gripper_rows_matching_teacher": "121/128" }, "unseen_failures": { "n_decisions": 11, "decisions": "most-divergent replan of the 11 failed practice episodes used by neither the 6- nor a 12-decision build", "teacher_projection": -0.18859429292194552, "mean_shift_vs_unedited_cm": 2.615728955292566, "teacher_mean_shift_vs_unedited_cm": 2.032215103761039 } } }, { "experiment": "wsp_offline_check", "model": "GR00T N1.7 DROID (nvidia/GR00T-N1.7-DROID@05e7cc97)", "task": "T1 RubiksCubeThenBananaTask", "wording_pair": [ "Put the cube then the banana in the bowl", "Put the cube in the bowl then put the banana in the bowl" ], "edit_location": "action head", "notes": "teacher_projection = sum / sum |x-o|^2 over executed rows 0-7 (joints + gripper); shift = end-effector target of executed row 7. Only bb and dit were run closed loop.", "_ptask": null, "_task": "T1", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms-rtx3090", "edit": "dit", "grasp_phase": { "n_decisions": 16, "decisions": "k = 4, 8, 12, 16 of slots 30016-30019 (none used to build)", "mean_shift_vs_unedited_cm": 1.2063241505529732, "teacher_mean_shift_vs_unedited_cm": 0.8770811306021642, "teacher_projection": 0.3140669254802602, "gripper_rows_matching_teacher": "106/128", "unedited_gripper_rows_matching_teacher": "121/128" }, "unseen_failures": { "n_decisions": 11, "decisions": "most-divergent replan of the 11 failed practice episodes used by neither the 6- nor a 12-decision build", "teacher_projection": 0.21875615897501224, "mean_shift_vs_unedited_cm": 1.4456311039711265, "teacher_mean_shift_vs_unedited_cm": 2.034298603592271 } } }, { "experiment": "wsp_offline_check", "model": "GR00T N1.7 DROID (nvidia/GR00T-N1.7-DROID@05e7cc97)", "task": "T1 RubiksCubeThenBananaTask", "wording_pair": [ "Put the cube then the banana in the bowl", "Put the cube in the bowl then put the banana in the bowl" ], "edit_location": "backbone", "notes": "teacher_projection = sum / sum |x-o|^2 over executed rows 0-7 (joints + gripper); shift = end-effector target of executed row 7. Only bb and dit were run closed loop.", "_ptask": null, "_task": "T1", "_edit_task": null, "_prov": false, "extra": { "machine": "ali-ms-rtx3090", "edit": "bb_rand", "grasp_phase": null, "unseen_failures": { "n_decisions": 11, "decisions": "most-divergent replan of the 11 failed practice episodes used by neither the 6- nor a 12-decision build", "teacher_projection": -0.0026354435264788947, "mean_shift_vs_unedited_cm": 0.3309514915930446, "teacher_mean_shift_vs_unedited_cm": 2.034298603592271 } } } ], "pi05": [ { "experiment": "failure_map", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "RubiksCubeThenBananaTask", "code": "T1", "wording_pair": [ "Put the cube then the banana in the bowl" ], "slots": [ 90000, 90127 ], "n": 128, "unedited_k": 27, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "wilson95_pct": [ 14.9, 29.0 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "unedited policy, task's default instruction. Other unedited measurements: 4/32 on 70000-70031; 11/64 on 91008-91071; 19/128 on 90000-90127; 5/64 on 94016-94079. Different slot blocks and runs differ by more than their Wilson intervals suggest (identical reruns flip ~30% of pairs).", "_ptask": null, "_task": "T1", "_edit_task": null, "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": null, "values_changed_note": null, "source": "study/results/opus/heldout/cube-then-banana__v2mi-f-RubiksCubeThenBananaTask-90000-128.json (base arm)" } }, { "experiment": "failure_map", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "PickGlassesTask", "code": "T2", "wording_pair": [ "Pick up the eye glasses" ], "slots": [ 91008, 91071 ], "n": 64, "unedited_k": 18, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "wilson95_pct": [ 18.6, 40.1 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "unedited policy, task's default instruction. Other unedited measurements: 12/32 on 70000-70031; 22/64 on 90000-90063; 14/32 on 94016-94047; 26/64 on 60000-60063. Different slot blocks and runs differ by more than their Wilson intervals suggest (identical reruns flip ~30% of pairs).", "_ptask": null, "_task": "T2", "_edit_task": null, "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": null, "values_changed_note": null, "source": "study/results/ctrl/heldout (base arm of p1 held-out)" } }, { "experiment": "failure_map", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "RubiksCubeLeftOfBowlTask", "code": "T3", "wording_pair": [ "Put the rubiks cube to the left of the bowl" ], "slots": [ 90000, 90063 ], "n": 64, "unedited_k": 8, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "wilson95_pct": [ 6.5, 22.8 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "unedited policy, task's default instruction. Other unedited measurements: 5/32 on 70000-70031; 4/32 on 94016-94047; 3/32 on 60000-60031. Different slot blocks and runs differ by more than their Wilson intervals suggest (identical reruns flip ~30% of pairs).", "_ptask": null, "_task": "T3", "_edit_task": null, "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": null, "values_changed_note": null, "source": "study/results/opus/heldout/tx-rubikscubeleftofbowl__ctb-v2mi-RubiksCubeLeftOfBowlTask-90000-64.json (base arm)" } }, { "experiment": "failure_map", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "StackYellowOnRedTask", "code": "T4", "wording_pair": [ "Stack the yellow block on the red block" ], "slots": [ 70000, 70031 ], "n": 32, "unedited_k": 13, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "wilson95_pct": [ 25.5, 57.7 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "unedited policy, task's default instruction. Other unedited measurements: 14/32 on 20000-20031. Different slot blocks and runs differ by more than their Wilson intervals suggest (identical reruns flip ~30% of pairs).", "_ptask": null, "_task": "T4", "_edit_task": null, "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": null, "values_changed_note": null, "source": "study/results/ctrl/summary.json (controlled study, dev slots)" } }, { "experiment": "failure_map", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "MustardInLeftBinTask", "code": "T5", "wording_pair": [ "Put the mustard in the left bin" ], "slots": [ 91008, 91071 ], "n": 64, "unedited_k": 8, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "wilson95_pct": [ 6.5, 22.8 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "unedited policy, task's default instruction. Other unedited measurements: 6/32 on 70000-70031; 7/32 on 60000-60031. Different slot blocks and runs differ by more than their Wilson intervals suggest (identical reruns flip ~30% of pairs).", "_ptask": null, "_task": "T5", "_edit_task": null, "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": null, "values_changed_note": null, "source": "study/results/ctrl/heldout (base arm of p1 held-out)" } }, { "experiment": "failure_map", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "Stack3RubiksCubeTask", "code": "T6", "wording_pair": [ "Stack the rubiks cubes in a tower" ], "slots": [ 91008, 91071 ], "n": 64, "unedited_k": 35, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "wilson95_pct": [ 42.6, 66.3 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "unedited policy, task's default instruction. Other unedited measurements: 16/32 on 70000-70031; 14/32 on 60000-60031. Different slot blocks and runs differ by more than their Wilson intervals suggest (identical reruns flip ~30% of pairs).", "_ptask": null, "_task": "T6", "_edit_task": null, "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": null, "values_changed_note": null, "source": "study/results/ctrl/heldout (base arm of p1 held-out)" } }, { "experiment": "failure_map", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "PickUpGreenObjectTask", "code": "T7", "wording_pair": [ "Pick up the green vegetable block" ], "slots": [ 70000, 70031 ], "n": 32, "unedited_k": 0, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "wilson95_pct": [ 0.0, 10.7 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "unedited policy, task's default instruction. Other unedited measurements: 0/32 on 60000-60031. Different slot blocks and runs differ by more than their Wilson intervals suggest (identical reruns flip ~30% of pairs).", "_ptask": null, "_task": "T7", "_edit_task": null, "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": null, "values_changed_note": null, "source": "study/results/ctrl/summary.json (controlled study, dev slots)" } }, { "experiment": "failure_map", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "MustardRightOfRaisinRFTask", "code": "T8", "wording_pair": [ "Move the mustard bottle on the table so that the raisin box is to the left of the mustard bottle." ], "slots": [ 92000, 92063 ], "n": 64, "unedited_k": 0, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "wilson95_pct": [ 0.0, 5.7 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "unedited policy, task's default instruction. Other unedited measurements: 0/32 on 20000-20031. Different slot blocks and runs differ by more than their Wilson intervals suggest (identical reruns flip ~30% of pairs).", "_ptask": null, "_task": "T8", "_edit_task": null, "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": null, "values_changed_note": null, "source": "study/results/sgdw/results/heldout_pairs_92000-64.json" } }, { "experiment": "failure_map", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "RubiksCubeTask", "code": "R1", "wording_pair": [ "Put the cube in the bowl" ], "slots": [ 94016, 94047 ], "n": 32, "unedited_k": 32, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "wilson95_pct": [ 89.3, 100.0 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "unedited policy, task's default instruction. Other unedited measurements: 16/16 on 60000-60015. Different slot blocks and runs differ by more than their Wilson intervals suggest (identical reruns flip ~30% of pairs).", "_ptask": null, "_task": "R1", "_edit_task": null, "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": null, "values_changed_note": null, "source": "study/results/opus/where/results/transfer_cube_analysis.json" } }, { "experiment": "edit_target", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "RubiksCubeThenBananaTask", "code": "T1", "wording_pair": [ "Put the cube then the banana in the bowl", "Put the cube in the bowl then put the banana in the bowl" ], "slots": [ 90000, 90127 ], "n": 128, "unedited_k": 27, "edited_k": 106, "fixed": 81, "broke": 2, "p": 7.210946989931019e-22, "ci": { "wilson95_unedited_pct": [ 14.9, 29.0 ], "wilson95_edited_pct": [ 75.3, 88.4 ], "diff_pts": 61.7, "bootstrap95_diff_pts": [ 52.3, 70.3 ] }, "control_k": { "k": 9, "n": 64, "slots": [ 90000, 90063 ], "what": "random-direction update with v2mi-f's layers, rank and singular values (seed 0)" }, "teacher_k": { "k": 57, "n": 64, "slots": [ 90000, 90063 ], "wording": "Put the cube in the bowl then put the banana in the bowl" }, "recovery_pct": 84.1, "transfer": { "T2": { "n": 64, "unedited_k": 22, "edited_k": 44, "fixed": 31, "broke": 9, "p": 0.0006795482549932785, "ci": { "wilson95_unedited_pct": [ 23.9, 46.6 ], "wilson95_edited_pct": [ 56.6, 78.8 ], "diff_pts": 34.4, "bootstrap95_diff_pts": [ 17.2, 51.6 ] }, "paired_slots": [ 90000, 90063 ], "what": "same file on PickGlassesTask, held-out" }, "T3": { "n": 64, "unedited_k": 8, "edited_k": 36, "fixed": 33, "broke": 5, "p": 4.255962267052382e-06, "ci": { "wilson95_unedited_pct": [ 6.5, 22.8 ], "wilson95_edited_pct": [ 44.1, 67.7 ], "diff_pts": 43.8, "bootstrap95_diff_pts": [ 28.1, 59.4 ] }, "paired_slots": [ 90000, 90063 ], "what": "same file on RubiksCubeLeftOfBowlTask, held-out" } }, "retention": { "R1": { "n": 32, "unedited_k": 32, "edited_k": 32, "fixed": 0, "broke": 0, "p": 1.0, "slots": [ 94016, 94047 ] } }, "edit_location": "VLM MLP down-projections language.blocks.00-16.mlp.linear (17 tensors), rank <= 72 per layer", "values_changed": 450468837, "notes": "held-out, weightsmith/heldout.py (order alternating by half). Recovery % on slots 90000-90063, where the control and teacher ran: (edit 50 - unedited 13) / (teacher 57 - unedited 13). Teacher vs edit there: 11 teacher-only, 4 edit-only, p = 0.12. Practice slots 20000-20031: 3/32 -> 25/32 (22 fixed, 0 broke).", "_ptask": null, "_task": "T1", "_edit_task": null, "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": { "name": "v2mi-f", "fingerprint": "63db0c52a51306c9" }, "values_changed_note": "450,468,837 stored values (13.4% of 3,353,433,872)", "source": "study/results/opus/heldout/cube-then-banana__v2mi-f-RubiksCubeThenBananaTask-90000-128.json; study/results/opus/controls/ctl1-90000-64.analysis.json" } }, { "experiment": "edit_target", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "RubiksCubeThenBananaTask", "code": "T1", "wording_pair": [ "Put the cube then the banana in the bowl", "Put the cube in the bowl then put the banana in the bowl" ], "slots": [ 90000, 90127 ], "n": 128, "unedited_k": 19, "edited_k": 102, "fixed": 86, "broke": 3, "p": 3.7988915866351353e-22, "ci": { "wilson95_unedited_pct": [ 9.7, 22.0 ], "wilson95_edited_pct": [ 71.9, 85.7 ], "diff_pts": 64.8, "bootstrap95_diff_pts": [ 55.5, 73.4 ] }, "control_k": { "k": 9, "n": 64, "slots": [ 90000, 90063 ], "what": "random-direction control of v2mi-f" }, "teacher_k": { "k": 57, "n": 64, "slots": [ 90000, 90063 ], "wording": "Put the cube in the bowl then put the banana in the bowl" }, "recovery_pct": 93.8, "transfer": null, "retention": null, "edit_location": "VLM MLP down-projections language.blocks.08-16.mlp.linear (9 tensors), image-cell keys only, rank <= 32 per layer", "values_changed": 248057499, "notes": "held-out, heldout.py, against its own unedited run on the same slots. Recovery % on 90000-90063: (edit 54 - unedited 9) / (teacher 57 - unedited 9). On slots 94016-94079 (where-to-edit) it scored 41/64 against v2mi-f's 55/64 (5 / 19, p = 0.0066).", "_ptask": null, "_task": "T1", "_edit_task": null, "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": { "name": "abl-imgl8", "fingerprint": "5bf8c4329c1e416a" }, "values_changed_note": "248,057,499 stored values (7.4%); 5.3M numbers as low-rank factors (0.16%)", "source": "study/results/opus/heldout/cube-then-banana__abl-imgl8-RubiksCubeThenBananaTask-90000-128.json; study/results/opus/controls/ctl1-90000-64.analysis.json" } }, { "experiment": "where_to_edit", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "RubiksCubeThenBananaTask", "code": "T1", "wording_pair": [ "Put the cube then the banana in the bowl", "Put the cube in the bowl then put the banana in the bowl" ], "slots": [ 94016, 94079 ], "n": 64, "unedited_k": 5, "edited_k": 55, "fixed": 50, "broke": 0, "p": 1.7763568394002505e-15, "ci": { "wilson95_unedited_pct": [ 3.4, 17.0 ], "wilson95_edited_pct": [ 75.4, 92.4 ], "bootstrap95_diff_pts": [ 67.2, 87.5 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "VLM MLP down-projections language.blocks.00-16.mlp.linear (17 tensors), rank <= 72 per layer", "values_changed": 450468837, "notes": "VLM (v2mi-f); fingerprint 63db0c52a51306c9; all arms paired with one unedited run; 12 of 14 chunks ran on one lane, so their second 16 slots started from other scenes (sensitivity analysis on the 32 unaffected slots in where/README.md)", "_ptask": null, "_task": "T1", "_edit_task": null, "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": { "name": "VLM (v2mi-f)", "fingerprint": "63db0c52a51306c9" }, "values_changed_note": "stored values changed (where/README.md table)", "source": "study/results/opus/where/results/heldout_analysis.json" } }, { "experiment": "where_to_edit", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "RubiksCubeThenBananaTask", "code": "T1", "wording_pair": [ "Put the cube then the banana in the bowl", "Put the cube in the bowl then put the banana in the bowl" ], "slots": [ 94016, 94079 ], "n": 64, "unedited_k": 5, "edited_k": 41, "fixed": 38, "broke": 2, "p": 1.4933903003111482e-09, "ci": { "wilson95_unedited_pct": [ 3.4, 17.0 ], "wilson95_edited_pct": [ 51.8, 74.7 ], "bootstrap95_diff_pts": [ 42.2, 68.8 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "VLM MLP down-projections language.blocks.08-16.mlp.linear (9 tensors), image-cell keys only, rank <= 32 per layer", "values_changed": 248057499, "notes": "VLM image cells (abl-imgl8); fingerprint 5bf8c4329c1e416a; all arms paired with one unedited run; 12 of 14 chunks ran on one lane, so their second 16 slots started from other scenes (sensitivity analysis on the 32 unaffected slots in where/README.md)", "_ptask": null, "_task": "T1", "_edit_task": null, "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": { "name": "VLM image cells (abl-imgl8)", "fingerprint": "5bf8c4329c1e416a" }, "values_changed_note": "stored values changed (where/README.md table)", "source": "study/results/opus/where/results/heldout_analysis.json" } }, { "experiment": "where_to_edit", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "RubiksCubeThenBananaTask", "code": "T1", "wording_pair": [ "Put the cube then the banana in the bowl", "Put the cube in the bowl then put the banana in the bowl" ], "slots": [ 94016, 94079 ], "n": 64, "unedited_k": 5, "edited_k": 20, "fixed": 20, "broke": 5, "p": 0.004077315330505371, "ci": { "wilson95_unedited_pct": [ 3.4, 17.0 ], "wilson95_edited_pct": [ 21.2, 43.4 ], "bootstrap95_diff_pts": [ 9.4, 37.5 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "action-expert MLP down-projections action.blocks.00-17.mlp_1.linear, solved through the adaRMS gate", "values_changed": 60378838, "notes": "action expert (A1); fingerprint 09b7c9f5bfe26912; all arms paired with one unedited run; 12 of 14 chunks ran on one lane, so their second 16 slots started from other scenes (sensitivity analysis on the 32 unaffected slots in where/README.md)", "_ptask": null, "_task": "T1", "_edit_task": null, "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": { "name": "action expert (A1)", "fingerprint": "09b7c9f5bfe26912" }, "values_changed_note": "stored values changed (where/README.md table)", "source": "study/results/opus/where/results/heldout_analysis.json" } }, { "experiment": "where_to_edit", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "RubiksCubeThenBananaTask", "code": "T1", "wording_pair": [ "Put the cube then the banana in the bowl", "Put the cube in the bowl then put the banana in the bowl" ], "slots": [ 94016, 94079 ], "n": 64, "unedited_k": 5, "edited_k": 0, "fixed": 0, "broke": 5, "p": 0.0625, "ci": { "wilson95_unedited_pct": [ 3.4, 17.0 ], "wilson95_edited_pct": [ 0.0, 5.7 ], "bootstrap95_diff_pts": [ -15.6, -1.6 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "action head action.action_out_proj.kernel, 8 used columns, least-norm", "values_changed": 8056, "notes": "action head, least-norm (H1); fingerprint d965cd706aed9dec; all arms paired with one unedited run; 12 of 14 chunks ran on one lane, so their second 16 slots started from other scenes (sensitivity analysis on the 32 unaffected slots in where/README.md)", "_ptask": null, "_task": "T1", "_edit_task": null, "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": { "name": "action head, least-norm (H1)", "fingerprint": "d965cd706aed9dec" }, "values_changed_note": "stored values changed (where/README.md table)", "source": "study/results/opus/where/results/heldout_analysis.json" } }, { "experiment": "where_to_edit", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "RubiksCubeThenBananaTask", "code": "T1", "wording_pair": [ "Put the cube then the banana in the bowl", "Put the cube in the bowl then put the banana in the bowl" ], "slots": [ 94016, 94079 ], "n": 64, "unedited_k": 5, "edited_k": 30, "fixed": 28, "broke": 3, "p": 4.649162292480469e-06, "ci": { "wilson95_unedited_pct": [ 3.4, 17.0 ], "wilson95_edited_pct": [ 35.2, 58.9 ], "bootstrap95_diff_pts": [ 25.0, 53.1 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "action head gripper column, gated rank-1 (blind editor's accepted edit)", "values_changed": 1024, "notes": "action head, gated rank-1 (P1, blind editor); fingerprint 4ca8023b0592b79f; all arms paired with one unedited run; 12 of 14 chunks ran on one lane, so their second 16 slots started from other scenes (sensitivity analysis on the 32 unaffected slots in where/README.md)", "_ptask": null, "_task": "T1", "_edit_task": null, "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": { "name": "action head, gated rank-1 (P1, blind editor)", "fingerprint": "4ca8023b0592b79f" }, "values_changed_note": "stored values changed (where/README.md table)", "source": "study/results/opus/where/results/heldout_analysis.json" } }, { "experiment": "where_to_edit", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "RubiksCubeThenBananaTask", "code": "T1", "wording_pair": [ "Put the cube then the banana in the bowl", "Put the cube in the bowl then put the banana in the bowl" ], "slots": [ 94016, 94079 ], "n": 64, "unedited_k": 5, "edited_k": 43, "fixed": 41, "broke": 3, "p": 1.6183321349672042e-09, "ci": { "wilson95_unedited_pct": [ 3.4, 17.0 ], "wilson95_edited_pct": [ 55.0, 77.4 ], "bootstrap95_diff_pts": [ 45.3, 73.4 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "v2mi-f plus the action-expert update", "values_changed": 505694140, "notes": "VLM + action expert (VA); fingerprint 5ee25e8d2e78a555; all arms paired with one unedited run; 12 of 14 chunks ran on one lane, so their second 16 slots started from other scenes (sensitivity analysis on the 32 unaffected slots in where/README.md)", "_ptask": null, "_task": "T1", "_edit_task": null, "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": { "name": "VLM + action expert (VA)", "fingerprint": "5ee25e8d2e78a555" }, "values_changed_note": "stored values changed (where/README.md table)", "source": "study/results/opus/where/results/heldout_analysis.json" } }, { "experiment": "side_effect", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "PickGlassesTask", "code": "T2", "wording_pair": [ "Pick up the eye glasses" ], "slots": [ 90000, 90063 ], "n": 64, "unedited_k": 22, "edited_k": 44, "fixed": 31, "broke": 9, "p": 0.0006795482549932785, "ci": { "wilson95_unedited_pct": [ 23.9, 46.6 ], "wilson95_edited_pct": [ 56.6, 78.8 ], "diff_pts": 34.4, "bootstrap95_diff_pts": [ 17.2, 51.6 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "VLM MLP down-projections language.blocks.00-16.mlp.linear (17 tensors), rank <= 72 per layer", "values_changed": 450468837, "notes": "the T1 edit, unchanged, on this task's own instruction; weightsmith/heldout.py", "_ptask": null, "_task": "T2", "_edit_task": "T1", "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": { "name": "v2mi-f", "fingerprint": "63db0c52a51306c9" }, "values_changed_note": "450,468,837 stored values (13.4% of 3,353,433,872)", "source": "study/results/opus/heldout/ (held-out 90000-90063)" } }, { "experiment": "side_effect", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "RubiksCubeLeftOfBowlTask", "code": "T3", "wording_pair": [ "Put the rubiks cube to the left of the bowl" ], "slots": [ 90000, 90063 ], "n": 64, "unedited_k": 8, "edited_k": 36, "fixed": 33, "broke": 5, "p": 4.255962267052382e-06, "ci": { "wilson95_unedited_pct": [ 6.5, 22.8 ], "wilson95_edited_pct": [ 44.1, 67.7 ], "diff_pts": 43.8, "bootstrap95_diff_pts": [ 28.1, 59.4 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "VLM MLP down-projections language.blocks.00-16.mlp.linear (17 tensors), rank <= 72 per layer", "values_changed": 450468837, "notes": "the T1 edit, unchanged, on this task's own instruction; weightsmith/heldout.py", "_ptask": null, "_task": "T3", "_edit_task": "T1", "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": { "name": "v2mi-f", "fingerprint": "63db0c52a51306c9" }, "values_changed_note": "450,468,837 stored values (13.4% of 3,353,433,872)", "source": "study/results/opus/heldout/ (held-out 90000-90063)" } }, { "experiment": "side_effect", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "PickGlassesTask", "code": "T2", "wording_pair": [ "Pick up the eye glasses" ], "slots": [ 94016, 94047 ], "n": 32, "unedited_k": 14, "edited_k": 25, "fixed": 15, "broke": 4, "p": 0.0192108154296875, "ci": { "wilson95_unedited_pct": [ 28.2, 60.7 ], "wilson95_edited_pct": [ 61.2, 89.0 ], "bootstrap95_diff_pts": [ 9.4, 56.2 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "VLM MLP down-projections language.blocks.00-16.mlp.linear (17 tensors), rank <= 72 per layer", "values_changed": 450468837, "notes": "where-to-edit transfer run; every slot the first episode of its Isaac process", "_ptask": null, "_task": "T2", "_edit_task": "T1", "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": { "name": "v2mi-f", "fingerprint": "63db0c52a51306c9" }, "values_changed_note": "450,468,837 stored values (13.4% of 3,353,433,872)", "source": "study/results/opus/where/results/transfer_glasses_analysis.json" } }, { "experiment": "side_effect", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "RubiksCubeLeftOfBowlTask", "code": "T3", "wording_pair": [ "Put the rubiks cube to the left of the bowl" ], "slots": [ 94016, 94047 ], "n": 32, "unedited_k": 4, "edited_k": 20, "fixed": 16, "broke": 0, "p": 3.0517578125e-05, "ci": { "wilson95_unedited_pct": [ 5.0, 28.1 ], "wilson95_edited_pct": [ 45.3, 77.1 ], "bootstrap95_diff_pts": [ 34.4, 68.8 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "VLM MLP down-projections language.blocks.00-16.mlp.linear (17 tensors), rank <= 72 per layer", "values_changed": 450468837, "notes": "where-to-edit transfer run; every slot the first episode of its Isaac process", "_ptask": null, "_task": "T3", "_edit_task": "T1", "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": { "name": "v2mi-f", "fingerprint": "63db0c52a51306c9" }, "values_changed_note": "450,468,837 stored values (13.4% of 3,353,433,872)", "source": "study/results/opus/where/results/transfer_left_analysis.json" } }, { "experiment": "side_effect", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "RubiksCubeTask", "code": "R1", "wording_pair": [ "Put the cube in the bowl" ], "slots": [ 94016, 94047 ], "n": 32, "unedited_k": 32, "edited_k": 32, "fixed": 0, "broke": 0, "p": 1.0, "ci": { "wilson95_unedited_pct": [ 89.3, 100.0 ], "wilson95_edited_pct": [ 89.3, 100.0 ], "bootstrap95_diff_pts": [ 0.0, 0.0 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "VLM MLP down-projections language.blocks.00-16.mlp.linear (17 tensors), rank <= 72 per layer", "values_changed": 450468837, "notes": "where-to-edit transfer run; every slot the first episode of its Isaac process", "_ptask": null, "_task": "R1", "_edit_task": "T1", "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": { "name": "v2mi-f", "fingerprint": "63db0c52a51306c9" }, "values_changed_note": "450,468,837 stored values (13.4% of 3,353,433,872)", "source": "study/results/opus/where/results/transfer_cube_analysis.json" } }, { "experiment": "side_effect", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "MustardInLeftBinTask", "code": "T5", "wording_pair": [ "Put the mustard in the left bin" ], "slots": [ 60000, 60031 ], "n": 32, "unedited_k": 7, "edited_k": 12, "fixed": 11, "broke": 6, "p": 0.332305908203125, "ci": { "wilson95_unedited_pct": [ 11.0, 38.8 ], "wilson95_edited_pct": [ 22.9, 54.7 ], "diff_pts": 15.6, "bootstrap95_diff_pts": [ -9.4, 40.6 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "VLM MLP down-projections language.blocks.00-16.mlp.linear (17 tensors), rank <= 72 per layer", "values_changed": 450468837, "notes": "practice slots, owsp eval, edit run after the unedited arm in the same project; multi-group chunks may start later groups from other scenes", "_ptask": null, "_task": "T5", "_edit_task": "T1", "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": { "name": "v2mi-f", "fingerprint": "63db0c52a51306c9" }, "values_changed_note": "450,468,837 stored values (13.4% of 3,353,433,872)", "source": "ali-ms projects/mustard-grasp (study/results/cross/pi05/results/prior_pairs_ali-ms.json)" } }, { "experiment": "side_effect", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "PickGlassesTask", "code": "T2", "wording_pair": [ "Pick up the eye glasses" ], "slots": [ 60000, 60063 ], "n": 64, "unedited_k": 26, "edited_k": 46, "fixed": 28, "broke": 8, "p": 0.0011932429624721408, "ci": { "wilson95_unedited_pct": [ 29.5, 52.9 ], "wilson95_edited_pct": [ 59.9, 81.4 ], "diff_pts": 31.2, "bootstrap95_diff_pts": [ 14.1, 46.9 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "VLM MLP down-projections language.blocks.00-16.mlp.linear (17 tensors), rank <= 72 per layer", "values_changed": 450468837, "notes": "practice slots, owsp eval, edit run after the unedited arm in the same project; multi-group chunks may start later groups from other scenes", "_ptask": null, "_task": "T2", "_edit_task": "T1", "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": { "name": "v2mi-f", "fingerprint": "63db0c52a51306c9" }, "values_changed_note": "450,468,837 stored values (13.4% of 3,353,433,872)", "source": "ali-ms projects/thr-pickglasses (study/results/cross/pi05/results/prior_pairs_ali-ms.json)" } }, { "experiment": "side_effect", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "RubiksCubeTask", "code": "R1", "wording_pair": [ "Put the cube in the bowl" ], "slots": [ 60000, 60015 ], "n": 16, "unedited_k": 16, "edited_k": 16, "fixed": 0, "broke": 0, "p": 1.0, "ci": { "wilson95_unedited_pct": [ 80.6, 100.0 ], "wilson95_edited_pct": [ 80.6, 100.0 ], "diff_pts": 0.0, "bootstrap95_diff_pts": [ 0.0, 0.0 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "VLM MLP down-projections language.blocks.00-16.mlp.linear (17 tensors), rank <= 72 per layer", "values_changed": 450468837, "notes": "practice slots, owsp eval, edit run after the unedited arm in the same project; multi-group chunks may start later groups from other scenes", "_ptask": null, "_task": "R1", "_edit_task": "T1", "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": { "name": "v2mi-f", "fingerprint": "63db0c52a51306c9" }, "values_changed_note": "450,468,837 stored values (13.4% of 3,353,433,872)", "source": "ali-ms projects/thr-rubikscube (study/results/cross/pi05/results/prior_pairs_ali-ms.json)" } }, { "experiment": "side_effect", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "StackYellowOnRedTask", "code": "T4", "wording_pair": [ "Stack the yellow block on the red block" ], "slots": [ 20000, 20031 ], "n": 32, "unedited_k": 14, "edited_k": 17, "fixed": 9, "broke": 6, "p": 0.60723876953125, "ci": { "wilson95_unedited_pct": [ 28.2, 60.7 ], "wilson95_edited_pct": [ 36.4, 69.1 ], "diff_pts": 9.4, "bootstrap95_diff_pts": [ -15.6, 34.4 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "VLM MLP down-projections language.blocks.00-16.mlp.linear (17 tensors), rank <= 72 per layer", "values_changed": 450468837, "notes": "practice slots, owsp eval, edit run after the unedited arm in the same project; multi-group chunks may start later groups from other scenes", "_ptask": null, "_task": "T4", "_edit_task": "T1", "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": { "name": "v2mi-f", "fingerprint": "63db0c52a51306c9" }, "values_changed_note": "450,468,837 stored values (13.4% of 3,353,433,872)", "source": "ali-ms projects/thr-stack (study/results/cross/pi05/results/prior_pairs_ali-ms.json)" } }, { "experiment": "side_effect", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "PickUpGreenObjectTask", "code": "T7", "wording_pair": [ "Pick up the green vegetable block" ], "slots": [ 60000, 60031 ], "n": 32, "unedited_k": 0, "edited_k": 0, "fixed": 0, "broke": 0, "p": 1.0, "ci": { "wilson95_unedited_pct": [ 0.0, 10.7 ], "wilson95_edited_pct": [ 0.0, 10.7 ], "diff_pts": 0.0, "bootstrap95_diff_pts": [ 0.0, 0.0 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "VLM MLP down-projections language.blocks.00-16.mlp.linear (17 tensors), rank <= 72 per layer", "values_changed": 450468837, "notes": "practice slots, owsp eval, edit run after the unedited arm in the same project; multi-group chunks may start later groups from other scenes", "_ptask": null, "_task": "T7", "_edit_task": "T1", "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": { "name": "v2mi-f", "fingerprint": "63db0c52a51306c9" }, "values_changed_note": "450,468,837 stored values (13.4% of 3,353,433,872)", "source": "ali-ms projects/tx-pickupgreenobject (study/results/cross/pi05/results/prior_pairs_ali-ms.json)" } }, { "experiment": "side_effect", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "RubiksCubeLeftOfBowlTask", "code": "T3", "wording_pair": [ "Put the rubiks cube to the left of the bowl" ], "slots": [ 60000, 60031 ], "n": 32, "unedited_k": 3, "edited_k": 22, "fixed": 20, "broke": 1, "p": 2.09808349609375e-05, "ci": { "wilson95_unedited_pct": [ 3.2, 24.2 ], "wilson95_edited_pct": [ 51.4, 82.0 ], "diff_pts": 59.4, "bootstrap95_diff_pts": [ 40.6, 78.1 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "VLM MLP down-projections language.blocks.00-16.mlp.linear (17 tensors), rank <= 72 per layer", "values_changed": 450468837, "notes": "practice slots, owsp eval, edit run after the unedited arm in the same project; multi-group chunks may start later groups from other scenes", "_ptask": null, "_task": "T3", "_edit_task": "T1", "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": { "name": "v2mi-f", "fingerprint": "63db0c52a51306c9" }, "values_changed_note": "450,468,837 stored values (13.4% of 3,353,433,872)", "source": "ali-ms projects/tx-rubikscubeleftofbowl (study/results/cross/pi05/results/prior_pairs_ali-ms.json)" } }, { "experiment": "side_effect", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "Stack3RubiksCubeTask", "code": "T6", "wording_pair": [ "Stack the rubiks cubes in a tower" ], "slots": [ 60000, 60031 ], "n": 32, "unedited_k": 14, "edited_k": 19, "fixed": 9, "broke": 4, "p": 0.266845703125, "ci": { "wilson95_unedited_pct": [ 28.2, 60.7 ], "wilson95_edited_pct": [ 42.3, 74.5 ], "diff_pts": 15.6, "bootstrap95_diff_pts": [ -6.2, 37.5 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "VLM MLP down-projections language.blocks.00-16.mlp.linear (17 tensors), rank <= 72 per layer", "values_changed": 450468837, "notes": "practice slots, owsp eval, edit run after the unedited arm in the same project; multi-group chunks may start later groups from other scenes", "_ptask": null, "_task": "T6", "_edit_task": "T1", "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": { "name": "v2mi-f", "fingerprint": "63db0c52a51306c9" }, "values_changed_note": "450,468,837 stored values (13.4% of 3,353,433,872)", "source": "ali-ms projects/tx-stack3rubikscube (study/results/cross/pi05/results/prior_pairs_ali-ms.json)" } }, { "experiment": "side_effect", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "BananaOnPlateTask", "code": null, "wording_pair": [ "Pick up the banana and put it on the plate" ], "slots": [ 60000, 60015 ], "n": 16, "unedited_k": 16, "edited_k": 16, "fixed": 0, "broke": 0, "p": 1.0, "ci": { "wilson95_unedited_pct": [ 80.6, 100.0 ], "wilson95_edited_pct": [ 80.6, 100.0 ], "diff_pts": 0.0, "bootstrap95_diff_pts": [ 0.0, 0.0 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "VLM MLP down-projections language.blocks.00-16.mlp.linear (17 tensors), rank <= 72 per layer", "values_changed": 450468837, "notes": "practice slots, owsp eval, edit run after the unedited arm in the same project; multi-group chunks may start later groups from other scenes", "_ptask": null, "_task": null, "_edit_task": "T1", "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": { "name": "v2mi-f", "fingerprint": "63db0c52a51306c9" }, "values_changed_note": "450,468,837 stored values (13.4% of 3,353,433,872)", "source": "ali-ms projects/thr-bananaonplate (study/results/cross/pi05/results/prior_pairs_ali-ms.json)" } }, { "experiment": "side_effect", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "BowlStackingLeftOnRightTask", "code": null, "wording_pair": [ "Stack the left bowl on the right bowl" ], "slots": [ 60000, 60031 ], "n": 32, "unedited_k": 19, "edited_k": 21, "fixed": 8, "broke": 6, "p": 0.79052734375, "ci": { "wilson95_unedited_pct": [ 42.3, 74.5 ], "wilson95_edited_pct": [ 48.3, 79.6 ], "diff_pts": 6.2, "bootstrap95_diff_pts": [ -15.6, 28.1 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "VLM MLP down-projections language.blocks.00-16.mlp.linear (17 tensors), rank <= 72 per layer", "values_changed": 450468837, "notes": "practice slots, owsp eval, edit run after the unedited arm in the same project; multi-group chunks may start later groups from other scenes", "_ptask": null, "_task": null, "_edit_task": "T1", "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": { "name": "v2mi-f", "fingerprint": "63db0c52a51306c9" }, "values_changed_note": "450,468,837 stored values (13.4% of 3,353,433,872)", "source": "ali-ms projects/thr-bowlstackingleftonright (study/results/cross/pi05/results/prior_pairs_ali-ms.json)" } }, { "experiment": "side_effect", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "ButterAboveRaisinTask", "code": null, "wording_pair": [ "Pick up the butter box and place it on top of the raisin box" ], "slots": [ 60000, 60015 ], "n": 16, "unedited_k": 14, "edited_k": 10, "fixed": 0, "broke": 4, "p": 0.125, "ci": { "wilson95_unedited_pct": [ 64.0, 96.5 ], "wilson95_edited_pct": [ 38.6, 81.5 ], "diff_pts": -25.0, "bootstrap95_diff_pts": [ -50.0, -6.2 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "VLM MLP down-projections language.blocks.00-16.mlp.linear (17 tensors), rank <= 72 per layer", "values_changed": 450468837, "notes": "practice slots, owsp eval, edit run after the unedited arm in the same project; multi-group chunks may start later groups from other scenes", "_ptask": null, "_task": null, "_edit_task": "T1", "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": { "name": "v2mi-f", "fingerprint": "63db0c52a51306c9" }, "values_changed_note": "450,468,837 stored values (13.4% of 3,353,433,872)", "source": "ali-ms projects/thr-butteraboveraisin (study/results/cross/pi05/results/prior_pairs_ali-ms.json)" } }, { "experiment": "side_effect", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "BlockStackingOrderAgnosticTask", "code": null, "wording_pair": [ "Stack the blocks into a tower" ], "slots": [ 60000, 60031 ], "n": 32, "unedited_k": 15, "edited_k": 14, "fixed": 7, "broke": 8, "p": 1.0, "ci": { "wilson95_unedited_pct": [ 30.9, 63.6 ], "wilson95_edited_pct": [ 28.2, 60.7 ], "diff_pts": -3.1, "bootstrap95_diff_pts": [ -28.1, 21.9 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "VLM MLP down-projections language.blocks.00-16.mlp.linear (17 tensors), rank <= 72 per layer", "values_changed": 450468837, "notes": "practice slots, owsp eval, edit run after the unedited arm in the same project; multi-group chunks may start later groups from other scenes", "_ptask": null, "_task": null, "_edit_task": "T1", "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": { "name": "v2mi-f", "fingerprint": "63db0c52a51306c9" }, "values_changed_note": "450,468,837 stored values (13.4% of 3,353,433,872)", "source": "ali-ms projects/tx-blockstackingorderagnostic (study/results/cross/pi05/results/prior_pairs_ali-ms.json)" } }, { "experiment": "side_effect", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "GrabABagelTask", "code": null, "wording_pair": [ "Grab a bagel" ], "slots": [ 60000, 60031 ], "n": 32, "unedited_k": 0, "edited_k": 1, "fixed": 1, "broke": 0, "p": 1.0, "ci": { "wilson95_unedited_pct": [ 0.0, 10.7 ], "wilson95_edited_pct": [ 0.6, 15.7 ], "diff_pts": 3.1, "bootstrap95_diff_pts": [ 0.0, 9.4 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "VLM MLP down-projections language.blocks.00-16.mlp.linear (17 tensors), rank <= 72 per layer", "values_changed": 450468837, "notes": "practice slots, owsp eval, edit run after the unedited arm in the same project; multi-group chunks may start later groups from other scenes", "_ptask": null, "_task": null, "_edit_task": "T1", "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": { "name": "v2mi-f", "fingerprint": "63db0c52a51306c9" }, "values_changed_note": "450,468,837 stored values (13.4% of 3,353,433,872)", "source": "ali-ms projects/tx-grababagel (study/results/cross/pi05/results/prior_pairs_ali-ms.json)" } }, { "experiment": "side_effect", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "MarkerInMugTask", "code": null, "wording_pair": [ "Put the whiteboard marker in the mug" ], "slots": [ 60000, 60031 ], "n": 32, "unedited_k": 0, "edited_k": 1, "fixed": 1, "broke": 0, "p": 1.0, "ci": { "wilson95_unedited_pct": [ 0.0, 10.7 ], "wilson95_edited_pct": [ 0.6, 15.7 ], "diff_pts": 3.1, "bootstrap95_diff_pts": [ 0.0, 9.4 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "VLM MLP down-projections language.blocks.00-16.mlp.linear (17 tensors), rank <= 72 per layer", "values_changed": 450468837, "notes": "practice slots, owsp eval, edit run after the unedited arm in the same project; multi-group chunks may start later groups from other scenes", "_ptask": null, "_task": null, "_edit_task": "T1", "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": { "name": "v2mi-f", "fingerprint": "63db0c52a51306c9" }, "values_changed_note": "450,468,837 stored values (13.4% of 3,353,433,872)", "source": "ali-ms projects/tx-markerinmug (study/results/cross/pi05/results/prior_pairs_ali-ms.json)" } }, { "experiment": "side_effect", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "RubiksCubeRightOfBowlTask", "code": null, "wording_pair": [ "Put the rubiks cube to the right of the bowl" ], "slots": [ 60000, 60031 ], "n": 32, "unedited_k": 0, "edited_k": 0, "fixed": 0, "broke": 0, "p": 1.0, "ci": { "wilson95_unedited_pct": [ 0.0, 10.7 ], "wilson95_edited_pct": [ 0.0, 10.7 ], "diff_pts": 0.0, "bootstrap95_diff_pts": [ 0.0, 0.0 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "VLM MLP down-projections language.blocks.00-16.mlp.linear (17 tensors), rank <= 72 per layer", "values_changed": 450468837, "notes": "practice slots, owsp eval, edit run after the unedited arm in the same project; multi-group chunks may start later groups from other scenes", "_ptask": null, "_task": null, "_edit_task": "T1", "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": { "name": "v2mi-f", "fingerprint": "63db0c52a51306c9" }, "values_changed_note": "450,468,837 stored values (13.4% of 3,353,433,872)", "source": "ali-ms projects/tx-rubikscuberightofbowl (study/results/cross/pi05/results/prior_pairs_ali-ms.json)" } }, { "experiment": "side_effect", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "YogurtInBowlTask", "code": null, "wording_pair": [ "Put the small red yogurt in the red bowl" ], "slots": [ 60000, 60031 ], "n": 32, "unedited_k": 3, "edited_k": 2, "fixed": 2, "broke": 3, "p": 1.0, "ci": { "wilson95_unedited_pct": [ 3.2, 24.2 ], "wilson95_edited_pct": [ 1.7, 20.1 ], "diff_pts": -3.1, "bootstrap95_diff_pts": [ -15.6, 9.4 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "VLM MLP down-projections language.blocks.00-16.mlp.linear (17 tensors), rank <= 72 per layer", "values_changed": 450468837, "notes": "practice slots, owsp eval, edit run after the unedited arm in the same project; multi-group chunks may start later groups from other scenes", "_ptask": null, "_task": null, "_edit_task": "T1", "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": { "name": "v2mi-f", "fingerprint": "63db0c52a51306c9" }, "values_changed_note": "450,468,837 stored values (13.4% of 3,353,433,872)", "source": "ali-ms projects/tx-yogurtinbowl (study/results/cross/pi05/results/prior_pairs_ali-ms.json)" } }, { "experiment": "wording", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "MustardRightOfRaisinRFTask", "code": "T8", "wording_pair": [ "Move the mustard bottle on the table so that the raisin box is to the left of the mustard bottle.", "Move the mustard bottle on the table so that the mustard bottle is to the right of the raisin box." ], "slots": [ 20000, 20031 ], "n": 32, "unedited_k": 0, "edited_k": null, "fixed": 27, "broke": 0, "p": 1.4901161193847656e-08, "ci": { "wilson95_unedited_pct": [ 0.0, 10.7 ], "wilson95_edited_pct": [ 68.2, 93.1 ], "diff_pts": 84.4, "bootstrap95_diff_pts": [ 71.9, 96.9 ] }, "control_k": null, "teacher_k": { "k": 27, "n": 32, "slots": [ 20000, 20031 ], "wording": "Move the mustard bottle on the table so that the mustard bottle is to the right of the raisin box." }, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "unedited weights; RF (reference-first, the task's own wording) = unedited_k, TF (target-first, prompt override) = teacher_k; fixed = TF-only pairs, broke = RF-only pairs", "_ptask": null, "_task": "T8", "_edit_task": null, "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": null, "values_changed_note": null, "source": "study/results/sgdw/results/screen_s4r_tf_vs_rf.json" } }, { "experiment": "edit_target", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "MustardRightOfRaisinRFTask", "code": "T8", "wording_pair": [ "Move the mustard bottle on the table so that the raisin box is to the left of the mustard bottle.", "Move the mustard bottle on the table so that the mustard bottle is to the right of the raisin box." ], "slots": [ 92000, 92063 ], "n": 64, "unedited_k": 0, "edited_k": 27, "fixed": 27, "broke": 0, "p": 1.4901161193847656e-08, "ci": { "wilson95_unedited_pct": [ 0.0, 5.7 ], "wilson95_edited_pct": [ 30.9, 54.4 ], "diff_pts": 42.2, "bootstrap95_diff_pts": [ 29.7, 54.7 ] }, "control_k": { "k": 0, "n": 64, "slots": [ 92000, 92063 ], "what": "random-direction control with s4r-grp3's layers, rank and singular values" }, "teacher_k": { "k": 27, "n": 32, "slots": [ 20000, 20031 ], "wording": "Move the mustard bottle on the table so that the mustard bottle is to the right of the raisin box.", "note": "TF was not run on the held-out slots" }, "recovery_pct": 59.3, "transfer": null, "retention": { "T8 under TF (no harm)": { "n": 32, "unedited_k": 27, "edited_k": 23, "fixed": 5, "broke": 9, "p": 0.4239501953125, "slots": [ 20000, 20031 ] } }, "edit_location": "VLM MLP down-projections language.blocks.08-16.mlp.linear (9 tensors), 3 key groups (approach, grasp, transport), rank <= 96 per layer", "values_changed": null, "notes": "T8's own distilled edit (TF into the weights), not v2mi-f. Recovery % on practice slots 20000-20031, the only slots with a teacher arm: RF + s4r-grp3 16/32 vs RF 0/32 vs TF 27/32. s4r-grp2 (2 key groups): 19/64 on the same held-out slots.", "_ptask": null, "_task": "T8", "_edit_task": null, "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "edit": { "name": "s4r-grp3", "fingerprint": "9e6f2378e28be6a3" }, "values_changed_note": "15.9M numbers as low-rank factors (0.48% of parameters); dense count not recorded", "source": "study/results/sgdw/results/heldout_ctl_92000-64.json, heldout_grp2_92000-64.json (unedited arm), heldout_pairs_92000-64.json" } }, { "experiment": "control", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "RubiksCubeThenBananaTask", "code": "T1", "wording_pair": [ "Put the cube then the banana in the bowl" ], "slots": [ 90000, 90063 ], "n": 64, "unedited_k": 13, "edited_k": 9, "fixed": 6, "broke": 10, "p": 0.454498291015625, "ci": { "wilson95_unedited_pct": [ 12.3, 31.7 ], "wilson95_control_pct": [ 7.6, 24.6 ], "diff_pts": -6.2, "bootstrap95_diff_pts": [ -18.8, 6.2 ] }, "control_k": { "k": 9, "n": 64, "slots": [ 90000, 90063 ], "arm": "ctl-rand", "control_vs_unedited": { "fixed": 6, "broke": 10, "p": 0.454498291015625 }, "edit_vs_control": { "edit_only": 41, "control_only": 0, "p": 9.094947017729282e-13, "edit_k": 50 }, "edit_vs_unedited": { "fixed": 38, "broke": 1, "p": 1.4551915228366852e-10, "edit_k": 50 } }, "teacher_k": { "k": 57, "n": 64, "slots": [ 90000, 90063 ], "wording": "Put the cube in the bowl then put the banana in the bowl", "teacher_vs_unedited": { "fixed": 45, "broke": 1, "p": 1.3358203432289883e-12 }, "edit_vs_teacher": { "edit_only": 4, "teacher_only": 11, "p": 0.11846923828125 } }, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "same tensors, ranks and singular values as v2mi-f; random orthonormal directions (numpy default_rng(0))", "values_changed": 502291925, "notes": "PROVISIONAL (own slots, earlier run): random-direction control of v2mi-f; unedited_k / edited_k = unedited / control on the same slots; recomputed from the per-slot records in results/prior_slots_ali-ms.json (held-out runs of cube-then-banana (v2mi-f base/cand 90000, ctl1))", "_ptask": null, "_task": "T1", "_edit_task": null, "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "render_preset": "performance", "edit": { "name": "ctl-rand", "fingerprint": "1efd3512a92642f7", "built_for": "T1", "control_of": "v2mi-f" }, "values_changed_note": "about 502.3M stored values (from the control's build report)", "per_slot": { "90000": [ 0, 0 ], "90001": [ 0, 0 ], "90002": [ 0, 0 ], "90003": [ 0, 0 ], "90004": [ 0, 0 ], "90005": [ 0, 0 ], "90006": [ 0, 1 ], "90007": [ 0, 0 ], "90008": [ 0, 0 ], "90009": [ 0, 0 ], "90010": [ 0, 0 ], "90011": [ 0, 1 ], "90012": [ 0, 0 ], "90013": [ 1, 0 ], "90014": [ 0, 0 ], "90015": [ 0, 0 ], "90016": [ 0, 0 ], "90017": [ 0, 0 ], "90018": [ 0, 0 ], "90019": [ 1, 0 ], "90020": [ 0, 0 ], "90021": [ 0, 0 ], "90022": [ 0, 0 ], "90023": [ 0, 0 ], "90024": [ 0, 0 ], "90025": [ 0, 1 ], "90026": [ 0, 0 ], "90027": [ 0, 0 ], "90028": [ 0, 0 ], "90029": [ 0, 0 ], "90030": [ 1, 1 ], "90031": [ 0, 0 ], "90032": [ 0, 0 ], "90033": [ 0, 0 ], "90034": [ 0, 0 ], "90035": [ 0, 0 ], "90036": [ 1, 1 ], "90037": [ 1, 1 ], "90038": [ 0, 1 ], "90039": [ 1, 0 ], "90040": [ 1, 0 ], "90041": [ 1, 0 ], "90042": [ 0, 0 ], "90043": [ 0, 0 ], "90044": [ 0, 0 ], "90045": [ 1, 0 ], "90046": [ 1, 0 ], "90047": [ 0, 0 ], "90048": [ 0, 0 ], "90049": [ 1, 0 ], "90050": [ 0, 0 ], "90051": [ 0, 1 ], "90052": [ 0, 0 ], "90053": [ 0, 0 ], "90054": [ 1, 0 ], "90055": [ 0, 0 ], "90056": [ 0, 0 ], "90057": [ 0, 0 ], "90058": [ 0, 0 ], "90059": [ 0, 0 ], "90060": [ 1, 0 ], "90061": [ 0, 0 ], "90062": [ 0, 0 ], "90063": [ 0, 1 ] }, "source": "results/prior_slots_ali-ms.json: cube-then-banana/heldout/v2mi-f-RubiksCubeThenBananaTask-90000-128/logs-base-90000; cube-then-banana/heldout/v2mi-f-RubiksCubeThenBananaTask-90000-128/logs-cand-90000; cube-then-banana/heldout/ctl1-90000-64/logs-rand-90000; cube-then-banana/heldout/ctl1-90000-64/logs-teacher-90000" } }, { "experiment": "control", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "MustardRightOfRaisinRFTask", "code": "T8", "wording_pair": [ "Move the mustard bottle on the table so that the raisin box is to the left of the mustard bottle." ], "slots": [ 92000, 92063 ], "n": 64, "unedited_k": 0, "edited_k": 0, "fixed": 0, "broke": 0, "p": 1.0, "ci": { "wilson95_unedited_pct": [ 0.0, 5.7 ], "wilson95_control_pct": [ 0.0, 5.7 ], "diff_pts": 0.0, "bootstrap95_diff_pts": [ 0.0, 0.0 ] }, "control_k": { "k": 0, "n": 64, "slots": [ 92000, 92063 ], "arm": "ctl-rand3", "control_vs_unedited": { "fixed": 0, "broke": 0, "p": 1.0 }, "edit_vs_control": { "edit_only": 27, "control_only": 0, "p": 1.4901161193847656e-08, "edit_k": 27 }, "edit_vs_unedited": { "fixed": 27, "broke": 0, "p": 1.4901161193847656e-08, "edit_k": 27 } }, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": "same tensors, ranks and singular values as s4r-grp3; random orthonormal directions", "values_changed": 290322032, "notes": "PROVISIONAL (own slots, earlier run): random-direction control of s4r-grp3; unedited_k / edited_k = unedited / control on the same slots; recomputed from the per-slot records in results/prior_slots_ali-ms.json (held-out runs of sgdw-s4r (s4r-grp2 base arm; ctl-92000-64))", "_ptask": null, "_task": "T8", "_edit_task": null, "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090 (weightsmith runtime; earlier studies)", "render_preset": "performance", "edit": { "name": "ctl-rand3", "fingerprint": "fbc8349de4182099", "built_for": "T8", "control_of": "s4r-grp3" }, "values_changed_note": "about 290.3M stored values (from the control's build report)", "per_slot": { "92000": [ 0, 0 ], "92001": [ 0, 0 ], "92002": [ 0, 0 ], "92003": [ 0, 0 ], "92004": [ 0, 0 ], "92005": [ 0, 0 ], "92006": [ 0, 0 ], "92007": [ 0, 0 ], "92008": [ 0, 0 ], "92009": [ 0, 0 ], "92010": [ 0, 0 ], "92011": [ 0, 0 ], "92012": [ 0, 0 ], "92013": [ 0, 0 ], "92014": [ 0, 0 ], "92015": [ 0, 0 ], "92016": [ 0, 0 ], "92017": [ 0, 0 ], "92018": [ 0, 0 ], "92019": [ 0, 0 ], "92020": [ 0, 0 ], "92021": [ 0, 0 ], "92022": [ 0, 0 ], "92023": [ 0, 0 ], "92024": [ 0, 0 ], "92025": [ 0, 0 ], "92026": [ 0, 0 ], "92027": [ 0, 0 ], "92028": [ 0, 0 ], "92029": [ 0, 0 ], "92030": [ 0, 0 ], "92031": [ 0, 0 ], "92032": [ 0, 0 ], "92033": [ 0, 0 ], "92034": [ 0, 0 ], "92035": [ 0, 0 ], "92036": [ 0, 0 ], "92037": [ 0, 0 ], "92038": [ 0, 0 ], "92039": [ 0, 0 ], "92040": [ 0, 0 ], "92041": [ 0, 0 ], "92042": [ 0, 0 ], "92043": [ 0, 0 ], "92044": [ 0, 0 ], "92045": [ 0, 0 ], "92046": [ 0, 0 ], "92047": [ 0, 0 ], "92048": [ 0, 0 ], "92049": [ 0, 0 ], "92050": [ 0, 0 ], "92051": [ 0, 0 ], "92052": [ 0, 0 ], "92053": [ 0, 0 ], "92054": [ 0, 0 ], "92055": [ 0, 0 ], "92056": [ 0, 0 ], "92057": [ 0, 0 ], "92058": [ 0, 0 ], "92059": [ 0, 0 ], "92060": [ 0, 0 ], "92061": [ 0, 0 ], "92062": [ 0, 0 ], "92063": [ 0, 0 ] }, "source": "results/prior_slots_ali-ms.json: sgdw-s4r/heldout/s4r-grp2-MustardRightOfRaisinRFTask-92000-64/logs-base-92000; sgdw-s4r/heldout/s4r-grp2-MustardRightOfRaisinRFTask-92000-64/logs-base-92032; sgdw-s4r/heldout/ctl-92000-64/logs-g3-92000; sgdw-s4r/heldout/ctl-92000-64/logs-g3-92032; sgdw-s4r/heldout/ctl-92000-64/logs-rand3-92000; sgdw-s4r/heldout/ctl-92000-64/logs-rand3-92032" } }, { "experiment": "failure_map_partial", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "RubiksCubeThenBananaTask", "code": "T1", "wording_pair": [ "Put the cube then the banana in the bowl" ], "slots": [ 30000, 30015 ], "n": 16, "unedited_k": 5, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "wilson95_pct": [ 14.2, 55.6 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "FRESH but partial: the only pi0.5 episodes on the shared slots, one 16-slot group (first and only run of a fresh Isaac process, environment seed 0) on ali-ms GPU1 at 05:03 UTC before the pi0.5 queue moved to the pod; RoboLab performance preset (Tracks A and B use the default preset), so not a like-for-like failure-map entry; relabelled provisional on 2026-11-08 20:30 UTC: superseded by the Track C' default-preset failure map on the same slots (run C-prime-default)", "_ptask": null, "_task": "T1", "_edit_task": null, "_prov": true, "extra": { "status": "provisional", "hardware": "ali-ms, RTX 3090; weightsmith @ ad31044", "render_preset": "performance", "edit": null, "values_changed_note": null, "per_slot": { "30000": [ 0 ], "30001": [ 1 ], "30002": [ 1 ], "30003": [ 0 ], "30004": [ 0 ], "30005": [ 0 ], "30006": [ 1 ], "30007": [ 1 ], "30008": [ 1 ], "30009": [ 0 ], "30010": [ 0 ], "30011": [ 0 ], "30012": [ 0 ], "30013": [ 0 ], "30014": [ 0 ], "30015": [ 0 ] }, "run": "C-ali-ms-performance-0503" } }, { "experiment": "failure_map", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "RubiksCubeThenBananaTask", "code": "T1", "wording_pair": [ "Put the cube then the banana in the bowl" ], "slots": [ 30000, 30015 ], "n": 16, "unedited_k": 3, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "wilson95_pct": [ 6.6, 43.0 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "unedited pi0.5, task's default instruction, practice slots, default render preset; every 16-slot group the first and only run of a fresh Isaac process", "_ptask": null, "_task": "T1", "_edit_task": null, "_prov": false, "extra": { "status": "fresh", "run": "C-prime-default", "hardware": "ali-ms RTX 3090 24 GB (GPU index per record in `gpus`; both arms of every pair on one GPU); weightsmith @ ad31044 + OWSP_ENV_OFFSET patch in batched_client.py", "render_preset": "default", "envs_per_process": 8, "process_layout": "8 envs per Isaac process, two fresh processes per 16-slot group (env e of the process with offset o holds slot 16 g + o + e; the OWSP_ENV_OFFSET patch in weightsmith/batched_client.py keeps each slot's noise seeds ((g * 64 + o + e) * 100 + k) identical to a 16-env process), as Track A runs Cosmos3 Edge. 16 envs beside the 9.8 GB policy server ran out of GPU memory under the default preset (20:02 UTC, not counted)", "env_seed": 0, "edit": null, "values_changed_note": null, "per_slot": { "30000": 0, "30001": 0, "30002": 1, "30003": 0, "30004": 0, "30005": 0, "30006": 1, "30007": 0, "30008": 0, "30009": 1, "30010": 0, "30011": 0, "30012": 0, "30013": 0, "30014": 0, "30015": 0 }, "gpus": [ 0 ] } }, { "experiment": "failure_map", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "PickGlassesTask", "code": "T2", "wording_pair": [ "Pick up the eye glasses" ], "slots": [ 30000, 30015 ], "n": 16, "unedited_k": 7, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "wilson95_pct": [ 23.1, 66.8 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "unedited pi0.5, task's default instruction, practice slots, default render preset; every 16-slot group the first and only run of a fresh Isaac process", "_ptask": null, "_task": "T2", "_edit_task": null, "_prov": false, "extra": { "status": "fresh", "run": "C-prime-default", "hardware": "ali-ms RTX 3090 24 GB (GPU index per record in `gpus`; both arms of every pair on one GPU); weightsmith @ ad31044 + OWSP_ENV_OFFSET patch in batched_client.py", "render_preset": "default", "envs_per_process": 8, "process_layout": "8 envs per Isaac process, two fresh processes per 16-slot group (env e of the process with offset o holds slot 16 g + o + e; the OWSP_ENV_OFFSET patch in weightsmith/batched_client.py keeps each slot's noise seeds ((g * 64 + o + e) * 100 + k) identical to a 16-env process), as Track A runs Cosmos3 Edge. 16 envs beside the 9.8 GB policy server ran out of GPU memory under the default preset (20:02 UTC, not counted)", "env_seed": 0, "edit": null, "values_changed_note": null, "per_slot": { "30000": 0, "30001": 1, "30002": 0, "30003": 0, "30004": 0, "30005": 1, "30006": 0, "30007": 1, "30008": 1, "30009": 0, "30010": 0, "30011": 1, "30012": 1, "30013": 1, "30014": 0, "30015": 0 }, "gpus": [ 0 ] } }, { "experiment": "failure_map", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "RubiksCubeLeftOfBowlTask", "code": "T3", "wording_pair": [ "Put the rubiks cube to the left of the bowl" ], "slots": [ 30000, 30015 ], "n": 16, "unedited_k": 0, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "wilson95_pct": [ 0.0, 19.4 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "unedited pi0.5, task's default instruction, practice slots, default render preset; every 16-slot group the first and only run of a fresh Isaac process", "_ptask": null, "_task": "T3", "_edit_task": null, "_prov": false, "extra": { "status": "fresh", "run": "C-prime-default", "hardware": "ali-ms RTX 3090 24 GB (GPU index per record in `gpus`; both arms of every pair on one GPU); weightsmith @ ad31044 + OWSP_ENV_OFFSET patch in batched_client.py", "render_preset": "default", "envs_per_process": 8, "process_layout": "8 envs per Isaac process, two fresh processes per 16-slot group (env e of the process with offset o holds slot 16 g + o + e; the OWSP_ENV_OFFSET patch in weightsmith/batched_client.py keeps each slot's noise seeds ((g * 64 + o + e) * 100 + k) identical to a 16-env process), as Track A runs Cosmos3 Edge. 16 envs beside the 9.8 GB policy server ran out of GPU memory under the default preset (20:02 UTC, not counted)", "env_seed": 0, "edit": null, "values_changed_note": null, "per_slot": { "30000": 0, "30001": 0, "30002": 0, "30003": 0, "30004": 0, "30005": 0, "30006": 0, "30007": 0, "30008": 0, "30009": 0, "30010": 0, "30011": 0, "30012": 0, "30013": 0, "30014": 0, "30015": 0 }, "gpus": [ 0 ] } }, { "experiment": "failure_map", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "StackYellowOnRedTask", "code": "T4", "wording_pair": [ "Stack the yellow block on the red block" ], "slots": [ 30000, 30015 ], "n": 16, "unedited_k": 9, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "wilson95_pct": [ 33.2, 76.9 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "unedited pi0.5, task's default instruction, practice slots, default render preset; every 16-slot group the first and only run of a fresh Isaac process", "_ptask": null, "_task": "T4", "_edit_task": null, "_prov": false, "extra": { "status": "fresh", "run": "C-prime-default", "hardware": "ali-ms RTX 3090 24 GB (GPU index per record in `gpus`; both arms of every pair on one GPU); weightsmith @ ad31044 + OWSP_ENV_OFFSET patch in batched_client.py", "render_preset": "default", "envs_per_process": 8, "process_layout": "8 envs per Isaac process, two fresh processes per 16-slot group (env e of the process with offset o holds slot 16 g + o + e; the OWSP_ENV_OFFSET patch in weightsmith/batched_client.py keeps each slot's noise seeds ((g * 64 + o + e) * 100 + k) identical to a 16-env process), as Track A runs Cosmos3 Edge. 16 envs beside the 9.8 GB policy server ran out of GPU memory under the default preset (20:02 UTC, not counted)", "env_seed": 0, "edit": null, "values_changed_note": null, "per_slot": { "30000": 1, "30001": 1, "30002": 1, "30003": 1, "30004": 0, "30005": 0, "30006": 1, "30007": 1, "30008": 1, "30009": 0, "30010": 0, "30011": 1, "30012": 0, "30013": 1, "30014": 0, "30015": 0 }, "gpus": [ 0 ] } }, { "experiment": "failure_map", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "MustardInLeftBinTask", "code": "T5", "wording_pair": [ "Put the mustard in the left bin" ], "slots": [ 30000, 30015 ], "n": 16, "unedited_k": 3, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "wilson95_pct": [ 6.6, 43.0 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "unedited pi0.5, task's default instruction, practice slots, default render preset; every 16-slot group the first and only run of a fresh Isaac process", "_ptask": null, "_task": "T5", "_edit_task": null, "_prov": false, "extra": { "status": "fresh", "run": "C-prime-default", "hardware": "ali-ms RTX 3090 24 GB (GPU index per record in `gpus`; both arms of every pair on one GPU); weightsmith @ ad31044 + OWSP_ENV_OFFSET patch in batched_client.py", "render_preset": "default", "envs_per_process": 8, "process_layout": "8 envs per Isaac process, two fresh processes per 16-slot group (env e of the process with offset o holds slot 16 g + o + e; the OWSP_ENV_OFFSET patch in weightsmith/batched_client.py keeps each slot's noise seeds ((g * 64 + o + e) * 100 + k) identical to a 16-env process), as Track A runs Cosmos3 Edge. 16 envs beside the 9.8 GB policy server ran out of GPU memory under the default preset (20:02 UTC, not counted)", "env_seed": 0, "edit": null, "values_changed_note": null, "per_slot": { "30000": 0, "30001": 0, "30002": 1, "30003": 1, "30004": 0, "30005": 0, "30006": 0, "30007": 0, "30008": 0, "30009": 0, "30010": 0, "30011": 0, "30012": 0, "30013": 1, "30014": 0, "30015": 0 }, "gpus": [ 0 ] } }, { "experiment": "failure_map", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "Stack3RubiksCubeTask", "code": "T6", "wording_pair": [ "Stack the rubiks cubes in a tower" ], "slots": [ 30000, 30015 ], "n": 16, "unedited_k": 6, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "wilson95_pct": [ 18.5, 61.4 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "unedited pi0.5, task's default instruction, practice slots, default render preset; every 16-slot group the first and only run of a fresh Isaac process", "_ptask": null, "_task": "T6", "_edit_task": null, "_prov": false, "extra": { "status": "fresh", "run": "C-prime-default", "hardware": "ali-ms RTX 3090 24 GB (GPU index per record in `gpus`; both arms of every pair on one GPU); weightsmith @ ad31044 + OWSP_ENV_OFFSET patch in batched_client.py", "render_preset": "default", "envs_per_process": 8, "process_layout": "8 envs per Isaac process, two fresh processes per 16-slot group (env e of the process with offset o holds slot 16 g + o + e; the OWSP_ENV_OFFSET patch in weightsmith/batched_client.py keeps each slot's noise seeds ((g * 64 + o + e) * 100 + k) identical to a 16-env process), as Track A runs Cosmos3 Edge. 16 envs beside the 9.8 GB policy server ran out of GPU memory under the default preset (20:02 UTC, not counted)", "env_seed": 0, "edit": null, "values_changed_note": null, "per_slot": { "30000": 1, "30001": 1, "30002": 1, "30003": 1, "30004": 0, "30005": 0, "30006": 0, "30007": 0, "30008": 0, "30009": 0, "30010": 1, "30011": 0, "30012": 0, "30013": 0, "30014": 0, "30015": 1 }, "gpus": [ 0 ] } }, { "experiment": "failure_map", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "PickUpGreenObjectTask", "code": "T7", "wording_pair": [ "Pick up the green vegetable block" ], "slots": [ 30000, 30015 ], "n": 16, "unedited_k": 0, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "wilson95_pct": [ 0.0, 19.4 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "unedited pi0.5, task's default instruction, practice slots, default render preset; every 16-slot group the first and only run of a fresh Isaac process", "_ptask": null, "_task": "T7", "_edit_task": null, "_prov": false, "extra": { "status": "fresh", "run": "C-prime-default", "hardware": "ali-ms RTX 3090 24 GB (GPU index per record in `gpus`; both arms of every pair on one GPU); weightsmith @ ad31044 + OWSP_ENV_OFFSET patch in batched_client.py", "render_preset": "default", "envs_per_process": 8, "process_layout": "8 envs per Isaac process, two fresh processes per 16-slot group (env e of the process with offset o holds slot 16 g + o + e; the OWSP_ENV_OFFSET patch in weightsmith/batched_client.py keeps each slot's noise seeds ((g * 64 + o + e) * 100 + k) identical to a 16-env process), as Track A runs Cosmos3 Edge. 16 envs beside the 9.8 GB policy server ran out of GPU memory under the default preset (20:02 UTC, not counted)", "env_seed": 0, "edit": null, "values_changed_note": null, "per_slot": { "30000": 0, "30001": 0, "30002": 0, "30003": 0, "30004": 0, "30005": 0, "30006": 0, "30007": 0, "30008": 0, "30009": 0, "30010": 0, "30011": 0, "30012": 0, "30013": 0, "30014": 0, "30015": 0 }, "gpus": [ 0 ] } }, { "experiment": "failure_map", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "MustardRightOfRaisinRFTask", "code": "T8", "wording_pair": [ "Move the mustard bottle on the table so that the raisin box is to the left of the mustard bottle." ], "slots": [ 30000, 30015 ], "n": 16, "unedited_k": 0, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "wilson95_pct": [ 0.0, 19.4 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "unedited pi0.5, task's default instruction, practice slots, default render preset; every 16-slot group the first and only run of a fresh Isaac process", "_ptask": null, "_task": "T8", "_edit_task": null, "_prov": false, "extra": { "status": "fresh", "run": "C-prime-default", "hardware": "ali-ms RTX 3090 24 GB (GPU index per record in `gpus`; both arms of every pair on one GPU); weightsmith @ ad31044 + OWSP_ENV_OFFSET patch in batched_client.py", "render_preset": "default", "envs_per_process": 8, "process_layout": "8 envs per Isaac process, two fresh processes per 16-slot group (env e of the process with offset o holds slot 16 g + o + e; the OWSP_ENV_OFFSET patch in weightsmith/batched_client.py keeps each slot's noise seeds ((g * 64 + o + e) * 100 + k) identical to a 16-env process), as Track A runs Cosmos3 Edge. 16 envs beside the 9.8 GB policy server ran out of GPU memory under the default preset (20:02 UTC, not counted)", "env_seed": 0, "edit": null, "values_changed_note": null, "per_slot": { "30000": 0, "30001": 0, "30002": 0, "30003": 0, "30004": 0, "30005": 0, "30006": 0, "30007": 0, "30008": 0, "30009": 0, "30010": 0, "30011": 0, "30012": 0, "30013": 0, "30014": 0, "30015": 0 }, "gpus": [ 0 ] } }, { "experiment": "failure_map", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "RubiksCubeTask", "code": "R1", "wording_pair": [ "Put the cube in the bowl" ], "slots": [ 30000, 30015 ], "n": 16, "unedited_k": 16, "edited_k": null, "fixed": null, "broke": null, "p": null, "ci": { "wilson95_pct": [ 80.6, 100.0 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "unedited pi0.5, task's default instruction, practice slots, default render preset; every 16-slot group the first and only run of a fresh Isaac process", "_ptask": null, "_task": "R1", "_edit_task": null, "_prov": false, "extra": { "status": "fresh", "run": "C-prime-default", "hardware": "ali-ms RTX 3090 24 GB (GPU index per record in `gpus`; both arms of every pair on one GPU); weightsmith @ ad31044 + OWSP_ENV_OFFSET patch in batched_client.py", "render_preset": "default", "envs_per_process": 8, "process_layout": "8 envs per Isaac process, two fresh processes per 16-slot group (env e of the process with offset o holds slot 16 g + o + e; the OWSP_ENV_OFFSET patch in weightsmith/batched_client.py keeps each slot's noise seeds ((g * 64 + o + e) * 100 + k) identical to a 16-env process), as Track A runs Cosmos3 Edge. 16 envs beside the 9.8 GB policy server ran out of GPU memory under the default preset (20:02 UTC, not counted)", "env_seed": 0, "edit": null, "values_changed_note": null, "per_slot": { "30000": 1, "30001": 1, "30002": 1, "30003": 1, "30004": 1, "30005": 1, "30006": 1, "30007": 1, "30008": 1, "30009": 1, "30010": 1, "30011": 1, "30012": 1, "30013": 1, "30014": 1, "30015": 1 }, "gpus": [ 0 ] } }, { "experiment": "edit_target", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "RubiksCubeThenBananaTask", "code": "T1", "wording_pair": [ "Put the cube then the banana in the bowl" ], "slots": [ 96000, 96063 ], "n": 64, "unedited_k": 2, "edited_k": 40, "fixed": 38, "broke": 0, "p": 7.276e-12, "ci": { "unedited_wilson95_pct": [ 0.9, 10.7 ], "edited_wilson95_pct": [ 50.3, 73.3 ], "paired_diff_bootstrap95_pts": [ 46.9, 71.9 ] }, "control_k": 2, "teacher_k": 36, "recovery_pct": 111.8, "transfer": null, "retention": null, "edit_location": "VLM (PaliGemma) MLPs, layers 0-16: prompt + state + image keys", "values_changed": 450468837, "notes": "held-out slots (never used for any choice); instruction unchanged (task default); per_slot = [unedited, edited, control]", "_ptask": null, "_task": "T1", "_edit_task": null, "_prov": false, "extra": { "status": "fresh", "run": "C-prime-default", "hardware": "ali-ms RTX 3090 24 GB (GPU index per record in `gpus`; both arms of every pair on one GPU); weightsmith @ ad31044 + OWSP_ENV_OFFSET patch in batched_client.py", "render_preset": "default", "envs_per_process": 8, "process_layout": "8 envs per Isaac process, two fresh processes per 16-slot group (env e of the process with offset o holds slot 16 g + o + e; the OWSP_ENV_OFFSET patch in weightsmith/batched_client.py keeps each slot's noise seeds ((g * 64 + o + e) * 100 + k) identical to a 16-env process), as Track A runs Cosmos3 Edge. 16 envs beside the 9.8 GB policy server ran out of GPU memory under the default preset (20:02 UTC, not counted)", "env_seed": 0, "edit": { "name": "v2mi-f", "fingerprint": "63db0c52a51306c9" }, "values_changed_note": "13.4% of pi0.5's parameters as dense entries; closed-form distillation of the explicit wording 'Put the cube in the bowl then put the banana in the bowl'", "per_slot": { "96000": [ 0, 1, 0 ], "96001": [ 0, 0, 0 ], "96002": [ 0, 1, 0 ], "96003": [ 1, 1, 0 ], "96004": [ 0, 1, 0 ], "96005": [ 0, 1, 0 ], "96006": [ 0, 1, 0 ], "96007": [ 0, 0, 0 ], "96008": [ 0, 0, 0 ], "96009": [ 0, 1, 0 ], "96010": [ 0, 1, 0 ], "96011": [ 0, 0, 0 ], "96012": [ 0, 0, 0 ], "96013": [ 0, 0, 0 ], "96014": [ 0, 0, 0 ], "96015": [ 0, 1, 0 ], "96016": [ 0, 0, 0 ], "96017": [ 0, 0, 0 ], "96018": [ 0, 1, 0 ], "96019": [ 0, 0, 0 ], "96020": [ 0, 1, 0 ], "96021": [ 0, 0, 0 ], "96022": [ 0, 1, 0 ], "96023": [ 0, 1, 0 ], "96024": [ 0, 1, 0 ], "96025": [ 0, 1, 0 ], "96026": [ 1, 1, 0 ], "96027": [ 0, 1, 0 ], "96028": [ 0, 1, 0 ], "96029": [ 0, 0, 0 ], "96030": [ 0, 1, 0 ], "96031": [ 0, 1, 0 ], "96032": [ 0, 1, 0 ], "96033": [ 0, 1, 0 ], "96034": [ 0, 1, 0 ], "96035": [ 0, 1, 0 ], "96036": [ 0, 1, 0 ], "96037": [ 0, 0, 0 ], "96038": [ 0, 1, 1 ], "96039": [ 0, 1, 0 ], "96040": [ 0, 1, 0 ], "96041": [ 0, 0, 0 ], "96042": [ 0, 0, 0 ], "96043": [ 0, 1, 0 ], "96044": [ 0, 0, 1 ], "96045": [ 0, 1, 0 ], "96046": [ 0, 1, 0 ], "96047": [ 0, 1, 0 ], "96048": [ 0, 1, 0 ], "96049": [ 0, 0, 0 ], "96050": [ 0, 0, 0 ], "96051": [ 0, 1, 0 ], "96052": [ 0, 0, 0 ], "96053": [ 0, 0, 0 ], "96054": [ 0, 1, 0 ], "96055": [ 0, 1, 0 ], "96056": [ 0, 0, 0 ], "96057": [ 0, 1, 0 ], "96058": [ 0, 1, 0 ], "96059": [ 0, 0, 0 ], "96060": [ 0, 0, 0 ], "96061": [ 0, 1, 0 ], "96062": [ 0, 0, 0 ], "96063": [ 0, 1, 0 ] }, "control": { "name": "ctl-rand", "fingerprint": "1efd3512a92642f7", "slots": [ 96000, 96063 ], "n": 64, "k": 2, "wilson95_pct": [ 0.9, 10.7 ], "unedited_k_same_slots": 2, "edited_k_same_slots": 40, "vs_unedited": { "fixed": 2, "broke": 2, "p": 1.0 }, "edit_vs_control": { "edit_only": 39, "control_only": 1, "p": 7.457856554538012e-11 }, "edit_location": "same layers as v2mi-f", "values_changed": 502291925, "values_changed_note": "random-direction control: v2mi-f's layers, rank and singular values, random orthonormal directions (seed 0)" }, "teacher": { "wording": "Put the cube in the bowl then put the banana in the bowl", "slots": [ 96000, 96063 ], "n": 64, "k": 36, "vs_edit": { "fixed": 11, "broke": 15, "p": 0.557197093963623 }, "vs_unedited": { "fixed": 34, "broke": 0, "p": 1.1641532182693481e-10 }, "unedited_k_same_slots": 2, "edited_k_same_slots": 40, "per_slot": { "96000": 0, "96001": 0, "96002": 0, "96003": 1, "96004": 1, "96005": 0, "96006": 0, "96007": 1, "96008": 0, "96009": 1, "96010": 0, "96011": 0, "96012": 1, "96013": 1, "96014": 1, "96015": 1, "96016": 0, "96017": 0, "96018": 0, "96019": 1, "96020": 1, "96021": 0, "96022": 1, "96023": 0, "96024": 1, "96025": 0, "96026": 1, "96027": 0, "96028": 1, "96029": 1, "96030": 1, "96031": 0, "96032": 0, "96033": 1, "96034": 0, "96035": 0, "96036": 1, "96037": 1, "96038": 1, "96039": 1, "96040": 1, "96041": 0, "96042": 1, "96043": 1, "96044": 1, "96045": 1, "96046": 1, "96047": 0, "96048": 1, "96049": 0, "96050": 0, "96051": 1, "96052": 1, "96053": 0, "96054": 0, "96055": 1, "96056": 1, "96057": 1, "96058": 1, "96059": 0, "96060": 0, "96061": 1, "96062": 0, "96063": 1 } }, "gpus": [ 0 ], "control_n": 64, "control_slots": [ 96000, 96063 ], "control_unedited_k": 2, "edit_vs_control": { "cand_k": 40, "ref_k": 2, "p": 7.457856554538012e-11, "slots": [ 96000, 96063 ] }, "teacher_n": 64, "teacher_slots": [ 96000, 96063 ], "teacher_unedited_k": 2, "teacher_edited_k": 40, "teacher_p": 1.1641532182693481e-10, "teacher_fixed": 34, "teacher_broke": 0, "edit_vs_teacher": { "p": 0.557197093963623, "ref_k": 36, "cand_k": 40 } } }, { "experiment": "edit_target", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "MustardRightOfRaisinRFTask", "code": "T8", "wording_pair": [ "Move the mustard bottle on the table so that the raisin box is to the left of the mustard bottle." ], "slots": [ 96000, 96063 ], "n": 64, "unedited_k": 0, "edited_k": 28, "fixed": 28, "broke": 0, "p": 7.451e-09, "ci": { "unedited_wilson95_pct": [ 0.0, 5.7 ], "edited_wilson95_pct": [ 32.3, 55.9 ], "paired_diff_bootstrap95_pts": [ 31.2, 56.2 ] }, "control_k": 0, "teacher_k": 52, "recovery_pct": 53.8, "transfer": null, "retention": null, "edit_location": "VLM MLP down-projections language.blocks.08-16.mlp.linear (9 tensors), 3 key groups (approach, grasp, transport), rank <= 96 per layer", "values_changed": 275482328, "notes": "held-out slots (never used for any choice); instruction unchanged (task default); per_slot = [unedited, edited, control]", "_ptask": null, "_task": "T8", "_edit_task": null, "_prov": false, "extra": { "status": "fresh", "run": "C-prime-default", "hardware": "ali-ms RTX 3090 24 GB (GPU index per record in `gpus`; both arms of every pair on one GPU); weightsmith @ ad31044 + OWSP_ENV_OFFSET patch in batched_client.py", "render_preset": "default", "envs_per_process": 8, "process_layout": "8 envs per Isaac process, two fresh processes per 16-slot group (env e of the process with offset o holds slot 16 g + o + e; the OWSP_ENV_OFFSET patch in weightsmith/batched_client.py keeps each slot's noise seeds ((g * 64 + o + e) * 100 + k) identical to a 16-env process), as Track A runs Cosmos3 Edge. 16 envs beside the 9.8 GB policy server ran out of GPU memory under the default preset (20:02 UTC, not counted)", "env_seed": 0, "edit": { "name": "s4r-grp3", "fingerprint": "9e6f2378e28be6a3" }, "values_changed_note": "T8's own edit from the sgdw study: the target-first wording distilled into the reference-first one; dense count from the policy server's load report", "per_slot": { "96000": [ 0, 0, 0 ], "96001": [ 0, 0, 0 ], "96002": [ 0, 1, 0 ], "96003": [ 0, 0, 0 ], "96004": [ 0, 0, 0 ], "96005": [ 0, 1, 0 ], "96006": [ 0, 1, 0 ], "96007": [ 0, 0, 0 ], "96008": [ 0, 1, 0 ], "96009": [ 0, 0, 0 ], "96010": [ 0, 1, 0 ], "96011": [ 0, 1, 0 ], "96012": [ 0, 1, 0 ], "96013": [ 0, 0, 0 ], "96014": [ 0, 1, 0 ], "96015": [ 0, 1, 0 ], "96016": [ 0, 1, 0 ], "96017": [ 0, 1, 0 ], "96018": [ 0, 0, 0 ], "96019": [ 0, 0, 0 ], "96020": [ 0, 1, 0 ], "96021": [ 0, 1, 0 ], "96022": [ 0, 0, 0 ], "96023": [ 0, 0, 0 ], "96024": [ 0, 0, 0 ], "96025": [ 0, 1, 0 ], "96026": [ 0, 1, 0 ], "96027": [ 0, 0, 0 ], "96028": [ 0, 0, 0 ], "96029": [ 0, 0, 0 ], "96030": [ 0, 0, 0 ], "96031": [ 0, 1, 0 ], "96032": [ 0, 1, 0 ], "96033": [ 0, 0, 0 ], "96034": [ 0, 0, 0 ], "96035": [ 0, 0, 0 ], "96036": [ 0, 0, 0 ], "96037": [ 0, 0, 0 ], "96038": [ 0, 1, 0 ], "96039": [ 0, 0, 0 ], "96040": [ 0, 0, 0 ], "96041": [ 0, 0, 0 ], "96042": [ 0, 0, 0 ], "96043": [ 0, 1, 0 ], "96044": [ 0, 0, 0 ], "96045": [ 0, 0, 0 ], "96046": [ 0, 1, 0 ], "96047": [ 0, 0, 0 ], "96048": [ 0, 0, 0 ], "96049": [ 0, 1, 0 ], "96050": [ 0, 1, 0 ], "96051": [ 0, 1, 0 ], "96052": [ 0, 0, 0 ], "96053": [ 0, 1, 0 ], "96054": [ 0, 0, 0 ], "96055": [ 0, 1, 0 ], "96056": [ 0, 1, 0 ], "96057": [ 0, 0, 0 ], "96058": [ 0, 0, 0 ], "96059": [ 0, 0, 0 ], "96060": [ 0, 0, 0 ], "96061": [ 0, 0, 0 ], "96062": [ 0, 1, 0 ], "96063": [ 0, 1, 0 ] }, "control": { "name": "ctl-rand3", "fingerprint": "fbc8349de4182099", "slots": [ 96000, 96063 ], "n": 64, "k": 0, "wilson95_pct": [ 0.0, 5.7 ], "unedited_k_same_slots": 0, "edited_k_same_slots": 28, "vs_unedited": { "fixed": 0, "broke": 0, "p": 1.0 }, "edit_vs_control": { "edit_only": 28, "control_only": 0, "p": 7.450580596923828e-09 }, "edit_location": "same tensors as s4r-grp3", "values_changed": 290322032, "values_changed_note": "random-direction control: s4r-grp3's tensors, ranks and singular values, random orthonormal directions (default_rng(0))" }, "teacher": { "wording": "Move the mustard bottle on the table so that the mustard bottle is to the right of the raisin box.", "slots": [ 96000, 96063 ], "n": 64, "k": 52, "vs_edit": { "fixed": 31, "broke": 7, "p": 0.00011616700794547796 }, "vs_unedited": { "fixed": 52, "broke": 0, "p": 4.440892098500626e-16 }, "unedited_k_same_slots": 0, "edited_k_same_slots": 28, "per_slot": { "96000": 1, "96001": 1, "96002": 0, "96003": 0, "96004": 1, "96005": 1, "96006": 1, "96007": 1, "96008": 1, "96009": 1, "96010": 1, "96011": 1, "96012": 1, "96013": 1, "96014": 1, "96015": 0, "96016": 0, "96017": 1, "96018": 1, "96019": 1, "96020": 1, "96021": 1, "96022": 1, "96023": 1, "96024": 0, "96025": 0, "96026": 1, "96027": 1, "96028": 1, "96029": 1, "96030": 1, "96031": 1, "96032": 1, "96033": 1, "96034": 1, "96035": 1, "96036": 1, "96037": 0, "96038": 1, "96039": 1, "96040": 0, "96041": 1, "96042": 1, "96043": 0, "96044": 1, "96045": 1, "96046": 0, "96047": 1, "96048": 1, "96049": 1, "96050": 1, "96051": 1, "96052": 1, "96053": 1, "96054": 1, "96055": 0, "96056": 1, "96057": 1, "96058": 1, "96059": 0, "96060": 1, "96061": 1, "96062": 1, "96063": 1 } }, "gpus": [ 0, 1 ], "control_n": 64, "control_slots": [ 96000, 96063 ], "control_unedited_k": 0, "edit_vs_control": { "cand_k": 28, "ref_k": 0, "p": 7.450580596923828e-09, "slots": [ 96000, 96063 ] }, "teacher_n": 64, "teacher_slots": [ 96000, 96063 ], "teacher_unedited_k": 0, "teacher_edited_k": 28, "teacher_p": 4.440892098500626e-16, "teacher_fixed": 52, "teacher_broke": 0, "edit_vs_teacher": { "p": 0.00011616700794547796, "ref_k": 52, "cand_k": 28 } } }, { "experiment": "side_effect", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "PickGlassesTask", "code": "T2", "wording_pair": [ "Pick up the eye glasses" ], "slots": [ 96000, 96031 ], "n": 32, "unedited_k": 12, "edited_k": 21, "fixed": 12, "broke": 3, "p": 0.03516, "ci": { "unedited_wilson95_pct": [ 22.9, 54.7 ], "edited_wilson95_pct": [ 48.3, 79.6 ], "paired_diff_bootstrap95_pts": [ 6.2, 50.0 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": { "unedited_k": 12, "edited_k": 21, "n": 32 }, "retention": null, "edit_location": "VLM (PaliGemma) MLPs, layers 0-16: prompt + state + image keys", "values_changed": 450468837, "notes": "side effect of the T1 edit v2mi-f on another task, held-out slots, the task's own default instruction; per_slot = [unedited, edited]", "_ptask": null, "_task": "T2", "_edit_task": "T1", "_prov": false, "extra": { "status": "fresh", "run": "C-prime-default", "hardware": "ali-ms RTX 3090 24 GB (GPU index per record in `gpus`; both arms of every pair on one GPU); weightsmith @ ad31044 + OWSP_ENV_OFFSET patch in batched_client.py", "render_preset": "default", "envs_per_process": 8, "process_layout": "8 envs per Isaac process, two fresh processes per 16-slot group (env e of the process with offset o holds slot 16 g + o + e; the OWSP_ENV_OFFSET patch in weightsmith/batched_client.py keeps each slot's noise seeds ((g * 64 + o + e) * 100 + k) identical to a 16-env process), as Track A runs Cosmos3 Edge. 16 envs beside the 9.8 GB policy server ran out of GPU memory under the default preset (20:02 UTC, not counted)", "env_seed": 0, "edit": { "name": "v2mi-f", "fingerprint": "63db0c52a51306c9" }, "values_changed_note": "13.4% of pi0.5's parameters as dense entries; closed-form distillation of the explicit wording 'Put the cube in the bowl then put the banana in the bowl'", "per_slot": { "96000": [ 1, 1 ], "96001": [ 0, 0 ], "96002": [ 0, 1 ], "96003": [ 0, 1 ], "96004": [ 0, 1 ], "96005": [ 0, 0 ], "96006": [ 0, 0 ], "96007": [ 1, 1 ], "96008": [ 0, 1 ], "96009": [ 1, 1 ], "96010": [ 1, 1 ], "96011": [ 1, 1 ], "96012": [ 1, 0 ], "96013": [ 0, 1 ], "96014": [ 0, 1 ], "96015": [ 1, 1 ], "96016": [ 0, 0 ], "96017": [ 1, 0 ], "96018": [ 1, 1 ], "96019": [ 0, 1 ], "96020": [ 0, 1 ], "96021": [ 1, 0 ], "96022": [ 0, 1 ], "96023": [ 1, 1 ], "96024": [ 0, 1 ], "96025": [ 0, 0 ], "96026": [ 0, 0 ], "96027": [ 0, 1 ], "96028": [ 1, 1 ], "96029": [ 0, 1 ], "96030": [ 0, 0 ], "96031": [ 0, 0 ] }, "gpus": [ 1 ] } }, { "experiment": "side_effect", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "RubiksCubeLeftOfBowlTask", "code": "T3", "wording_pair": [ "Put the rubiks cube to the left of the bowl" ], "slots": [ 96000, 96031 ], "n": 32, "unedited_k": 1, "edited_k": 6, "fixed": 6, "broke": 1, "p": 0.125, "ci": { "unedited_wilson95_pct": [ 0.6, 15.7 ], "edited_wilson95_pct": [ 8.9, 35.3 ], "paired_diff_bootstrap95_pts": [ 0.0, 31.2 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": { "unedited_k": 1, "edited_k": 6, "n": 32 }, "retention": null, "edit_location": "VLM (PaliGemma) MLPs, layers 0-16: prompt + state + image keys", "values_changed": 450468837, "notes": "side effect of the T1 edit v2mi-f on another task, held-out slots, the task's own default instruction; per_slot = [unedited, edited]", "_ptask": null, "_task": "T3", "_edit_task": "T1", "_prov": false, "extra": { "status": "fresh", "run": "C-prime-default", "hardware": "ali-ms RTX 3090 24 GB (GPU index per record in `gpus`; both arms of every pair on one GPU); weightsmith @ ad31044 + OWSP_ENV_OFFSET patch in batched_client.py", "render_preset": "default", "envs_per_process": 8, "process_layout": "8 envs per Isaac process, two fresh processes per 16-slot group (env e of the process with offset o holds slot 16 g + o + e; the OWSP_ENV_OFFSET patch in weightsmith/batched_client.py keeps each slot's noise seeds ((g * 64 + o + e) * 100 + k) identical to a 16-env process), as Track A runs Cosmos3 Edge. 16 envs beside the 9.8 GB policy server ran out of GPU memory under the default preset (20:02 UTC, not counted)", "env_seed": 0, "edit": { "name": "v2mi-f", "fingerprint": "63db0c52a51306c9" }, "values_changed_note": "13.4% of pi0.5's parameters as dense entries; closed-form distillation of the explicit wording 'Put the cube in the bowl then put the banana in the bowl'", "per_slot": { "96000": [ 0, 1 ], "96001": [ 0, 0 ], "96002": [ 0, 0 ], "96003": [ 0, 0 ], "96004": [ 1, 0 ], "96005": [ 0, 0 ], "96006": [ 0, 0 ], "96007": [ 0, 0 ], "96008": [ 0, 1 ], "96009": [ 0, 0 ], "96010": [ 0, 1 ], "96011": [ 0, 0 ], "96012": [ 0, 1 ], "96013": [ 0, 0 ], "96014": [ 0, 0 ], "96015": [ 0, 0 ], "96016": [ 0, 1 ], "96017": [ 0, 0 ], "96018": [ 0, 0 ], "96019": [ 0, 0 ], "96020": [ 0, 0 ], "96021": [ 0, 0 ], "96022": [ 0, 0 ], "96023": [ 0, 0 ], "96024": [ 0, 0 ], "96025": [ 0, 1 ], "96026": [ 0, 0 ], "96027": [ 0, 0 ], "96028": [ 0, 0 ], "96029": [ 0, 0 ], "96030": [ 0, 0 ], "96031": [ 0, 0 ] }, "gpus": [ 1 ] } }, { "experiment": "side_effect", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "StackYellowOnRedTask", "code": "T4", "wording_pair": [ "Stack the yellow block on the red block" ], "slots": [ 96000, 96031 ], "n": 32, "unedited_k": 21, "edited_k": 27, "fixed": 9, "broke": 3, "p": 0.146, "ci": { "unedited_wilson95_pct": [ 48.3, 79.6 ], "edited_wilson95_pct": [ 68.2, 93.1 ], "paired_diff_bootstrap95_pts": [ 0.0, 40.6 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": { "unedited_k": 21, "edited_k": 27, "n": 32 }, "retention": null, "edit_location": "VLM (PaliGemma) MLPs, layers 0-16: prompt + state + image keys", "values_changed": 450468837, "notes": "side effect of the T1 edit v2mi-f on another task, held-out slots, the task's own default instruction; per_slot = [unedited, edited]", "_ptask": null, "_task": "T4", "_edit_task": "T1", "_prov": false, "extra": { "status": "fresh", "run": "C-prime-default", "hardware": "ali-ms RTX 3090 24 GB (GPU index per record in `gpus`; both arms of every pair on one GPU); weightsmith @ ad31044 + OWSP_ENV_OFFSET patch in batched_client.py", "render_preset": "default", "envs_per_process": 8, "process_layout": "8 envs per Isaac process, two fresh processes per 16-slot group (env e of the process with offset o holds slot 16 g + o + e; the OWSP_ENV_OFFSET patch in weightsmith/batched_client.py keeps each slot's noise seeds ((g * 64 + o + e) * 100 + k) identical to a 16-env process), as Track A runs Cosmos3 Edge. 16 envs beside the 9.8 GB policy server ran out of GPU memory under the default preset (20:02 UTC, not counted)", "env_seed": 0, "edit": { "name": "v2mi-f", "fingerprint": "63db0c52a51306c9" }, "values_changed_note": "13.4% of pi0.5's parameters as dense entries; closed-form distillation of the explicit wording 'Put the cube in the bowl then put the banana in the bowl'", "per_slot": { "96000": [ 1, 1 ], "96001": [ 1, 1 ], "96002": [ 1, 1 ], "96003": [ 1, 1 ], "96004": [ 1, 1 ], "96005": [ 1, 1 ], "96006": [ 1, 0 ], "96007": [ 1, 1 ], "96008": [ 1, 0 ], "96009": [ 0, 1 ], "96010": [ 1, 1 ], "96011": [ 1, 1 ], "96012": [ 0, 1 ], "96013": [ 1, 0 ], "96014": [ 1, 1 ], "96015": [ 0, 0 ], "96016": [ 1, 1 ], "96017": [ 0, 1 ], "96018": [ 1, 1 ], "96019": [ 0, 1 ], "96020": [ 1, 1 ], "96021": [ 0, 1 ], "96022": [ 1, 1 ], "96023": [ 0, 1 ], "96024": [ 1, 1 ], "96025": [ 0, 1 ], "96026": [ 0, 1 ], "96027": [ 1, 1 ], "96028": [ 1, 1 ], "96029": [ 0, 0 ], "96030": [ 0, 1 ], "96031": [ 1, 1 ] }, "gpus": [ 0, 1 ] } }, { "experiment": "side_effect", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "MustardInLeftBinTask", "code": "T5", "wording_pair": [ "Put the mustard in the left bin" ], "slots": [ 96000, 96031 ], "n": 32, "unedited_k": 7, "edited_k": 6, "fixed": 6, "broke": 7, "p": 1.0, "ci": { "unedited_wilson95_pct": [ 11.0, 38.8 ], "edited_wilson95_pct": [ 8.9, 35.3 ], "paired_diff_bootstrap95_pts": [ -25.0, 18.8 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": { "unedited_k": 7, "edited_k": 6, "n": 32 }, "retention": null, "edit_location": "VLM (PaliGemma) MLPs, layers 0-16: prompt + state + image keys", "values_changed": 450468837, "notes": "side effect of the T1 edit v2mi-f on another task, held-out slots, the task's own default instruction; per_slot = [unedited, edited]", "_ptask": null, "_task": "T5", "_edit_task": "T1", "_prov": false, "extra": { "status": "fresh", "run": "C-prime-default", "hardware": "ali-ms RTX 3090 24 GB (GPU index per record in `gpus`; both arms of every pair on one GPU); weightsmith @ ad31044 + OWSP_ENV_OFFSET patch in batched_client.py", "render_preset": "default", "envs_per_process": 8, "process_layout": "8 envs per Isaac process, two fresh processes per 16-slot group (env e of the process with offset o holds slot 16 g + o + e; the OWSP_ENV_OFFSET patch in weightsmith/batched_client.py keeps each slot's noise seeds ((g * 64 + o + e) * 100 + k) identical to a 16-env process), as Track A runs Cosmos3 Edge. 16 envs beside the 9.8 GB policy server ran out of GPU memory under the default preset (20:02 UTC, not counted)", "env_seed": 0, "edit": { "name": "v2mi-f", "fingerprint": "63db0c52a51306c9" }, "values_changed_note": "13.4% of pi0.5's parameters as dense entries; closed-form distillation of the explicit wording 'Put the cube in the bowl then put the banana in the bowl'", "per_slot": { "96000": [ 0, 0 ], "96001": [ 0, 0 ], "96002": [ 0, 0 ], "96003": [ 0, 0 ], "96004": [ 0, 0 ], "96005": [ 0, 1 ], "96006": [ 1, 0 ], "96007": [ 1, 0 ], "96008": [ 1, 0 ], "96009": [ 0, 1 ], "96010": [ 0, 1 ], "96011": [ 1, 0 ], "96012": [ 0, 0 ], "96013": [ 0, 0 ], "96014": [ 0, 0 ], "96015": [ 0, 1 ], "96016": [ 0, 1 ], "96017": [ 1, 0 ], "96018": [ 0, 0 ], "96019": [ 0, 0 ], "96020": [ 0, 0 ], "96021": [ 0, 0 ], "96022": [ 1, 0 ], "96023": [ 0, 0 ], "96024": [ 0, 0 ], "96025": [ 0, 1 ], "96026": [ 0, 0 ], "96027": [ 0, 0 ], "96028": [ 0, 0 ], "96029": [ 1, 0 ], "96030": [ 0, 0 ], "96031": [ 0, 0 ] }, "gpus": [ 1 ] } }, { "experiment": "side_effect", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "Stack3RubiksCubeTask", "code": "T6", "wording_pair": [ "Stack the rubiks cubes in a tower" ], "slots": [ 96000, 96031 ], "n": 32, "unedited_k": 11, "edited_k": 11, "fixed": 7, "broke": 7, "p": 1.0, "ci": { "unedited_wilson95_pct": [ 20.4, 51.7 ], "edited_wilson95_pct": [ 20.4, 51.7 ], "paired_diff_bootstrap95_pts": [ -21.9, 21.9 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": { "unedited_k": 11, "edited_k": 11, "n": 32 }, "retention": null, "edit_location": "VLM (PaliGemma) MLPs, layers 0-16: prompt + state + image keys", "values_changed": 450468837, "notes": "side effect of the T1 edit v2mi-f on another task, held-out slots, the task's own default instruction; per_slot = [unedited, edited]", "_ptask": null, "_task": "T6", "_edit_task": "T1", "_prov": false, "extra": { "status": "fresh", "run": "C-prime-default", "hardware": "ali-ms RTX 3090 24 GB (GPU index per record in `gpus`; both arms of every pair on one GPU); weightsmith @ ad31044 + OWSP_ENV_OFFSET patch in batched_client.py", "render_preset": "default", "envs_per_process": 8, "process_layout": "8 envs per Isaac process, two fresh processes per 16-slot group (env e of the process with offset o holds slot 16 g + o + e; the OWSP_ENV_OFFSET patch in weightsmith/batched_client.py keeps each slot's noise seeds ((g * 64 + o + e) * 100 + k) identical to a 16-env process), as Track A runs Cosmos3 Edge. 16 envs beside the 9.8 GB policy server ran out of GPU memory under the default preset (20:02 UTC, not counted)", "env_seed": 0, "edit": { "name": "v2mi-f", "fingerprint": "63db0c52a51306c9" }, "values_changed_note": "13.4% of pi0.5's parameters as dense entries; closed-form distillation of the explicit wording 'Put the cube in the bowl then put the banana in the bowl'", "per_slot": { "96000": [ 0, 0 ], "96001": [ 0, 0 ], "96002": [ 0, 1 ], "96003": [ 0, 0 ], "96004": [ 0, 1 ], "96005": [ 0, 1 ], "96006": [ 0, 0 ], "96007": [ 1, 0 ], "96008": [ 1, 0 ], "96009": [ 1, 1 ], "96010": [ 1, 1 ], "96011": [ 0, 0 ], "96012": [ 0, 0 ], "96013": [ 0, 0 ], "96014": [ 1, 0 ], "96015": [ 1, 1 ], "96016": [ 1, 1 ], "96017": [ 0, 0 ], "96018": [ 0, 1 ], "96019": [ 0, 0 ], "96020": [ 0, 1 ], "96021": [ 0, 1 ], "96022": [ 0, 0 ], "96023": [ 0, 0 ], "96024": [ 0, 0 ], "96025": [ 1, 0 ], "96026": [ 1, 0 ], "96027": [ 1, 0 ], "96028": [ 0, 1 ], "96029": [ 0, 0 ], "96030": [ 0, 0 ], "96031": [ 1, 0 ] }, "gpus": [ 0, 1 ] } }, { "experiment": "side_effect", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "PickUpGreenObjectTask", "code": "T7", "wording_pair": [ "Pick up the green vegetable block" ], "slots": [ 96000, 96031 ], "n": 32, "unedited_k": 0, "edited_k": 0, "fixed": 0, "broke": 0, "p": 1.0, "ci": { "unedited_wilson95_pct": [ 0.0, 10.7 ], "edited_wilson95_pct": [ 0.0, 10.7 ], "paired_diff_bootstrap95_pts": [ 0.0, 0.0 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": { "unedited_k": 0, "edited_k": 0, "n": 32 }, "retention": null, "edit_location": "VLM (PaliGemma) MLPs, layers 0-16: prompt + state + image keys", "values_changed": 450468837, "notes": "side effect of the T1 edit v2mi-f on another task, held-out slots, the task's own default instruction; per_slot = [unedited, edited]", "_ptask": null, "_task": "T7", "_edit_task": "T1", "_prov": false, "extra": { "status": "fresh", "run": "C-prime-default", "hardware": "ali-ms RTX 3090 24 GB (GPU index per record in `gpus`; both arms of every pair on one GPU); weightsmith @ ad31044 + OWSP_ENV_OFFSET patch in batched_client.py", "render_preset": "default", "envs_per_process": 8, "process_layout": "8 envs per Isaac process, two fresh processes per 16-slot group (env e of the process with offset o holds slot 16 g + o + e; the OWSP_ENV_OFFSET patch in weightsmith/batched_client.py keeps each slot's noise seeds ((g * 64 + o + e) * 100 + k) identical to a 16-env process), as Track A runs Cosmos3 Edge. 16 envs beside the 9.8 GB policy server ran out of GPU memory under the default preset (20:02 UTC, not counted)", "env_seed": 0, "edit": { "name": "v2mi-f", "fingerprint": "63db0c52a51306c9" }, "values_changed_note": "13.4% of pi0.5's parameters as dense entries; closed-form distillation of the explicit wording 'Put the cube in the bowl then put the banana in the bowl'", "per_slot": { "96000": [ 0, 0 ], "96001": [ 0, 0 ], "96002": [ 0, 0 ], "96003": [ 0, 0 ], "96004": [ 0, 0 ], "96005": [ 0, 0 ], "96006": [ 0, 0 ], "96007": [ 0, 0 ], "96008": [ 0, 0 ], "96009": [ 0, 0 ], "96010": [ 0, 0 ], "96011": [ 0, 0 ], "96012": [ 0, 0 ], "96013": [ 0, 0 ], "96014": [ 0, 0 ], "96015": [ 0, 0 ], "96016": [ 0, 0 ], "96017": [ 0, 0 ], "96018": [ 0, 0 ], "96019": [ 0, 0 ], "96020": [ 0, 0 ], "96021": [ 0, 0 ], "96022": [ 0, 0 ], "96023": [ 0, 0 ], "96024": [ 0, 0 ], "96025": [ 0, 0 ], "96026": [ 0, 0 ], "96027": [ 0, 0 ], "96028": [ 0, 0 ], "96029": [ 0, 0 ], "96030": [ 0, 0 ], "96031": [ 0, 0 ] }, "gpus": [ 1 ] } }, { "experiment": "side_effect", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "MustardRightOfRaisinRFTask", "code": "T8", "wording_pair": [ "Move the mustard bottle on the table so that the raisin box is to the left of the mustard bottle." ], "slots": [ 96000, 96031 ], "n": 32, "unedited_k": 0, "edited_k": 0, "fixed": 0, "broke": 0, "p": 1.0, "ci": { "unedited_wilson95_pct": [ 0.0, 10.7 ], "edited_wilson95_pct": [ 0.0, 10.7 ], "paired_diff_bootstrap95_pts": [ 0.0, 0.0 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": { "unedited_k": 0, "edited_k": 0, "n": 32 }, "retention": null, "edit_location": "VLM (PaliGemma) MLPs, layers 0-16: prompt + state + image keys", "values_changed": 450468837, "notes": "side effect of the T1 edit v2mi-f on another task, held-out slots, the task's own default instruction; per_slot = [unedited, edited]", "_ptask": null, "_task": "T8", "_edit_task": "T1", "_prov": false, "extra": { "status": "fresh", "run": "C-prime-default", "hardware": "ali-ms RTX 3090 24 GB (GPU index per record in `gpus`; both arms of every pair on one GPU); weightsmith @ ad31044 + OWSP_ENV_OFFSET patch in batched_client.py", "render_preset": "default", "envs_per_process": 8, "process_layout": "8 envs per Isaac process, two fresh processes per 16-slot group (env e of the process with offset o holds slot 16 g + o + e; the OWSP_ENV_OFFSET patch in weightsmith/batched_client.py keeps each slot's noise seeds ((g * 64 + o + e) * 100 + k) identical to a 16-env process), as Track A runs Cosmos3 Edge. 16 envs beside the 9.8 GB policy server ran out of GPU memory under the default preset (20:02 UTC, not counted)", "env_seed": 0, "edit": { "name": "v2mi-f", "fingerprint": "63db0c52a51306c9" }, "values_changed_note": "13.4% of pi0.5's parameters as dense entries; closed-form distillation of the explicit wording 'Put the cube in the bowl then put the banana in the bowl'", "per_slot": { "96000": [ 0, 0 ], "96001": [ 0, 0 ], "96002": [ 0, 0 ], "96003": [ 0, 0 ], "96004": [ 0, 0 ], "96005": [ 0, 0 ], "96006": [ 0, 0 ], "96007": [ 0, 0 ], "96008": [ 0, 0 ], "96009": [ 0, 0 ], "96010": [ 0, 0 ], "96011": [ 0, 0 ], "96012": [ 0, 0 ], "96013": [ 0, 0 ], "96014": [ 0, 0 ], "96015": [ 0, 0 ], "96016": [ 0, 0 ], "96017": [ 0, 0 ], "96018": [ 0, 0 ], "96019": [ 0, 0 ], "96020": [ 0, 0 ], "96021": [ 0, 0 ], "96022": [ 0, 0 ], "96023": [ 0, 0 ], "96024": [ 0, 0 ], "96025": [ 0, 0 ], "96026": [ 0, 0 ], "96027": [ 0, 0 ], "96028": [ 0, 0 ], "96029": [ 0, 0 ], "96030": [ 0, 0 ], "96031": [ 0, 0 ] }, "gpus": [ 0, 1 ] } }, { "experiment": "side_effect", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "RubiksCubeTask", "code": "R1", "wording_pair": [ "Put the cube in the bowl" ], "slots": [ 96000, 96031 ], "n": 32, "unedited_k": 31, "edited_k": 31, "fixed": 0, "broke": 0, "p": 1.0, "ci": { "unedited_wilson95_pct": [ 84.3, 99.4 ], "edited_wilson95_pct": [ 84.3, 99.4 ], "paired_diff_bootstrap95_pts": [ 0.0, 0.0 ] }, "control_k": null, "teacher_k": null, "recovery_pct": null, "transfer": null, "retention": { "unedited_k": 31, "edited_k": 31, "n": 32 }, "edit_location": "VLM (PaliGemma) MLPs, layers 0-16: prompt + state + image keys", "values_changed": 450468837, "notes": "side effect of the T1 edit v2mi-f on another task, held-out slots, the task's own default instruction; per_slot = [unedited, edited]", "_ptask": null, "_task": "R1", "_edit_task": "T1", "_prov": false, "extra": { "status": "fresh", "run": "C-prime-default", "hardware": "ali-ms RTX 3090 24 GB (GPU index per record in `gpus`; both arms of every pair on one GPU); weightsmith @ ad31044 + OWSP_ENV_OFFSET patch in batched_client.py", "render_preset": "default", "envs_per_process": 8, "process_layout": "8 envs per Isaac process, two fresh processes per 16-slot group (env e of the process with offset o holds slot 16 g + o + e; the OWSP_ENV_OFFSET patch in weightsmith/batched_client.py keeps each slot's noise seeds ((g * 64 + o + e) * 100 + k) identical to a 16-env process), as Track A runs Cosmos3 Edge. 16 envs beside the 9.8 GB policy server ran out of GPU memory under the default preset (20:02 UTC, not counted)", "env_seed": 0, "edit": { "name": "v2mi-f", "fingerprint": "63db0c52a51306c9" }, "values_changed_note": "13.4% of pi0.5's parameters as dense entries; closed-form distillation of the explicit wording 'Put the cube in the bowl then put the banana in the bowl'", "per_slot": { "96000": [ 1, 1 ], "96001": [ 1, 1 ], "96002": [ 1, 1 ], "96003": [ 1, 1 ], "96004": [ 1, 1 ], "96005": [ 1, 1 ], "96006": [ 1, 1 ], "96007": [ 1, 1 ], "96008": [ 1, 1 ], "96009": [ 1, 1 ], "96010": [ 0, 0 ], "96011": [ 1, 1 ], "96012": [ 1, 1 ], "96013": [ 1, 1 ], "96014": [ 1, 1 ], "96015": [ 1, 1 ], "96016": [ 1, 1 ], "96017": [ 1, 1 ], "96018": [ 1, 1 ], "96019": [ 1, 1 ], "96020": [ 1, 1 ], "96021": [ 1, 1 ], "96022": [ 1, 1 ], "96023": [ 1, 1 ], "96024": [ 1, 1 ], "96025": [ 1, 1 ], "96026": [ 1, 1 ], "96027": [ 1, 1 ], "96028": [ 1, 1 ], "96029": [ 1, 1 ], "96030": [ 1, 1 ], "96031": [ 1, 1 ] }, "gpus": [ 1 ] } }, { "experiment": "wording", "model": "pi05_droid_jointpos (OpenPI pi0.5 DROID, joint position; gs://openpi-assets-simeval/pi05_droid_jointpos)", "task": "MustardRightOfRaisinRFTask", "code": "T8", "wording_pair": [ "Move the mustard bottle on the table so that the raisin box is to the left of the mustard bottle.", "Move the mustard bottle on the table so that the mustard bottle is to the right of the raisin box." ], "slots": [ 30000, 30015 ], "n": 16, "unedited_k": 0, "edited_k": null, "fixed": 12, "broke": 0, "p": 0.0004883, "ci": { "rf_wilson95_pct": [ 0.0, 19.4 ], "tf_wilson95_pct": [ 50.5, 89.8 ] }, "control_k": null, "teacher_k": 12, "recovery_pct": null, "transfer": null, "retention": null, "edit_location": null, "values_changed": null, "notes": "unedited weights; reference-first (RF, task default) vs target-first (TF) wording on the same starts; fixed = TF-only, broke = RF-only; per_slot = [RF, TF]", "_ptask": null, "_task": "T8", "_edit_task": null, "_prov": false, "extra": { "status": "fresh", "run": "C-prime-default", "hardware": "ali-ms RTX 3090 24 GB (GPU index per record in `gpus`; both arms of every pair on one GPU); weightsmith @ ad31044 + OWSP_ENV_OFFSET patch in batched_client.py", "render_preset": "default", "envs_per_process": 8, "process_layout": "8 envs per Isaac process, two fresh processes per 16-slot group (env e of the process with offset o holds slot 16 g + o + e; the OWSP_ENV_OFFSET patch in weightsmith/batched_client.py keeps each slot's noise seeds ((g * 64 + o + e) * 100 + k) identical to a 16-env process), as Track A runs Cosmos3 Edge. 16 envs beside the 9.8 GB policy server ran out of GPU memory under the default preset (20:02 UTC, not counted)", "env_seed": 0, "edit": null, "values_changed_note": null, "per_slot": { "30000": [ 0, 1 ], "30001": [ 0, 1 ], "30002": [ 0, 0 ], "30003": [ 0, 0 ], "30004": [ 0, 1 ], "30005": [ 0, 1 ], "30006": [ 0, 1 ], "30007": [ 0, 1 ], "30008": [ 0, 1 ], "30009": [ 0, 1 ], "30010": [ 0, 0 ], "30011": [ 0, 1 ], "30012": [ 0, 1 ], "30013": [ 0, 0 ], "30014": [ 0, 1 ], "30015": [ 0, 1 ] }, "gpus": [ 0 ] } } ] }, "sources": { "pi05_earlier_records": { "study/blog/failure-levels.md": "working tree", "study/results/ctrl/heldout/ctrl-ctb__p1-RubiksCubeThenBananaTask-91008-64.json": "working tree", "study/results/ctrl/heldout/ctrl-glasses__p1-PickGlassesTask-91008-64.json": "working tree", "study/results/ctrl/heldout/ctrl-mustard__p1-MustardInLeftBinTask-91008-64.json": "working tree", "study/results/ctrl/heldout/ctrl-stack3__p1-Stack3RubiksCubeTask-91008-64.json": "working tree", "study/results/ctrl/summary.json": "working tree", "study/results/opus/comparisons.json": "working tree", "study/results/opus/controls/ctl1-90000-64.analysis.json": "working tree", "study/results/opus/controls/ctl1-90000-64.results.json": "working tree", "study/results/opus/heldout/cube-then-banana__abl-imgl8-RubiksCubeThenBananaTask-90000-128.json": "working tree", "study/results/opus/heldout/cube-then-banana__v2mi-f-RubiksCubeThenBananaTask-90000-128.json": "working tree", "study/results/opus/heldout/thr-pickglasses__ctb-v2mi-PickGlassesTask-90000-64.json": "working tree", "study/results/opus/heldout/tx-rubikscubeleftofbowl__ctb-v2mi-RubiksCubeLeftOfBowlTask-90000-64.json": "working tree", "study/results/opus/where/results/diff_a1.json": "working tree", "study/results/opus/where/results/diff_h1.json": "working tree", "study/results/opus/where/results/diff_p1-blind.json": "working tree", "study/results/opus/where/results/diff_v1-full.json": "working tree", "study/results/opus/where/results/diff_v1-img816.json": "working tree", "study/results/opus/where/results/diff_va.json": "working tree", "study/results/opus/where/results/heldout_analysis.json": "working tree", "study/results/opus/where/results/heldout_results.json": "working tree", "study/results/opus/where/results/size_summary.json": "working tree", "study/results/opus/where/results/transfer_cube_analysis.json": "working tree", "study/results/opus/where/results/transfer_glasses_analysis.json": "working tree", "study/results/opus/where/results/transfer_left_analysis.json": "working tree", "study/results/sgdw/results/heldout_pairs_92000-64.json": "working tree", "study/results/sgdw/results/practice_rf_grp3_vs_base.json": "working tree", "study/results/sgdw/results/practice_tf_grp3_vs_base.json": "working tree", "study/results/sgdw/results/screen_s4r_tf_vs_rf.json": "working tree", "study/results/sgdw/runs/s4r/build_grp3.json": "working tree", "study/results/sgdw/runs/s4r/build_grp3.report.txt": "working tree", "study/results/tutorial/tutorial-ctb__ctb-first-95008-64.results.json": "working tree" }, "cosmos": { "kind": "branch", "branch": "cross-cosmos/v1", "path": "study/results/cross/metrics_cosmos.json", "url": "https://github.com/adeeb10abbas/weightsmith/blob/cross-cosmos/v1/study/results/cross/metrics_cosmos.json", "commit": "2444083b32c2" }, "groot": { "kind": "branch", "branch": "cross-groot/v1", "path": "study/results/cross/metrics_groot.json", "url": "https://github.com/adeeb10abbas/weightsmith/blob/cross-groot/v1/study/results/cross/metrics_groot.json", "commit": "f1dfb229b095" }, "pi05": { "kind": "branch", "branch": "cross-pi05/v1", "path": "study/results/cross/metrics_pi05.json", "url": "https://github.com/adeeb10abbas/weightsmith/blob/cross-pi05/v1/study/results/cross/metrics_pi05.json", "commit": "05b5a28fd7b5" } }, "method": "Exact two-sided McNemar on fixed vs broke pairs; paired bootstrap 95% interval of the difference (20,000 multinomial draws over the four paired outcome types, seed 0); Wilson 95% intervals.", "content_hash": "212e8ea37d8add18", "numbers_hash": "790835e243aec5e9", "updated": "2026-11-09T06:06Z", "updated_text": "9 Nov 2026, 06:06 UTC", "status_text": "Cosmos3 Edge Policy: complete · GR00T N1.7: complete · π0.5: complete" }