{ "what": "The Inspect binding of blind-spot: every recorded `inspect eval` run re-scored by this package, the human-baselines file pinned and graded by the same verifier. Every number a page or README states about the Inspect runs is read from here.", "scorer": "blind_spot.adapters_v0._decide — the function the verifiers rubric calls; a KILL is verified by simulating the netlist under the mutation, EQUIVALENT against the SAT proof; no answer key, no judge, no tolerance", "ladder": { "claude-sonnet-5@low": { "model": "anthropic/claude-sonnet-5", "effort": "low", "rungs": { "blind": { "solved": 3, "n": 12, "wrong": 3, "unreadable": 1, "mean_reward": 0.0 }, "located": { "solved": 8, "n": 12, "wrong": 1, "unreadable": 0, "mean_reward": 0.5833 }, "profile": { "solved": 6, "n": 12, "wrong": 0, "unreadable": 0, "mean_reward": 0.5 } }, "costUsd": 0.2996 }, "claude-sonnet-5@medium": { "model": "anthropic/claude-sonnet-5", "effort": "medium", "rungs": { "blind": { "solved": 5, "n": 12, "wrong": 3, "unreadable": 0, "mean_reward": 0.1667 }, "located": { "solved": 10, "n": 12, "wrong": 0, "unreadable": 0, "mean_reward": 0.8333 }, "profile": { "solved": 8, "n": 12, "wrong": 0, "unreadable": 0, "mean_reward": 0.6667 } }, "costUsd": 0.8273 }, "claude-sonnet-5@high": { "model": "anthropic/claude-sonnet-5", "effort": "high", "rungs": { "blind": { "solved": 5, "n": 12, "wrong": 3, "unreadable": 1, "mean_reward": 0.1667 }, "located": { "solved": 12, "n": 12, "wrong": 0, "unreadable": 0, "mean_reward": 1.0 }, "profile": { "solved": 6, "n": 12, "wrong": 1, "unreadable": 3, "mean_reward": 0.4167 } }, "costUsd": 2.0771 }, "claude-opus-5@low": { "model": "anthropic/claude-opus-5", "effort": "low", "rungs": { "located": { "solved": 10, "n": 12, "wrong": 0, "unreadable": 2, "mean_reward": 0.8333 }, "profile": { "solved": 2, "n": 12, "wrong": 0, "unreadable": 10, "mean_reward": 0.1667 }, "blind": { "solved": 4, "n": 12, "wrong": 3, "unreadable": 2, "mean_reward": 0.0833 } }, "costUsd": 0.3995 }, "claude-haiku-4-5-20251001@default": { "model": "anthropic/claude-haiku-4-5-20251001", "effort": "default", "rungs": { "blind": { "solved": 4, "n": 12, "wrong": 3, "unreadable": 0, "mean_reward": 0.0833 }, "located": { "solved": 3, "n": 12, "wrong": 2, "unreadable": 0, "mean_reward": 0.0833 }, "profile": { "solved": 4, "n": 12, "wrong": 2, "unreadable": 0, "mean_reward": 0.1667 } }, "costUsd": 0.2545 } }, "taskset": { "seed": 2027, "rule": "index i serves rung RUNGS[i % 3]; a rung variant is the indices congruent to its offset mod 3; the sample id is `seed-index` = taskset.Task.id" }, "frontier": { "models_run": [ "anthropic/claude-haiku-4-5-20251001", "anthropic/claude-opus-5", "anthropic/claude-sonnet-5" ], "status": "3 model(s) run end to end" }, "runs": [ { "log": "environments/blind_spot/inspect/logs/2026-09-21T22-07-00-00-00_blind-spot_GeEYUW8Z3Fax5rX3Q94JsG.json", "sha256": "45781b5dd4190793483863138844f66032011773b145a65c3860b90a444ad732", "task": "blind_spot", "task_args": { "num_tasks": 36, "seed": 2027, "start": 0 }, "model": "mockllm/model", "control": true, "policy": "sat", "inspect_ai": "0.3.266", "status": "success", "ran": "2026-09-21T22:07:00+00:00", "config": { "effort": null, "max_tokens": null, "reasoning_effort": null }, "concord": { "rollouts": 36, "disagreements": 0 }, "by_rung": { "located": { "n": 12, "solved": 12, "wrong": 0, "missed": 0, "undecided": 0, "refused_parse": 0, "false_claims": 0, "out_of_box_kills": 4, "mean_reward": 1.0, "mean_reward_answered": 1.0, "errors": 0 }, "profile": { "n": 12, "solved": 12, "wrong": 0, "missed": 0, "undecided": 0, "refused_parse": 0, "false_claims": 0, "out_of_box_kills": 2, "mean_reward": 1.0, "mean_reward_answered": 1.0, "errors": 0 }, "blind": { "n": 12, "solved": 12, "wrong": 0, "missed": 0, "undecided": 0, "refused_parse": 0, "false_claims": 0, "out_of_box_kills": 3, "mean_reward": 1.0, "mean_reward_answered": 1.0, "errors": 0 } }, "by_class": { "ALIGNED_ONLY": { "n": 5, "solved": 5 }, "COVERED": { "n": 15, "solved": 15 }, "IDENTITY": { "n": 4, "solved": 4 }, "MINT_ONLY": { "n": 2, "solved": 2 }, "NOCHANGE": { "n": 4, "solved": 4 }, "OUTBOX_ONLY": { "n": 6, "solved": 6 } }, "totals": { "n": 36, "solved": 36, "wrong": 0, "missed": 0, "undecided": 0, "refused_parse": 0, "false_claims": 0, "out_of_box_kills": 9, "mean_reward": 1.0, "mean_reward_answered": 1.0, "errors": 0 }, "usage": { "input": 0, "output": 0 }, "costUsd": null, "costNote": "errored rollouts report no usage, so this is a floor, not the bill", "rows": [ { "id": "2027-0", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 306, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": false, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.9 }, { "id": "2027-1", "rung": "profile", "klass": "COVERED", "truth": "KILL", "mutant": 329, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.9 }, { "id": "2027-10", "rung": "profile", "klass": "MINT_ONLY", "truth": "KILL", "mutant": 319, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.9 }, { "id": "2027-11", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 145, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.9 }, { "id": "2027-12", "rung": "located", "klass": "MINT_ONLY", "truth": "KILL", "mutant": 315, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.9 }, { "id": "2027-13", "rung": "profile", "klass": "COVERED", "truth": "KILL", "mutant": 56, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.9 }, { "id": "2027-14", "rung": "blind", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 45, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.9 }, { "id": "2027-15", "rung": "located", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 60, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": false, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.9 }, { "id": "2027-16", "rung": "profile", "klass": "COVERED", "truth": "KILL", "mutant": 7, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.9 }, { "id": "2027-17", "rung": "blind", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 268, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": false, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.9 }, { "id": "2027-18", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 356, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.8 }, { "id": "2027-19", "rung": "profile", "klass": "NOCHANGE", "truth": "EQUIVALENT", "mutant": 383, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.8 }, { "id": "2027-2", "rung": "blind", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 61, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.9 }, { "id": "2027-20", "rung": "blind", "klass": "NOCHANGE", "truth": "EQUIVALENT", "mutant": 97, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.8 }, { "id": "2027-21", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 360, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.8 }, { "id": "2027-22", "rung": "profile", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 244, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.8 }, { "id": "2027-23", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 380, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.8 }, { "id": "2027-24", "rung": "located", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 95, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": false, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.8 }, { "id": "2027-25", "rung": "profile", "klass": "NOCHANGE", "truth": "EQUIVALENT", "mutant": 344, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-26", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 260, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": false, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.1 }, { "id": "2027-27", "rung": "located", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 350, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": false, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.1 }, { "id": "2027-28", "rung": "profile", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 45, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.1 }, { "id": "2027-29", "rung": "blind", "klass": "IDENTITY", "truth": "EQUIVALENT", "mutant": 1, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.1 }, { "id": "2027-3", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 161, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.9 }, { "id": "2027-30", "rung": "located", "klass": "IDENTITY", "truth": "EQUIVALENT", "mutant": 1, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.2 }, { "id": "2027-31", "rung": "profile", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 60, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": false, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.2 }, { "id": "2027-32", "rung": "blind", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 45, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.2 }, { "id": "2027-33", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 148, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-34", "rung": "profile", "klass": "IDENTITY", "truth": "EQUIVALENT", "mutant": 1, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-35", "rung": "blind", "klass": "IDENTITY", "truth": "EQUIVALENT", "mutant": 1, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-4", "rung": "profile", "klass": "COVERED", "truth": "KILL", "mutant": 29, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.9 }, { "id": "2027-5", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 15, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.9 }, { "id": "2027-6", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 76, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.9 }, { "id": "2027-7", "rung": "profile", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 88, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": false, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.9 }, { "id": "2027-8", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 347, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": false, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.9 }, { "id": "2027-9", "rung": "located", "klass": "NOCHANGE", "truth": "EQUIVALENT", "mutant": 338, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.9 } ] }, { "log": "environments/blind_spot/inspect/logs/2026-09-21T22-07-02-00-00_blind-spot_Ca3WwQCoPRRP3a7ACABpPG.json", "sha256": "67028de999c8db92ca46eeb392e8427a7347a41e655aedb38c9ac764b158af1f", "task": "blind_spot", "task_args": { "num_tasks": 36, "seed": 2027, "start": 0 }, "model": "mockllm/model", "control": true, "policy": "never", "inspect_ai": "0.3.266", "status": "success", "ran": "2026-09-21T22:07:02+00:00", "config": { "effort": null, "max_tokens": null, "reasoning_effort": null }, "concord": { "rollouts": 36, "disagreements": 0 }, "by_rung": { "located": { "n": 12, "solved": 2, "wrong": 10, "missed": 0, "undecided": 0, "refused_parse": 0, "false_claims": 10, "out_of_box_kills": 0, "mean_reward": -0.6667, "mean_reward_answered": -0.6667, "errors": 0 }, "profile": { "n": 12, "solved": 3, "wrong": 9, "missed": 0, "undecided": 0, "refused_parse": 0, "false_claims": 9, "out_of_box_kills": 0, "mean_reward": -0.5, "mean_reward_answered": -0.5, "errors": 0 }, "blind": { "n": 12, "solved": 3, "wrong": 9, "missed": 0, "undecided": 0, "refused_parse": 0, "false_claims": 9, "out_of_box_kills": 0, "mean_reward": -0.5, "mean_reward_answered": -0.5, "errors": 0 } }, "by_class": { "ALIGNED_ONLY": { "n": 5, "solved": 0 }, "COVERED": { "n": 15, "solved": 0 }, "IDENTITY": { "n": 4, "solved": 4 }, "MINT_ONLY": { "n": 2, "solved": 0 }, "NOCHANGE": { "n": 4, "solved": 4 }, "OUTBOX_ONLY": { "n": 6, "solved": 0 } }, "totals": { "n": 36, "solved": 8, "wrong": 28, "missed": 0, "undecided": 0, "refused_parse": 0, "false_claims": 28, "out_of_box_kills": 0, "mean_reward": -0.5556, "mean_reward_answered": -0.5556, "errors": 0 }, "usage": { "input": 0, "output": 0 }, "costUsd": null, "costNote": "errored rollouts report no usage, so this is a floor, not the bill", "rows": [ { "id": "2027-0", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 306, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-1", "rung": "profile", "klass": "COVERED", "truth": "KILL", "mutant": 329, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-10", "rung": "profile", "klass": "MINT_ONLY", "truth": "KILL", "mutant": 319, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-11", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 145, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-12", "rung": "located", "klass": "MINT_ONLY", "truth": "KILL", "mutant": 315, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-13", "rung": "profile", "klass": "COVERED", "truth": "KILL", "mutant": 56, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-14", "rung": "blind", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 45, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-15", "rung": "located", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 60, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-16", "rung": "profile", "klass": "COVERED", "truth": "KILL", "mutant": 7, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-17", "rung": "blind", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 268, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-18", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 356, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-19", "rung": "profile", "klass": "NOCHANGE", "truth": "EQUIVALENT", "mutant": 383, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-2", "rung": "blind", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 61, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-20", "rung": "blind", "klass": "NOCHANGE", "truth": "EQUIVALENT", "mutant": 97, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-21", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 360, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-22", "rung": "profile", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 244, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-23", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 380, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-24", "rung": "located", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 95, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-25", "rung": "profile", "klass": "NOCHANGE", "truth": "EQUIVALENT", "mutant": 344, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-26", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 260, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-27", "rung": "located", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 350, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-28", "rung": "profile", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 45, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-29", "rung": "blind", "klass": "IDENTITY", "truth": "EQUIVALENT", "mutant": 1, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-3", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 161, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-30", "rung": "located", "klass": "IDENTITY", "truth": "EQUIVALENT", "mutant": 1, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-31", "rung": "profile", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 60, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-32", "rung": "blind", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 45, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-33", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 148, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-34", "rung": "profile", "klass": "IDENTITY", "truth": "EQUIVALENT", "mutant": 1, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-35", "rung": "blind", "klass": "IDENTITY", "truth": "EQUIVALENT", "mutant": 1, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-4", "rung": "profile", "klass": "COVERED", "truth": "KILL", "mutant": 29, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-5", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 15, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-6", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 76, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-7", "rung": "profile", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 88, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-8", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 347, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-9", "rung": "located", "klass": "NOCHANGE", "truth": "EQUIVALENT", "mutant": 338, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 1.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 } ] }, { "log": "environments/blind_spot/inspect/logs/2026-09-21T22-07-02-00-00_blind-spot_XcEZxxzHPCU2HZgLkCRNhp.json", "sha256": "da2b1f9e867ef8b45335854b5ec216c3ef8ef80c2864ae6d4b19d30281718144", "task": "blind_spot", "task_args": { "num_tasks": 36, "seed": 2027, "start": 0 }, "model": "mockllm/model", "control": true, "policy": "abstain", "inspect_ai": "0.3.266", "status": "success", "ran": "2026-09-21T22:07:02+00:00", "config": { "effort": null, "max_tokens": null, "reasoning_effort": null }, "concord": { "rollouts": 36, "disagreements": 0 }, "by_rung": { "located": { "n": 12, "solved": 0, "wrong": 0, "missed": 0, "undecided": 12, "refused_parse": 0, "false_claims": 0, "out_of_box_kills": 0, "mean_reward": 0.0, "mean_reward_answered": 0.0, "errors": 0 }, "profile": { "n": 12, "solved": 0, "wrong": 0, "missed": 0, "undecided": 12, "refused_parse": 0, "false_claims": 0, "out_of_box_kills": 0, "mean_reward": 0.0, "mean_reward_answered": 0.0, "errors": 0 }, "blind": { "n": 12, "solved": 0, "wrong": 0, "missed": 0, "undecided": 12, "refused_parse": 0, "false_claims": 0, "out_of_box_kills": 0, "mean_reward": 0.0, "mean_reward_answered": 0.0, "errors": 0 } }, "by_class": { "ALIGNED_ONLY": { "n": 5, "solved": 0 }, "COVERED": { "n": 15, "solved": 0 }, "IDENTITY": { "n": 4, "solved": 0 }, "MINT_ONLY": { "n": 2, "solved": 0 }, "NOCHANGE": { "n": 4, "solved": 0 }, "OUTBOX_ONLY": { "n": 6, "solved": 0 } }, "totals": { "n": 36, "solved": 0, "wrong": 0, "missed": 0, "undecided": 36, "refused_parse": 0, "false_claims": 0, "out_of_box_kills": 0, "mean_reward": 0.0, "mean_reward_answered": 0.0, "errors": 0 }, "usage": { "input": 0, "output": 0 }, "costUsd": null, "costNote": "errored rollouts report no usage, so this is a floor, not the bill", "rows": [ { "id": "2027-0", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 306, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-1", "rung": "profile", "klass": "COVERED", "truth": "KILL", "mutant": 329, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-10", "rung": "profile", "klass": "MINT_ONLY", "truth": "KILL", "mutant": 319, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-11", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 145, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-12", "rung": "located", "klass": "MINT_ONLY", "truth": "KILL", "mutant": 315, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-13", "rung": "profile", "klass": "COVERED", "truth": "KILL", "mutant": 56, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-14", "rung": "blind", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 45, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-15", "rung": "located", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 60, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-16", "rung": "profile", "klass": "COVERED", "truth": "KILL", "mutant": 7, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-17", "rung": "blind", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 268, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-18", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 356, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-19", "rung": "profile", "klass": "NOCHANGE", "truth": "EQUIVALENT", "mutant": 383, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-2", "rung": "blind", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 61, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-20", "rung": "blind", "klass": "NOCHANGE", "truth": "EQUIVALENT", "mutant": 97, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-21", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 360, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-22", "rung": "profile", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 244, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-23", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 380, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-24", "rung": "located", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 95, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-25", "rung": "profile", "klass": "NOCHANGE", "truth": "EQUIVALENT", "mutant": 344, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-26", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 260, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-27", "rung": "located", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 350, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-28", "rung": "profile", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 45, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-29", "rung": "blind", "klass": "IDENTITY", "truth": "EQUIVALENT", "mutant": 1, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-3", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 161, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-30", "rung": "located", "klass": "IDENTITY", "truth": "EQUIVALENT", "mutant": 1, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-31", "rung": "profile", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 60, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-32", "rung": "blind", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 45, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-33", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 148, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-34", "rung": "profile", "klass": "IDENTITY", "truth": "EQUIVALENT", "mutant": 1, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-35", "rung": "blind", "klass": "IDENTITY", "truth": "EQUIVALENT", "mutant": 1, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-4", "rung": "profile", "klass": "COVERED", "truth": "KILL", "mutant": 29, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-5", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 15, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-6", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 76, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-7", "rung": "profile", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 88, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-8", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 347, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 }, { "id": "2027-9", "rung": "located", "klass": "NOCHANGE", "truth": "EQUIVALENT", "mutant": 338, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 0, "out_tokens": 0, "seconds": 0.0 } ] }, { "log": "environments/blind_spot/inspect/logs/2026-09-25T17-22-47-00-00_blind-spot-blind_nhxEY3MY3fgLdfAvVskWSX.json", "sha256": "0ea624ee8378a664acb9bb8e7af815347870ade82764f8b0fd7157a757b4a368", "task": "blind_spot_blind", "task_args": { "num_tasks": 12, "seed": 2027, "start": 0 }, "model": "anthropic/claude-sonnet-5", "control": false, "policy": null, "inspect_ai": "0.3.266", "status": "success", "ran": "2026-09-25T17:22:47+00:00", "config": { "effort": "low", "max_tokens": 12000, "reasoning_effort": null }, "concord": { "rollouts": 12, "disagreements": 0 }, "by_rung": { "blind": { "n": 12, "solved": 3, "wrong": 3, "missed": 5, "undecided": 0, "refused_parse": 1, "false_claims": 3, "out_of_box_kills": 0, "mean_reward": 0.0, "mean_reward_answered": 0.0, "errors": 0 } }, "by_class": { "ALIGNED_ONLY": { "n": 3, "solved": 0 }, "COVERED": { "n": 5, "solved": 3 }, "IDENTITY": { "n": 2, "solved": 0 }, "NOCHANGE": { "n": 1, "solved": 0 }, "OUTBOX_ONLY": { "n": 1, "solved": 0 } }, "totals": { "n": 12, "solved": 3, "wrong": 3, "missed": 5, "undecided": 0, "refused_parse": 1, "false_claims": 3, "out_of_box_kills": 0, "mean_reward": 0.0, "mean_reward_answered": 0.0, "errors": 0 }, "usage": { "input": 24, "output": 7700 }, "costUsd": 0.077, "costNote": "errored rollouts report no usage, so this is a floor, not the bill", "rows": [ { "id": "2027-11", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 145, "outcome": "REFUSED_PARSE", "reward": 0.0, "well_formed": 0.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 557, "seconds": 7.6 }, { "id": "2027-14", "rung": "blind", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 45, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 579, "seconds": 5.5 }, { "id": "2027-17", "rung": "blind", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 268, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 576, "seconds": 5.7 }, { "id": "2027-2", "rung": "blind", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 61, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 752, "seconds": 7.5 }, { "id": "2027-20", "rung": "blind", "klass": "NOCHANGE", "truth": "EQUIVALENT", "mutant": 97, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 2, "out_tokens": 1160, "seconds": 11.5 }, { "id": "2027-23", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 380, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 557, "seconds": 5.8 }, { "id": "2027-26", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 260, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 553, "seconds": 5.4 }, { "id": "2027-29", "rung": "blind", "klass": "IDENTITY", "truth": "EQUIVALENT", "mutant": 1, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 2, "out_tokens": 554, "seconds": 5.6 }, { "id": "2027-32", "rung": "blind", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 45, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 601, "seconds": 5.8 }, { "id": "2027-35", "rung": "blind", "klass": "IDENTITY", "truth": "EQUIVALENT", "mutant": 1, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 2, "out_tokens": 589, "seconds": 5.6 }, { "id": "2027-5", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 15, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 613, "seconds": 7.8 }, { "id": "2027-8", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 347, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 609, "seconds": 5.9 } ] }, { "log": "environments/blind_spot/inspect/logs/2026-09-25T17-22-47-00-00_blind-spot-located_id36sTPegphcYKpM3H8AwA.json", "sha256": "2df8b0925b280393bca7c9e8d9db0915c26fed2e37cc0538f27dfddf9ddb4b4c", "task": "blind_spot_located", "task_args": { "num_tasks": 12, "seed": 2027, "start": 0 }, "model": "anthropic/claude-sonnet-5", "control": false, "policy": null, "inspect_ai": "0.3.266", "status": "success", "ran": "2026-09-25T17:22:47+00:00", "config": { "effort": "low", "max_tokens": 12000, "reasoning_effort": null }, "concord": { "rollouts": 12, "disagreements": 0 }, "by_rung": { "located": { "n": 12, "solved": 8, "wrong": 1, "missed": 3, "undecided": 0, "refused_parse": 0, "false_claims": 1, "out_of_box_kills": 0, "mean_reward": 0.5833, "mean_reward_answered": 0.5833, "errors": 0 } }, "by_class": { "COVERED": { "n": 6, "solved": 5 }, "IDENTITY": { "n": 1, "solved": 1 }, "MINT_ONLY": { "n": 1, "solved": 1 }, "NOCHANGE": { "n": 1, "solved": 1 }, "OUTBOX_ONLY": { "n": 3, "solved": 0 } }, "totals": { "n": 12, "solved": 8, "wrong": 1, "missed": 3, "undecided": 0, "refused_parse": 0, "false_claims": 1, "out_of_box_kills": 0, "mean_reward": 0.5833, "mean_reward_answered": 0.5833, "errors": 0 }, "usage": { "input": 24, "output": 11506 }, "costUsd": 0.1151, "costNote": "errored rollouts report no usage, so this is a floor, not the bill", "rows": [ { "id": "2027-0", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 306, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 827, "seconds": 10.9 }, { "id": "2027-12", "rung": "located", "klass": "MINT_ONLY", "truth": "KILL", "mutant": 315, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 2397, "seconds": 25.1 }, { "id": "2027-15", "rung": "located", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 60, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 831, "seconds": 10.8 }, { "id": "2027-18", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 356, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 597, "seconds": 7.4 }, { "id": "2027-21", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 360, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 645, "seconds": 7.5 }, { "id": "2027-24", "rung": "located", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 95, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 431, "seconds": 6.1 }, { "id": "2027-27", "rung": "located", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 350, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 725, "seconds": 9.2 }, { "id": "2027-3", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 161, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 997, "seconds": 11.6 }, { "id": "2027-30", "rung": "located", "klass": "IDENTITY", "truth": "EQUIVALENT", "mutant": 1, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 48, "seconds": 1.5 }, { "id": "2027-33", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 148, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 3243, "seconds": 35.7 }, { "id": "2027-6", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 76, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 2, "out_tokens": 274, "seconds": 4.2 }, { "id": "2027-9", "rung": "located", "klass": "NOCHANGE", "truth": "EQUIVALENT", "mutant": 338, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 491, "seconds": 6.5 } ] }, { "log": "environments/blind_spot/inspect/logs/2026-09-25T17-22-47-00-00_blind-spot-profile_6QHtgs6ejELWgrxu7UhRbN.json", "sha256": "b9a1340fb457f89b709f167fa720b22d8ce709177cce1d4f4769a5311f5b7287", "task": "blind_spot_profile", "task_args": { "num_tasks": 12, "seed": 2027, "start": 0 }, "model": "anthropic/claude-sonnet-5", "control": false, "policy": null, "inspect_ai": "0.3.266", "status": "success", "ran": "2026-09-25T17:22:47+00:00", "config": { "effort": "low", "max_tokens": 12000, "reasoning_effort": null }, "concord": { "rollouts": 12, "disagreements": 0 }, "by_rung": { "profile": { "n": 12, "solved": 6, "wrong": 0, "missed": 3, "undecided": 3, "refused_parse": 0, "false_claims": 0, "out_of_box_kills": 2, "mean_reward": 0.5, "mean_reward_answered": 0.5, "errors": 0 } }, "by_class": { "ALIGNED_ONLY": { "n": 2, "solved": 0 }, "COVERED": { "n": 4, "solved": 4 }, "IDENTITY": { "n": 1, "solved": 0 }, "MINT_ONLY": { "n": 1, "solved": 0 }, "NOCHANGE": { "n": 2, "solved": 0 }, "OUTBOX_ONLY": { "n": 2, "solved": 2 } }, "totals": { "n": 12, "solved": 6, "wrong": 0, "missed": 3, "undecided": 3, "refused_parse": 0, "false_claims": 0, "out_of_box_kills": 2, "mean_reward": 0.5, "mean_reward_answered": 0.5, "errors": 0 }, "usage": { "input": 24, "output": 10742 }, "costUsd": 0.1075, "costNote": "errored rollouts report no usage, so this is a floor, not the bill", "rows": [ { "id": "2027-1", "rung": "profile", "klass": "COVERED", "truth": "KILL", "mutant": 329, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 1858, "seconds": 19.3 }, { "id": "2027-10", "rung": "profile", "klass": "MINT_ONLY", "truth": "KILL", "mutant": 319, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 1531, "seconds": 13.4 }, { "id": "2027-13", "rung": "profile", "klass": "COVERED", "truth": "KILL", "mutant": 56, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 1366, "seconds": 13.8 }, { "id": "2027-16", "rung": "profile", "klass": "COVERED", "truth": "KILL", "mutant": 7, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 1136, "seconds": 12.1 }, { "id": "2027-19", "rung": "profile", "klass": "NOCHANGE", "truth": "EQUIVALENT", "mutant": 383, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 18, "seconds": 2.4 }, { "id": "2027-22", "rung": "profile", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 244, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 727, "seconds": 7.6 }, { "id": "2027-25", "rung": "profile", "klass": "NOCHANGE", "truth": "EQUIVALENT", "mutant": 344, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 18, "seconds": 1.8 }, { "id": "2027-28", "rung": "profile", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 45, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 750, "seconds": 7.7 }, { "id": "2027-31", "rung": "profile", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 60, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": false, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 441, "seconds": 4.7 }, { "id": "2027-34", "rung": "profile", "klass": "IDENTITY", "truth": "EQUIVALENT", "mutant": 1, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 18, "seconds": 2.0 }, { "id": "2027-4", "rung": "profile", "klass": "COVERED", "truth": "KILL", "mutant": 29, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 1873, "seconds": 19.9 }, { "id": "2027-7", "rung": "profile", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 88, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": false, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 1006, "seconds": 8.1 } ] }, { "log": "environments/blind_spot/inspect/logs/2026-09-25T17-24-41-00-00_blind-spot-blind_eQAVvuKbVJGnwApZm2TYhX.json", "sha256": "b8fb66d3ada2245b90888685fafa9f9c95d4b279ea17526d756cb5d09591b7d1", "task": "blind_spot_blind", "task_args": { "num_tasks": 12, "seed": 2027, "start": 0 }, "model": "anthropic/claude-sonnet-5", "control": false, "policy": null, "inspect_ai": "0.3.266", "status": "success", "ran": "2026-09-25T17:24:41+00:00", "config": { "effort": "medium", "max_tokens": 12000, "reasoning_effort": null }, "concord": { "rollouts": 12, "disagreements": 0 }, "by_rung": { "blind": { "n": 12, "solved": 5, "wrong": 3, "missed": 4, "undecided": 0, "refused_parse": 0, "false_claims": 3, "out_of_box_kills": 1, "mean_reward": 0.1667, "mean_reward_answered": 0.1667, "errors": 0 } }, "by_class": { "ALIGNED_ONLY": { "n": 3, "solved": 0 }, "COVERED": { "n": 5, "solved": 5 }, "IDENTITY": { "n": 2, "solved": 0 }, "NOCHANGE": { "n": 1, "solved": 0 }, "OUTBOX_ONLY": { "n": 1, "solved": 0 } }, "totals": { "n": 12, "solved": 5, "wrong": 3, "missed": 4, "undecided": 0, "refused_parse": 0, "false_claims": 3, "out_of_box_kills": 1, "mean_reward": 0.1667, "mean_reward_answered": 0.1667, "errors": 0 }, "usage": { "input": 24, "output": 29713 }, "costUsd": 0.2972, "costNote": "errored rollouts report no usage, so this is a floor, not the bill", "rows": [ { "id": "2027-11", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 145, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 2081, "seconds": 19.1 }, { "id": "2027-14", "rung": "blind", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 45, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 2575, "seconds": 23.4 }, { "id": "2027-17", "rung": "blind", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 268, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 1698, "seconds": 14.8 }, { "id": "2027-2", "rung": "blind", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 61, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 2418, "seconds": 22.2 }, { "id": "2027-20", "rung": "blind", "klass": "NOCHANGE", "truth": "EQUIVALENT", "mutant": 97, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 2, "out_tokens": 4103, "seconds": 41.0 }, { "id": "2027-23", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 380, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 2561, "seconds": 21.9 }, { "id": "2027-26", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 260, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": false, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 2253, "seconds": 22.4 }, { "id": "2027-29", "rung": "blind", "klass": "IDENTITY", "truth": "EQUIVALENT", "mutant": 1, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 2, "out_tokens": 2785, "seconds": 26.0 }, { "id": "2027-32", "rung": "blind", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 45, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 2450, "seconds": 22.9 }, { "id": "2027-35", "rung": "blind", "klass": "IDENTITY", "truth": "EQUIVALENT", "mutant": 1, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 2, "out_tokens": 1731, "seconds": 17.6 }, { "id": "2027-5", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 15, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 2542, "seconds": 23.6 }, { "id": "2027-8", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 347, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 2516, "seconds": 23.5 } ] }, { "log": "environments/blind_spot/inspect/logs/2026-09-25T17-24-41-00-00_blind-spot-located_RnxSdoCh89MwN3zVXb4ikR.json", "sha256": "c974377ce5e0afbd474d7eac6e86fa56781eac38622d537ec75c3b121e5d1fbf", "task": "blind_spot_located", "task_args": { "num_tasks": 12, "seed": 2027, "start": 0 }, "model": "anthropic/claude-sonnet-5", "control": false, "policy": null, "inspect_ai": "0.3.266", "status": "success", "ran": "2026-09-25T17:24:41+00:00", "config": { "effort": "medium", "max_tokens": 12000, "reasoning_effort": null }, "concord": { "rollouts": 12, "disagreements": 0 }, "by_rung": { "located": { "n": 12, "solved": 10, "wrong": 0, "missed": 2, "undecided": 0, "refused_parse": 0, "false_claims": 0, "out_of_box_kills": 1, "mean_reward": 0.8333, "mean_reward_answered": 0.8333, "errors": 0 } }, "by_class": { "COVERED": { "n": 6, "solved": 6 }, "IDENTITY": { "n": 1, "solved": 1 }, "MINT_ONLY": { "n": 1, "solved": 1 }, "NOCHANGE": { "n": 1, "solved": 1 }, "OUTBOX_ONLY": { "n": 3, "solved": 1 } }, "totals": { "n": 12, "solved": 10, "wrong": 0, "missed": 2, "undecided": 0, "refused_parse": 0, "false_claims": 0, "out_of_box_kills": 1, "mean_reward": 0.8333, "mean_reward_answered": 0.8333, "errors": 0 }, "usage": { "input": 24, "output": 27020 }, "costUsd": 0.2702, "costNote": "errored rollouts report no usage, so this is a floor, not the bill", "rows": [ { "id": "2027-0", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 306, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 2796, "seconds": 28.9 }, { "id": "2027-12", "rung": "located", "klass": "MINT_ONLY", "truth": "KILL", "mutant": 315, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 3230, "seconds": 32.4 }, { "id": "2027-15", "rung": "located", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 60, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 2047, "seconds": 21.2 }, { "id": "2027-18", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 356, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 999, "seconds": 12.5 }, { "id": "2027-21", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 360, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 3793, "seconds": 39.0 }, { "id": "2027-24", "rung": "located", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 95, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 4682, "seconds": 52.6 }, { "id": "2027-27", "rung": "located", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 350, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": false, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 2020, "seconds": 21.2 }, { "id": "2027-3", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 161, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 1469, "seconds": 16.9 }, { "id": "2027-30", "rung": "located", "klass": "IDENTITY", "truth": "EQUIVALENT", "mutant": 1, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 63, "seconds": 1.9 }, { "id": "2027-33", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 148, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 2763, "seconds": 29.3 }, { "id": "2027-6", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 76, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 2199, "seconds": 22.4 }, { "id": "2027-9", "rung": "located", "klass": "NOCHANGE", "truth": "EQUIVALENT", "mutant": 338, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 959, "seconds": 12.2 } ] }, { "log": "environments/blind_spot/inspect/logs/2026-09-25T17-24-41-00-00_blind-spot-profile_Mh5YE9Nd3WBp7s8mST3HFa.json", "sha256": "ede2750982a083bb476c4cef0a1f3f19224657abc8e5aeee72fa749543e4ff63", "task": "blind_spot_profile", "task_args": { "num_tasks": 12, "seed": 2027, "start": 0 }, "model": "anthropic/claude-sonnet-5", "control": false, "policy": null, "inspect_ai": "0.3.266", "status": "success", "ran": "2026-09-25T17:24:41+00:00", "config": { "effort": "medium", "max_tokens": 12000, "reasoning_effort": null }, "concord": { "rollouts": 12, "disagreements": 0 }, "by_rung": { "profile": { "n": 12, "solved": 8, "wrong": 0, "missed": 3, "undecided": 1, "refused_parse": 0, "false_claims": 0, "out_of_box_kills": 2, "mean_reward": 0.6667, "mean_reward_answered": 0.6667, "errors": 0 } }, "by_class": { "ALIGNED_ONLY": { "n": 2, "solved": 0 }, "COVERED": { "n": 4, "solved": 4 }, "IDENTITY": { "n": 1, "solved": 1 }, "MINT_ONLY": { "n": 1, "solved": 0 }, "NOCHANGE": { "n": 2, "solved": 1 }, "OUTBOX_ONLY": { "n": 2, "solved": 2 } }, "totals": { "n": 12, "solved": 8, "wrong": 0, "missed": 3, "undecided": 1, "refused_parse": 0, "false_claims": 0, "out_of_box_kills": 2, "mean_reward": 0.6667, "mean_reward_answered": 0.6667, "errors": 0 }, "usage": { "input": 24, "output": 25987 }, "costUsd": 0.2599, "costNote": "errored rollouts report no usage, so this is a floor, not the bill", "rows": [ { "id": "2027-1", "rung": "profile", "klass": "COVERED", "truth": "KILL", "mutant": 329, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 2100, "seconds": 20.3 }, { "id": "2027-10", "rung": "profile", "klass": "MINT_ONLY", "truth": "KILL", "mutant": 319, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 2566, "seconds": 24.2 }, { "id": "2027-13", "rung": "profile", "klass": "COVERED", "truth": "KILL", "mutant": 56, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 1579, "seconds": 15.3 }, { "id": "2027-16", "rung": "profile", "klass": "COVERED", "truth": "KILL", "mutant": 7, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 6362, "seconds": 61.4 }, { "id": "2027-19", "rung": "profile", "klass": "NOCHANGE", "truth": "EQUIVALENT", "mutant": 383, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 794, "seconds": 10.1 }, { "id": "2027-22", "rung": "profile", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 244, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 3008, "seconds": 31.3 }, { "id": "2027-25", "rung": "profile", "klass": "NOCHANGE", "truth": "EQUIVALENT", "mutant": 344, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 1251, "seconds": 16.5 }, { "id": "2027-28", "rung": "profile", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 45, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 2106, "seconds": 20.1 }, { "id": "2027-31", "rung": "profile", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 60, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": false, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 1012, "seconds": 10.4 }, { "id": "2027-34", "rung": "profile", "klass": "IDENTITY", "truth": "EQUIVALENT", "mutant": 1, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 1197, "seconds": 15.3 }, { "id": "2027-4", "rung": "profile", "klass": "COVERED", "truth": "KILL", "mutant": 29, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 3185, "seconds": 31.4 }, { "id": "2027-7", "rung": "profile", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 88, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": false, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 827, "seconds": 9.2 } ] }, { "log": "environments/blind_spot/inspect/logs/2026-09-25T17-28-54-00-00_blind-spot-blind_mjMbix5tfVdyqBSoYstmNC.json", "sha256": "01c4b1388c528970be544c3d25665cf3d5f90228efebb2a8a862c859463754c7", "task": "blind_spot_blind", "task_args": { "num_tasks": 12, "seed": 2027, "start": 0 }, "model": "anthropic/claude-sonnet-5", "control": false, "policy": null, "inspect_ai": "0.3.266", "status": "success", "ran": "2026-09-25T17:28:54+00:00", "config": { "effort": "high", "max_tokens": 12000, "reasoning_effort": null }, "concord": { "rollouts": 12, "disagreements": 0 }, "by_rung": { "blind": { "n": 12, "solved": 5, "wrong": 3, "missed": 3, "undecided": 0, "refused_parse": 1, "false_claims": 3, "out_of_box_kills": 1, "mean_reward": 0.1667, "mean_reward_answered": 0.1818, "errors": 0 } }, "by_class": { "ALIGNED_ONLY": { "n": 3, "solved": 0 }, "COVERED": { "n": 5, "solved": 5 }, "IDENTITY": { "n": 2, "solved": 0 }, "NOCHANGE": { "n": 1, "solved": 0 }, "OUTBOX_ONLY": { "n": 1, "solved": 0 } }, "totals": { "n": 12, "solved": 5, "wrong": 3, "missed": 3, "undecided": 0, "refused_parse": 1, "false_claims": 3, "out_of_box_kills": 1, "mean_reward": 0.1667, "mean_reward_answered": 0.1818, "errors": 0 }, "usage": { "input": 24, "output": 54121 }, "costUsd": 0.5413, "costNote": "errored rollouts report no usage, so this is a floor, not the bill", "rows": [ { "id": "2027-11", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 145, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 4994, "seconds": 47.1 }, { "id": "2027-14", "rung": "blind", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 45, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 5117, "seconds": 48.9 }, { "id": "2027-17", "rung": "blind", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 268, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 2737, "seconds": 26.0 }, { "id": "2027-2", "rung": "blind", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 61, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 6513, "seconds": 61.6 }, { "id": "2027-20", "rung": "blind", "klass": "NOCHANGE", "truth": "EQUIVALENT", "mutant": 97, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 2, "out_tokens": 4806, "seconds": 48.2 }, { "id": "2027-23", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 380, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 5230, "seconds": 48.6 }, { "id": "2027-26", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 260, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": false, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 4008, "seconds": 37.7 }, { "id": "2027-29", "rung": "blind", "klass": "IDENTITY", "truth": "EQUIVALENT", "mutant": 1, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 2, "out_tokens": 4936, "seconds": 48.6 }, { "id": "2027-32", "rung": "blind", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 45, "outcome": "REFUSED_PARSE", "reward": 0.0, "well_formed": 0.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 4617, "seconds": 42.7 }, { "id": "2027-35", "rung": "blind", "klass": "IDENTITY", "truth": "EQUIVALENT", "mutant": 1, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 2, "out_tokens": 3153, "seconds": 28.2 }, { "id": "2027-5", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 15, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 5927, "seconds": 57.9 }, { "id": "2027-8", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 347, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 2083, "seconds": 21.3 } ] }, { "log": "environments/blind_spot/inspect/logs/2026-09-25T17-28-54-00-00_blind-spot-located_Nz4S6hftZb9qkoyU7XrD38.json", "sha256": "d9eaffc4f56a3602724fd9b908a07534f3d61707dc2e856942ea6dc3adce3a83", "task": "blind_spot_located", "task_args": { "num_tasks": 12, "seed": 2027, "start": 0 }, "model": "anthropic/claude-sonnet-5", "control": false, "policy": null, "inspect_ai": "0.3.266", "status": "success", "ran": "2026-09-25T17:28:54+00:00", "config": { "effort": "high", "max_tokens": 12000, "reasoning_effort": null }, "concord": { "rollouts": 12, "disagreements": 0 }, "by_rung": { "located": { "n": 12, "solved": 12, "wrong": 0, "missed": 0, "undecided": 0, "refused_parse": 0, "false_claims": 0, "out_of_box_kills": 3, "mean_reward": 1.0, "mean_reward_answered": 1.0, "errors": 0 } }, "by_class": { "COVERED": { "n": 6, "solved": 6 }, "IDENTITY": { "n": 1, "solved": 1 }, "MINT_ONLY": { "n": 1, "solved": 1 }, "NOCHANGE": { "n": 1, "solved": 1 }, "OUTBOX_ONLY": { "n": 3, "solved": 3 } }, "totals": { "n": 12, "solved": 12, "wrong": 0, "missed": 0, "undecided": 0, "refused_parse": 0, "false_claims": 0, "out_of_box_kills": 3, "mean_reward": 1.0, "mean_reward_answered": 1.0, "errors": 0 }, "usage": { "input": 24, "output": 52335 }, "costUsd": 0.5234, "costNote": "errored rollouts report no usage, so this is a floor, not the bill", "rows": [ { "id": "2027-0", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 306, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 5046, "seconds": 51.3 }, { "id": "2027-12", "rung": "located", "klass": "MINT_ONLY", "truth": "KILL", "mutant": 315, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 6545, "seconds": 65.7 }, { "id": "2027-15", "rung": "located", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 60, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": false, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 3059, "seconds": 31.2 }, { "id": "2027-18", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 356, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 8108, "seconds": 86.4 }, { "id": "2027-21", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 360, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 7629, "seconds": 72.2 }, { "id": "2027-24", "rung": "located", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 95, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": false, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 2725, "seconds": 27.3 }, { "id": "2027-27", "rung": "located", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 350, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": false, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 3201, "seconds": 31.1 }, { "id": "2027-3", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 161, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 4673, "seconds": 45.7 }, { "id": "2027-30", "rung": "located", "klass": "IDENTITY", "truth": "EQUIVALENT", "mutant": 1, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 64, "seconds": 1.8 }, { "id": "2027-33", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 148, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 5365, "seconds": 52.2 }, { "id": "2027-6", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 76, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 4289, "seconds": 45.0 }, { "id": "2027-9", "rung": "located", "klass": "NOCHANGE", "truth": "EQUIVALENT", "mutant": 338, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 1631, "seconds": 17.1 } ] }, { "log": "environments/blind_spot/inspect/logs/2026-09-25T17-28-54-00-00_blind-spot-profile_UWoLQ3b3Hy4uUWgu4JD7ut.json", "sha256": "f35ee2437601da44e181493cfcecde5417118eb72c14d552b140e88330aaea9b", "task": "blind_spot_profile", "task_args": { "num_tasks": 12, "seed": 2027, "start": 0 }, "model": "anthropic/claude-sonnet-5", "control": false, "policy": null, "inspect_ai": "0.3.266", "status": "success", "ran": "2026-09-25T17:28:54+00:00", "config": { "effort": "high", "max_tokens": 12000, "reasoning_effort": null }, "concord": { "rollouts": 12, "disagreements": 0 }, "by_rung": { "profile": { "n": 12, "solved": 6, "wrong": 1, "missed": 2, "undecided": 0, "refused_parse": 3, "false_claims": 1, "out_of_box_kills": 1, "mean_reward": 0.4167, "mean_reward_answered": 0.5556, "errors": 0 } }, "by_class": { "ALIGNED_ONLY": { "n": 2, "solved": 0 }, "COVERED": { "n": 4, "solved": 3 }, "IDENTITY": { "n": 1, "solved": 0 }, "MINT_ONLY": { "n": 1, "solved": 0 }, "NOCHANGE": { "n": 2, "solved": 2 }, "OUTBOX_ONLY": { "n": 2, "solved": 1 } }, "totals": { "n": 12, "solved": 6, "wrong": 1, "missed": 2, "undecided": 0, "refused_parse": 3, "false_claims": 1, "out_of_box_kills": 1, "mean_reward": 0.4167, "mean_reward_answered": 0.5556, "errors": 0 }, "usage": { "input": 24, "output": 101236 }, "costUsd": 1.0124, "costNote": "errored rollouts report no usage, so this is a floor, not the bill", "rows": [ { "id": "2027-1", "rung": "profile", "klass": "COVERED", "truth": "KILL", "mutant": 329, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 10186, "seconds": 106.4 }, { "id": "2027-10", "rung": "profile", "klass": "MINT_ONLY", "truth": "KILL", "mutant": 319, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 7560, "seconds": 69.7 }, { "id": "2027-13", "rung": "profile", "klass": "COVERED", "truth": "KILL", "mutant": 56, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 8589, "seconds": 82.7 }, { "id": "2027-16", "rung": "profile", "klass": "COVERED", "truth": "KILL", "mutant": 7, "outcome": "REFUSED_PARSE", "reward": 0.0, "well_formed": 0.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 12000, "seconds": 136.0 }, { "id": "2027-19", "rung": "profile", "klass": "NOCHANGE", "truth": "EQUIVALENT", "mutant": 383, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 5730, "seconds": 66.4 }, { "id": "2027-22", "rung": "profile", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 244, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 5446, "seconds": 55.4 }, { "id": "2027-25", "rung": "profile", "klass": "NOCHANGE", "truth": "EQUIVALENT", "mutant": 344, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 5642, "seconds": 64.6 }, { "id": "2027-28", "rung": "profile", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 45, "outcome": "REFUSED_PARSE", "reward": 0.0, "well_formed": 0.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 12000, "seconds": 135.1 }, { "id": "2027-31", "rung": "profile", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 60, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": false, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 1975, "seconds": 21.8 }, { "id": "2027-34", "rung": "profile", "klass": "IDENTITY", "truth": "EQUIVALENT", "mutant": 1, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 2, "out_tokens": 11270, "seconds": 122.8 }, { "id": "2027-4", "rung": "profile", "klass": "COVERED", "truth": "KILL", "mutant": 29, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 8838, "seconds": 93.7 }, { "id": "2027-7", "rung": "profile", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 88, "outcome": "REFUSED_PARSE", "reward": 0.0, "well_formed": 0.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 12000, "seconds": 138.9 } ] }, { "log": "environments/blind_spot/inspect/logs/2026-09-25T17-39-45-00-00_blind-spot-located_eSo3S6bWMMYM8TTaeg7M5j.json", "sha256": "aaf578ef7b0bdbbc126baab9c7ba9ed332e222803b76c104f9551af42053f2bd", "task": "blind_spot_located", "task_args": { "num_tasks": 12, "seed": 2027, "start": 0 }, "model": "anthropic/claude-opus-5", "control": false, "policy": null, "inspect_ai": "0.3.266", "status": "success", "ran": "2026-09-25T17:39:45+00:00", "config": { "effort": "low", "max_tokens": 12000, "reasoning_effort": null }, "concord": { "rollouts": 12, "disagreements": 0 }, "by_rung": { "located": { "n": 12, "solved": 10, "wrong": 0, "missed": 0, "undecided": 0, "refused_parse": 2, "false_claims": 0, "out_of_box_kills": 3, "mean_reward": 0.8333, "mean_reward_answered": 1.0, "errors": 0 } }, "by_class": { "COVERED": { "n": 6, "solved": 4 }, "IDENTITY": { "n": 1, "solved": 1 }, "MINT_ONLY": { "n": 1, "solved": 1 }, "NOCHANGE": { "n": 1, "solved": 1 }, "OUTBOX_ONLY": { "n": 3, "solved": 3 } }, "totals": { "n": 12, "solved": 10, "wrong": 0, "missed": 0, "undecided": 0, "refused_parse": 2, "false_claims": 0, "out_of_box_kills": 3, "mean_reward": 0.8333, "mean_reward_answered": 1.0, "errors": 0 }, "usage": { "input": 24, "output": 3480 }, "costUsd": 0.0871, "costNote": "errored rollouts report no usage, so this is a floor, not the bill", "rows": [ { "id": "2027-0", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 306, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 377, "seconds": 6.8 }, { "id": "2027-12", "rung": "located", "klass": "MINT_ONLY", "truth": "KILL", "mutant": 315, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 554, "seconds": 7.8 }, { "id": "2027-15", "rung": "located", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 60, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": false, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 329, "seconds": 4.9 }, { "id": "2027-18", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 356, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 323, "seconds": 4.9 }, { "id": "2027-21", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 360, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 403, "seconds": 5.6 }, { "id": "2027-24", "rung": "located", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 95, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": false, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 273, "seconds": 5.3 }, { "id": "2027-27", "rung": "located", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 350, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": false, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 405, "seconds": 6.2 }, { "id": "2027-3", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 161, "outcome": "REFUSED_PARSE", "reward": 0.0, "well_formed": 0.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 121, "seconds": 3.3 }, { "id": "2027-30", "rung": "located", "klass": "IDENTITY", "truth": "EQUIVALENT", "mutant": 1, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 46, "seconds": 2.3 }, { "id": "2027-33", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 148, "outcome": "REFUSED_PARSE", "reward": 0.0, "well_formed": 0.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 40, "seconds": 3.0 }, { "id": "2027-6", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 76, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 387, "seconds": 6.8 }, { "id": "2027-9", "rung": "located", "klass": "NOCHANGE", "truth": "EQUIVALENT", "mutant": 338, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 222, "seconds": 4.5 } ] }, { "log": "environments/blind_spot/inspect/logs/2026-09-25T17-39-45-00-00_blind-spot-profile_CkWh4uYeFodZfMDPCgx8Qp.json", "sha256": "167feac5491cdb8cf28c2f92809cc265adca2441e718eef3be37a811da7ab99c", "task": "blind_spot_profile", "task_args": { "num_tasks": 12, "seed": 2027, "start": 0 }, "model": "anthropic/claude-opus-5", "control": false, "policy": null, "inspect_ai": "0.3.266", "status": "success", "ran": "2026-09-25T17:39:45+00:00", "config": { "effort": "low", "max_tokens": 12000, "reasoning_effort": null }, "concord": { "rollouts": 12, "disagreements": 0 }, "by_rung": { "profile": { "n": 12, "solved": 2, "wrong": 0, "missed": 0, "undecided": 0, "refused_parse": 10, "false_claims": 0, "out_of_box_kills": 0, "mean_reward": 0.1667, "mean_reward_answered": 1.0, "errors": 0 } }, "by_class": { "ALIGNED_ONLY": { "n": 2, "solved": 0 }, "COVERED": { "n": 4, "solved": 0 }, "IDENTITY": { "n": 1, "solved": 0 }, "MINT_ONLY": { "n": 1, "solved": 0 }, "NOCHANGE": { "n": 2, "solved": 2 }, "OUTBOX_ONLY": { "n": 2, "solved": 0 } }, "totals": { "n": 12, "solved": 2, "wrong": 0, "missed": 0, "undecided": 0, "refused_parse": 10, "false_claims": 0, "out_of_box_kills": 0, "mean_reward": 0.1667, "mean_reward_answered": 1.0, "errors": 0 }, "usage": { "input": 24, "output": 955 }, "costUsd": 0.024, "costNote": "errored rollouts report no usage, so this is a floor, not the bill", "rows": [ { "id": "2027-1", "rung": "profile", "klass": "COVERED", "truth": "KILL", "mutant": 329, "outcome": "REFUSED_PARSE", "reward": 0.0, "well_formed": 0.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 0, "seconds": 1.3 }, { "id": "2027-10", "rung": "profile", "klass": "MINT_ONLY", "truth": "KILL", "mutant": 319, "outcome": "REFUSED_PARSE", "reward": 0.0, "well_formed": 0.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 0, "seconds": 1.4 }, { "id": "2027-13", "rung": "profile", "klass": "COVERED", "truth": "KILL", "mutant": 56, "outcome": "REFUSED_PARSE", "reward": 0.0, "well_formed": 0.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 0, "seconds": 1.3 }, { "id": "2027-16", "rung": "profile", "klass": "COVERED", "truth": "KILL", "mutant": 7, "outcome": "REFUSED_PARSE", "reward": 0.0, "well_formed": 0.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 0, "seconds": 1.7 }, { "id": "2027-19", "rung": "profile", "klass": "NOCHANGE", "truth": "EQUIVALENT", "mutant": 383, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 434, "seconds": 7.0 }, { "id": "2027-22", "rung": "profile", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 244, "outcome": "REFUSED_PARSE", "reward": 0.0, "well_formed": 0.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 0, "seconds": 1.6 }, { "id": "2027-25", "rung": "profile", "klass": "NOCHANGE", "truth": "EQUIVALENT", "mutant": 344, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 185, "seconds": 3.4 }, { "id": "2027-28", "rung": "profile", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 45, "outcome": "REFUSED_PARSE", "reward": 0.0, "well_formed": 0.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 0, "seconds": 1.4 }, { "id": "2027-31", "rung": "profile", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 60, "outcome": "REFUSED_PARSE", "reward": 0.0, "well_formed": 0.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 0, "seconds": 1.2 }, { "id": "2027-34", "rung": "profile", "klass": "IDENTITY", "truth": "EQUIVALENT", "mutant": 1, "outcome": "REFUSED_PARSE", "reward": 0.0, "well_formed": 0.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 336, "seconds": 5.4 }, { "id": "2027-4", "rung": "profile", "klass": "COVERED", "truth": "KILL", "mutant": 29, "outcome": "REFUSED_PARSE", "reward": 0.0, "well_formed": 0.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 0, "seconds": 1.3 }, { "id": "2027-7", "rung": "profile", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 88, "outcome": "REFUSED_PARSE", "reward": 0.0, "well_formed": 0.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 0, "seconds": 1.1 } ] }, { "log": "environments/blind_spot/inspect/logs/2026-09-25T17-39-46-00-00_blind-spot-blind_LwtTHv42KjtToKY9hoxv5U.json", "sha256": "6923758cc581008b299f511d0a2e01c36aecb2295b4c95a166a8e906d85c8999", "task": "blind_spot_blind", "task_args": { "num_tasks": 12, "seed": 2027, "start": 0 }, "model": "anthropic/claude-opus-5", "control": false, "policy": null, "inspect_ai": "0.3.266", "status": "success", "ran": "2026-09-25T17:39:46+00:00", "config": { "effort": "low", "max_tokens": 12000, "reasoning_effort": null }, "concord": { "rollouts": 12, "disagreements": 0 }, "by_rung": { "blind": { "n": 12, "solved": 4, "wrong": 3, "missed": 3, "undecided": 0, "refused_parse": 2, "false_claims": 3, "out_of_box_kills": 1, "mean_reward": 0.0833, "mean_reward_answered": 0.1, "errors": 0 } }, "by_class": { "ALIGNED_ONLY": { "n": 3, "solved": 0 }, "COVERED": { "n": 5, "solved": 4 }, "IDENTITY": { "n": 2, "solved": 0 }, "NOCHANGE": { "n": 1, "solved": 0 }, "OUTBOX_ONLY": { "n": 1, "solved": 0 } }, "totals": { "n": 12, "solved": 4, "wrong": 3, "missed": 3, "undecided": 0, "refused_parse": 2, "false_claims": 3, "out_of_box_kills": 1, "mean_reward": 0.0833, "mean_reward_answered": 0.1, "errors": 0 }, "usage": { "input": 24, "output": 11532 }, "costUsd": 0.2884, "costNote": "errored rollouts report no usage, so this is a floor, not the bill", "rows": [ { "id": "2027-11", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 145, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 888, "seconds": 10.3 }, { "id": "2027-14", "rung": "blind", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 45, "outcome": "REFUSED_PARSE", "reward": 0.0, "well_formed": 0.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 515, "seconds": 8.8 }, { "id": "2027-17", "rung": "blind", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 268, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 1187, "seconds": 14.3 }, { "id": "2027-2", "rung": "blind", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 61, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 1080, "seconds": 13.3 }, { "id": "2027-20", "rung": "blind", "klass": "NOCHANGE", "truth": "EQUIVALENT", "mutant": 97, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 2, "out_tokens": 1063, "seconds": 13.0 }, { "id": "2027-23", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 380, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 1071, "seconds": 13.3 }, { "id": "2027-26", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 260, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": false, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 1113, "seconds": 13.2 }, { "id": "2027-29", "rung": "blind", "klass": "IDENTITY", "truth": "EQUIVALENT", "mutant": 1, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 2, "out_tokens": 891, "seconds": 11.4 }, { "id": "2027-32", "rung": "blind", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 45, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 904, "seconds": 10.8 }, { "id": "2027-35", "rung": "blind", "klass": "IDENTITY", "truth": "EQUIVALENT", "mutant": 1, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 2, "out_tokens": 923, "seconds": 11.5 }, { "id": "2027-5", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 15, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 2, "out_tokens": 1137, "seconds": 13.8 }, { "id": "2027-8", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 347, "outcome": "REFUSED_PARSE", "reward": 0.0, "well_formed": 0.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 2, "out_tokens": 760, "seconds": 10.4 } ] }, { "log": "environments/blind_spot/inspect/logs/2026-09-25T17-40-57-00-00_blind-spot-blind_TG7qpuuXeb2n73tWcXkRqK.json", "sha256": "0c650f6a54deb7fadcd77f6f709719e2f8fdc7ae3f52992da7232629d0231624", "task": "blind_spot_blind", "task_args": { "num_tasks": 12, "seed": 2027, "start": 0 }, "model": "anthropic/claude-haiku-4-5-20251001", "control": false, "policy": null, "inspect_ai": "0.3.266", "status": "success", "ran": "2026-09-25T17:40:57+00:00", "config": { "effort": null, "max_tokens": 12000, "reasoning_effort": null }, "concord": { "rollouts": 12, "disagreements": 0 }, "by_rung": { "blind": { "n": 12, "solved": 4, "wrong": 3, "missed": 5, "undecided": 0, "refused_parse": 0, "false_claims": 3, "out_of_box_kills": 0, "mean_reward": 0.0833, "mean_reward_answered": 0.0833, "errors": 0 } }, "by_class": { "ALIGNED_ONLY": { "n": 3, "solved": 0 }, "COVERED": { "n": 5, "solved": 4 }, "IDENTITY": { "n": 2, "solved": 0 }, "NOCHANGE": { "n": 1, "solved": 0 }, "OUTBOX_ONLY": { "n": 1, "solved": 0 } }, "totals": { "n": 12, "solved": 4, "wrong": 3, "missed": 5, "undecided": 0, "refused_parse": 0, "false_claims": 3, "out_of_box_kills": 0, "mean_reward": 0.0833, "mean_reward_answered": 0.0833, "errors": 0 }, "usage": { "input": 10296, "output": 17838 }, "costUsd": 0.0995, "costNote": "errored rollouts report no usage, so this is a floor, not the bill", "rows": [ { "id": "2027-11", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 145, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 858, "out_tokens": 1524, "seconds": 10.6 }, { "id": "2027-14", "rung": "blind", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 45, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 858, "out_tokens": 1634, "seconds": 11.6 }, { "id": "2027-17", "rung": "blind", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 268, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 858, "out_tokens": 1568, "seconds": 10.5 }, { "id": "2027-2", "rung": "blind", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 61, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 858, "out_tokens": 1457, "seconds": 10.8 }, { "id": "2027-20", "rung": "blind", "klass": "NOCHANGE", "truth": "EQUIVALENT", "mutant": 97, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 858, "out_tokens": 1500, "seconds": 10.6 }, { "id": "2027-23", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 380, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 858, "out_tokens": 1281, "seconds": 9.7 }, { "id": "2027-26", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 260, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 858, "out_tokens": 1564, "seconds": 11.7 }, { "id": "2027-29", "rung": "blind", "klass": "IDENTITY", "truth": "EQUIVALENT", "mutant": 1, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 858, "out_tokens": 1574, "seconds": 10.3 }, { "id": "2027-32", "rung": "blind", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 45, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 858, "out_tokens": 1440, "seconds": 12.0 }, { "id": "2027-35", "rung": "blind", "klass": "IDENTITY", "truth": "EQUIVALENT", "mutant": 1, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 858, "out_tokens": 1472, "seconds": 10.2 }, { "id": "2027-5", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 15, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 858, "out_tokens": 1463, "seconds": 10.2 }, { "id": "2027-8", "rung": "blind", "klass": "COVERED", "truth": "KILL", "mutant": 347, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 858, "out_tokens": 1361, "seconds": 10.3 } ] }, { "log": "environments/blind_spot/inspect/logs/2026-09-25T17-40-57-00-00_blind-spot-located_mwmoxaSgCX3fSBCxxsiqhc.json", "sha256": "88d9c296ae20e07e682548edb3ed5d66235a198d3e073ba5e41cfe7e8032f6f2", "task": "blind_spot_located", "task_args": { "num_tasks": 12, "seed": 2027, "start": 0 }, "model": "anthropic/claude-haiku-4-5-20251001", "control": false, "policy": null, "inspect_ai": "0.3.266", "status": "success", "ran": "2026-09-25T17:40:57+00:00", "config": { "effort": null, "max_tokens": 12000, "reasoning_effort": null }, "concord": { "rollouts": 12, "disagreements": 0 }, "by_rung": { "located": { "n": 12, "solved": 3, "wrong": 2, "missed": 6, "undecided": 1, "refused_parse": 0, "false_claims": 2, "out_of_box_kills": 0, "mean_reward": 0.0833, "mean_reward_answered": 0.0833, "errors": 0 } }, "by_class": { "COVERED": { "n": 6, "solved": 2 }, "IDENTITY": { "n": 1, "solved": 1 }, "MINT_ONLY": { "n": 1, "solved": 0 }, "NOCHANGE": { "n": 1, "solved": 0 }, "OUTBOX_ONLY": { "n": 3, "solved": 0 } }, "totals": { "n": 12, "solved": 3, "wrong": 2, "missed": 6, "undecided": 1, "refused_parse": 0, "false_claims": 2, "out_of_box_kills": 0, "mean_reward": 0.0833, "mean_reward_answered": 0.0833, "errors": 0 }, "usage": { "input": 12888, "output": 11385 }, "costUsd": 0.0698, "costNote": "errored rollouts report no usage, so this is a floor, not the bill", "rows": [ { "id": "2027-0", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 306, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 1429, "out_tokens": 967, "seconds": 10.5 }, { "id": "2027-12", "rung": "located", "klass": "MINT_ONLY", "truth": "KILL", "mutant": 315, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 1134, "out_tokens": 973, "seconds": 10.0 }, { "id": "2027-15", "rung": "located", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 60, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 1127, "out_tokens": 1085, "seconds": 11.7 }, { "id": "2027-18", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 356, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 982, "out_tokens": 813, "seconds": 8.2 }, { "id": "2027-21", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 360, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 1025, "out_tokens": 1047, "seconds": 9.4 }, { "id": "2027-24", "rung": "located", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 95, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 1127, "out_tokens": 1221, "seconds": 12.4 }, { "id": "2027-27", "rung": "located", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 350, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 998, "out_tokens": 1011, "seconds": 10.5 }, { "id": "2027-3", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 161, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 998, "out_tokens": 928, "seconds": 9.7 }, { "id": "2027-30", "rung": "located", "klass": "IDENTITY", "truth": "EQUIVALENT", "mutant": 1, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 1.0, "error": null, "in_tokens": 861, "out_tokens": 66, "seconds": 1.3 }, { "id": "2027-33", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 148, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 1025, "out_tokens": 1351, "seconds": 13.4 }, { "id": "2027-6", "rung": "located", "klass": "COVERED", "truth": "KILL", "mutant": 76, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 1090, "out_tokens": 683, "seconds": 8.1 }, { "id": "2027-9", "rung": "located", "klass": "NOCHANGE", "truth": "EQUIVALENT", "mutant": 338, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 1092, "out_tokens": 1240, "seconds": 11.0 } ] }, { "log": "environments/blind_spot/inspect/logs/2026-09-25T17-40-57-00-00_blind-spot-profile_o2dwboFMpCrb4y8CXGjCiq.json", "sha256": "b08787a864aea94878075dd6e16d6f0e1718552e9c616cb6a7d42d00e4a01926", "task": "blind_spot_profile", "task_args": { "num_tasks": 12, "seed": 2027, "start": 0 }, "model": "anthropic/claude-haiku-4-5-20251001", "control": false, "policy": null, "inspect_ai": "0.3.266", "status": "success", "ran": "2026-09-25T17:40:57+00:00", "config": { "effort": null, "max_tokens": 12000, "reasoning_effort": null }, "concord": { "rollouts": 12, "disagreements": 0 }, "by_rung": { "profile": { "n": 12, "solved": 4, "wrong": 2, "missed": 5, "undecided": 1, "refused_parse": 0, "false_claims": 2, "out_of_box_kills": 2, "mean_reward": 0.1667, "mean_reward_answered": 0.1667, "errors": 0 } }, "by_class": { "ALIGNED_ONLY": { "n": 2, "solved": 0 }, "COVERED": { "n": 4, "solved": 2 }, "IDENTITY": { "n": 1, "solved": 0 }, "MINT_ONLY": { "n": 1, "solved": 0 }, "NOCHANGE": { "n": 2, "solved": 0 }, "OUTBOX_ONLY": { "n": 2, "solved": 2 } }, "totals": { "n": 12, "solved": 4, "wrong": 2, "missed": 5, "undecided": 1, "refused_parse": 0, "false_claims": 2, "out_of_box_kills": 2, "mean_reward": 0.1667, "mean_reward_answered": 0.1667, "errors": 0 }, "usage": { "input": 12552, "output": 14531 }, "costUsd": 0.0852, "costNote": "errored rollouts report no usage, so this is a floor, not the bill", "rows": [ { "id": "2027-1", "rung": "profile", "klass": "COVERED", "truth": "KILL", "mutant": 329, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 1046, "out_tokens": 1459, "seconds": 14.1 }, { "id": "2027-10", "rung": "profile", "klass": "MINT_ONLY", "truth": "KILL", "mutant": 319, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 1046, "out_tokens": 1085, "seconds": 10.4 }, { "id": "2027-13", "rung": "profile", "klass": "COVERED", "truth": "KILL", "mutant": 56, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 1046, "out_tokens": 1067, "seconds": 10.9 }, { "id": "2027-16", "rung": "profile", "klass": "COVERED", "truth": "KILL", "mutant": 7, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 1046, "out_tokens": 896, "seconds": 9.2 }, { "id": "2027-19", "rung": "profile", "klass": "NOCHANGE", "truth": "EQUIVALENT", "mutant": 383, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 1046, "out_tokens": 1454, "seconds": 13.2 }, { "id": "2027-22", "rung": "profile", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 244, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 1046, "out_tokens": 1300, "seconds": 10.5 }, { "id": "2027-25", "rung": "profile", "klass": "NOCHANGE", "truth": "EQUIVALENT", "mutant": 344, "outcome": "WRONG", "reward": -1.0, "well_formed": 1.0, "false_claim": 1.0, "in_box_kill": null, "logged_reward": -1.0, "error": null, "in_tokens": 1046, "out_tokens": 1528, "seconds": 14.1 }, { "id": "2027-28", "rung": "profile", "klass": "ALIGNED_ONLY", "truth": "KILL", "mutant": 45, "outcome": "MISSED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 1046, "out_tokens": 1276, "seconds": 11.6 }, { "id": "2027-31", "rung": "profile", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 60, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": false, "logged_reward": 1.0, "error": null, "in_tokens": 1046, "out_tokens": 978, "seconds": 8.2 }, { "id": "2027-34", "rung": "profile", "klass": "IDENTITY", "truth": "EQUIVALENT", "mutant": 1, "outcome": "UNDECIDED", "reward": 0.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": null, "logged_reward": 0.0, "error": null, "in_tokens": 1046, "out_tokens": 1259, "seconds": 12.0 }, { "id": "2027-4", "rung": "profile", "klass": "COVERED", "truth": "KILL", "mutant": 29, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": true, "logged_reward": 1.0, "error": null, "in_tokens": 1046, "out_tokens": 1211, "seconds": 12.1 }, { "id": "2027-7", "rung": "profile", "klass": "OUTBOX_ONLY", "truth": "KILL", "mutant": 88, "outcome": "SOLVED", "reward": 1.0, "well_formed": 1.0, "false_claim": 0.0, "in_box_kill": false, "logged_reward": 1.0, "error": null, "in_tokens": 1046, "out_tokens": 1018, "seconds": 8.3 } ] } ], "blocked": [ { "log": "environments/blind_spot/inspect/logs/blocked/2026-09-21T22-01-56-00-00_blind-spot-located_cuNxXNELEce2tWLeg2Kz2R.json", "sha256": "bc8f1e59f1b1e1272911c28c6cd633437086b3a9a413c263ac5aa9e919e7c550", "model": "anthropic/claude-sonnet-5", "task": "blind_spot_located", "status": "error", "ran": "2026-09-21T22:01:56+00:00", "samples_completed": 1, "error": "Error code: 400 - {\\'type\\': \\'error\\', \\'error\\': {\\'type\\': \\'invalid_request_error\\', \\'message\\': \\'Your credit balance is too low to access the Anthropic API. Please go to Plans & Billing to upgrade or purchase cred" } ], "baselines": { "path": "environments/blind_spot/baselines.json", "attempts": 0, "baseliners": 0, "status": "NO DATA", "by_rung": { "located": { "attempts": 0, "solved": 0, "baseliners": 0, "median_seconds": null, "median_seconds_solved": null }, "profile": { "attempts": 0, "solved": 0, "baseliners": 0, "median_seconds": null, "median_seconds_solved": null }, "blind": { "attempts": 0, "solved": 0, "baseliners": 0, "median_seconds": null, "median_seconds_solved": null } }, "tasks": 18 }, "pins": { "environments/blind_spot/inspect/task.py": "6187edad84ca59b504419cfe368b3f1017aa289e2311258142b2ebe5113272e3", "environments/blind_spot/baselines.json": "d014b488cf1754f06031366567aafee02445e7576e5818c589bfe781287fe3db", "environments/blind_spot/inspect/logs/2026-09-21T22-07-00-00-00_blind-spot_GeEYUW8Z3Fax5rX3Q94JsG.json": "45781b5dd4190793483863138844f66032011773b145a65c3860b90a444ad732", "environments/blind_spot/inspect/logs/2026-09-21T22-07-02-00-00_blind-spot_Ca3WwQCoPRRP3a7ACABpPG.json": "67028de999c8db92ca46eeb392e8427a7347a41e655aedb38c9ac764b158af1f", "environments/blind_spot/inspect/logs/2026-09-21T22-07-02-00-00_blind-spot_XcEZxxzHPCU2HZgLkCRNhp.json": "da2b1f9e867ef8b45335854b5ec216c3ef8ef80c2864ae6d4b19d30281718144", "environments/blind_spot/inspect/logs/2026-09-25T17-22-47-00-00_blind-spot-blind_nhxEY3MY3fgLdfAvVskWSX.json": "0ea624ee8378a664acb9bb8e7af815347870ade82764f8b0fd7157a757b4a368", "environments/blind_spot/inspect/logs/2026-09-25T17-22-47-00-00_blind-spot-located_id36sTPegphcYKpM3H8AwA.json": "2df8b0925b280393bca7c9e8d9db0915c26fed2e37cc0538f27dfddf9ddb4b4c", "environments/blind_spot/inspect/logs/2026-09-25T17-22-47-00-00_blind-spot-profile_6QHtgs6ejELWgrxu7UhRbN.json": "b9a1340fb457f89b709f167fa720b22d8ce709177cce1d4f4769a5311f5b7287", "environments/blind_spot/inspect/logs/2026-09-25T17-24-41-00-00_blind-spot-blind_eQAVvuKbVJGnwApZm2TYhX.json": "b8fb66d3ada2245b90888685fafa9f9c95d4b279ea17526d756cb5d09591b7d1", "environments/blind_spot/inspect/logs/2026-09-25T17-24-41-00-00_blind-spot-located_RnxSdoCh89MwN3zVXb4ikR.json": "c974377ce5e0afbd474d7eac6e86fa56781eac38622d537ec75c3b121e5d1fbf", "environments/blind_spot/inspect/logs/2026-09-25T17-24-41-00-00_blind-spot-profile_Mh5YE9Nd3WBp7s8mST3HFa.json": "ede2750982a083bb476c4cef0a1f3f19224657abc8e5aeee72fa749543e4ff63", "environments/blind_spot/inspect/logs/2026-09-25T17-28-54-00-00_blind-spot-blind_mjMbix5tfVdyqBSoYstmNC.json": "01c4b1388c528970be544c3d25665cf3d5f90228efebb2a8a862c859463754c7", "environments/blind_spot/inspect/logs/2026-09-25T17-28-54-00-00_blind-spot-located_Nz4S6hftZb9qkoyU7XrD38.json": "d9eaffc4f56a3602724fd9b908a07534f3d61707dc2e856942ea6dc3adce3a83", "environments/blind_spot/inspect/logs/2026-09-25T17-28-54-00-00_blind-spot-profile_UWoLQ3b3Hy4uUWgu4JD7ut.json": "f35ee2437601da44e181493cfcecde5417118eb72c14d552b140e88330aaea9b", "environments/blind_spot/inspect/logs/2026-09-25T17-39-45-00-00_blind-spot-located_eSo3S6bWMMYM8TTaeg7M5j.json": "aaf578ef7b0bdbbc126baab9c7ba9ed332e222803b76c104f9551af42053f2bd", "environments/blind_spot/inspect/logs/2026-09-25T17-39-45-00-00_blind-spot-profile_CkWh4uYeFodZfMDPCgx8Qp.json": "167feac5491cdb8cf28c2f92809cc265adca2441e718eef3be37a811da7ab99c", "environments/blind_spot/inspect/logs/2026-09-25T17-39-46-00-00_blind-spot-blind_LwtTHv42KjtToKY9hoxv5U.json": "6923758cc581008b299f511d0a2e01c36aecb2295b4c95a166a8e906d85c8999", "environments/blind_spot/inspect/logs/2026-09-25T17-40-57-00-00_blind-spot-blind_TG7qpuuXeb2n73tWcXkRqK.json": "0c650f6a54deb7fadcd77f6f709719e2f8fdc7ae3f52992da7232629d0231624", "environments/blind_spot/inspect/logs/2026-09-25T17-40-57-00-00_blind-spot-located_mwmoxaSgCX3fSBCxxsiqhc.json": "88d9c296ae20e07e682548edb3ed5d66235a198d3e073ba5e41cfe7e8032f6f2", "environments/blind_spot/inspect/logs/2026-09-25T17-40-57-00-00_blind-spot-profile_o2dwboFMpCrb4y8CXGjCiq.json": "b08787a864aea94878075dd6e16d6f0e1718552e9c616cb6a7d42d00e4a01926", "environments/blind_spot/inspect/logs/blocked/2026-09-21T22-01-56-00-00_blind-spot-located_cuNxXNELEce2tWLeg2Kz2R.json": "bc8f1e59f1b1e1272911c28c6cd633437086b3a9a413c263ac5aa9e919e7c550" }, "readme_must_quote": [ "`sat`", "+1.000", "+1.000", "+1.000", "`never`", "-0.667", "-0.500", "-0.500", "`abstain`", "+0.000", "+0.000", "+0.000", "claude-sonnet-5", "3/12", "claude-sonnet-5", "8/12", "claude-sonnet-5", "6/12", "claude-sonnet-5", "5/12", "claude-sonnet-5", "10/12", "claude-sonnet-5", "8/12", "claude-sonnet-5", "5/12", "claude-sonnet-5", "12/12", "claude-sonnet-5", "6/12", "claude-opus-5", "10/12", "claude-opus-5", "2/12", "claude-opus-5", "4/12", "claude-haiku-4-5-20251001", "4/12", "claude-haiku-4-5-20251001", "3/12", "claude-haiku-4-5-20251001", "4/12" ], "git": "fe73145", "builtOn": "2026-09-25 14:43:45 -0300" }