{
 "what": "verify-gym, the verifier in the loop (notes/verify-gym-preregistration-2026-10-31.md, deviation 7): for every model row in certs/verify-gym-scorecard.json, the 8 stored replies per held-out task turned into one answer three ways (first, majority, CERTIFIED-only and tightest), nothing new sampled; every chosen answer and every CERTIFIED sample decided by both deciders. Re-derived by environments/verify_gym/filter.py --check.",
 "preregistration": "notes/verify-gym-preregistration-2026-10-31.md (deviation 7)",
 "scorecard": "certs/verify-gym-scorecard.json",
 "strategies": {
  "first": "sample 0",
  "majority": "the bracket stated most often among the replies that state one (exact rationals), ties to the earliest first occurrence; no verifier, no answer key",
  "verify": "only samples both deciders call CERTIFIED; the most progress, then the narrower bracket, then the earlier sample; none certified: no answer"
 },
 "note": "the reward is the verifier's own number, so verify is the best of the 8 under this reward by definition; what it shows is that the best is reached with no answer key, and that first and majority state false brackets a reader cannot tell from true ones",
 "rule": "a reply cut off at the caller's max_tokens is no answer under every strategy, as in the scorecard",
 "headline": {
  "weakest": "deepseek-v4-pro",
  "strongest": "deepseek-v4.1-flash",
  "chosen_by": "the lowest and the highest mean reward per reply in the scorecard"
 },
 "crosscheck": {
  "decider2": "environments/verify_gym/check.js",
  "distinct_answers_compared": 2451,
  "disagreements": 0
 },
 "rows": [
  {
   "id": "deepseek-v4-pro",
   "model": "deepseek/deepseek-v4-pro",
   "thinking": "off (chat_template_kwargs enable_thinking false)",
   "row_mean_reward_per_reply": 0.049479,
   "strategies": {
    "first": {
     "tasks": 120,
     "certified": 37,
     "false_brackets_let_through": 70,
     "refused": 0,
     "malformed": 13,
     "no_answer": 0,
     "mean_reward": "197/4800",
     "mean_reward_float": 0.041042,
     "by_family": {
      "xx": {
       "tasks": 40,
       "certified": 7,
       "false_brackets_let_through": 24,
       "refused": 0,
       "malformed": 9,
       "no_answer": 0,
       "mean_reward": "13/320",
       "mean_reward_float": 0.040625
      },
      "kroot": {
       "tasks": 40,
       "certified": 17,
       "false_brackets_let_through": 23,
       "refused": 0,
       "malformed": 0,
       "no_answer": 0,
       "mean_reward": "9/160",
       "mean_reward_float": 0.05625
      },
      "poly": {
       "tasks": 40,
       "certified": 13,
       "false_brackets_let_through": 23,
       "refused": 0,
       "malformed": 4,
       "no_answer": 0,
       "mean_reward": "21/800",
       "mean_reward_float": 0.02625
      }
     }
    },
    "majority": {
     "tasks": 120,
     "certified": 54,
     "false_brackets_let_through": 56,
     "refused": 0,
     "malformed": 10,
     "no_answer": 0,
     "mean_reward": "31/480",
     "mean_reward_float": 0.064583,
     "by_family": {
      "xx": {
       "tasks": 40,
       "certified": 13,
       "false_brackets_let_through": 20,
       "refused": 0,
       "malformed": 7,
       "no_answer": 0,
       "mean_reward": "7/80",
       "mean_reward_float": 0.0875
      },
      "kroot": {
       "tasks": 40,
       "certified": 22,
       "false_brackets_let_through": 18,
       "refused": 0,
       "malformed": 0,
       "no_answer": 0,
       "mean_reward": "7/100",
       "mean_reward_float": 0.07
      },
      "poly": {
       "tasks": 40,
       "certified": 19,
       "false_brackets_let_through": 18,
       "refused": 0,
       "malformed": 3,
       "no_answer": 0,
       "mean_reward": "29/800",
       "mean_reward_float": 0.03625
      }
     }
    },
    "verify": {
     "tasks": 120,
     "certified": 112,
     "false_brackets_let_through": 0,
     "refused": 0,
     "malformed": 0,
     "no_answer": 8,
     "mean_reward": "839/4800",
     "mean_reward_float": 0.174792,
     "by_family": {
      "xx": {
       "tasks": 40,
       "certified": 38,
       "false_brackets_let_through": 0,
       "refused": 0,
       "malformed": 0,
       "no_answer": 2,
       "mean_reward": "89/320",
       "mean_reward_float": 0.278125
      },
      "kroot": {
       "tasks": 40,
       "certified": 39,
       "false_brackets_let_through": 0,
       "refused": 0,
       "malformed": 0,
       "no_answer": 1,
       "mean_reward": "63/400",
       "mean_reward_float": 0.1575
      },
      "poly": {
       "tasks": 40,
       "certified": 35,
       "false_brackets_let_through": 0,
       "refused": 0,
       "malformed": 0,
       "no_answer": 5,
       "mean_reward": "71/800",
       "mean_reward_float": 0.08875
      }
     }
    }
   },
   "chosen_sample": {
    "first": [
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0
    ],
    "majority": [
     0,
     0,
     0,
     0,
     0,
     0,
     5,
     0,
     0,
     0,
     2,
     3,
     0,
     4,
     0,
     0,
     0,
     0,
     0,
     3,
     0,
     2,
     0,
     0,
     0,
     0,
     3,
     4,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     1,
     0,
     2,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     5,
     1,
     1,
     2,
     0,
     0,
     0,
     0,
     0,
     3,
     0,
     0,
     0,
     0,
     0,
     1,
     0,
     0,
     0,
     0,
     1,
     1,
     0,
     2,
     0,
     2,
     5,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     1,
     0,
     5,
     0,
     0,
     0,
     0,
     0,
     0,
     1,
     0,
     0,
     0,
     0,
     2,
     0,
     0,
     0,
     0,
     2,
     2,
     0,
     0,
     2,
     0,
     0,
     1,
     0,
     0,
     0,
     1,
     2
    ],
    "verify": [
     2,
     null,
     2,
     7,
     3,
     null,
     5,
     0,
     6,
     0,
     1,
     3,
     5,
     3,
     1,
     2,
     7,
     1,
     5,
     3,
     1,
     2,
     2,
     5,
     5,
     6,
     3,
     2,
     4,
     4,
     2,
     6,
     4,
     6,
     0,
     4,
     2,
     1,
     7,
     2,
     null,
     7,
     2,
     2,
     2,
     4,
     0,
     5,
     2,
     1,
     4,
     7,
     5,
     1,
     1,
     6,
     1,
     2,
     7,
     1,
     6,
     0,
     0,
     4,
     0,
     0,
     1,
     4,
     0,
     1,
     4,
     3,
     1,
     5,
     7,
     2,
     3,
     5,
     1,
     7,
     1,
     7,
     2,
     3,
     7,
     0,
     1,
     0,
     0,
     4,
     null,
     0,
     5,
     7,
     null,
     null,
     0,
     null,
     0,
     6,
     1,
     6,
     2,
     6,
     null,
     0,
     6,
     1,
     7,
     4,
     0,
     1,
     5,
     1,
     4,
     5,
     5,
     6,
     1,
     1
    ]
   },
   "log": {
    "file": "corpus/verify-gym/runs/deepseek-v4-pro/rollouts.jsonl",
    "sha256": "a3387682e7e5b9b1781759326e76ee4646d904c4a553c5ad2058716eb39dbb2a"
   }
  },
  {
   "id": "deepseek-v4.1-flash",
   "model": "deepseek/deepseek-v4.1-flash",
   "thinking": "off (chat_template_kwargs enable_thinking false)",
   "row_mean_reward_per_reply": 0.099036,
   "strategies": {
    "first": {
     "tasks": 120,
     "certified": 40,
     "false_brackets_let_through": 71,
     "refused": 0,
     "malformed": 9,
     "no_answer": 0,
     "mean_reward": "107/1200",
     "mean_reward_float": 0.089167,
     "by_family": {
      "xx": {
       "tasks": 40,
       "certified": 10,
       "false_brackets_let_through": 22,
       "refused": 0,
       "malformed": 8,
       "no_answer": 0,
       "mean_reward": "3/32",
       "mean_reward_float": 0.09375
      },
      "kroot": {
       "tasks": 40,
       "certified": 15,
       "false_brackets_let_through": 25,
       "refused": 0,
       "malformed": 0,
       "no_answer": 0,
       "mean_reward": "99/800",
       "mean_reward_float": 0.12375
      },
      "poly": {
       "tasks": 40,
       "certified": 15,
       "false_brackets_let_through": 24,
       "refused": 0,
       "malformed": 1,
       "no_answer": 0,
       "mean_reward": "1/20",
       "mean_reward_float": 0.05
      }
     }
    },
    "majority": {
     "tasks": 120,
     "certified": 55,
     "false_brackets_let_through": 55,
     "refused": 0,
     "malformed": 10,
     "no_answer": 0,
     "mean_reward": "41/320",
     "mean_reward_float": 0.128125,
     "by_family": {
      "xx": {
       "tasks": 40,
       "certified": 12,
       "false_brackets_let_through": 19,
       "refused": 0,
       "malformed": 9,
       "no_answer": 0,
       "mean_reward": "41/320",
       "mean_reward_float": 0.128125
      },
      "kroot": {
       "tasks": 40,
       "certified": 24,
       "false_brackets_let_through": 16,
       "refused": 0,
       "malformed": 0,
       "no_answer": 0,
       "mean_reward": "3/16",
       "mean_reward_float": 0.1875
      },
      "poly": {
       "tasks": 40,
       "certified": 19,
       "false_brackets_let_through": 20,
       "refused": 0,
       "malformed": 1,
       "no_answer": 0,
       "mean_reward": "11/160",
       "mean_reward_float": 0.06875
      }
     }
    },
    "verify": {
     "tasks": 120,
     "certified": 114,
     "false_brackets_let_through": 0,
     "refused": 0,
     "malformed": 0,
     "no_answer": 6,
     "mean_reward": "763/2400",
     "mean_reward_float": 0.317917,
     "by_family": {
      "xx": {
       "tasks": 40,
       "certified": 35,
       "false_brackets_let_through": 0,
       "refused": 0,
       "malformed": 0,
       "no_answer": 5,
       "mean_reward": "33/80",
       "mean_reward_float": 0.4125
      },
      "kroot": {
       "tasks": 40,
       "certified": 40,
       "false_brackets_let_through": 0,
       "refused": 0,
       "malformed": 0,
       "no_answer": 0,
       "mean_reward": "73/200",
       "mean_reward_float": 0.365
      },
      "poly": {
       "tasks": 40,
       "certified": 39,
       "false_brackets_let_through": 0,
       "refused": 0,
       "malformed": 0,
       "no_answer": 1,
       "mean_reward": "141/800",
       "mean_reward_float": 0.17625
      }
     }
    }
   },
   "chosen_sample": {
    "first": [
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0
    ],
    "majority": [
     0,
     0,
     3,
     0,
     0,
     0,
     0,
     0,
     1,
     0,
     5,
     0,
     0,
     0,
     0,
     0,
     2,
     0,
     0,
     0,
     1,
     0,
     0,
     0,
     1,
     0,
     4,
     1,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     1,
     0,
     3,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     5,
     3,
     0,
     2,
     0,
     0,
     0,
     4,
     0,
     2,
     0,
     0,
     0,
     1,
     0,
     0,
     0,
     0,
     5,
     2,
     4,
     4,
     0,
     0,
     0,
     0,
     1,
     0,
     0,
     0,
     2,
     3,
     0,
     0,
     0,
     0,
     0,
     0,
     3,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     5,
     0,
     0,
     0,
     0,
     0,
     2,
     0,
     2,
     0,
     0,
     1,
     0,
     0,
     0,
     0,
     1,
     0,
     0,
     6
    ],
    "verify": [
     2,
     3,
     0,
     2,
     1,
     6,
     4,
     null,
     1,
     6,
     2,
     0,
     5,
     null,
     5,
     3,
     4,
     0,
     1,
     0,
     4,
     3,
     2,
     7,
     1,
     3,
     4,
     null,
     2,
     2,
     7,
     4,
     0,
     7,
     null,
     5,
     null,
     6,
     5,
     3,
     6,
     3,
     5,
     0,
     7,
     3,
     2,
     1,
     1,
     4,
     2,
     2,
     5,
     4,
     4,
     1,
     6,
     1,
     6,
     3,
     4,
     5,
     3,
     0,
     3,
     3,
     3,
     4,
     6,
     2,
     7,
     5,
     5,
     2,
     2,
     2,
     4,
     2,
     4,
     4,
     6,
     2,
     1,
     1,
     0,
     3,
     0,
     2,
     1,
     3,
     2,
     7,
     7,
     6,
     null,
     4,
     1,
     7,
     7,
     6,
     5,
     3,
     2,
     3,
     5,
     7,
     5,
     5,
     2,
     1,
     7,
     1,
     7,
     0,
     4,
     1,
     6,
     5,
     3,
     0
    ]
   },
   "log": {
    "file": "corpus/verify-gym/runs/deepseek-v4.1-flash/rollouts.jsonl",
    "sha256": "bb69dcf95542448095d35d24dd55be9fd6a5631095df9386d90e6f1a7af85f79"
   }
  },
  {
   "id": "gemma-4-26b-a4b-it",
   "model": "google/gemma-4-26b-a4b-it",
   "thinking": "off (chat_template_kwargs enable_thinking false)",
   "row_mean_reward_per_reply": 0.083073,
   "strategies": {
    "first": {
     "tasks": 120,
     "certified": 38,
     "false_brackets_let_through": 70,
     "refused": 0,
     "malformed": 11,
     "no_answer": 1,
     "mean_reward": "331/4800",
     "mean_reward_float": 0.068958,
     "by_family": {
      "xx": {
       "tasks": 40,
       "certified": 10,
       "false_brackets_let_through": 20,
       "refused": 0,
       "malformed": 10,
       "no_answer": 0,
       "mean_reward": "33/320",
       "mean_reward_float": 0.103125
      },
      "kroot": {
       "tasks": 40,
       "certified": 10,
       "false_brackets_let_through": 29,
       "refused": 0,
       "malformed": 1,
       "no_answer": 0,
       "mean_reward": "17/400",
       "mean_reward_float": 0.0425
      },
      "poly": {
       "tasks": 40,
       "certified": 18,
       "false_brackets_let_through": 21,
       "refused": 0,
       "malformed": 0,
       "no_answer": 1,
       "mean_reward": "49/800",
       "mean_reward_float": 0.06125
      }
     }
    },
    "majority": {
     "tasks": 120,
     "certified": 62,
     "false_brackets_let_through": 53,
     "refused": 0,
     "malformed": 5,
     "no_answer": 0,
     "mean_reward": "49/400",
     "mean_reward_float": 0.1225,
     "by_family": {
      "xx": {
       "tasks": 40,
       "certified": 17,
       "false_brackets_let_through": 18,
       "refused": 0,
       "malformed": 5,
       "no_answer": 0,
       "mean_reward": "3/16",
       "mean_reward_float": 0.1875
      },
      "kroot": {
       "tasks": 40,
       "certified": 18,
       "false_brackets_let_through": 22,
       "refused": 0,
       "malformed": 0,
       "no_answer": 0,
       "mean_reward": "7/80",
       "mean_reward_float": 0.0875
      },
      "poly": {
       "tasks": 40,
       "certified": 27,
       "false_brackets_let_through": 13,
       "refused": 0,
       "malformed": 0,
       "no_answer": 0,
       "mean_reward": "37/400",
       "mean_reward_float": 0.0925
      }
     }
    },
    "verify": {
     "tasks": 120,
     "certified": 103,
     "false_brackets_let_through": 0,
     "refused": 0,
     "malformed": 0,
     "no_answer": 17,
     "mean_reward": "1111/4800",
     "mean_reward_float": 0.231458,
     "by_family": {
      "xx": {
       "tasks": 40,
       "certified": 34,
       "false_brackets_let_through": 0,
       "refused": 0,
       "malformed": 0,
       "no_answer": 6,
       "mean_reward": "25/64",
       "mean_reward_float": 0.390625
      },
      "kroot": {
       "tasks": 40,
       "certified": 32,
       "false_brackets_let_through": 0,
       "refused": 0,
       "malformed": 0,
       "no_answer": 8,
       "mean_reward": "4/25",
       "mean_reward_float": 0.16
      },
      "poly": {
       "tasks": 40,
       "certified": 37,
       "false_brackets_let_through": 0,
       "refused": 0,
       "malformed": 0,
       "no_answer": 3,
       "mean_reward": "23/160",
       "mean_reward_float": 0.14375
      }
     }
    }
   },
   "chosen_sample": {
    "first": [
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0
    ],
    "majority": [
     6,
     2,
     3,
     0,
     0,
     0,
     0,
     0,
     0,
     2,
     4,
     0,
     4,
     1,
     0,
     1,
     0,
     0,
     0,
     2,
     0,
     0,
     6,
     6,
     1,
     5,
     0,
     3,
     0,
     1,
     0,
     1,
     4,
     2,
     1,
     5,
     2,
     6,
     0,
     1,
     0,
     0,
     0,
     5,
     0,
     5,
     0,
     4,
     1,
     0,
     3,
     0,
     2,
     3,
     0,
     0,
     1,
     0,
     0,
     0,
     0,
     3,
     2,
     4,
     0,
     2,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     3,
     2,
     0,
     2,
     0,
     2,
     0,
     2,
     3,
     3,
     1,
     0,
     1,
     0,
     0,
     0,
     1,
     0,
     0,
     0,
     0,
     0,
     1,
     0,
     2,
     0,
     0,
     0,
     2,
     0,
     0,
     1,
     1,
     0,
     0,
     2,
     0,
     0,
     1,
     0,
     1,
     4,
     1,
     1,
     0,
     3
    ],
    "verify": [
     6,
     2,
     2,
     6,
     4,
     3,
     7,
     6,
     null,
     3,
     4,
     5,
     4,
     1,
     0,
     0,
     null,
     6,
     null,
     null,
     5,
     0,
     1,
     5,
     1,
     5,
     0,
     4,
     null,
     0,
     3,
     1,
     4,
     null,
     2,
     5,
     2,
     6,
     2,
     4,
     2,
     6,
     6,
     7,
     null,
     null,
     2,
     1,
     4,
     2,
     3,
     null,
     2,
     3,
     1,
     0,
     null,
     6,
     5,
     0,
     null,
     0,
     1,
     4,
     4,
     7,
     null,
     4,
     1,
     7,
     1,
     6,
     null,
     4,
     3,
     2,
     null,
     2,
     4,
     0,
     3,
     2,
     2,
     3,
     3,
     null,
     1,
     3,
     2,
     6,
     1,
     2,
     7,
     5,
     null,
     3,
     1,
     0,
     2,
     0,
     5,
     0,
     0,
     3,
     3,
     0,
     0,
     0,
     4,
     0,
     1,
     7,
     3,
     null,
     6,
     7,
     1,
     3,
     0,
     7
    ]
   },
   "log": {
    "file": "corpus/verify-gym/runs/gemma-4-26b-a4b-it/rollouts.jsonl",
    "sha256": "541aded25d5bfdeb778c35321d2ab5a58212baf18e30ab8e1c2153b0dcb5b467"
   }
  },
  {
   "id": "qwen3.5-9b",
   "model": "qwen/qwen3.5-9b",
   "thinking": "off (chat_template_kwargs enable_thinking false)",
   "row_mean_reward_per_reply": 0.07526,
   "strategies": {
    "first": {
     "tasks": 120,
     "certified": 46,
     "false_brackets_let_through": 65,
     "refused": 0,
     "malformed": 7,
     "no_answer": 2,
     "mean_reward": "49/800",
     "mean_reward_float": 0.06125,
     "by_family": {
      "xx": {
       "tasks": 40,
       "certified": 6,
       "false_brackets_let_through": 29,
       "refused": 0,
       "malformed": 5,
       "no_answer": 0,
       "mean_reward": "9/160",
       "mean_reward_float": 0.05625
      },
      "kroot": {
       "tasks": 40,
       "certified": 17,
       "false_brackets_let_through": 21,
       "refused": 0,
       "malformed": 2,
       "no_answer": 0,
       "mean_reward": "27/400",
       "mean_reward_float": 0.0675
      },
      "poly": {
       "tasks": 40,
       "certified": 23,
       "false_brackets_let_through": 15,
       "refused": 0,
       "malformed": 0,
       "no_answer": 2,
       "mean_reward": "3/50",
       "mean_reward_float": 0.06
      }
     }
    },
    "majority": {
     "tasks": 120,
     "certified": 62,
     "false_brackets_let_through": 53,
     "refused": 0,
     "malformed": 5,
     "no_answer": 0,
     "mean_reward": "247/2400",
     "mean_reward_float": 0.102917,
     "by_family": {
      "xx": {
       "tasks": 40,
       "certified": 8,
       "false_brackets_let_through": 28,
       "refused": 0,
       "malformed": 4,
       "no_answer": 0,
       "mean_reward": "7/80",
       "mean_reward_float": 0.0875
      },
      "kroot": {
       "tasks": 40,
       "certified": 27,
       "false_brackets_let_through": 12,
       "refused": 0,
       "malformed": 1,
       "no_answer": 0,
       "mean_reward": "101/800",
       "mean_reward_float": 0.12625
      },
      "poly": {
       "tasks": 40,
       "certified": 27,
       "false_brackets_let_through": 13,
       "refused": 0,
       "malformed": 0,
       "no_answer": 0,
       "mean_reward": "19/200",
       "mean_reward_float": 0.095
      }
     }
    },
    "verify": {
     "tasks": 120,
     "certified": 107,
     "false_brackets_let_through": 0,
     "refused": 0,
     "malformed": 0,
     "no_answer": 13,
     "mean_reward": "1171/4800",
     "mean_reward_float": 0.243958,
     "by_family": {
      "xx": {
       "tasks": 40,
       "certified": 28,
       "false_brackets_let_through": 0,
       "refused": 0,
       "malformed": 0,
       "no_answer": 12,
       "mean_reward": "97/320",
       "mean_reward_float": 0.303125
      },
      "kroot": {
       "tasks": 40,
       "certified": 39,
       "false_brackets_let_through": 0,
       "refused": 0,
       "malformed": 0,
       "no_answer": 1,
       "mean_reward": "213/800",
       "mean_reward_float": 0.26625
      },
      "poly": {
       "tasks": 40,
       "certified": 40,
       "false_brackets_let_through": 0,
       "refused": 0,
       "malformed": 0,
       "no_answer": 0,
       "mean_reward": "13/80",
       "mean_reward_float": 0.1625
      }
     }
    }
   },
   "chosen_sample": {
    "first": [
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0
    ],
    "majority": [
     0,
     0,
     4,
     2,
     2,
     0,
     0,
     4,
     4,
     0,
     0,
     0,
     3,
     0,
     0,
     0,
     0,
     0,
     3,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     3,
     0,
     0,
     0,
     1,
     0,
     1,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     2,
     0,
     3,
     2,
     3,
     0,
     2,
     0,
     5,
     0,
     0,
     0,
     0,
     0,
     0,
     1,
     0,
     0,
     0,
     1,
     4,
     1,
     0,
     1,
     0,
     2,
     1,
     1,
     1,
     0,
     0,
     0,
     0,
     4,
     4,
     0,
     0,
     1,
     0,
     0,
     3,
     3,
     1,
     2,
     2,
     2,
     0,
     1,
     0,
     0,
     4,
     0,
     1,
     0,
     1,
     0,
     1,
     0,
     0,
     0,
     2,
     4,
     1,
     0,
     1,
     2,
     0,
     1,
     2,
     0,
     6,
     1,
     0,
     4,
     0,
     0,
     0,
     2
    ],
    "verify": [
     null,
     3,
     null,
     null,
     1,
     4,
     4,
     5,
     2,
     1,
     3,
     3,
     null,
     null,
     null,
     0,
     0,
     5,
     3,
     7,
     5,
     null,
     null,
     3,
     3,
     1,
     3,
     0,
     5,
     2,
     3,
     null,
     1,
     null,
     5,
     null,
     3,
     null,
     2,
     4,
     0,
     5,
     2,
     7,
     3,
     2,
     3,
     2,
     3,
     0,
     1,
     2,
     0,
     4,
     1,
     6,
     2,
     1,
     3,
     4,
     2,
     2,
     7,
     1,
     1,
     6,
     7,
     1,
     1,
     7,
     5,
     4,
     2,
     4,
     0,
     4,
     5,
     2,
     null,
     6,
     2,
     0,
     1,
     0,
     4,
     3,
     4,
     2,
     0,
     1,
     0,
     5,
     5,
     6,
     5,
     6,
     1,
     3,
     1,
     3,
     4,
     7,
     0,
     3,
     2,
     6,
     7,
     6,
     6,
     7,
     2,
     1,
     4,
     1,
     6,
     4,
     5,
     3,
     0,
     3
    ]
   },
   "log": {
    "file": "corpus/verify-gym/runs/qwen3.5-9b/rollouts.jsonl",
    "sha256": "f4f71ee8f1302f815f8df930aaab239b6c6b0d07a9f7b2c616a15a6a6585f79e"
   }
  },
  {
   "id": "gpt-oss-20b",
   "model": "openai/gpt-oss-20b",
   "thinking": "low (reasoning_effort low; gpt-oss cannot be off)",
   "row_mean_reward_per_reply": 0.053542,
   "strategies": {
    "first": {
     "tasks": 120,
     "certified": 65,
     "false_brackets_let_through": 50,
     "refused": 0,
     "malformed": 5,
     "no_answer": 0,
     "mean_reward": "59/1200",
     "mean_reward_float": 0.049167,
     "by_family": {
      "xx": {
       "tasks": 40,
       "certified": 17,
       "false_brackets_let_through": 21,
       "refused": 0,
       "malformed": 2,
       "no_answer": 0,
       "mean_reward": "11/160",
       "mean_reward_float": 0.06875
      },
      "kroot": {
       "tasks": 40,
       "certified": 21,
       "false_brackets_let_through": 19,
       "refused": 0,
       "malformed": 0,
       "no_answer": 0,
       "mean_reward": "1/25",
       "mean_reward_float": 0.04
      },
      "poly": {
       "tasks": 40,
       "certified": 27,
       "false_brackets_let_through": 10,
       "refused": 0,
       "malformed": 3,
       "no_answer": 0,
       "mean_reward": "31/800",
       "mean_reward_float": 0.03875
      }
     }
    },
    "majority": {
     "tasks": 120,
     "certified": 90,
     "false_brackets_let_through": 27,
     "refused": 0,
     "malformed": 3,
     "no_answer": 0,
     "mean_reward": "59/800",
     "mean_reward_float": 0.07375,
     "by_family": {
      "xx": {
       "tasks": 40,
       "certified": 23,
       "false_brackets_let_through": 16,
       "refused": 0,
       "malformed": 1,
       "no_answer": 0,
       "mean_reward": "19/160",
       "mean_reward_float": 0.11875
      },
      "kroot": {
       "tasks": 40,
       "certified": 32,
       "false_brackets_let_through": 8,
       "refused": 0,
       "malformed": 0,
       "no_answer": 0,
       "mean_reward": "9/160",
       "mean_reward_float": 0.05625
      },
      "poly": {
       "tasks": 40,
       "certified": 35,
       "false_brackets_let_through": 3,
       "refused": 0,
       "malformed": 2,
       "no_answer": 0,
       "mean_reward": "37/800",
       "mean_reward_float": 0.04625
      }
     }
    },
    "verify": {
     "tasks": 120,
     "certified": 120,
     "false_brackets_let_through": 0,
     "refused": 0,
     "malformed": 0,
     "no_answer": 0,
     "mean_reward": "47/320",
     "mean_reward_float": 0.146875,
     "by_family": {
      "xx": {
       "tasks": 40,
       "certified": 40,
       "false_brackets_let_through": 0,
       "refused": 0,
       "malformed": 0,
       "no_answer": 0,
       "mean_reward": "15/64",
       "mean_reward_float": 0.234375
      },
      "kroot": {
       "tasks": 40,
       "certified": 40,
       "false_brackets_let_through": 0,
       "refused": 0,
       "malformed": 0,
       "no_answer": 0,
       "mean_reward": "49/400",
       "mean_reward_float": 0.1225
      },
      "poly": {
       "tasks": 40,
       "certified": 40,
       "false_brackets_let_through": 0,
       "refused": 0,
       "malformed": 0,
       "no_answer": 0,
       "mean_reward": "67/800",
       "mean_reward_float": 0.08375
      }
     }
    }
   },
   "chosen_sample": {
    "first": [
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0
    ],
    "majority": [
     0,
     0,
     1,
     0,
     0,
     3,
     0,
     1,
     0,
     2,
     1,
     0,
     0,
     0,
     0,
     1,
     0,
     3,
     2,
     0,
     0,
     2,
     0,
     0,
     4,
     1,
     3,
     3,
     2,
     0,
     0,
     0,
     0,
     0,
     4,
     0,
     2,
     0,
     0,
     2,
     2,
     0,
     5,
     0,
     2,
     3,
     0,
     0,
     0,
     0,
     2,
     0,
     0,
     4,
     5,
     0,
     2,
     0,
     0,
     0,
     0,
     0,
     0,
     1,
     0,
     1,
     0,
     0,
     4,
     1,
     0,
     2,
     0,
     0,
     1,
     6,
     0,
     3,
     0,
     1,
     2,
     0,
     1,
     0,
     0,
     1,
     0,
     2,
     2,
     3,
     2,
     0,
     2,
     0,
     0,
     6,
     0,
     0,
     5,
     1,
     4,
     1,
     0,
     0,
     0,
     0,
     3,
     6,
     0,
     3,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     3,
     0,
     2
    ],
    "verify": [
     6,
     5,
     1,
     3,
     4,
     0,
     0,
     7,
     4,
     3,
     5,
     5,
     6,
     7,
     3,
     7,
     5,
     3,
     7,
     1,
     0,
     2,
     2,
     7,
     4,
     6,
     3,
     1,
     7,
     0,
     2,
     2,
     0,
     6,
     3,
     2,
     4,
     0,
     1,
     2,
     2,
     4,
     3,
     7,
     6,
     1,
     2,
     0,
     2,
     6,
     2,
     6,
     3,
     1,
     7,
     3,
     3,
     7,
     4,
     0,
     5,
     0,
     7,
     2,
     2,
     2,
     7,
     7,
     4,
     1,
     1,
     1,
     7,
     6,
     1,
     3,
     7,
     6,
     6,
     1,
     2,
     3,
     5,
     7,
     6,
     4,
     6,
     2,
     4,
     4,
     5,
     6,
     2,
     7,
     4,
     5,
     2,
     7,
     1,
     5,
     0,
     3,
     2,
     2,
     3,
     0,
     4,
     2,
     5,
     7,
     6,
     2,
     0,
     6,
     1,
     2,
     0,
     0,
     6,
     7
    ]
   },
   "log": {
    "file": "corpus/verify-gym/runs/gpt-oss-20b/rollouts.jsonl",
    "sha256": "e7266a7b42808ac8674d4578ebb56be7b382ac1c8e64726c631057d1d983883d"
   }
  },
  {
   "id": "gpt-oss-120b",
   "model": "openai/gpt-oss-120b",
   "thinking": "low (reasoning_effort low; gpt-oss cannot be off)",
   "row_mean_reward_per_reply": 0.094714,
   "strategies": {
    "first": {
     "tasks": 120,
     "certified": 77,
     "false_brackets_let_through": 35,
     "refused": 0,
     "malformed": 3,
     "no_answer": 5,
     "mean_reward": "467/4800",
     "mean_reward_float": 0.097292,
     "by_family": {
      "xx": {
       "tasks": 40,
       "certified": 21,
       "false_brackets_let_through": 16,
       "refused": 0,
       "malformed": 2,
       "no_answer": 1,
       "mean_reward": "9/64",
       "mean_reward_float": 0.140625
      },
      "kroot": {
       "tasks": 40,
       "certified": 27,
       "false_brackets_let_through": 11,
       "refused": 0,
       "malformed": 1,
       "no_answer": 1,
       "mean_reward": "19/200",
       "mean_reward_float": 0.095
      },
      "poly": {
       "tasks": 40,
       "certified": 29,
       "false_brackets_let_through": 8,
       "refused": 0,
       "malformed": 0,
       "no_answer": 3,
       "mean_reward": "9/160",
       "mean_reward_float": 0.05625
      }
     }
    },
    "majority": {
     "tasks": 120,
     "certified": 94,
     "false_brackets_let_through": 26,
     "refused": 0,
     "malformed": 0,
     "no_answer": 0,
     "mean_reward": "37/320",
     "mean_reward_float": 0.115625,
     "by_family": {
      "xx": {
       "tasks": 40,
       "certified": 22,
       "false_brackets_let_through": 18,
       "refused": 0,
       "malformed": 0,
       "no_answer": 0,
       "mean_reward": "47/320",
       "mean_reward_float": 0.146875
      },
      "kroot": {
       "tasks": 40,
       "certified": 34,
       "false_brackets_let_through": 6,
       "refused": 0,
       "malformed": 0,
       "no_answer": 0,
       "mean_reward": "49/400",
       "mean_reward_float": 0.1225
      },
      "poly": {
       "tasks": 40,
       "certified": 38,
       "false_brackets_let_through": 2,
       "refused": 0,
       "malformed": 0,
       "no_answer": 0,
       "mean_reward": "31/400",
       "mean_reward_float": 0.0775
      }
     }
    },
    "verify": {
     "tasks": 120,
     "certified": 120,
     "false_brackets_let_through": 0,
     "refused": 0,
     "malformed": 0,
     "no_answer": 0,
     "mean_reward": "399/1600",
     "mean_reward_float": 0.249375,
     "by_family": {
      "xx": {
       "tasks": 40,
       "certified": 40,
       "false_brackets_let_through": 0,
       "refused": 0,
       "malformed": 0,
       "no_answer": 0,
       "mean_reward": "21/64",
       "mean_reward_float": 0.328125
      },
      "kroot": {
       "tasks": 40,
       "certified": 40,
       "false_brackets_let_through": 0,
       "refused": 0,
       "malformed": 0,
       "no_answer": 0,
       "mean_reward": "117/400",
       "mean_reward_float": 0.2925
      },
      "poly": {
       "tasks": 40,
       "certified": 40,
       "false_brackets_let_through": 0,
       "refused": 0,
       "malformed": 0,
       "no_answer": 0,
       "mean_reward": "51/400",
       "mean_reward_float": 0.1275
      }
     }
    }
   },
   "chosen_sample": {
    "first": [
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0
    ],
    "majority": [
     0,
     0,
     4,
     0,
     0,
     0,
     0,
     5,
     0,
     3,
     0,
     0,
     5,
     1,
     1,
     0,
     0,
     2,
     0,
     3,
     1,
     0,
     1,
     0,
     0,
     0,
     0,
     0,
     3,
     0,
     1,
     0,
     0,
     0,
     3,
     1,
     3,
     0,
     2,
     0,
     4,
     2,
     1,
     2,
     0,
     0,
     0,
     3,
     1,
     1,
     0,
     0,
     0,
     0,
     1,
     0,
     1,
     0,
     1,
     0,
     1,
     3,
     0,
     0,
     2,
     0,
     1,
     2,
     3,
     1,
     4,
     0,
     2,
     6,
     2,
     0,
     1,
     0,
     0,
     0,
     0,
     0,
     0,
     2,
     0,
     1,
     0,
     3,
     0,
     2,
     1,
     0,
     3,
     0,
     0,
     1,
     2,
     2,
     0,
     2,
     1,
     0,
     0,
     1,
     0,
     3,
     0,
     0,
     5,
     0,
     3,
     2,
     6,
     0,
     3,
     0,
     0,
     0,
     5,
     4
    ],
    "verify": [
     7,
     4,
     6,
     3,
     5,
     0,
     4,
     4,
     4,
     5,
     4,
     4,
     6,
     7,
     6,
     4,
     1,
     1,
     7,
     2,
     0,
     2,
     1,
     1,
     7,
     1,
     4,
     5,
     4,
     4,
     7,
     4,
     3,
     0,
     0,
     6,
     4,
     1,
     0,
     6,
     2,
     0,
     6,
     0,
     2,
     2,
     3,
     6,
     3,
     6,
     7,
     6,
     0,
     7,
     2,
     2,
     3,
     7,
     1,
     5,
     1,
     1,
     2,
     4,
     6,
     3,
     1,
     1,
     4,
     2,
     3,
     5,
     0,
     2,
     0,
     5,
     5,
     3,
     7,
     5,
     1,
     0,
     2,
     5,
     7,
     4,
     3,
     0,
     4,
     2,
     2,
     6,
     3,
     2,
     7,
     0,
     4,
     2,
     0,
     1,
     6,
     7,
     6,
     1,
     0,
     5,
     3,
     5,
     3,
     6,
     2,
     4,
     1,
     0,
     4,
     2,
     5,
     1,
     4,
     1
    ]
   },
   "log": {
    "file": "corpus/verify-gym/runs/gpt-oss-120b/rollouts.jsonl",
    "sha256": "37afbac2be7e400c44b63411aaf3508e8fca23bddab3ad63cb0729640d428eaa"
   }
  },
  {
   "id": "qwen3.5-35b-a3b",
   "model": "qwen/qwen3.5-35b-a3b",
   "thinking": "off (chat_template_kwargs enable_thinking false)",
   "row_mean_reward_per_reply": 0.078698,
   "strategies": {
    "first": {
     "tasks": 120,
     "certified": 38,
     "false_brackets_let_through": 76,
     "refused": 0,
     "malformed": 5,
     "no_answer": 1,
     "mean_reward": "307/4800",
     "mean_reward_float": 0.063958,
     "by_family": {
      "xx": {
       "tasks": 40,
       "certified": 4,
       "false_brackets_let_through": 31,
       "refused": 0,
       "malformed": 5,
       "no_answer": 0,
       "mean_reward": "3/64",
       "mean_reward_float": 0.046875
      },
      "kroot": {
       "tasks": 40,
       "certified": 14,
       "false_brackets_let_through": 25,
       "refused": 0,
       "malformed": 0,
       "no_answer": 1,
       "mean_reward": "7/100",
       "mean_reward_float": 0.07
      },
      "poly": {
       "tasks": 40,
       "certified": 20,
       "false_brackets_let_through": 20,
       "refused": 0,
       "malformed": 0,
       "no_answer": 0,
       "mean_reward": "3/40",
       "mean_reward_float": 0.075
      }
     }
    },
    "majority": {
     "tasks": 120,
     "certified": 54,
     "false_brackets_let_through": 62,
     "refused": 0,
     "malformed": 4,
     "no_answer": 0,
     "mean_reward": "33/320",
     "mean_reward_float": 0.103125,
     "by_family": {
      "xx": {
       "tasks": 40,
       "certified": 5,
       "false_brackets_let_through": 31,
       "refused": 0,
       "malformed": 4,
       "no_answer": 0,
       "mean_reward": "19/320",
       "mean_reward_float": 0.059375
      },
      "kroot": {
       "tasks": 40,
       "certified": 23,
       "false_brackets_let_through": 17,
       "refused": 0,
       "malformed": 0,
       "no_answer": 0,
       "mean_reward": "59/400",
       "mean_reward_float": 0.1475
      },
      "poly": {
       "tasks": 40,
       "certified": 26,
       "false_brackets_let_through": 14,
       "refused": 0,
       "malformed": 0,
       "no_answer": 0,
       "mean_reward": "41/400",
       "mean_reward_float": 0.1025
      }
     }
    },
    "verify": {
     "tasks": 120,
     "certified": 101,
     "false_brackets_let_through": 0,
     "refused": 0,
     "malformed": 0,
     "no_answer": 19,
     "mean_reward": "101/400",
     "mean_reward_float": 0.2525,
     "by_family": {
      "xx": {
       "tasks": 40,
       "certified": 22,
       "false_brackets_let_through": 0,
       "refused": 0,
       "malformed": 0,
       "no_answer": 18,
       "mean_reward": "21/80",
       "mean_reward_float": 0.2625
      },
      "kroot": {
       "tasks": 40,
       "certified": 40,
       "false_brackets_let_through": 0,
       "refused": 0,
       "malformed": 0,
       "no_answer": 0,
       "mean_reward": "49/160",
       "mean_reward_float": 0.30625
      },
      "poly": {
       "tasks": 40,
       "certified": 39,
       "false_brackets_let_through": 0,
       "refused": 0,
       "malformed": 0,
       "no_answer": 1,
       "mean_reward": "151/800",
       "mean_reward_float": 0.18875
      }
     }
    }
   },
   "chosen_sample": {
    "first": [
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0,
     0
    ],
    "majority": [
     0,
     0,
     2,
     0,
     0,
     0,
     1,
     0,
     1,
     0,
     0,
     0,
     2,
     0,
     0,
     0,
     0,
     0,
     3,
     1,
     5,
     3,
     0,
     0,
     0,
     0,
     3,
     0,
     2,
     0,
     1,
     0,
     0,
     0,
     0,
     1,
     0,
     0,
     0,
     2,
     0,
     3,
     0,
     5,
     0,
     0,
     0,
     0,
     1,
     0,
     0,
     0,
     1,
     0,
     0,
     0,
     0,
     2,
     0,
     0,
     0,
     0,
     0,
     1,
     0,
     4,
     0,
     0,
     3,
     1,
     0,
     0,
     0,
     3,
     0,
     2,
     5,
     3,
     0,
     0,
     1,
     0,
     0,
     2,
     1,
     0,
     2,
     1,
     0,
     0,
     0,
     5,
     1,
     0,
     1,
     0,
     0,
     0,
     0,
     1,
     0,
     0,
     1,
     1,
     0,
     0,
     2,
     3,
     0,
     0,
     0,
     5,
     0,
     0,
     1,
     0,
     0,
     0,
     0,
     0
    ],
    "verify": [
     1,
     null,
     null,
     4,
     4,
     null,
     null,
     1,
     null,
     3,
     null,
     5,
     2,
     7,
     3,
     7,
     6,
     1,
     3,
     null,
     2,
     2,
     2,
     null,
     2,
     null,
     null,
     null,
     null,
     null,
     0,
     7,
     5,
     0,
     null,
     null,
     null,
     null,
     2,
     null,
     3,
     3,
     2,
     3,
     6,
     1,
     7,
     0,
     4,
     2,
     4,
     2,
     2,
     6,
     0,
     0,
     0,
     2,
     1,
     2,
     4,
     1,
     0,
     2,
     1,
     4,
     7,
     4,
     6,
     3,
     7,
     2,
     5,
     3,
     5,
     2,
     5,
     0,
     1,
     7,
     1,
     2,
     1,
     null,
     2,
     1,
     7,
     5,
     2,
     5,
     4,
     0,
     2,
     1,
     1,
     3,
     3,
     5,
     5,
     1,
     6,
     6,
     2,
     0,
     5,
     0,
     2,
     3,
     7,
     7,
     3,
     0,
     2,
     5,
     1,
     1,
     3,
     3,
     1,
     2
    ]
   },
   "log": {
    "file": "corpus/verify-gym/runs/qwen3.5-35b-a3b/rollouts.jsonl",
    "sha256": "a4db54ef2e53e1efd6ff606552de7bcf45688ffc0cb7620a923042a321b2db5c"
   }
  }
 ],
 "dataset": {
  "file": "corpus/verify-gym/verified-v0.jsonl.gz",
  "rows": 2953,
  "paid_rows": 2775,
  "text_sha256": "f7362cb61b582941d6fe6733a4b472b8f87de9578b21ee5c911538352f10215e",
  "gzip_sha256": "a8db22059a66e3f7b62c182df78e98bf384c5fbcd53d1b9840449505cedbbf09",
  "gzip_bytes": 6527380,
  "text_bytes": 23562304,
  "by_model": {
   "deepseek/deepseek-v4-pro": 331,
   "deepseek/deepseek-v4.1-flash": 368,
   "google/gemma-4-26b-a4b-it": 363,
   "openai/gpt-oss-120b": 618,
   "openai/gpt-oss-20b": 569,
   "qwen/qwen3.5-35b-a3b": 335,
   "qwen/qwen3.5-9b": 369
  },
  "by_family": {
   "kroot": 1142,
   "poly": 1185,
   "xx": 626
  },
  "recheck": {
   "verdict": "AGREES",
   "samples": 2775,
   "paid": 2775,
   "rechecked": 2775,
   "disagreements": 0
  },
  "schema": "corpus/verify-gym/verified-v0.README.md",
  "split": "heldout: never train a model that is evaluated on heldout-v0 on this file",
  "published": false
 }
}