{
 "what": "verify-gym v0 scorecard: the reference policies on the frozen held-out set, every answer decided exactly twice (verify_gym/exact.py and check.js, no shared code). Policies only; a model row is added by its own run on the operator's word. Re-derived by environments/verify_gym/scorecard.py --check.",
 "preregistration": "notes/verify-gym-preregistration-2026-10-31.md",
 "heldout": {
  "file": "environments/verify_gym/verify_gym/heldout-v0.json",
  "sha256": "789cead61a5cb4612589b511562d9a6bfb4fe7d2b7dd2dcfc80b94193e71978d",
  "n": 120,
  "per_family": 40
 },
 "reward": "progress / cap, progress = min(cap, max(0, d - d0)), d = max(0, floor(-log10(hi - lo))) when both sides are CERTIFIED; 0 for a REFUTED or REFUSED side, lo >= hi, an unreadable endpoint, or one over the size limit",
 "families": [
  {
   "id": "xx",
   "cap": 8,
   "limit": "10^5"
  },
  {
   "id": "kroot",
   "cap": 20,
   "limit": "10^30"
  },
  {
   "id": "poly",
   "cap": 20,
   "limit": "10^30"
  }
 ],
 "gate": {
  "rows": 120,
  "planted": 2240,
  "leaked": 0,
  "misfiled": 0,
  "green": 240,
  "green_failed": 0
 },
 "crosscheck": {
  "decider2": "environments/verify_gym/check.js (node, BigInt; Sturm for the isolation)",
  "answers_compared": 2600,
  "disagreements": 0,
  "format_malformed_not_sent": 480,
  "fields": [
   "verdict",
   "sides",
   "decimals",
   "progress"
  ]
 },
 "policies": [
  {
   "id": "never",
   "does": "no answer",
   "n": 120,
   "mean_reward": 0.0,
   "mean_progress": 0.0,
   "verdicts": {
    "CERTIFIED": 0,
    "REFUTED": 0,
    "REFUSED": 0,
    "MALFORMED": 120
   },
   "by_family": {
    "xx": {
     "n": 40,
     "mean_reward": 0.0,
     "mean_progress": 0.0,
     "verdicts": {
      "CERTIFIED": 0,
      "REFUTED": 0,
      "REFUSED": 0,
      "MALFORMED": 40
     }
    },
    "kroot": {
     "n": 40,
     "mean_reward": 0.0,
     "mean_progress": 0.0,
     "verdicts": {
      "CERTIFIED": 0,
      "REFUTED": 0,
      "REFUSED": 0,
      "MALFORMED": 40
     }
    },
    "poly": {
     "n": 40,
     "mean_reward": 0.0,
     "mean_progress": 0.0,
     "verdicts": {
      "CERTIFIED": 0,
      "REFUTED": 0,
      "REFUSED": 0,
      "MALFORMED": 40
     }
    }
   }
  },
  {
   "id": "echo",
   "does": "repeat the stated bracket",
   "n": 120,
   "mean_reward": 0.0,
   "mean_progress": 0.0,
   "verdicts": {
    "CERTIFIED": 120,
    "REFUTED": 0,
    "REFUSED": 0,
    "MALFORMED": 0
   },
   "by_family": {
    "xx": {
     "n": 40,
     "mean_reward": 0.0,
     "mean_progress": 0.0,
     "verdicts": {
      "CERTIFIED": 40,
      "REFUTED": 0,
      "REFUSED": 0,
      "MALFORMED": 0
     }
    },
    "kroot": {
     "n": 40,
     "mean_reward": 0.0,
     "mean_progress": 0.0,
     "verdicts": {
      "CERTIFIED": 40,
      "REFUTED": 0,
      "REFUSED": 0,
      "MALFORMED": 0
     }
    },
    "poly": {
     "n": 40,
     "mean_reward": 0.0,
     "mean_progress": 0.0,
     "verdicts": {
      "CERTIFIED": 40,
      "REFUTED": 0,
      "REFUSED": 0,
      "MALFORMED": 0
     }
    }
   }
  },
  {
   "id": "float",
   "does": "float64 root: its 16 significant digits +-1 in the last place (xx: aim at the cap, round to the nearest fraction the limit allows)",
   "n": 120,
   "mean_reward": 0.530833,
   "mean_progress": 8.216667,
   "verdicts": {
    "CERTIFIED": 85,
    "REFUTED": 22,
    "REFUSED": 0,
    "MALFORMED": 13
   },
   "by_family": {
    "xx": {
     "n": 40,
     "mean_reward": 0.6,
     "mean_progress": 4.8,
     "verdicts": {
      "CERTIFIED": 26,
      "REFUTED": 1,
      "REFUSED": 0,
      "MALFORMED": 13
     }
    },
    "kroot": {
     "n": 40,
     "mean_reward": 0.43,
     "mean_progress": 8.6,
     "verdicts": {
      "CERTIFIED": 27,
      "REFUTED": 13,
      "REFUSED": 0,
      "MALFORMED": 0
     }
    },
    "poly": {
     "n": 40,
     "mean_reward": 0.5625,
     "mean_progress": 11.25,
     "verdicts": {
      "CERTIFIED": 32,
      "REFUTED": 8,
      "REFUSED": 0,
      "MALFORMED": 0
     }
    }
   }
  },
  {
   "id": "bisect",
   "does": "20 steps of exact bisection (each midpoint decided exactly), fewer where the size limit stops it",
   "n": 120,
   "mean_reward": 0.340417,
   "mean_progress": 5.008333,
   "verdicts": {
    "CERTIFIED": 120,
    "REFUTED": 0,
    "REFUSED": 0,
    "MALFORMED": 0
   },
   "by_family": {
    "xx": {
     "n": 40,
     "mean_reward": 0.45,
     "mean_progress": 3.6,
     "verdicts": {
      "CERTIFIED": 40,
      "REFUTED": 0,
      "REFUSED": 0,
      "MALFORMED": 0
     }
    },
    "kroot": {
     "n": 40,
     "mean_reward": 0.3,
     "mean_progress": 6.0,
     "verdicts": {
      "CERTIFIED": 40,
      "REFUTED": 0,
      "REFUSED": 0,
      "MALFORMED": 0
     }
    },
    "poly": {
     "n": 40,
     "mean_reward": 0.27125,
     "mean_progress": 5.425,
     "verdicts": {
      "CERTIFIED": 40,
      "REFUTED": 0,
      "REFUSED": 0,
      "MALFORMED": 0
     }
    }
   }
  },
  {
   "id": "cf",
   "does": "continued fraction of a 100-digit estimate: the first convergent pair as tight as the cap, else the Farey neighbours the size limit allows",
   "n": 120,
   "mean_reward": 0.98125,
   "mean_progress": 15.85,
   "verdicts": {
    "CERTIFIED": 120,
    "REFUTED": 0,
    "REFUSED": 0,
    "MALFORMED": 0
   },
   "by_family": {
    "xx": {
     "n": 40,
     "mean_reward": 0.94375,
     "mean_progress": 7.55,
     "verdicts": {
      "CERTIFIED": 40,
      "REFUTED": 0,
      "REFUSED": 0,
      "MALFORMED": 0
     }
    },
    "kroot": {
     "n": 40,
     "mean_reward": 1.0,
     "mean_progress": 20.0,
     "verdicts": {
      "CERTIFIED": 40,
      "REFUTED": 0,
      "REFUSED": 0,
      "MALFORMED": 0
     }
    },
    "poly": {
     "n": 40,
     "mean_reward": 1.0,
     "mean_progress": 20.0,
     "verdicts": {
      "CERTIFIED": 40,
      "REFUTED": 0,
      "REFUSED": 0,
      "MALFORMED": 0
     }
    }
   }
  }
 ],
 "per_task": {
  "order": [
   "xx/heldout/0",
   "xx/heldout/1",
   "xx/heldout/2",
   "xx/heldout/3",
   "xx/heldout/4",
   "xx/heldout/5",
   "xx/heldout/6",
   "xx/heldout/7",
   "xx/heldout/8",
   "xx/heldout/9",
   "xx/heldout/10",
   "xx/heldout/11",
   "xx/heldout/12",
   "xx/heldout/13",
   "xx/heldout/14",
   "xx/heldout/15",
   "xx/heldout/16",
   "xx/heldout/17",
   "xx/heldout/18",
   "xx/heldout/19",
   "xx/heldout/20",
   "xx/heldout/21",
   "xx/heldout/22",
   "xx/heldout/23",
   "xx/heldout/24",
   "xx/heldout/25",
   "xx/heldout/26",
   "xx/heldout/27",
   "xx/heldout/28",
   "xx/heldout/29",
   "xx/heldout/30",
   "xx/heldout/31",
   "xx/heldout/32",
   "xx/heldout/33",
   "xx/heldout/34",
   "xx/heldout/35",
   "xx/heldout/36",
   "xx/heldout/37",
   "xx/heldout/38",
   "xx/heldout/39",
   "kroot/heldout/0",
   "kroot/heldout/1",
   "kroot/heldout/2",
   "kroot/heldout/3",
   "kroot/heldout/4",
   "kroot/heldout/5",
   "kroot/heldout/6",
   "kroot/heldout/7",
   "kroot/heldout/8",
   "kroot/heldout/9",
   "kroot/heldout/10",
   "kroot/heldout/11",
   "kroot/heldout/12",
   "kroot/heldout/13",
   "kroot/heldout/14",
   "kroot/heldout/15",
   "kroot/heldout/16",
   "kroot/heldout/17",
   "kroot/heldout/18",
   "kroot/heldout/19",
   "kroot/heldout/20",
   "kroot/heldout/21",
   "kroot/heldout/22",
   "kroot/heldout/23",
   "kroot/heldout/24",
   "kroot/heldout/25",
   "kroot/heldout/26",
   "kroot/heldout/27",
   "kroot/heldout/28",
   "kroot/heldout/29",
   "kroot/heldout/30",
   "kroot/heldout/31",
   "kroot/heldout/32",
   "kroot/heldout/33",
   "kroot/heldout/34",
   "kroot/heldout/35",
   "kroot/heldout/36",
   "kroot/heldout/37",
   "kroot/heldout/38",
   "kroot/heldout/39",
   "poly/heldout/0",
   "poly/heldout/1",
   "poly/heldout/2",
   "poly/heldout/3",
   "poly/heldout/4",
   "poly/heldout/5",
   "poly/heldout/6",
   "poly/heldout/7",
   "poly/heldout/8",
   "poly/heldout/9",
   "poly/heldout/10",
   "poly/heldout/11",
   "poly/heldout/12",
   "poly/heldout/13",
   "poly/heldout/14",
   "poly/heldout/15",
   "poly/heldout/16",
   "poly/heldout/17",
   "poly/heldout/18",
   "poly/heldout/19",
   "poly/heldout/20",
   "poly/heldout/21",
   "poly/heldout/22",
   "poly/heldout/23",
   "poly/heldout/24",
   "poly/heldout/25",
   "poly/heldout/26",
   "poly/heldout/27",
   "poly/heldout/28",
   "poly/heldout/29",
   "poly/heldout/30",
   "poly/heldout/31",
   "poly/heldout/32",
   "poly/heldout/33",
   "poly/heldout/34",
   "poly/heldout/35",
   "poly/heldout/36",
   "poly/heldout/37",
   "poly/heldout/38",
   "poly/heldout/39"
  ],
  "order_sha256": "fe07ec9c264946e76d0238168e3688713be9cd76accb6daa90b5fe39d2cc589f",
  "progress": {
   "never": [
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0
   ],
   "echo": [
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0,
    0
   ],
   "float": [
    8,
    0,
    7,
    8,
    7,
    8,
    7,
    6,
    0,
    8,
    0,
    8,
    7,
    0,
    8,
    0,
    7,
    0,
    7,
    7,
    0,
    8,
    7,
    0,
    7,
    7,
    8,
    0,
    0,
    8,
    7,
    0,
    7,
    7,
    8,
    8,
    0,
    0,
    0,
    7,
    0,
    13,
    9,
    14,
    0,
    13,
    0,
    13,
    13,
    12,
    12,
    0,
    12,
    13,
    13,
    12,
    0,
    13,
    13,
    14,
    0,
    12,
    12,
    13,
    13,
    14,
    0,
    14,
    0,
    13,
    13,
    0,
    0,
    12,
    13,
    13,
    0,
    0,
    0,
    13,
    14,
    0,
    14,
    14,
    14,
    0,
    14,
    14,
    14,
    14,
    0,
    15,
    14,
    13,
    0,
    0,
    14,
    0,
    14,
    0,
    14,
    14,
    14,
    15,
    14,
    14,
    14,
    14,
    14,
    0,
    14,
    14,
    14,
    14,
    15,
    14,
    14,
    14,
    14,
    14
   ],
   "bisect": [
    3,
    3,
    3,
    4,
    4,
    4,
    3,
    3,
    3,
    4,
    4,
    4,
    4,
    3,
    4,
    4,
    3,
    3,
    3,
    4,
    3,
    4,
    3,
    3,
    4,
    4,
    4,
    4,
    3,
    4,
    4,
    4,
    4,
    4,
    4,
    4,
    3,
    3,
    4,
    4,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    6,
    5,
    6,
    6,
    5,
    5,
    6,
    6,
    5,
    5,
    5,
    7,
    5,
    5,
    6,
    6,
    6,
    6,
    6,
    5,
    6,
    6,
    5,
    5,
    5,
    6,
    5,
    5,
    5,
    5,
    6,
    5,
    5,
    5,
    5,
    5,
    5,
    6,
    5,
    6,
    5
   ],
   "cf": [
    8,
    7,
    7,
    8,
    8,
    8,
    7,
    6,
    7,
    8,
    7,
    8,
    8,
    7,
    8,
    8,
    7,
    8,
    8,
    7,
    8,
    8,
    7,
    7,
    7,
    8,
    8,
    8,
    7,
    8,
    8,
    8,
    7,
    8,
    8,
    8,
    7,
    6,
    8,
    8,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20,
    20
   ]
  }
 },
 "models": [
  {
   "id": "deepseek-v4-pro",
   "model": "deepseek/deepseek-v4-pro",
   "provider": "Prime Inference",
   "endpoint": "https://api.pinference.ai/api/v1",
   "date": "2026-10-31",
   "started": "2026-10-31T17:04:30Z",
   "finished": "2026-10-31T17:15:18Z",
   "sampling": {
    "temperature": 1.0,
    "top_p": 1.0,
    "max_tokens": 16384,
    "thinking": "off (chat_template_kwargs enable_thinking false)",
    "tools": "none",
    "extra_body": {
     "chat_template_kwargs": {
      "enable_thinking": false
     }
    }
   },
   "samples_per_task": 8,
   "n": 960,
   "mean_reward": 0.049479,
   "mean_progress": 0.698958,
   "verdicts": {
    "CERTIFIED": 331,
    "REFUTED": 546,
    "REFUSED": 0,
    "MALFORMED": 83
   },
   "truncated": 0,
   "by_family": {
    "xx": {
     "n": 320,
     "mean_reward": 0.072656,
     "mean_progress": 0.58125,
     "verdicts": {
      "CERTIFIED": 90,
      "REFUTED": 186,
      "REFUSED": 0,
      "MALFORMED": 44
     },
     "truncated": 0
    },
    "kroot": {
     "n": 320,
     "mean_reward": 0.050781,
     "mean_progress": 1.015625,
     "verdicts": {
      "CERTIFIED": 139,
      "REFUTED": 166,
      "REFUSED": 0,
      "MALFORMED": 15
     },
     "truncated": 0
    },
    "poly": {
     "n": 320,
     "mean_reward": 0.025,
     "mean_progress": 0.5,
     "verdicts": {
      "CERTIFIED": 102,
      "REFUTED": 194,
      "REFUSED": 0,
      "MALFORMED": 24
     },
     "truncated": 0
    }
   },
   "refuted_share": 0.56875,
   "crosscheck": {
    "distinct_answers_compared": 910,
    "disagreements": 0
   },
   "spend": {
    "usage": {
     "prompt_tokens": 361672,
     "completion_tokens": 1670753,
     "cost_usd_reported": 6.7849,
     "samples_with_usage": 960
    },
    "prices_usd_per_1m": {
     "input": 1.91,
     "output": 3.83
    },
    "cost_usd_at_listed_prices": 7.0898
   },
   "log": {
    "file": "corpus/verify-gym/runs/deepseek-v4-pro/rollouts.jsonl",
    "sha256": "a3387682e7e5b9b1781759326e76ee4646d904c4a553c5ad2058716eb39dbb2a"
   },
   "per_task_mean_reward": [
    0.03125,
    0.0,
    0.109375,
    0.03125,
    0.171875,
    0.0,
    0.125,
    0.09375,
    0.0625,
    0.0625,
    0.234375,
    0.078125,
    0.078125,
    0.046875,
    0.0625,
    0.078125,
    0.0625,
    0.03125,
    0.109375,
    0.125,
    0.03125,
    0.078125,
    0.03125,
    0.03125,
    0.109375,
    0.046875,
    0.140625,
    0.09375,
    0.046875,
    0.09375,
    0.046875,
    0.0625,
    0.078125,
    0.046875,
    0.03125,
    0.078125,
    0.03125,
    0.046875,
    0.03125,
    0.15625,
    0.0,
    0.04375,
    0.09375,
    0.03125,
    0.03125,
    0.0875,
    0.0125,
    0.11875,
    0.025,
    0.025,
    0.0625,
    0.025,
    0.03125,
    0.06875,
    0.06875,
    0.05,
    0.05625,
    0.025,
    0.125,
    0.0375,
    0.01875,
    0.15625,
    0.01875,
    0.0875,
    0.1,
    0.08125,
    0.00625,
    0.0125,
    0.025,
    0.04375,
    0.06875,
    0.0375,
    0.05,
    0.08125,
    0.0375,
    0.05625,
    0.01875,
    0.0375,
    0.03125,
    0.04375,
    0.0375,
    0.025,
    0.01875,
    0.0125,
    0.01875,
    0.00625,
    0.00625,
    0.0375,
    0.05,
    0.01875,
    0.0,
    0.0125,
    0.0375,
    0.0,
    0.0,
    0.0,
    0.04375,
    0.0,
    0.0125,
    0.0125,
    0.0,
    0.01875,
    0.025,
    0.03125,
    0.0,
    0.05,
    0.0125,
    0.08125,
    0.06875,
    0.00625,
    0.025,
    0.075,
    0.01875,
    0.0625,
    0.0375,
    0.025,
    0.0375,
    0.01875,
    0.025,
    0.03125
   ]
  },
  {
   "id": "deepseek-v4.1-flash",
   "model": "deepseek/deepseek-v4.1-flash",
   "provider": "Prime Inference",
   "endpoint": "https://api.pinference.ai/api/v1",
   "date": "2026-10-31",
   "started": "2026-10-31T17:04:30Z",
   "finished": "2026-10-31T17:12:34Z",
   "sampling": {
    "temperature": 1.0,
    "top_p": 1.0,
    "max_tokens": 16384,
    "thinking": "off (chat_template_kwargs enable_thinking false)",
    "tools": "none",
    "extra_body": {
     "chat_template_kwargs": {
      "enable_thinking": false
     }
    }
   },
   "samples_per_task": 8,
   "n": 960,
   "mean_reward": 0.099036,
   "mean_progress": 1.613542,
   "verdicts": {
    "CERTIFIED": 368,
    "REFUTED": 517,
    "REFUSED": 0,
    "MALFORMED": 73
   },
   "truncated": 2,
   "by_family": {
    "xx": {
     "n": 320,
     "mean_reward": 0.091797,
     "mean_progress": 0.734375,
     "verdicts": {
      "CERTIFIED": 67,
      "REFUTED": 187,
      "REFUSED": 0,
      "MALFORMED": 65
     },
     "truncated": 1
    },
    "kroot": {
     "n": 320,
     "mean_reward": 0.145469,
     "mean_progress": 2.909375,
     "verdicts": {
      "CERTIFIED": 163,
      "REFUTED": 153,
      "REFUSED": 0,
      "MALFORMED": 3
     },
     "truncated": 1
    },
    "poly": {
     "n": 320,
     "mean_reward": 0.059844,
     "mean_progress": 1.196875,
     "verdicts": {
      "CERTIFIED": 138,
      "REFUTED": 177,
      "REFUSED": 0,
      "MALFORMED": 5
     },
     "truncated": 0
    }
   },
   "refuted_share": 0.538542,
   "crosscheck": {
    "distinct_answers_compared": 910,
    "disagreements": 0
   },
   "spend": {
    "usage": {
     "prompt_tokens": 361672,
     "completion_tokens": 3458329,
     "cost_usd_reported": 4.1919,
     "samples_with_usage": 960
    },
    "prices_usd_per_1m": {
     "input": 0.3,
     "output": 1.2
    },
    "cost_usd_at_listed_prices": 4.2585
   },
   "log": {
    "file": "corpus/verify-gym/runs/deepseek-v4.1-flash/rollouts.jsonl",
    "sha256": "bb69dcf95542448095d35d24dd55be9fd6a5631095df9386d90e6f1a7af85f79"
   },
   "per_task_mean_reward": [
    0.125,
    0.046875,
    0.109375,
    0.078125,
    0.046875,
    0.046875,
    0.109375,
    0.0,
    0.21875,
    0.046875,
    0.0625,
    0.171875,
    0.140625,
    0.0,
    0.109375,
    0.1875,
    0.03125,
    0.0625,
    0.09375,
    0.125,
    0.0625,
    0.046875,
    0.046875,
    0.046875,
    0.125,
    0.09375,
    0.125,
    0.0,
    0.0625,
    0.125,
    0.25,
    0.0625,
    0.078125,
    0.046875,
    0.0,
    0.15625,
    0.0,
    0.25,
    0.125,
    0.15625,
    0.09375,
    0.05625,
    0.14375,
    0.28125,
    0.275,
    0.0625,
    0.05625,
    0.63125,
    0.15625,
    0.10625,
    0.05625,
    0.04375,
    0.0625,
    0.09375,
    0.08125,
    0.0875,
    0.08125,
    0.15625,
    0.2375,
    0.15,
    0.2125,
    0.10625,
    0.125,
    0.04375,
    0.0875,
    0.775,
    0.15,
    0.3125,
    0.05,
    0.075,
    0.0375,
    0.0375,
    0.15625,
    0.11875,
    0.025,
    0.075,
    0.0875,
    0.025,
    0.19375,
    0.2125,
    0.075,
    0.05625,
    0.08125,
    0.03125,
    0.05,
    0.025,
    0.04375,
    0.075,
    0.0125,
    0.04375,
    0.00625,
    0.05,
    0.08125,
    0.0375,
    0.0,
    0.025,
    0.075,
    0.1125,
    0.0875,
    0.01875,
    0.06875,
    0.1125,
    0.0375,
    0.075,
    0.0125,
    0.06875,
    0.04375,
    0.0875,
    0.05,
    0.0125,
    0.0375,
    0.14375,
    0.03125,
    0.09375,
    0.0625,
    0.13125,
    0.1125,
    0.05625,
    0.05,
    0.11875
   ]
  },
  {
   "id": "gemma-4-26b-a4b-it",
   "model": "google/gemma-4-26b-a4b-it",
   "provider": "Prime Inference",
   "endpoint": "https://api.pinference.ai/api/v1",
   "date": "2026-10-31",
   "started": "2026-10-31T17:04:30Z",
   "finished": "2026-10-31T18:21:10Z",
   "sampling": {
    "temperature": 1.0,
    "top_p": 1.0,
    "max_tokens": 16384,
    "thinking": "off (chat_template_kwargs enable_thinking false)",
    "tools": "none",
    "extra_body": {
     "chat_template_kwargs": {
      "enable_thinking": false
     }
    }
   },
   "samples_per_task": 8,
   "n": 960,
   "mean_reward": 0.083073,
   "mean_progress": 1.183333,
   "verdicts": {
    "CERTIFIED": 363,
    "REFUTED": 520,
    "REFUSED": 0,
    "MALFORMED": 67
   },
   "truncated": 10,
   "by_family": {
    "xx": {
     "n": 320,
     "mean_reward": 0.119531,
     "mean_progress": 0.95625,
     "verdicts": {
      "CERTIFIED": 88,
      "REFUTED": 173,
      "REFUSED": 0,
      "MALFORMED": 58
     },
     "truncated": 1
    },
    "kroot": {
     "n": 320,
     "mean_reward": 0.059219,
     "mean_progress": 1.184375,
     "verdicts": {
      "CERTIFIED": 102,
      "REFUTED": 207,
      "REFUSED": 0,
      "MALFORMED": 8
     },
     "truncated": 3
    },
    "poly": {
     "n": 320,
     "mean_reward": 0.070469,
     "mean_progress": 1.409375,
     "verdicts": {
      "CERTIFIED": 173,
      "REFUTED": 140,
      "REFUSED": 0,
      "MALFORMED": 1
     },
     "truncated": 6
    }
   },
   "refuted_share": 0.541667,
   "crosscheck": {
    "distinct_answers_compared": 787,
    "disagreements": 0
   },
   "spend": {
    "usage": {
     "prompt_tokens": 406080,
     "completion_tokens": 2920682,
     "cost_usd_reported": 0.2436,
     "samples_with_usage": 960
    },
    "prices_usd_per_1m": {
     "input": 0.021,
     "output": 0.064
    },
    "cost_usd_at_listed_prices": 0.1955
   },
   "log": {
    "file": "corpus/verify-gym/runs/gemma-4-26b-a4b-it/rollouts.jsonl",
    "sha256": "541aded25d5bfdeb778c35321d2ab5a58212baf18e30ab8e1c2153b0dcb5b467"
   },
   "per_task_mean_reward": [
    0.171875,
    0.09375,
    0.0625,
    0.15625,
    0.203125,
    0.109375,
    0.109375,
    0.046875,
    0.0,
    0.125,
    0.21875,
    0.046875,
    0.09375,
    0.1875,
    0.125,
    0.09375,
    0.0,
    0.0625,
    0.0,
    0.0,
    0.109375,
    0.28125,
    0.0625,
    0.109375,
    0.09375,
    0.09375,
    0.296875,
    0.109375,
    0.0,
    0.171875,
    0.21875,
    0.28125,
    0.125,
    0.0,
    0.125,
    0.125,
    0.09375,
    0.171875,
    0.375,
    0.03125,
    0.01875,
    0.01875,
    0.04375,
    0.1875,
    0.0,
    0.0,
    0.0125,
    0.2125,
    0.0625,
    0.0125,
    0.05625,
    0.0,
    0.0375,
    0.075,
    0.04375,
    0.03125,
    0.0,
    0.01875,
    0.1125,
    0.03125,
    0.0,
    0.0125,
    0.1,
    0.08125,
    0.16875,
    0.275,
    0.0,
    0.025,
    0.01875,
    0.14375,
    0.13125,
    0.01875,
    0.0,
    0.08125,
    0.05,
    0.09375,
    0.0,
    0.05625,
    0.01875,
    0.11875,
    0.0875,
    0.05625,
    0.0625,
    0.075,
    0.10625,
    0.0,
    0.05625,
    0.0875,
    0.05625,
    0.1,
    0.0375,
    0.01875,
    0.14375,
    0.0375,
    0.0,
    0.0125,
    0.1375,
    0.06875,
    0.0875,
    0.075,
    0.01875,
    0.09375,
    0.05625,
    0.05625,
    0.05,
    0.1625,
    0.1125,
    0.10625,
    0.1,
    0.01875,
    0.05,
    0.1,
    0.05,
    0.0,
    0.075,
    0.13125,
    0.075,
    0.08125,
    0.14375,
    0.03125
   ]
  },
  {
   "id": "qwen3.5-9b",
   "model": "qwen/qwen3.5-9b",
   "provider": "Prime Inference",
   "endpoint": "https://api.pinference.ai/api/v1",
   "date": "2026-10-31",
   "started": "2026-10-31T17:04:30Z",
   "finished": "2026-10-31T19:15:10Z",
   "sampling": {
    "temperature": 1.0,
    "top_p": 1.0,
    "max_tokens": 16384,
    "thinking": "off (chat_template_kwargs enable_thinking false)",
    "tools": "none",
    "extra_body": {
     "chat_template_kwargs": {
      "enable_thinking": false
     }
    }
   },
   "samples_per_task": 8,
   "n": 960,
   "mean_reward": 0.07526,
   "mean_progress": 1.248958,
   "verdicts": {
    "CERTIFIED": 369,
    "REFUTED": 506,
    "REFUSED": 2,
    "MALFORMED": 59
   },
   "truncated": 24,
   "by_family": {
    "xx": {
     "n": 320,
     "mean_reward": 0.064062,
     "mean_progress": 0.5125,
     "verdicts": {
      "CERTIFIED": 50,
      "REFUTED": 218,
      "REFUSED": 2,
      "MALFORMED": 47
     },
     "truncated": 3
    },
    "kroot": {
     "n": 320,
     "mean_reward": 0.096562,
     "mean_progress": 1.93125,
     "verdicts": {
      "CERTIFIED": 156,
      "REFUTED": 145,
      "REFUSED": 0,
      "MALFORMED": 8
     },
     "truncated": 11
    },
    "poly": {
     "n": 320,
     "mean_reward": 0.065156,
     "mean_progress": 1.303125,
     "verdicts": {
      "CERTIFIED": 163,
      "REFUTED": 143,
      "REFUSED": 0,
      "MALFORMED": 4
     },
     "truncated": 10
    }
   },
   "refuted_share": 0.527083,
   "crosscheck": {
    "distinct_answers_compared": 856,
    "disagreements": 0
   },
   "spend": {
    "usage": {
     "prompt_tokens": 399840,
     "completion_tokens": 10860787,
     "cost_usd_reported": 2.5346,
     "samples_with_usage": 960
    },
    "prices_usd_per_1m": {
     "input": 0.17,
     "output": 0.25
    },
    "cost_usd_at_listed_prices": 2.7832
   },
   "log": {
    "file": "corpus/verify-gym/runs/qwen3.5-9b/rollouts.jsonl",
    "sha256": "f4f71ee8f1302f815f8df930aaab239b6c6b0d07a9f7b2c616a15a6a6585f79e"
   },
   "per_task_mean_reward": [
    0.0,
    0.046875,
    0.0,
    0.0,
    0.03125,
    0.109375,
    0.09375,
    0.0625,
    0.03125,
    0.046875,
    0.0625,
    0.0625,
    0.0,
    0.0,
    0.0,
    0.125,
    0.09375,
    0.03125,
    0.21875,
    0.046875,
    0.03125,
    0.0,
    0.0,
    0.046875,
    0.0625,
    0.078125,
    0.1875,
    0.1875,
    0.078125,
    0.15625,
    0.140625,
    0.0,
    0.171875,
    0.0,
    0.0625,
    0.0,
    0.109375,
    0.0,
    0.125,
    0.0625,
    0.0875,
    0.00625,
    0.15,
    0.09375,
    0.05625,
    0.075,
    0.0875,
    0.19375,
    0.1,
    0.0125,
    0.0625,
    0.01875,
    0.0125,
    0.10625,
    0.04375,
    0.00625,
    0.00625,
    0.10625,
    0.2375,
    0.04375,
    0.08125,
    0.2,
    0.225,
    0.1125,
    0.03125,
    0.31875,
    0.08125,
    0.05625,
    0.0125,
    0.09375,
    0.10625,
    0.04375,
    0.05625,
    0.3,
    0.01875,
    0.1,
    0.0875,
    0.075,
    0.0,
    0.35625,
    0.075,
    0.00625,
    0.10625,
    0.06875,
    0.09375,
    0.01875,
    0.05625,
    0.125,
    0.1,
    0.09375,
    0.01875,
    0.01875,
    0.0875,
    0.05625,
    0.0,
    0.00625,
    0.11875,
    0.00625,
    0.08125,
    0.04375,
    0.075,
    0.075,
    0.0875,
    0.1,
    0.0125,
    0.09375,
    0.0125,
    0.1,
    0.0875,
    0.0125,
    0.1125,
    0.09375,
    0.08125,
    0.06875,
    0.1,
    0.025,
    0.0625,
    0.11875,
    0.025,
    0.08125
   ]
  },
  {
   "id": "gpt-oss-20b",
   "model": "openai/gpt-oss-20b",
   "provider": "Prime Inference",
   "endpoint": "https://api.pinference.ai/api/v1",
   "date": "2026-11-01",
   "started": "2026-11-01T02:24:50Z",
   "finished": "2026-11-01T02:36:56Z",
   "sampling": {
    "temperature": 1.0,
    "top_p": 1.0,
    "max_tokens": 16384,
    "thinking": "low (reasoning_effort low; gpt-oss cannot be off)",
    "tools": "none",
    "extra_body": {
     "reasoning_effort": "low"
    }
   },
   "samples_per_task": 8,
   "n": 960,
   "mean_reward": 0.053542,
   "mean_progress": 0.755208,
   "verdicts": {
    "CERTIFIED": 569,
    "REFUTED": 354,
    "REFUSED": 0,
    "MALFORMED": 37
   },
   "truncated": 0,
   "by_family": {
    "xx": {
     "n": 320,
     "mean_reward": 0.078906,
     "mean_progress": 0.63125,
     "verdicts": {
      "CERTIFIED": 149,
      "REFUTED": 162,
      "REFUSED": 0,
      "MALFORMED": 9
     },
     "truncated": 0
    },
    "kroot": {
     "n": 320,
     "mean_reward": 0.046094,
     "mean_progress": 0.921875,
     "verdicts": {
      "CERTIFIED": 202,
      "REFUTED": 101,
      "REFUSED": 0,
      "MALFORMED": 17
     },
     "truncated": 0
    },
    "poly": {
     "n": 320,
     "mean_reward": 0.035625,
     "mean_progress": 0.7125,
     "verdicts": {
      "CERTIFIED": 218,
      "REFUTED": 91,
      "REFUSED": 0,
      "MALFORMED": 11
     },
     "truncated": 0
    }
   },
   "refuted_share": 0.36875,
   "crosscheck": {
    "distinct_answers_compared": 882,
    "disagreements": 0
   },
   "spend": {
    "usage": {
     "prompt_tokens": 418034,
     "completion_tokens": 1099310,
     "cost_usd_reported": 0.1844,
     "samples_with_usage": 960
    },
    "prices_usd_per_1m": {
     "input": 0.07,
     "output": 0.3
    },
    "cost_usd_at_listed_prices": 0.3591
   },
   "log": {
    "file": "corpus/verify-gym/runs/gpt-oss-20b/rollouts.jsonl",
    "sha256": "e7266a7b42808ac8674d4578ebb56be7b382ac1c8e64726c631057d1d983883d"
   },
   "per_task_mean_reward": [
    0.03125,
    0.015625,
    0.09375,
    0.140625,
    0.09375,
    0.15625,
    0.0625,
    0.0625,
    0.03125,
    0.0625,
    0.15625,
    0.015625,
    0.046875,
    0.015625,
    0.125,
    0.046875,
    0.046875,
    0.09375,
    0.09375,
    0.109375,
    0.0625,
    0.078125,
    0.03125,
    0.046875,
    0.078125,
    0.0625,
    0.078125,
    0.1875,
    0.109375,
    0.125,
    0.03125,
    0.09375,
    0.109375,
    0.15625,
    0.0625,
    0.0625,
    0.046875,
    0.0625,
    0.109375,
    0.0625,
    0.05,
    0.03125,
    0.04375,
    0.13125,
    0.05625,
    0.03125,
    0.05,
    0.13125,
    0.075,
    0.00625,
    0.0,
    0.025,
    0.01875,
    0.03125,
    0.025,
    0.025,
    0.04375,
    0.025,
    0.04375,
    0.075,
    0.03125,
    0.0625,
    0.0875,
    0.0375,
    0.05625,
    0.1375,
    0.025,
    0.0625,
    0.0,
    0.03125,
    0.01875,
    0.0,
    0.03125,
    0.06875,
    0.05625,
    0.06875,
    0.0125,
    0.01875,
    0.025,
    0.09375,
    0.03125,
    0.03125,
    0.075,
    0.00625,
    0.03125,
    0.01875,
    0.025,
    0.05625,
    0.03125,
    0.04375,
    0.0125,
    0.025,
    0.01875,
    0.00625,
    0.0,
    0.00625,
    0.05,
    0.025,
    0.075,
    0.0125,
    0.025,
    0.0875,
    0.06875,
    0.0125,
    0.0,
    0.0625,
    0.04375,
    0.05,
    0.06875,
    0.00625,
    0.04375,
    0.0375,
    0.0125,
    0.03125,
    0.03125,
    0.04375,
    0.05,
    0.08125,
    0.05,
    0.0375
   ]
  },
  {
   "id": "gpt-oss-120b",
   "model": "openai/gpt-oss-120b",
   "provider": "Prime Inference",
   "endpoint": "https://api.pinference.ai/api/v1",
   "date": "2026-11-01",
   "started": "2026-11-01T02:24:50Z",
   "finished": "2026-11-01T03:10:11Z",
   "sampling": {
    "temperature": 1.0,
    "top_p": 1.0,
    "max_tokens": 16384,
    "thinking": "low (reasoning_effort low; gpt-oss cannot be off)",
    "tools": "none",
    "extra_body": {
     "reasoning_effort": "low"
    }
   },
   "samples_per_task": 8,
   "n": 960,
   "mean_reward": 0.094714,
   "mean_progress": 1.420833,
   "verdicts": {
    "CERTIFIED": 618,
    "REFUTED": 295,
    "REFUSED": 0,
    "MALFORMED": 20
   },
   "truncated": 27,
   "by_family": {
    "xx": {
     "n": 320,
     "mean_reward": 0.118359,
     "mean_progress": 0.946875,
     "verdicts": {
      "CERTIFIED": 150,
      "REFUTED": 151,
      "REFUSED": 0,
      "MALFORMED": 15
     },
     "truncated": 4
    },
    "kroot": {
     "n": 320,
     "mean_reward": 0.1075,
     "mean_progress": 2.15,
     "verdicts": {
      "CERTIFIED": 243,
      "REFUTED": 73,
      "REFUSED": 0,
      "MALFORMED": 2
     },
     "truncated": 2
    },
    "poly": {
     "n": 320,
     "mean_reward": 0.058281,
     "mean_progress": 1.165625,
     "verdicts": {
      "CERTIFIED": 225,
      "REFUTED": 71,
      "REFUSED": 0,
      "MALFORMED": 3
     },
     "truncated": 21
    }
   },
   "refuted_share": 0.307292,
   "crosscheck": {
    "distinct_answers_compared": 823,
    "disagreements": 0
   },
   "spend": {
    "usage": {
     "prompt_tokens": 421042,
     "completion_tokens": 2974539,
     "cost_usd_reported": 1.7008,
     "samples_with_usage": 960
    },
    "prices_usd_per_1m": {
     "input": 0.35,
     "output": 0.75
    },
    "cost_usd_at_listed_prices": 2.3783
   },
   "log": {
    "file": "corpus/verify-gym/runs/gpt-oss-120b/rollouts.jsonl",
    "sha256": "37afbac2be7e400c44b63411aaf3508e8fca23bddab3ad63cb0729640d428eaa"
   },
   "per_task_mean_reward": [
    0.0625,
    0.015625,
    0.140625,
    0.09375,
    0.140625,
    0.125,
    0.078125,
    0.09375,
    0.125,
    0.140625,
    0.140625,
    0.03125,
    0.125,
    0.09375,
    0.28125,
    0.125,
    0.15625,
    0.109375,
    0.125,
    0.234375,
    0.046875,
    0.21875,
    0.109375,
    0.109375,
    0.109375,
    0.109375,
    0.125,
    0.1875,
    0.140625,
    0.09375,
    0.203125,
    0.109375,
    0.078125,
    0.125,
    0.109375,
    0.0625,
    0.140625,
    0.078125,
    0.078125,
    0.0625,
    0.0375,
    0.06875,
    0.15,
    0.225,
    0.125,
    0.03125,
    0.15,
    0.15625,
    0.0875,
    0.10625,
    0.025,
    0.04375,
    0.05,
    0.08125,
    0.025,
    0.1,
    0.1,
    0.05625,
    0.33125,
    0.08125,
    0.06875,
    0.125,
    0.2,
    0.075,
    0.09375,
    0.35625,
    0.04375,
    0.1125,
    0.0375,
    0.05625,
    0.1125,
    0.05,
    0.10625,
    0.225,
    0.06875,
    0.1,
    0.04375,
    0.025,
    0.10625,
    0.2625,
    0.075,
    0.0125,
    0.08125,
    0.025,
    0.05,
    0.00625,
    0.06875,
    0.09375,
    0.075,
    0.04375,
    0.03125,
    0.0125,
    0.0875,
    0.04375,
    0.00625,
    0.01875,
    0.04375,
    0.04375,
    0.05,
    0.05625,
    0.0875,
    0.09375,
    0.10625,
    0.025,
    0.0625,
    0.06875,
    0.0625,
    0.0875,
    0.0375,
    0.0375,
    0.0625,
    0.1,
    0.04375,
    0.05625,
    0.0375,
    0.09375,
    0.05625,
    0.1,
    0.11875,
    0.06875
   ]
  },
  {
   "id": "qwen3.5-35b-a3b",
   "model": "qwen/qwen3.5-35b-a3b",
   "provider": "Prime Inference",
   "endpoint": "https://api.pinference.ai/api/v1",
   "date": "2026-11-01",
   "started": "2026-11-01T02:24:50Z",
   "finished": "2026-11-01T03:40:37Z",
   "sampling": {
    "temperature": 1.0,
    "top_p": 1.0,
    "max_tokens": 16384,
    "thinking": "off (chat_template_kwargs enable_thinking false)",
    "tools": "none",
    "extra_body": {
     "chat_template_kwargs": {
      "enable_thinking": false
     }
    }
   },
   "samples_per_task": 8,
   "n": 960,
   "mean_reward": 0.078698,
   "mean_progress": 1.380208,
   "verdicts": {
    "CERTIFIED": 335,
    "REFUTED": 549,
    "REFUSED": 0,
    "MALFORMED": 73
   },
   "truncated": 3,
   "by_family": {
    "xx": {
     "n": 320,
     "mean_reward": 0.048438,
     "mean_progress": 0.3875,
     "verdicts": {
      "CERTIFIED": 32,
      "REFUTED": 223,
      "REFUSED": 0,
      "MALFORMED": 64
     },
     "truncated": 1
    },
    "kroot": {
     "n": 320,
     "mean_reward": 0.107969,
     "mean_progress": 2.159375,
     "verdicts": {
      "CERTIFIED": 137,
      "REFUTED": 176,
      "REFUSED": 0,
      "MALFORMED": 5
     },
     "truncated": 2
    },
    "poly": {
     "n": 320,
     "mean_reward": 0.079687,
     "mean_progress": 1.59375,
     "verdicts": {
      "CERTIFIED": 166,
      "REFUTED": 150,
      "REFUSED": 0,
      "MALFORMED": 4
     },
     "truncated": 0
    }
   },
   "refuted_share": 0.571875,
   "crosscheck": {
    "distinct_answers_compared": 883,
    "disagreements": 0
   },
   "spend": {
    "usage": {
     "prompt_tokens": 399840,
     "completion_tokens": 10265364,
     "cost_usd_reported": 13.4442,
     "samples_with_usage": 960
    },
    "prices_usd_per_1m": {
     "input": 0.25,
     "output": 1.3
    },
    "cost_usd_at_listed_prices": 13.4449
   },
   "log": {
    "file": "corpus/verify-gym/runs/qwen3.5-35b-a3b/rollouts.jsonl",
    "sha256": "a4db54ef2e53e1efd6ff606552de7bcf45688ffc0cb7620a923042a321b2db5c"
   },
   "per_task_mean_reward": [
    0.0625,
    0.0,
    0.0,
    0.125,
    0.0625,
    0.0,
    0.0,
    0.0625,
    0.0,
    0.0625,
    0.0,
    0.046875,
    0.1875,
    0.0625,
    0.046875,
    0.0625,
    0.0625,
    0.0625,
    0.125,
    0.0,
    0.046875,
    0.046875,
    0.0625,
    0.0,
    0.046875,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.125,
    0.0625,
    0.0625,
    0.125,
    0.0,
    0.0,
    0.0,
    0.0,
    0.328125,
    0.0,
    0.0625,
    0.05,
    0.03125,
    0.1875,
    0.16875,
    0.1125,
    0.05625,
    0.2625,
    0.1125,
    0.05625,
    0.05,
    0.04375,
    0.05625,
    0.075,
    0.05625,
    0.03125,
    0.09375,
    0.06875,
    0.25,
    0.0625,
    0.0375,
    0.09375,
    0.175,
    0.08125,
    0.01875,
    0.79375,
    0.05625,
    0.125,
    0.03125,
    0.125,
    0.10625,
    0.05625,
    0.0625,
    0.2875,
    0.0125,
    0.08125,
    0.0375,
    0.1125,
    0.0625,
    0.075,
    0.13125,
    0.0875,
    0.1125,
    0.0,
    0.125,
    0.01875,
    0.0875,
    0.1375,
    0.01875,
    0.0875,
    0.03125,
    0.08125,
    0.1,
    0.09375,
    0.025,
    0.00625,
    0.1,
    0.01875,
    0.1125,
    0.10625,
    0.025,
    0.1,
    0.125,
    0.09375,
    0.075,
    0.0875,
    0.06875,
    0.04375,
    0.06875,
    0.04375,
    0.13125,
    0.075,
    0.1,
    0.09375,
    0.09375,
    0.10625,
    0.0375,
    0.08125,
    0.13125,
    0.125
   ]
  }
 ]
}