{
  "interval": "95% Wilson score interval; low and high are fractions",
  "done3": {
    "measured_at": "2026-10-11",
    "files": "https://github.com/Ludion-ai/ludion/tree/main/docs/bench/2026-10-11-done3",
    "questions": "FreshBench B questions, one per change",
    "tools": "Claude Code's default read-only tools, in a project with the new versions installed",
    "columns": [
      "agents",
      "vitest",
      "wrangler",
      "all"
    ],
    "rows": [
      {
        "model": "claude-opus-5-5",
        "name": "Claude Opus 5.5",
        "measured": true,
        "run": "B-project",
        "sync": false,
        "web_search": true,
        "cells": {
          "agents": {
            "correct": 11,
            "total": 15,
            "low": 0.48049565944019434,
            "high": 0.8910254667430763
          },
          "vitest": {
            "correct": 6,
            "total": 15,
            "low": 0.1982449614134158,
            "high": 0.642531698793754
          },
          "wrangler": {
            "correct": 7,
            "total": 15,
            "low": 0.24809535365737387,
            "high": 0.698830199745016
          },
          "all": {
            "correct": 24,
            "total": 45,
            "low": 0.3907745474012537,
            "high": 0.670648679349887
          }
        }
      },
      {
        "model": "claude-opus-5-5",
        "name": "Claude Opus 5.5",
        "measured": true,
        "run": "B-project-sync",
        "sync": true,
        "web_search": true,
        "cells": {
          "agents": {
            "correct": 14,
            "total": 15,
            "low": 0.7018347012621998,
            "high": 0.9881331045067316
          },
          "vitest": {
            "correct": 11,
            "total": 15,
            "low": 0.48049565944019434,
            "high": 0.8910254667430763
          },
          "wrangler": {
            "correct": 11,
            "total": 15,
            "low": 0.48049565944019434,
            "high": 0.8910254667430763
          },
          "all": {
            "correct": 36,
            "total": 45,
            "low": 0.6617703088385799,
            "high": 0.8910387319216871
          }
        }
      },
      {
        "model": "claude-haiku-5-5",
        "name": "Claude Haiku 5.5",
        "measured": false
      },
      {
        "model": "claude-sonnet-5-5",
        "name": "Claude Sonnet 5.5",
        "measured": false
      }
    ]
  },
  "without_tools": {
    "measured_at": "2026-10-11",
    "files": "https://github.com/Ludion-ai/ludion/tree/main/docs/bench/2026-10-11-models/",
    "method": "Each model answered each change's FreshBench A question twice, without tools (claude -p, no tools); an opus judge graded each answer against the verified key (docs/bench/2026-10-11-keys/). knows = both answers correct; misses = otherwise.",
    "unit": "lessons whose change the model knows (both answers correct) out of measured, active lessons",
    "columns": [
      "agents",
      "vitest",
      "wrangler",
      "all"
    ],
    "rows": [
      {
        "model": "claude-haiku-5-5",
        "name": "Claude Haiku 5.5",
        "measured": true,
        "run": "A-none",
        "sync": false,
        "web_search": false,
        "cells": {
          "agents": {
            "correct": 1,
            "total": 9,
            "low": 0.019890887638544918,
            "high": 0.434999705576656
          },
          "vitest": {
            "correct": 1,
            "total": 9,
            "low": 0.019890887638544918,
            "high": 0.434999705576656
          },
          "wrangler": {
            "correct": 0,
            "total": 11,
            "low": 0,
            "high": 0.2588329669680317
          },
          "all": {
            "correct": 2,
            "total": 29,
            "low": 0.01912126616943336,
            "high": 0.21964580150275695
          }
        }
      },
      {
        "model": "claude-sonnet-5-5",
        "name": "Claude Sonnet 5.5",
        "measured": true,
        "run": "A-none",
        "sync": false,
        "web_search": false,
        "cells": {
          "agents": {
            "correct": 1,
            "total": 9,
            "low": 0.019890887638544918,
            "high": 0.434999705576656
          },
          "vitest": {
            "correct": 0,
            "total": 9,
            "low": 0,
            "high": 0.2991450484195441
          },
          "wrangler": {
            "correct": 0,
            "total": 11,
            "low": 0,
            "high": 0.2588329669680317
          },
          "all": {
            "correct": 1,
            "total": 29,
            "low": 0.006113214292762649,
            "high": 0.17175521879320288
          }
        }
      },
      {
        "model": "claude-opus-5-5",
        "name": "Claude Opus 5.5",
        "measured": true,
        "run": "A-none",
        "sync": false,
        "web_search": false,
        "cells": {
          "agents": {
            "correct": 1,
            "total": 9,
            "low": 0.019890887638544918,
            "high": 0.434999705576656
          },
          "vitest": {
            "correct": 0,
            "total": 9,
            "low": 0,
            "high": 0.2991450484195441
          },
          "wrangler": {
            "correct": 0,
            "total": 11,
            "low": 0,
            "high": 0.2588329669680317
          },
          "all": {
            "correct": 1,
            "total": 29,
            "low": 0.006113214292762649,
            "high": 0.17175521879320288
          }
        }
      }
    ]
  }
}