{
 "schema_version": 1,
 "generated": "2026-10-26",
 "id": "noisyxl-brier--see-calibration-gap-in-four-seconds",
 "repo": "Noisyxl/brier",
 "repo_url": "https://github.com/Noisyxl/brier",
 "repo_slug": "noisyxl-brier",
 "title": "See a forecaster's calibration gap in four seconds",
 "page_title": "How to see a forecaster's calibration gap in four seconds with Brier",
 "summary": "Run the bundled demo and score it to see how stated confidence compares with actual hit rate.",
 "who": "A developer or analyst who wants to see how a forecaster's stated probabilities line up with real outcomes before trusting it.",
 "problem": "Forecasts are usually judged from memory, and a confident model can look right while being consistently overconfident.",
 "outcome": "A scoreboard showing each forecaster's said percentage, how often it was right, the gap between them, and its skill against the base rate.",
 "steps": [
  "Open a terminal in an empty folder with Node.js installed.",
  "Run `npx brier demo && npx brier score`.",
  "Read the row for each forecaster and compare the said column with the right column.",
  "Check the always-yes row to see how a trivial forecaster scores on the same question set."
 ],
 "agent_prompt": "Use the Noisyxl/brier repo. Run `npx brier demo && npx brier score` in a clean directory and report the said, right, gap and skill columns for each forecaster. Verify that the output includes an always-yes row before summarizing.",
 "agents": [
  "any"
 ],
 "difficulty": "Intermediate",
 "time": "15 minutes",
 "basis": "readme",
 "caveat": "The demo shows the ledger's own bundled questions and forecasters, so its numbers describe that set and do not generalize to your domain.",
 "category": "Research",
 "tags": [
  "Research"
 ],
 "lang": "html",
 "requirements": [
  "A terminal with Node.js and npx, since the README's commands run through npx",
  "An API key in a .env file, only if you want your own models on the board",
  "A wallet you control, only if you want to anchor the head hash on Robinhood Chain"
 ],
 "risk": "none",
 "seen_in_the_wild": false,
 "mentioned_publicly": false,
 "risks": [],
 "best": true,
 "score": 1.0,
 "evidence": [],
 "url": "https://dedupedai.com/use-cases/noisyxl-brier/see-calibration-gap-in-four-seconds/",
 "md": "https://dedupedai.com/use-cases/noisyxl-brier/see-calibration-gap-in-four-seconds/index.md",
 "disclaimer": "Deduped lists open-source repositories that belong to other people. We do not own, maintain, audit, or vouch for any of them, and a repo appearing here is not an endorsement or a safety check. The use cases, steps, and prompts are written by AI and may be incomplete, outdated, or just wrong. Where a use case is marked \"Seen in the wild\", we link to public posts where someone described something similar, but we have not verified those posts and the use case itself is still our AI's own write-up. Use all of this at your own risk. Only run code, commands, or agents on systems and accounts you are authorized to use, and read the code and every command before you run it, especially anything that touches credentials, files, or the network. Nothing here is professional, security, legal, or financial advice. All trademarks and project names belong to their respective owners, and their use here does not imply affiliation. Content is provided as is, without warranties of any kind. Found a mistake, a safety problem, or something you want removed? Email hello@dedupedai.com and we will look at it promptly.",
 "api": "https://dedupedai.com/api/v1/usecases/noisyxl-brier--see-calibration-gap-in-four-seconds.json",
 "api_version": "v1"
}