{
 "schema_version": 1,
 "generated": "2026-10-26",
 "id": "phuryn-bug-hunt-bench--audit-model-fix-claims-against-receipts",
 "repo": "phuryn/bug-hunt-bench",
 "repo_url": "https://github.com/phuryn/bug-hunt-bench",
 "repo_slug": "phuryn-bug-hunt-bench",
 "title": "Audit a model's bug-fix claims against the receipts",
 "page_title": "How to audit a model's bug-fix claims against the receipts with Bug Hunt Bench",
 "summary": "Check a leaderboard score against the raw run data before you trust it.",
 "who": "A reviewer or buyer who needs evidence behind a claim that one model fixes more bugs than another.",
 "problem": "A single number on a leaderboard can be hard to trust without seeing the runs that produced it.",
 "outcome": "A verified view of how a score was produced, using the raw results and the stated grading method.",
 "steps": [
  "Open the Method page at bughunt.productcompass.pm/method to read how diffs are graded blind against the withheld answer key.",
  "Open the raw data in the results/ folder on GitHub.",
  "Find the rows for the model you are checking and compare their run counts and scores with the README table.",
  "Note any model whose score comes from few runs, such as one run, before you rely on it."
 ],
 "agent_prompt": "In the bug-hunt-bench repo, open the results/ folder and find the raw data for <model name>. Report its run count, Fixed /105 score, and per-repo scores, and state whether they match the README leaderboard row. Verify the totals before you report.",
 "agents": [
  "any"
 ],
 "difficulty": "Intermediate",
 "time": "30 minutes",
 "basis": "readme",
 "caveat": "The answer key is withheld, so you can check the scores against the raw data but not re-grade the diffs yourself.",
 "category": "Software and DevOps",
 "tags": [
  "Software and DevOps"
 ],
 "lang": "javascript",
 "requirements": [
  "A web browser to open the live leaderboard at bughunt.productcompass.pm",
  "Access to the results/ folder on GitHub or a local copy of the repo for the raw data"
 ],
 "risk": "none",
 "seen_in_the_wild": false,
 "mentioned_publicly": false,
 "risks": [],
 "best": false,
 "score": 0.7,
 "evidence": [],
 "url": "https://dedupedai.com/use-cases/phuryn-bug-hunt-bench/audit-model-fix-claims-against-receipts/",
 "md": "https://dedupedai.com/use-cases/phuryn-bug-hunt-bench/audit-model-fix-claims-against-receipts/index.md",
 "disclaimer": "Deduped lists open-source repositories that belong to other people. We do not own, maintain, audit, or vouch for any of them, and a repo appearing here is not an endorsement or a safety check. The use cases, steps, and prompts are written by AI and may be incomplete, outdated, or just wrong. Where a use case is marked \"Seen in the wild\", we link to public posts where someone described something similar, but we have not verified those posts and the use case itself is still our AI's own write-up. Use all of this at your own risk. Only run code, commands, or agents on systems and accounts you are authorized to use, and read the code and every command before you run it, especially anything that touches credentials, files, or the network. Nothing here is professional, security, legal, or financial advice. All trademarks and project names belong to their respective owners, and their use here does not imply affiliation. Content is provided as is, without warranties of any kind. Found a mistake, a safety problem, or something you want removed? Email hello@dedupedai.com and we will look at it promptly.",
 "api": "https://dedupedai.com/api/v1/usecases/phuryn-bug-hunt-bench--audit-model-fix-claims-against-receipts.json",
 "api_version": "v1"
}