{
 "schema_version": 1,
 "generated": "2026-10-20",
 "repo": "xinxuxin/keystone-bench",
 "name": "keystone-bench",
 "owner": "xinxuxin",
 "slug": "xinxuxin-keystone-bench",
 "github": "https://github.com/xinxuxin/keystone-bench",
 "what": "Keystone is a Python benchmark that tests whether a clinical chat assistant changes its action when one decisive fact is removed, replaced, or contradicted, built on HealthBench conversations.",
 "why": "",
 "language": "Python",
 "requirements": [
  "Python with pip to install keystone-bench from PyPI",
  "An API key for an OpenAI-compatible endpoint, such as OpenRouter, to run models and a judge from a different vendor",
  "Network access so that keystone build can fetch HealthBench from OpenAI's public copy and the label files",
  "Enough disk space and time for the local rebuild of the twins, since the release ships span references rather than benchmark prose"
 ],
 "audience_note": "Built for AI safety and evaluation researchers, clinical AI teams, and rubric authors who need to check whether a model's medical advice follows the evidence in the message.",
 "risk": "sensitive",
 "section": "earlier",
 "weekly_rank": null,
 "category": "Research",
 "url": "https://dedupedai.com/use-cases/xinxuxin-keystone-bench/",
 "use_case_count": 4,
 "use_cases": [
  {
   "id": "xinxuxin-keystone-bench--audit-assistant-decision-evidence",
   "title": "Audit a clinical chat assistant's response to missing evidence",
   "best": true,
   "url": "https://dedupedai.com/use-cases/xinxuxin-keystone-bench/audit-assistant-decision-evidence/",
   "json": "https://dedupedai.com/use-cases/u/xinxuxin-keystone-bench/audit-assistant-decision-evidence.json"
  },
  {
   "id": "xinxuxin-keystone-bench--check-what-a-new-version-regressed",
   "title": "Find which items a new assistant version fixed or broke",
   "best": false,
   "url": "https://dedupedai.com/use-cases/xinxuxin-keystone-bench/check-what-a-new-version-regressed/",
   "json": "https://dedupedai.com/use-cases/u/xinxuxin-keystone-bench/check-what-a-new-version-regressed.json"
  },
  {
   "id": "xinxuxin-keystone-bench--verify-benchmark-rebuild-hashes",
   "title": "Confirm a local rebuild matches the published benchmark",
   "best": false,
   "url": "https://dedupedai.com/use-cases/xinxuxin-keystone-bench/verify-benchmark-rebuild-hashes/",
   "json": "https://dedupedai.com/use-cases/u/xinxuxin-keystone-bench/verify-benchmark-rebuild-hashes.json"
  },
  {
   "id": "xinxuxin-keystone-bench--test-judge-agreement-on-replies",
   "title": "Measure agreement between two judges on the same replies",
   "best": false,
   "url": "https://dedupedai.com/use-cases/xinxuxin-keystone-bench/test-judge-agreement-on-replies/",
   "json": "https://dedupedai.com/use-cases/u/xinxuxin-keystone-bench/test-judge-agreement-on-replies.json"
  }
 ],
 "disclaimer": "Third-party repos we don't own or vouch for. AI-generated use cases can be wrong: review everything and run at your own risk."
}