{
 "schema_version": 1,
 "generated": "2026-10-20",
 "id": "fstandhartinger-jevbench--test-option-order-sensitivity-before-deploy",
 "repo": "fstandhartinger/jevbench",
 "repo_url": "https://github.com/fstandhartinger/jevbench",
 "repo_slug": "fstandhartinger-jevbench",
 "title": "Test a decision model for option-order sensitivity",
 "page_title": "How to test a decision model for option-order sensitivity with Jevbench",
 "summary": "Run the public hard items with options in two orders to see whether your model's answers flip.",
 "who": "A machine learning engineer who is about to ship a small decision model and needs to know if its answers depend on how options are listed.",
 "problem": "The README reports that one small model scored 21 % instead of 72 % on answer-judging items when option order was reversed, so order can silently change results.",
 "outcome": "A comparison of answers under original and reversed option order, with a list of items where the model's choice changed.",
 "steps": [
  "Clone the repo with `git clone https://github.com/fstandhartinger/jevbench` and open the folder. (inferred, check the README)",
  "Open datasets/public/hard.jsonl, which holds 111 public hard items.",
  "Run your decision model on every item twice: once with the options in the original order and once with the options reversed.",
  "Compare the two sets of answers and count the items where the chosen answer changed.",
  "Compare your results with the reference runs in results/v1.2/runs/open-alternative-jev/ and the analysis in issue #40."
 ],
 "agent_prompt": "Clone github.com/fstandhartinger/jevbench and load datasets/public/hard.jsonl. Run <your decision model> on each item twice, once with the original option order and once with the options reversed, and output a table of items whose chosen answer changed with the total percentage of flips. Verify that the reference runs in results/v1.2/runs/open-alternative-jev/ are used only for comparison and that your report says the public split can be trained on, so these results are not a held-out score.",
 "agents": [
  "any"
 ],
 "difficulty": "Intermediate",
 "time": "1 hour",
 "basis": "readme",
 "caveat": "The public items can be trained on or selected against, so a flip test on them shows sensitivity but does not give a held-out accuracy score.",
 "category": "Research",
 "tags": [
  "Research"
 ],
 "lang": "python",
 "requirements": [
  "A web browser to read the public boards and release manifest",
  "Git and Python for the local repo steps",
  "Your decision model reachable from your own machine if you run the option-order test"
 ],
 "risk": "none",
 "seen_in_the_wild": false,
 "mentioned_publicly": false,
 "risks": [],
 "best": false,
 "score": 0.9,
 "evidence": [],
 "url": "https://dedupedai.com/use-cases/fstandhartinger-jevbench/test-option-order-sensitivity-before-deploy/",
 "md": "https://dedupedai.com/use-cases/fstandhartinger-jevbench/test-option-order-sensitivity-before-deploy/index.md",
 "disclaimer": "Deduped lists open-source repositories that belong to other people. We do not own, maintain, audit, or vouch for any of them, and a repo appearing here is not an endorsement or a safety check. The use cases, steps, and prompts are written by AI and may be incomplete, outdated, or just wrong. Where a use case is marked \"Seen in the wild\", we link to public posts where someone described something similar, but we have not verified those posts and the use case itself is still our AI's own write-up. Use all of this at your own risk. Only run code, commands, or agents on systems and accounts you are authorized to use, and read the code and every command before you run it, especially anything that touches credentials, files, or the network. Nothing here is professional, security, legal, or financial advice. All trademarks and project names belong to their respective owners, and their use here does not imply affiliation. Content is provided as is, without warranties of any kind. Found a mistake, a safety problem, or something you want removed? Email hello@dedupedai.com and we will look at it promptly.",
 "api": "https://dedupedai.com/api/v1/usecases/fstandhartinger-jevbench--test-option-order-sensitivity-before-deploy.json",
 "api_version": "v1"
}