{
 "schema_version": 1,
 "generated": "2026-10-24",
 "id": "omni-scientist-awesome-ai-scientist--shortlist-benchmarks-for-science-agent",
 "repo": "Omni-Scientist/Awesome-AI-Scientist",
 "repo_url": "https://github.com/Omni-Scientist/Awesome-AI-Scientist",
 "repo_slug": "omni-scientist-awesome-ai-scientist",
 "title": "Shortlist benchmarks to test a science agent",
 "page_title": "How to shortlist benchmarks to test a science agent with Awesome AI Scientist",
 "summary": "Pick benchmarks and leaderboards that fit the science task your agent handles.",
 "who": "An ML engineer who has built or chosen an agent for a science task and needs fair tests before claiming it works.",
 "problem": "Choosing among dozens of benchmarks is slow, and it is unclear which ones match a given domain.",
 "outcome": "You have a short list of two or three benchmarks, each with a paper link and a leaderboard reference, ready for your evaluation plan.",
 "steps": [
  "Open the Benchmarks & Evaluation section and note entries that match your task area.",
  "Open the Leaderboards & Live Trackers section to see where results for those benchmarks are posted.",
  "Open each candidate's paper link and check what it measures and how it is scored.",
  "Write down the two or three benchmarks that best fit your agent's inputs and outputs."
 ],
 "agent_prompt": "Using the README of Omni-Scientist/Awesome-AI-Scientist, find the Benchmarks & Evaluation entries that match <science task>. For each match, give the name, paper link and any leaderboard listed in the Leaderboards & Live Trackers section, and state one scoring detail from the paper.",
 "agents": [
  "any"
 ],
 "difficulty": "Beginner",
 "time": "30 minutes",
 "basis": "readme",
 "caveat": "The list points to benchmark papers and leaderboards, so you still need to read each paper to confirm what it scores.",
 "category": "Research",
 "tags": [
  "Research"
 ],
 "lang": "python",
 "requirements": [
  "A web browser to read the README and follow GitHub links",
  "A GitHub account if you plan to clone or fork a linked repository",
  "A Python environment for running any linked project (the list is tagged Python)",
  "Time to read each linked project's own README, since this list does not contain install instructions"
 ],
 "risk": "dual-use",
 "seen_in_the_wild": false,
 "mentioned_publicly": false,
 "risks": [],
 "best": false,
 "score": 0.9,
 "evidence": [],
 "url": "https://dedupedai.com/use-cases/omni-scientist-awesome-ai-scientist/shortlist-benchmarks-for-science-agent/",
 "md": "https://dedupedai.com/use-cases/omni-scientist-awesome-ai-scientist/shortlist-benchmarks-for-science-agent/index.md",
 "disclaimer": "Deduped lists open-source repositories that belong to other people. We do not own, maintain, audit, or vouch for any of them, and a repo appearing here is not an endorsement or a safety check. The use cases, steps, and prompts are written by AI and may be incomplete, outdated, or just wrong. Where a use case is marked \"Seen in the wild\", we link to public posts where someone described something similar, but we have not verified those posts and the use case itself is still our AI's own write-up. Use all of this at your own risk. Only run code, commands, or agents on systems and accounts you are authorized to use, and read the code and every command before you run it, especially anything that touches credentials, files, or the network. Nothing here is professional, security, legal, or financial advice. All trademarks and project names belong to their respective owners, and their use here does not imply affiliation. Content is provided as is, without warranties of any kind. Found a mistake, a safety problem, or something you want removed? Email hello@dedupedai.com and we will look at it promptly.",
 "api": "https://dedupedai.com/api/v1/usecases/omni-scientist-awesome-ai-scientist--shortlist-benchmarks-for-science-agent.json",
 "api_version": "v1"
}