{
 "schema_version": 1,
 "generated": "2026-10-20",
 "repo": "Simreal-AI/Simreal-MLBench",
 "name": "Simreal-MLBench",
 "owner": "Simreal-AI",
 "slug": "simreal-ai-simreal-mlbench",
 "github": "https://github.com/Simreal-AI/Simreal-MLBench",
 "what": "Simreal-MLBench is a public Python package with a 60-task agentic ML research benchmark, its scoring code, and a client SDK, while the official evaluation runs on a Simreal-operated service.",
 "why": "",
 "language": "Python",
 "requirements": [
  "Python 3.11 or newer (Python 3.12 is the locally tested version)",
  "A local copy of the repository from github.com/Simreal-AI/Simreal-MLBench",
  "A terminal with a shell that supports the commands in the README",
  "For full agent evaluation only: contact with business@simreal.co to get access to the Simreal evaluation service"
 ],
 "audience_note": "Written for ML researchers, benchmark reviewers, and AI labs that want to check or run agentic ML research evaluations.",
 "risk": "none",
 "section": "earlier",
 "weekly_rank": null,
 "category": "Research",
 "url": "https://dedupedai.com/use-cases/simreal-ai-simreal-mlbench/",
 "use_case_count": 4,
 "use_cases": [
  {
   "id": "simreal-ai-simreal-mlbench--validate-benchmark-catalog",
   "title": "Validate the 60-task benchmark catalog locally",
   "best": true,
   "url": "https://dedupedai.com/use-cases/simreal-ai-simreal-mlbench/validate-benchmark-catalog/",
   "json": "https://dedupedai.com/use-cases/u/simreal-ai-simreal-mlbench/validate-benchmark-catalog.json"
  },
  {
   "id": "simreal-ai-simreal-mlbench--recompute-published-scores",
   "title": "Recompute a published benchmark score from saved counts",
   "best": false,
   "url": "https://dedupedai.com/use-cases/simreal-ai-simreal-mlbench/recompute-published-scores/",
   "json": "https://dedupedai.com/use-cases/u/simreal-ai-simreal-mlbench/recompute-published-scores.json"
  },
  {
   "id": "simreal-ai-simreal-mlbench--evaluate-agent-via-simreal-service",
   "title": "Get an agent officially scored on the 60 tasks",
   "best": false,
   "url": "https://dedupedai.com/use-cases/simreal-ai-simreal-mlbench/evaluate-agent-via-simreal-service/",
   "json": "https://dedupedai.com/use-cases/u/simreal-ai-simreal-mlbench/evaluate-agent-via-simreal-service.json"
  },
  {
   "id": "simreal-ai-simreal-mlbench--check-scoring-code-with-tests",
   "title": "Run the scoring test suite before changing the rule",
   "best": false,
   "url": "https://dedupedai.com/use-cases/simreal-ai-simreal-mlbench/check-scoring-code-with-tests/",
   "json": "https://dedupedai.com/use-cases/u/simreal-ai-simreal-mlbench/check-scoring-code-with-tests.json"
  }
 ],
 "disclaimer": "Third-party repos we don't own or vouch for. AI-generated use cases can be wrong: review everything and run at your own risk."
}