{
 "schema_version": 1,
 "generated": "2026-10-26",
 "id": "kkakkkka-awesome-code-as-x--benchmark-coding-agent-on-robot-tasks",
 "repo": "kkakkkka/awesome-code-as-x",
 "repo_url": "https://github.com/kkakkkka/awesome-code-as-x",
 "repo_slug": "kkakkkka-awesome-code-as-x",
 "title": "Test a coding agent on robot tasks with CaP-X or RLE-Bench",
 "page_title": "How to test a coding agent on robot tasks with CaP-X or RLE-Bench with Awesome Code as X",
 "summary": "Pick a benchmark from the list to measure how well a coding agent writes robot code.",
 "who": "An ML researcher who wants to compare coding agents on robot learning or manipulation tasks before choosing one for a project.",
 "problem": "There is no single agreed test for whether a coding agent can write, run, and revise robot programs.",
 "outcome": "A chosen benchmark with its project page open, a clear list of its task types, and a plan for running the first evaluation.",
 "steps": [
  "Read the Benchmark section and the Robot evaluation infrastructure subsection.",
  "Open the RLE-Bench project page at rle-bench.github.io and note what the coding agent must do.",
  "Open the CaP-X project page at capgym.github.io and compare its robot manipulation tasks with RLE-Bench.",
  "Pick one benchmark and follow its own setup instructions, since the list itself gives no install steps."
 ],
 "agent_prompt": "Read the Benchmark section of the README in kkakkkka/awesome-code-as-x and compare RLE-Bench and CaP-X for a coding agent test on <robot task>. Expected output: a short comparison of task type, required inputs, and evaluation method for each benchmark. Verify that each claim is taken from the linked project page or the README.",
 "agents": [
  "any"
 ],
 "difficulty": "Intermediate",
 "time": "PT2H",
 "basis": "readme",
 "caveat": "The README describes the benchmarks only in one line each, so you must read the linked project pages for setup and metrics.",
 "category": "Research",
 "tags": [
  "Research"
 ],
 "lang": "",
 "requirements": [
  "A web browser to open the linked papers and project pages",
  "Basic familiarity with robot manipulation or simulation terms",
  "For reproduction: a GitHub account, git, and the hardware or simulator that a chosen project requires (the list does not state these)"
 ],
 "risk": "dual-use",
 "seen_in_the_wild": false,
 "mentioned_publicly": false,
 "risks": [],
 "best": false,
 "score": 0.9,
 "evidence": [],
 "url": "https://dedupedai.com/use-cases/kkakkkka-awesome-code-as-x/benchmark-coding-agent-on-robot-tasks/",
 "md": "https://dedupedai.com/use-cases/kkakkkka-awesome-code-as-x/benchmark-coding-agent-on-robot-tasks/index.md",
 "disclaimer": "Deduped lists open-source repositories that belong to other people. We do not own, maintain, audit, or vouch for any of them, and a repo appearing here is not an endorsement or a safety check. The use cases, steps, and prompts are written by AI and may be incomplete, outdated, or just wrong. Where a use case is marked \"Seen in the wild\", we link to public posts where someone described something similar, but we have not verified those posts and the use case itself is still our AI's own write-up. Use all of this at your own risk. Only run code, commands, or agents on systems and accounts you are authorized to use, and read the code and every command before you run it, especially anything that touches credentials, files, or the network. Nothing here is professional, security, legal, or financial advice. All trademarks and project names belong to their respective owners, and their use here does not imply affiliation. Content is provided as is, without warranties of any kind. Found a mistake, a safety problem, or something you want removed? Email hello@dedupedai.com and we will look at it promptly.",
 "api": "https://dedupedai.com/api/v1/usecases/kkakkkka-awesome-code-as-x--benchmark-coding-agent-on-robot-tasks.json",
 "api_version": "v1"
}