{
 "schema_version": 1,
 "generated": "2026-10-26",
 "id": "mll-lab-nu-embodied-first-exam--test-one-benchmark-suite",
 "repo": "mll-lab-nu/embodied-first-exam",
 "repo_url": "https://github.com/mll-lab-nu/embodied-first-exam",
 "repo_slug": "mll-lab-nu-embodied-first-exam",
 "title": "Test one benchmark suite with a single command",
 "page_title": "How to test one benchmark suite with a single command with Embodied First Exam",
 "summary": "Run only one suite, such as RoboARC, to check a setup before the full exam.",
 "who": "An engineer who has a GPU workstation and wants to confirm the setup works on one suite before spending time on all 2224 task directories.",
 "problem": "The full exam is large, and a setup problem found late wastes hours of image pulls and runs.",
 "outcome": "A short, verified run on one suite that shows whether the host, Docker, GPU, and model key are working.",
 "steps": [
  "Install and configure as in the core exam: clone the repository, run `uv tool install --force -e . --with-executables-from harbor`, and set your key in `.env`",
  "Check the GPU rules for your suite in the README table, for RoboARC an NVIDIA driver 580 or newer with one GPU per container",
  "Run one suite, here RoboARC with 196 task directories: `harbor run -d embodied-first-exam@all -p tasks/roboarc -a codex -m openai/gpt-6-astra`"
 ],
 "agent_prompt": "In the repo mll-lab-nu/embodied-first-exam, run only the suite at tasks/<suite folder> with the agent codex and the model <model id>. Report the pass count and any setup error from the Harbor log. Verify that the task count matches the number of task directories in that suite.",
 "agents": [
  "codex",
  "claude-code"
 ],
 "difficulty": "Intermediate",
 "time": "1 hour 30 minutes",
 "basis": "readme",
 "caveat": "RoboARC needs a 24 GB GPU for knot_untying and rope_threading, so a smaller card may fail on those tasks even when the rest of the suite runs.",
 "category": "Research",
 "tags": [
  "Research"
 ],
 "lang": "python",
 "requirements": [
  "Linux x86_64 host with git, uv, and Docker Engine with Compose v2 and BuildKit (or rootless podman with -e podman)",
  "NVIDIA driver and NVIDIA Container Toolkit for any GPU task",
  "An API key for the agent's model in .env, or a ChatGPT or Claude subscription",
  "Enough free disk for the benchmark images (1 to 150 GB depending on the suite)"
 ],
 "risk": "none",
 "seen_in_the_wild": false,
 "mentioned_publicly": false,
 "risks": [],
 "best": false,
 "score": 0.9,
 "evidence": [],
 "url": "https://dedupedai.com/use-cases/mll-lab-nu-embodied-first-exam/test-one-benchmark-suite/",
 "md": "https://dedupedai.com/use-cases/mll-lab-nu-embodied-first-exam/test-one-benchmark-suite/index.md",
 "disclaimer": "Deduped lists open-source repositories that belong to other people. We do not own, maintain, audit, or vouch for any of them, and a repo appearing here is not an endorsement or a safety check. The use cases, steps, and prompts are written by AI and may be incomplete, outdated, or just wrong. Where a use case is marked \"Seen in the wild\", we link to public posts where someone described something similar, but we have not verified those posts and the use case itself is still our AI's own write-up. Use all of this at your own risk. Only run code, commands, or agents on systems and accounts you are authorized to use, and read the code and every command before you run it, especially anything that touches credentials, files, or the network. Nothing here is professional, security, legal, or financial advice. All trademarks and project names belong to their respective owners, and their use here does not imply affiliation. Content is provided as is, without warranties of any kind. Found a mistake, a safety problem, or something you want removed? Email hello@dedupedai.com and we will look at it promptly.",
 "api": "https://dedupedai.com/api/v1/usecases/mll-lab-nu-embodied-first-exam--test-one-benchmark-suite.json",
 "api_version": "v1"
}