{
 "schema_version": 1,
 "generated": "2026-10-26",
 "id": "amap-ml-looparena--compare-controller-models-coding-tasks",
 "repo": "AMAP-ML/LoopArena",
 "repo_url": "https://github.com/AMAP-ML/LoopArena",
 "repo_slug": "amap-ml-looparena",
 "title": "Compare Controller models on paired coding tasks",
 "page_title": "How to compare Controller models on paired coding tasks with Loop Arena",
 "summary": "Run candidate Controllers on Type II and III coding tasks with a shared Worker.",
 "who": "A coding-agent team that must decide which model should act as the runtime Controller for long repository edits.",
 "problem": "Single-answer tests do not show whether a Controller can keep a Worker on track, verify progress, and stop at the right time across a full coding task.",
 "outcome": "Strict Success Rate results for each candidate Controller across the 80 paired tasks, each repeated 3 times, using source-native terminal evaluation.",
 "steps": [
  "Read docs/protocol.md to confirm the role boundaries, budgets, and scoring.",
  "Follow docs/setup.md to pin the upstream source revisions, prepare the task data and starting workspaces, and install the Docker runtimes.",
  "Set the Controller model and the shared Worker and Reporter model in the configuration described in docs/setup.md. (inferred, check the README)",
  "Run a single case first, then the full panel, using the single-case and panel run instructions in docs/setup.md.",
  "Compare the Strict Success Rate for each Controller against the Core-based scores in the README results table."
 ],
 "agent_prompt": "In the AMAP-ML/LoopArena repo, follow docs/setup.md to prepare the Type II coding task data and Docker runtimes, then run one single-case Type II test with a <Controller model ID> as Controller and the shared Worker and Reporter from the setup guide. Report the Strict Success Rate for that case and the path of the run output, and verify that the protocol and budget validity checks passed.",
 "agents": [
  "any"
 ],
 "difficulty": "Advanced",
 "time": "4 hours",
 "basis": "readme",
 "caveat": "The README does not give setup times or resource needs, and the upstream repositories and container images must be obtained from their original sources, so the time estimate is rough.",
 "category": "Research",
 "tags": [
  "Research"
 ],
 "lang": "python",
 "requirements": [
  "Python 3.10 or newer",
  "An OpenAI-compatible model endpoint, its base URL, and an API key",
  "The model ID you want to test",
  "For Type II/III only: the pinned upstream task data, starting workspaces, and Docker runtimes described in docs/setup.md"
 ],
 "risk": "none",
 "seen_in_the_wild": false,
 "mentioned_publicly": false,
 "risks": [],
 "best": false,
 "score": 0.9,
 "evidence": [],
 "url": "https://dedupedai.com/use-cases/amap-ml-looparena/compare-controller-models-coding-tasks/",
 "md": "https://dedupedai.com/use-cases/amap-ml-looparena/compare-controller-models-coding-tasks/index.md",
 "disclaimer": "Deduped lists open-source repositories that belong to other people. We do not own, maintain, audit, or vouch for any of them, and a repo appearing here is not an endorsement or a safety check. The use cases, steps, and prompts are written by AI and may be incomplete, outdated, or just wrong. Where a use case is marked \"Seen in the wild\", we link to public posts where someone described something similar, but we have not verified those posts and the use case itself is still our AI's own write-up. Use all of this at your own risk. Only run code, commands, or agents on systems and accounts you are authorized to use, and read the code and every command before you run it, especially anything that touches credentials, files, or the network. Nothing here is professional, security, legal, or financial advice. All trademarks and project names belong to their respective owners, and their use here does not imply affiliation. Content is provided as is, without warranties of any kind. Found a mistake, a safety problem, or something you want removed? Email hello@dedupedai.com and we will look at it promptly.",
 "api": "https://dedupedai.com/api/v1/usecases/amap-ml-looparena--compare-controller-models-coding-tasks.json",
 "api_version": "v1"
}