{
 "schema_version": 1,
 "generated": "2026-10-26",
 "id": "amap-ml-looparena--smoke-test-controller-model-type1",
 "repo": "AMAP-ML/LoopArena",
 "repo_url": "https://github.com/AMAP-ML/LoopArena",
 "repo_slug": "amap-ml-looparena",
 "title": "Check a Controller model on Type I questions",
 "page_title": "How to check a Controller model on Type I questions with Loop Arena",
 "summary": "Run one model on the 160 Type I Loop Contract questions without a Worker.",
 "who": "An AI engineer who wants to see whether a candidate model can pick the right Loop Contract before a longer benchmark run.",
 "problem": "Type II and III need Docker, task data, and many runs. A quick Type I check tells you if the model can answer the basic Controller decision.",
 "outcome": "A JSONL output file with the model's answer to a single Type I question, after a preflight check of the endpoint and configuration.",
 "steps": [
  "Install Python 3.10+ and run `python -m pip install -e '.[gateway]'` from the repo root.",
  "Set your endpoint details with `export OPENAI_API_KEY=YOUR_API_KEY` and `export OPENAI_BASE_URL=YOUR_BASE_URL`, or copy the values from `.env.example`.",
  "Run `looparena-type1-run --data benchmarks/type1/questions --model MODEL_ID --preflight-only` with your model ID in place of MODEL_ID. (inferred, check the README)",
  "Run `looparena-type1-run --data benchmarks/type1/questions --model MODEL_ID --output runs/type1-smoke.jsonl --limit 1`. (inferred, check the README)",
  "Open runs/type1-smoke.jsonl and confirm it contains one result for the chosen model."
 ],
 "agent_prompt": "In the AMAP-ML/LoopArena repo, set OPENAI_API_KEY and OPENAI_BASE_URL from the environment, then run looparena-type1-run with --data benchmarks/type1/questions, --model <model ID>, and --preflight-only, followed by a one-question run with --limit 1 writing to runs/type1-smoke.jsonl. Report whether preflight passed and the path of the output file, then confirm the file holds exactly one result.",
 "agents": [
  "any"
 ],
 "difficulty": "Intermediate",
 "time": "30 minutes",
 "basis": "readme",
 "caveat": "The README gives the model ID only as a placeholder, so you must supply a model your endpoint actually serves.",
 "category": "Research",
 "tags": [
  "Research"
 ],
 "lang": "python",
 "requirements": [
  "Python 3.10 or newer",
  "An OpenAI-compatible model endpoint, its base URL, and an API key",
  "The model ID you want to test",
  "For Type II/III only: the pinned upstream task data, starting workspaces, and Docker runtimes described in docs/setup.md"
 ],
 "risk": "none",
 "seen_in_the_wild": false,
 "mentioned_publicly": false,
 "risks": [
  {
   "tag": "credentials",
   "notice": "The setup uses an API key set in environment variables, so keep it out of committed files and shared logs."
  }
 ],
 "best": true,
 "score": 1.0,
 "evidence": [],
 "url": "https://dedupedai.com/use-cases/amap-ml-looparena/smoke-test-controller-model-type1/",
 "md": "https://dedupedai.com/use-cases/amap-ml-looparena/smoke-test-controller-model-type1/index.md",
 "disclaimer": "Deduped lists open-source repositories that belong to other people. We do not own, maintain, audit, or vouch for any of them, and a repo appearing here is not an endorsement or a safety check. The use cases, steps, and prompts are written by AI and may be incomplete, outdated, or just wrong. Where a use case is marked \"Seen in the wild\", we link to public posts where someone described something similar, but we have not verified those posts and the use case itself is still our AI's own write-up. Use all of this at your own risk. Only run code, commands, or agents on systems and accounts you are authorized to use, and read the code and every command before you run it, especially anything that touches credentials, files, or the network. Nothing here is professional, security, legal, or financial advice. All trademarks and project names belong to their respective owners, and their use here does not imply affiliation. Content is provided as is, without warranties of any kind. Found a mistake, a safety problem, or something you want removed? Email hello@dedupedai.com and we will look at it promptly.",
 "api": "https://dedupedai.com/api/v1/usecases/amap-ml-looparena--smoke-test-controller-model-type1.json",
 "api_version": "v1"
}