{
 "schema_version": 1,
 "generated": "2026-10-20",
 "id": "recurse-labs-recurse--run-agent-eval-tiers-headlessly",
 "repo": "Recurse-Labs/recurse",
 "repo_url": "https://github.com/Recurse-Labs/recurse",
 "repo_slug": "recurse-labs-recurse",
 "title": "Run the agent against crackme eval tiers",
 "page_title": "How to run the agent against crackme eval tiers with Recurse",
 "summary": "Fetch the eval binaries, run the harness self-tests and execute an eval tier to measure the agent.",
 "who": "A developer working on Recurse who wants to measure how well the agent solves crackme tiers after a change.",
 "problem": "Judging agent changes by hand is slow, and running the agent costs API calls.",
 "outcome": "Per-task traces and a run log that show how the agent handled each crackme in the chosen tier.",
 "steps": [
  "Copy crates/recurse-eval/.env.example to crates/recurse-eval/.env and add your endpoint and key.",
  "Run `just eval-fetch` to download the tier's binaries.",
  "Run `just eval-test` to run the harness self-tests, which make no LLM calls.",
  "Run `just eval-run` to execute the tier. This is the only way to execute an eval YAML.",
  "Open target/eval-traces/ and read the run.log and the per-task JSON files."
 ],
 "agent_prompt": "In the Recurse repo at <repo path>, fetch the eval binaries, run the harness self-tests, then run the eval tier. Report the number of tasks that finished and the path of the run.log. Verify that the trace folder contains one JSON file per task.",
 "agents": [
  "any"
 ],
 "difficulty": "Intermediate",
 "time": "1 hour",
 "basis": "readme",
 "caveat": "Running the eval tier calls the configured LLM endpoint and can cost money, so run the self-tests first.",
 "category": "Security and science",
 "tags": [
  "Security and science"
 ],
 "lang": "rust",
 "requirements": [
  "Node.js 20 or newer and npm 10 or newer",
  "Rust 1.77 or newer with cargo, installed through rustup",
  "Linux system packages for Tauri (libwebkit2gtk-4.1-dev and the others listed in the README) if you build the desktop app on Debian or Ubuntu",
  "A binary you are allowed to analyze",
  "An OpenAI-compatible API key (for example OPENROUTER_API_KEY) if you want the chat agent to run"
 ],
 "risk": "dual-use",
 "seen_in_the_wild": false,
 "mentioned_publicly": false,
 "risks": [
  {
   "tag": "credentials",
   "notice": "The eval endpoint and API key are stored in crates/recurse-eval/.env, so keep that file out of version control."
  }
 ],
 "best": false,
 "score": 0.7,
 "evidence": [],
 "url": "https://dedupedai.com/use-cases/recurse-labs-recurse/run-agent-eval-tiers-headlessly/",
 "md": "https://dedupedai.com/use-cases/recurse-labs-recurse/run-agent-eval-tiers-headlessly/index.md",
 "disclaimer": "Deduped lists open-source repositories that belong to other people. We do not own, maintain, audit, or vouch for any of them, and a repo appearing here is not an endorsement or a safety check. The use cases, steps, and prompts are written by AI and may be incomplete, outdated, or just wrong. Where a use case is marked \"Seen in the wild\", we link to public posts where someone described something similar, but we have not verified those posts and the use case itself is still our AI's own write-up. Use all of this at your own risk. Only run code, commands, or agents on systems and accounts you are authorized to use, and read the code and every command before you run it, especially anything that touches credentials, files, or the network. Nothing here is professional, security, legal, or financial advice. All trademarks and project names belong to their respective owners, and their use here does not imply affiliation. Content is provided as is, without warranties of any kind. Found a mistake, a safety problem, or something you want removed? Email hello@dedupedai.com and we will look at it promptly.",
 "api": "https://dedupedai.com/api/v1/usecases/recurse-labs-recurse--run-agent-eval-tiers-headlessly.json",
 "api_version": "v1"
}