{
 "schema_version": 1,
 "generated": "2026-10-26",
 "id": "proteus-evolve-proteus--score-evolution-against-hidden-benchmark",
 "repo": "proteus-evolve/Proteus",
 "repo_url": "https://github.com/proteus-evolve/Proteus",
 "repo_slug": "proteus-evolve-proteus",
 "title": "Score a harness against a hidden benchmark",
 "page_title": "How to score a harness against a hidden benchmark with Proteus",
 "summary": "Attach a verifier as a hidden evaluator and keep only the harness changes that do not regress.",
 "who": "An ML engineer with a benchmark verifier who wants a harness to improve on it without the agent seeing the score.",
 "problem": "When the agent can see the benchmark score, it may tune to the metric, and regressions can slip into the harness unnoticed.",
 "outcome": "A goal configuration with a hidden evaluator and accept_reject selection, plus run output that measures each condition with the same ruler.",
 "steps": [
  "`pip install proteus-evolve==0.4.0`",
  "Write your verifier as a callable (trace, ctx) -> EvalResult and wrap it in EvaluatorSpec with Visibility.HIDDEN",
  "Pass the spec to GoalConfig.of with selection=\"accept_reject\"",
  "`proteus check --harness mypkg.theirs_adapter:TheirsHarness --episode`",
  "`proteus run --harness mypkg.theirs_adapter:TheirsHarness --arm neutral ...` (inferred, check the README)"
 ],
 "agent_prompt": "In the Proteus repo (proteus-evolve/Proteus), write a verifier callable that returns an EvalResult, wrap it in EvaluatorSpec with Visibility.HIDDEN, and run GoalConfig.of with selection=\"accept_reject\" against the adapter at <harness path>. Expected output: a run folder with per-episode evaluator results and accepted or rejected snapshots. Verify that the hidden evaluator's score does not appear in any observe-phase prompt.",
 "agents": [
  "any"
 ],
 "difficulty": "Advanced",
 "time": "2 hours",
 "basis": "readme",
 "caveat": "This assumes you already have a working HarnessAdapter, and the README only shows the adapter path as a placeholder, so the check step must pass on your own adapter first.",
 "category": "Software and DevOps",
 "tags": [
  "Software and DevOps"
 ],
 "lang": "python",
 "requirements": [
  "Python 3.10 or newer",
  "pip to install proteus-evolve==0.4.0 (no model SDK is required for the minimal harness)",
  "For real harnesses: Docker and a DeepSeek or Codex key, as listed in the harness table"
 ],
 "risk": "none",
 "seen_in_the_wild": false,
 "mentioned_publicly": false,
 "risks": [],
 "best": false,
 "score": 0.7,
 "evidence": [],
 "url": "https://dedupedai.com/use-cases/proteus-evolve-proteus/score-evolution-against-hidden-benchmark/",
 "md": "https://dedupedai.com/use-cases/proteus-evolve-proteus/score-evolution-against-hidden-benchmark/index.md",
 "disclaimer": "Deduped lists open-source repositories that belong to other people. We do not own, maintain, audit, or vouch for any of them, and a repo appearing here is not an endorsement or a safety check. The use cases, steps, and prompts are written by AI and may be incomplete, outdated, or just wrong. Where a use case is marked \"Seen in the wild\", we link to public posts where someone described something similar, but we have not verified those posts and the use case itself is still our AI's own write-up. Use all of this at your own risk. Only run code, commands, or agents on systems and accounts you are authorized to use, and read the code and every command before you run it, especially anything that touches credentials, files, or the network. Nothing here is professional, security, legal, or financial advice. All trademarks and project names belong to their respective owners, and their use here does not imply affiliation. Content is provided as is, without warranties of any kind. Found a mistake, a safety problem, or something you want removed? Email hello@dedupedai.com and we will look at it promptly.",
 "api": "https://dedupedai.com/api/v1/usecases/proteus-evolve-proteus--score-evolution-against-hidden-benchmark.json",
 "api_version": "v1"
}