{
 "schema_version": 1,
 "generated": "2026-10-20",
 "id": "dicklesworthstone-skillranker--replay-labeled-evaluations-offline",
 "repo": "Dicklesworthstone/skillranker",
 "repo_url": "https://github.com/Dicklesworthstone/skillranker",
 "repo_slug": "dicklesworthstone-skillranker",
 "title": "Replay a labeled evaluation set without network calls",
 "page_title": "How to replay a labeled evaluation set without network calls with Skillranker",
 "summary": "Re-run a labeled ranking dataset locally to compare policies without spending provider calls.",
 "who": "A maintainer tuning SkillRanker policies who needs repeatable results on a fixed labeled dataset.",
 "problem": "Changing ranking settings and re-querying a live provider is slow, costly, and hard to reproduce.",
 "outcome": "Offline replay shows how the labeled artifact scores and explains each decision, with no network requests.",
 "steps": [
  "Install `sr` with `cargo install --locked --path . --bin sr`.",
  "Prepare a labeled evaluation artifact at `scratch/evaluation.json`, using the case capture described in the README.",
  "Replay it with `sr eval --dataset scratch/evaluation.json --explain`.",
  "Compare the explained decisions against your expected labels."
 ],
 "agent_prompt": "Using the Dicklesworthstone/skillranker repo, run `sr eval --dataset <evaluation file> --explain`. Expected output: replayed decisions with explanations and no network requests. Verify that the replay reports no provider calls and that the labels in the dataset match the outcomes you expect.",
 "agents": [
  "any"
 ],
 "difficulty": "Advanced",
 "time": "60 minutes",
 "basis": "readme",
 "caveat": "The README asks for a labeled evaluation artifact, so you must build or capture the dataset yourself before replay gives useful results.",
 "category": "Software and DevOps",
 "tags": [
  "Software and DevOps"
 ],
 "lang": "rust",
 "requirements": [
  "Linux or macOS with a local filesystem that supports Unix permissions",
  "Your own credential for a Jev-compatible provider, such as TypeSafe_API_KEY for TypeSafe.ai or Cloudflare account settings",
  "Rust toolchain for a source build, or the installer's prerequisites: Bash, Python 3, curl, and sha256sum or shasum",
  "Explicit consent to send selected session context over the network for live ranking"
 ],
 "risk": "sensitive",
 "seen_in_the_wild": false,
 "mentioned_publicly": false,
 "risks": [],
 "best": false,
 "score": 0.6,
 "evidence": [],
 "url": "https://dedupedai.com/use-cases/dicklesworthstone-skillranker/replay-labeled-evaluations-offline/",
 "md": "https://dedupedai.com/use-cases/dicklesworthstone-skillranker/replay-labeled-evaluations-offline/index.md",
 "disclaimer": "Deduped lists open-source repositories that belong to other people. We do not own, maintain, audit, or vouch for any of them, and a repo appearing here is not an endorsement or a safety check. The use cases, steps, and prompts are written by AI and may be incomplete, outdated, or just wrong. Where a use case is marked \"Seen in the wild\", we link to public posts where someone described something similar, but we have not verified those posts and the use case itself is still our AI's own write-up. Use all of this at your own risk. Only run code, commands, or agents on systems and accounts you are authorized to use, and read the code and every command before you run it, especially anything that touches credentials, files, or the network. Nothing here is professional, security, legal, or financial advice. All trademarks and project names belong to their respective owners, and their use here does not imply affiliation. Content is provided as is, without warranties of any kind. Found a mistake, a safety problem, or something you want removed? Email hello@dedupedai.com and we will look at it promptly.",
 "api": "https://dedupedai.com/api/v1/usecases/dicklesworthstone-skillranker--replay-labeled-evaluations-offline.json",
 "api_version": "v1"
}