{
 "schema_version": 1,
 "generated": "2026-10-24",
 "id": "aibuildai-inc-aibuildai-llm-posttrain-agent--grade-post-trained-model-with-task-grader",
 "repo": "aibuildai-inc/aibuildai-llm-posttrain-agent",
 "repo_url": "https://github.com/aibuildai-inc/aibuildai-llm-posttrain-agent",
 "repo_slug": "aibuildai-inc-aibuildai-llm-posttrain-agent",
 "title": "Grade a post-trained model with the task's own grader",
 "page_title": "How to grade a post-trained model with the task's own grader with Aibuildai LLM Posttrain Agent",
 "summary": "Score a deliverable model with the frozen grader that the run trained against.",
 "who": "A reviewer or ML engineer who needs an independent score for a model produced by an aibuildai run.",
 "problem": "A run's reported progress is only useful if the final model is measured with the same grader and on the full test split.",
 "outcome": "A grader result for the merged model on the full GSM8K test split (--limit -1), computed with the run's own Program environment.",
 "steps": [
  "Set the run home with `H=~/gsm8k-example/playground/gsm8k-qwen3-1.7b-base/`",
  "Confirm the deliverable exists at $H/deliverable/final_model",
  "Run `$H/program-environment/env/bin/python ~/gsm8k-example/task/grade/evaluate.py --model-path $H/deliverable/final_model --limit -1` (inferred, check the README)",
  "Compare the score with the base model score of 0.105 on the first 200 problems"
 ],
 "agent_prompt": "For the aibuildai-llm-posttrain-agent GSM8K example, run the grader in the run's Program environment against deliverable/final_model with --limit -1. Expected output: a grader score for the full test split. Verify that the model path exists and that the score is reported on the full split before stating the result.",
 "agents": [
  "any"
 ],
 "difficulty": "Intermediate",
 "time": "3 hours",
 "basis": "readme",
 "caveat": "The README only gives the base model's score on the first 200 problems, so compare the trained model on the same split or note the difference in split size.",
 "category": "Software and DevOps",
 "tags": [
  "Software and DevOps"
 ],
 "lang": "python",
 "requirements": [
  "Linux x86_64 with a systemd user session and Python 3.11 to 3.13",
  "A C toolchain (gcc, make) with zlib and readline headers, plus bwrap (bubblewrap) with unprivileged user namespaces enabled",
  "A Claude Code login or an Anthropic API key, or an Anthropic-compatible endpoint and its key",
  "A CUDA GPU is recommended",
  "Network access to GitHub releases and ftp.postgresql.org"
 ],
 "risk": "none",
 "seen_in_the_wild": false,
 "mentioned_publicly": false,
 "risks": [],
 "best": false,
 "score": 0.8,
 "evidence": [],
 "url": "https://dedupedai.com/use-cases/aibuildai-inc-aibuildai-llm-posttrain-agent/grade-post-trained-model-with-task-grader/",
 "md": "https://dedupedai.com/use-cases/aibuildai-inc-aibuildai-llm-posttrain-agent/grade-post-trained-model-with-task-grader/index.md",
 "disclaimer": "Deduped lists open-source repositories that belong to other people. We do not own, maintain, audit, or vouch for any of them, and a repo appearing here is not an endorsement or a safety check. The use cases, steps, and prompts are written by AI and may be incomplete, outdated, or just wrong. Where a use case is marked \"Seen in the wild\", we link to public posts where someone described something similar, but we have not verified those posts and the use case itself is still our AI's own write-up. Use all of this at your own risk. Only run code, commands, or agents on systems and accounts you are authorized to use, and read the code and every command before you run it, especially anything that touches credentials, files, or the network. Nothing here is professional, security, legal, or financial advice. All trademarks and project names belong to their respective owners, and their use here does not imply affiliation. Content is provided as is, without warranties of any kind. Found a mistake, a safety problem, or something you want removed? Email hello@dedupedai.com and we will look at it promptly.",
 "api": "https://dedupedai.com/api/v1/usecases/aibuildai-inc-aibuildai-llm-posttrain-agent--grade-post-trained-model-with-task-grader.json",
 "api_version": "v1"
}