{
 "schema_version": 1,
 "generated": "2026-10-31",
 "id": "huggingface-trl--align-a-model-to-preferences-with-trl-dpo",
 "repo": "huggingface/trl",
 "repo_url": "https://github.com/huggingface/trl",
 "repo_slug": "huggingface-trl",
 "title": "Align a model to preferences with DPO",
 "page_title": "How to align a model to preferences with DPO with TRL",
 "summary": "Train a model on chosen versus rejected answers with DPOTrainer or the TRL dpo command.",
 "who": "An engineer who has preference data and wants responses that match it.",
 "problem": "Supervised fine-tuning alone does not teach a model which of two answers is better.",
 "outcome": "A DPO-trained model saved to disk.",
 "steps": [
  "Install TRL with `pip install trl`",
  "Run `trl dpo --model_name_or_path Qwen/Qwen2.5-0.5B-Instruct` with `--dataset_name argilla/Capybara-Preferences` and `--output_dir Qwen2.5-0.5B-DPO`",
  "Or in Python load trl-lib/ultrafeedback_binarized and pass it to DPOTrainer, then call trainer.train()",
  "Compare answers from the base and trained models"
 ],
 "agent_prompt": "With huggingface/trl, train a DPO model using `trl dpo --model_name_or_path Qwen/Qwen2.5-0.5B-Instruct` and `--dataset_name argilla/Capybara-Preferences`, then write a small script that compares base and trained answers on 10 prompts from <my list>.",
 "agents": [
  "claude-code",
  "codex",
  "cursor",
  "gemini-cli"
 ],
 "difficulty": "Advanced",
 "time": "2 hours",
 "basis": "readme",
 "caveat": "This card is written from the huggingface/trl README, not from a test run by Deduped. Before you start, make sure you have Python with pip. Check the repo for newer instructions before you rely on these steps.",
 "category": "Software and DevOps",
 "tags": [
  "Software and DevOps"
 ],
 "lang": "python",
 "requirements": [
  "Python with pip",
  "A GPU is strongly recommended; the README covers NVIDIA, AMD and AWS Trainium setups",
  "Access to Hugging Face Hub models and datasets"
 ],
 "risk": "none",
 "seen_in_the_wild": false,
 "mentioned_publicly": false,
 "risks": [
  {
   "tag": "cost",
   "notice": "Preference training on a GPU can run for hours."
  }
 ],
 "best": false,
 "score": 0.9,
 "evidence": [],
 "url": "https://dedupedai.com/use-cases/huggingface-trl/align-a-model-to-preferences-with-trl-dpo/",
 "md": "https://dedupedai.com/use-cases/huggingface-trl/align-a-model-to-preferences-with-trl-dpo/index.md",
 "disclaimer": "Deduped lists open-source repositories that belong to other people. We do not own, maintain, audit, or vouch for any of them, and a repo appearing here is not an endorsement or a safety check. The use cases, steps, and prompts are written by AI and may be incomplete, outdated, or just wrong. Where a use case is marked \"Seen in the wild\", we link to public posts where someone described something similar, but we have not verified those posts and the use case itself is still our AI's own write-up. Use all of this at your own risk. Only run code, commands, or agents on systems and accounts you are authorized to use, and read the code and every command before you run it, especially anything that touches credentials, files, or the network. Nothing here is professional, security, legal, or financial advice. All trademarks and project names belong to their respective owners, and their use here does not imply affiliation. Content is provided as is, without warranties of any kind. Found a mistake, a safety problem, or something you want removed? Email hello@dedupedai.com and we will look at it promptly.",
 "api": "https://dedupedai.com/api/v1/usecases/huggingface-trl--align-a-model-to-preferences-with-trl-dpo.json",
 "api_version": "v1"
}