{
 "schema_version": 1,
 "generated": "2026-10-31",
 "repo": "EleutherAI/lm-evaluation-harness",
 "name": "lm-evaluation-harness",
 "owner": "EleutherAI",
 "slug": "eleutherai-lm-evaluation-harness",
 "github": "https://github.com/EleutherAI/lm-evaluation-harness",
 "what": "The LM Evaluation Harness is EleutherAI's framework for few-shot evaluation of language models. One lm_eval command scores a model on named tasks, with backends for Hugging Face transformers, vLLM, SGLang, llama.cpp servers, ONNX Runtime, NeMo, Megatron and any OpenAI compatible API. It supports multi GPU evaluation, result saving, response logging and pushing results to the Hugging Face Hub.",
 "why": "14k stars, picked up by one AI source we track",
 "language": "",
 "requirements": [
  "Python with pip; the base package no longer includes transformers or torch",
  "A model backend extra such as `pip install \"lm_eval[hf]\"`, `pip install \"lm_eval[vllm]\"` or `pip install \"lm_eval[api]\"`",
  "A CUDA GPU for the local model examples, which pass --device cuda:0",
  "An API key in the environment for hosted models"
 ],
 "audience_note": "ML engineers who need comparable benchmark numbers for the models they train, quantize or serve.",
 "risk": "none",
 "section": "earlier",
 "weekly_rank": null,
 "category": "Software and DevOps",
 "url": "https://dedupedai.com/use-cases/eleutherai-lm-evaluation-harness/",
 "use_case_count": 3,
 "use_cases": [
  {
   "id": "eleutherai-lm-evaluation-harness--benchmark-a-hugging-face-model-with-lm-eval",
   "title": "Score a model on standard benchmarks in one command",
   "best": true,
   "url": "https://dedupedai.com/use-cases/eleutherai-lm-evaluation-harness/benchmark-a-hugging-face-model-with-lm-eval/",
   "json": "https://dedupedai.com/use-cases/u/eleutherai-lm-evaluation-harness/benchmark-a-hugging-face-model-with-lm-eval.json"
  },
  {
   "id": "eleutherai-lm-evaluation-harness--evaluate-a-served-model-through-its-openai-api",
   "title": "Evaluate a model you serve behind an API",
   "best": false,
   "url": "https://dedupedai.com/use-cases/eleutherai-lm-evaluation-harness/evaluate-a-served-model-through-its-openai-api/",
   "json": "https://dedupedai.com/use-cases/u/eleutherai-lm-evaluation-harness/evaluate-a-served-model-through-its-openai-api.json"
  },
  {
   "id": "eleutherai-lm-evaluation-harness--save-and-share-lm-eval-results",
   "title": "Keep every eval run reproducible and shareable",
   "best": false,
   "url": "https://dedupedai.com/use-cases/eleutherai-lm-evaluation-harness/save-and-share-lm-eval-results/",
   "json": "https://dedupedai.com/use-cases/u/eleutherai-lm-evaluation-harness/save-and-share-lm-eval-results.json"
  }
 ],
 "disclaimer": "Third-party repos we don't own or vouch for. AI-generated use cases can be wrong: review everything and run at your own risk."
}