{
 "schema_version": 1,
 "generated": "2026-10-31",
 "id": "langfuse-langfuse--score-llm-outputs-with-evals-and-datasets",
 "repo": "langfuse/langfuse",
 "repo_url": "https://github.com/langfuse/langfuse",
 "repo_slug": "langfuse-langfuse",
 "title": "Score your app's answers with evals and datasets",
 "page_title": "How to score your app's answers with evals and datasets with Langfuse",
 "summary": "Build a dataset of test cases and score outputs with LLM-as-a-judge, code evaluators and user feedback.",
 "who": "A team about to swap models who needs proof the new one is not worse.",
 "problem": "Quality is judged by gut feel, so regressions slip into production.",
 "outcome": "Datasets and evaluation scores you can compare across prompt and model versions before deploying.",
 "steps": [
  "Run Langfuse and trace your app",
  "Create a Dataset of representative inputs and expected outputs",
  "Run experiments over the dataset for each prompt or model version",
  "Score results with LLM-as-a-judge, code evaluators, manual labeling or user feedback collection",
  "Use the API to wire these runs into your own LLMOps workflow"
 ],
 "agent_prompt": "Using langfuse/langfuse, create a dataset from <examples>, run it against <current model> and <new model>, and score both with an LLM-as-a-judge evaluator for <criteria>. Summarise which version wins and on which cases.",
 "agents": [
  "claude-code",
  "codex",
  "cursor"
 ],
 "difficulty": "Intermediate",
 "time": "1 hour",
 "basis": "readme",
 "caveat": "A judge model has its own biases, so spot-check its scores by hand.",
 "category": "Software and DevOps",
 "tags": [
  "Software and DevOps"
 ],
 "lang": "typescript",
 "requirements": [
  "Docker and Docker Compose for local self-hosting, or a Langfuse Cloud account",
  "The Python or JS/TS SDK, or a supported framework integration",
  "For production: a VM, Kubernetes with Helm, or Terraform on AWS, Azure or GCP"
 ],
 "risk": "none",
 "seen_in_the_wild": false,
 "mentioned_publicly": false,
 "risks": [
  {
   "tag": "cost",
   "notice": "LLM-as-a-judge evaluators call a model for every scored item, which costs money with a paid provider."
  }
 ],
 "best": false,
 "score": 0.8,
 "evidence": [],
 "url": "https://dedupedai.com/use-cases/langfuse-langfuse/score-llm-outputs-with-evals-and-datasets/",
 "md": "https://dedupedai.com/use-cases/langfuse-langfuse/score-llm-outputs-with-evals-and-datasets/index.md",
 "disclaimer": "Deduped lists open-source repositories that belong to other people. We do not own, maintain, audit, or vouch for any of them, and a repo appearing here is not an endorsement or a safety check. The use cases, steps, and prompts are written by AI and may be incomplete, outdated, or just wrong. Where a use case is marked \"Seen in the wild\", we link to public posts where someone described something similar, but we have not verified those posts and the use case itself is still our AI's own write-up. Use all of this at your own risk. Only run code, commands, or agents on systems and accounts you are authorized to use, and read the code and every command before you run it, especially anything that touches credentials, files, or the network. Nothing here is professional, security, legal, or financial advice. All trademarks and project names belong to their respective owners, and their use here does not imply affiliation. Content is provided as is, without warranties of any kind. Found a mistake, a safety problem, or something you want removed? Email hello@dedupedai.com and we will look at it promptly.",
 "api": "https://dedupedai.com/api/v1/usecases/langfuse-langfuse--score-llm-outputs-with-evals-and-datasets.json",
 "api_version": "v1"
}