{
 "schema_version": 1,
 "generated": "2026-10-31",
 "repo": "promptfoo/promptfoo",
 "name": "promptfoo",
 "owner": "promptfoo",
 "slug": "promptfoo-promptfoo",
 "github": "https://github.com/promptfoo/promptfoo",
 "what": "Promptfoo is a CLI and library for evaluating and red-teaming LLM-based apps. It runs automated evaluations of prompts and models, compares providers such as OpenAI, Anthropic, Azure, Bedrock and Ollama side by side, scans your own apps for vulnerabilities, automates checks in CI/CD, reviews pull requests for LLM security issues, and runs evals locally so prompts never leave your machine. It is now part of OpenAI and stays open source under MIT.",
 "why": "27k stars, talked about in 2 AI sources we track",
 "language": "TypeScript",
 "requirements": [
  "Node.js for npm, or Homebrew, or pip; npx works without installing",
  "API keys for the LLM providers you test, set as environment variables",
  "Only run red teaming against apps you own or are authorised to test"
 ],
 "audience_note": "Developers and teams shipping prompts, RAG systems or agents who want measured quality and security checks instead of trial and error.",
 "risk": "none",
 "section": "earlier",
 "weekly_rank": null,
 "category": "Software and DevOps",
 "url": "https://dedupedai.com/use-cases/promptfoo-promptfoo/",
 "use_case_count": 3,
 "use_cases": [
  {
   "id": "promptfoo-promptfoo--compare-prompts-and-models-with-evals",
   "title": "Compare prompts and models with automated evals",
   "best": true,
   "url": "https://dedupedai.com/use-cases/promptfoo-promptfoo/compare-prompts-and-models-with-evals/",
   "json": "https://dedupedai.com/use-cases/u/promptfoo-promptfoo/compare-prompts-and-models-with-evals.json"
  },
  {
   "id": "promptfoo-promptfoo--red-team-your-own-llm-app",
   "title": "Red team your own LLM app for vulnerabilities",
   "best": false,
   "url": "https://dedupedai.com/use-cases/promptfoo-promptfoo/red-team-your-own-llm-app/",
   "json": "https://dedupedai.com/use-cases/u/promptfoo-promptfoo/red-team-your-own-llm-app.json"
  },
  {
   "id": "promptfoo-promptfoo--block-prompt-regressions-in-ci",
   "title": "Block prompt regressions in CI",
   "best": false,
   "url": "https://dedupedai.com/use-cases/promptfoo-promptfoo/block-prompt-regressions-in-ci/",
   "json": "https://dedupedai.com/use-cases/u/promptfoo-promptfoo/block-prompt-regressions-in-ci.json"
  }
 ],
 "disclaimer": "Third-party repos we don't own or vouch for. AI-generated use cases can be wrong: review everything and run at your own risk."
}