{
 "schema_version": 1,
 "generated": "2026-10-23",
 "repo": "openlayer-ai/jevals",
 "name": "jevals",
 "owner": "openlayer-ai",
 "slug": "openlayer-ai-jevals",
 "github": "https://github.com/openlayer-ai/jevals",
 "what": "jevals is a Python library for running agent and quality evals and guardrails by sending all checks for a trace to a decision model in one request.",
 "why": "",
 "language": "Python",
 "requirements": [
  "Python 3.10 or newer",
  "An API key for one backend: AI_GATEWAY_API_KEY (Jev through Vercel AI Gateway), TYPESAFE_API_KEY, or OPENROUTER_API_KEY",
  "A trace to evaluate: a messages list in the OpenAI chat format plus the tools you gave the model"
 ],
 "audience_note": "Developers and teams building LLM agents or RAG apps who want to eval or guard every trace without paying for a frontier LLM judge on each one.",
 "risk": "none",
 "section": "earlier",
 "weekly_rank": null,
 "category": "Software and DevOps",
 "url": "https://dedupedai.com/use-cases/openlayer-ai-jevals/",
 "use_case_count": 5,
 "use_cases": [
  {
   "id": "openlayer-ai-jevals--flag-unsupported-agent-claims",
   "title": "Flag agent answers that invent facts the tools never returned",
   "best": true,
   "url": "https://dedupedai.com/use-cases/openlayer-ai-jevals/flag-unsupported-agent-claims/",
   "json": "https://dedupedai.com/use-cases/u/openlayer-ai-jevals/flag-unsupported-agent-claims.json"
  },
  {
   "id": "openlayer-ai-jevals--screen-tool-results-for-injected-instructions",
   "title": "Catch tool results that try to give the agent orders",
   "best": false,
   "url": "https://dedupedai.com/use-cases/openlayer-ai-jevals/screen-tool-results-for-injected-instructions/",
   "json": "https://dedupedai.com/use-cases/u/openlayer-ai-jevals/screen-tool-results-for-injected-instructions.json"
  },
  {
   "id": "openlayer-ai-jevals--gate-agent-loop-before-it-answers",
   "title": "Block agent replies that fail checks before they reach the user",
   "best": false,
   "url": "https://dedupedai.com/use-cases/openlayer-ai-jevals/gate-agent-loop-before-it-answers/",
   "json": "https://dedupedai.com/use-cases/u/openlayer-ai-jevals/gate-agent-loop-before-it-answers.json"
  },
  {
   "id": "openlayer-ai-jevals--write-and-run-evals-from-your-coding-agent",
   "title": "Let Cursor or Claude Code write and run agent evals for you",
   "best": false,
   "url": "https://dedupedai.com/use-cases/openlayer-ai-jevals/write-and-run-evals-from-your-coding-agent/",
   "json": "https://dedupedai.com/use-cases/u/openlayer-ai-jevals/write-and-run-evals-from-your-coding-agent.json"
  },
  {
   "id": "openlayer-ai-jevals--run-evals-fully-local-on-a-mac",
   "title": "Run agent evals on your own Mac with no API calls",
   "best": false,
   "url": "https://dedupedai.com/use-cases/openlayer-ai-jevals/run-evals-fully-local-on-a-mac/",
   "json": "https://dedupedai.com/use-cases/u/openlayer-ai-jevals/run-evals-fully-local-on-a-mac.json"
  }
 ],
 "disclaimer": "Third-party repos we don't own or vouch for. AI-generated use cases can be wrong: review everything and run at your own risk."
}