{
 "schema_version": 1,
 "generated": "2026-10-26",
 "id": "memoket-memoket-kite--reproduce-locomo-and-longmemeval-scores",
 "repo": "memoket/memoket-kite",
 "repo_url": "https://github.com/memoket/memoket-kite",
 "repo_slug": "memoket-memoket-kite",
 "title": "Rerun the LoCoMo and LongMemEval benchmark scores",
 "page_title": "How to rerun the LoCoMo and LongMemEval benchmark scores with Memoket Kite",
 "summary": "Rebuild the reported benchmark results locally to check KITE's scores before you adopt it.",
 "who": "A research engineer evaluating long-term memory options who needs to confirm published LoCoMo and LongMemEval-S scores under a shared protocol before choosing a memory engine.",
 "problem": "Published benchmark numbers are hard to trust without a way to reproduce them under the same model, judge, and dataset revisions.",
 "outcome": "You get a local run with manifests that record the commit, corpus digest, and scores, which you can verify offline.",
 "steps": [
  "Read the benchmark guide at benchmarks/README.md in the repo.",
  "Install the library: `python -m pip install memoket-kite`",
  "Set your key, and if you use another provider also set OPENAI_BASE_URL, following the README.",
  "Run the provided benchmark scripts named in benchmarks/README.md to rebuild the evaluation from the official datasets.",
  "Check the local manifest and judging manifest, then run the verifier described in docs/reproducibility.md."
 ],
 "agent_prompt": "Use the memoket-kite repo. Follow benchmarks/README.md to run the LoCoMo evaluation with a configured compatible LLM endpoint. Expected output: a local judging manifest with an overall accuracy and an average reader context. Verify that the manifest's commit and corpus digest match the run before you compare the score with the README's 93.51%.",
 "agents": [
  "any"
 ],
 "difficulty": "Advanced",
 "time": "2 hours",
 "basis": "readme",
 "caveat": "The README notes the benchmark corpora have their own licences in LICENSE-DATA.md, so check those terms before you redistribute any dataset files.",
 "category": "Software and DevOps",
 "tags": [
  "Software and DevOps"
 ],
 "lang": "python",
 "requirements": [
  "Python with pip available",
  "An OpenAI API key, or an OpenAI-compatible provider with OPENAI_BASE_URL set",
  "Network access to download the demo_codebook.xml file from GitHub",
  "Conversation text you are allowed to store in a local memory file"
 ],
 "risk": "sensitive",
 "seen_in_the_wild": false,
 "mentioned_publicly": false,
 "risks": [
  {
   "tag": "credentials",
   "notice": "The benchmark run calls a paid LLM endpoint with your key, so keep the key private and watch the usage on your account."
  }
 ],
 "best": false,
 "score": 0.8,
 "evidence": [],
 "url": "https://dedupedai.com/use-cases/memoket-memoket-kite/reproduce-locomo-and-longmemeval-scores/",
 "md": "https://dedupedai.com/use-cases/memoket-memoket-kite/reproduce-locomo-and-longmemeval-scores/index.md",
 "disclaimer": "Deduped lists open-source repositories that belong to other people. We do not own, maintain, audit, or vouch for any of them, and a repo appearing here is not an endorsement or a safety check. The use cases, steps, and prompts are written by AI and may be incomplete, outdated, or just wrong. Where a use case is marked \"Seen in the wild\", we link to public posts where someone described something similar, but we have not verified those posts and the use case itself is still our AI's own write-up. Use all of this at your own risk. Only run code, commands, or agents on systems and accounts you are authorized to use, and read the code and every command before you run it, especially anything that touches credentials, files, or the network. Nothing here is professional, security, legal, or financial advice. All trademarks and project names belong to their respective owners, and their use here does not imply affiliation. Content is provided as is, without warranties of any kind. Found a mistake, a safety problem, or something you want removed? Email hello@dedupedai.com and we will look at it promptly.",
 "api": "https://dedupedai.com/api/v1/usecases/memoket-memoket-kite--reproduce-locomo-and-longmemeval-scores.json",
 "api_version": "v1"
}