{
 "schema_version": 1,
 "generated": "2026-10-26",
 "id": "sherlockwz-ama--reproduce-am-paper-benchmarks",
 "repo": "Sherlockwz/AMA",
 "repo_url": "https://github.com/Sherlockwz/AMA",
 "repo_slug": "sherlockwz-ama",
 "title": "Reproduce AMA's LoCoMo and LongMemEval results",
 "page_title": "How to reproduce AMA's LoCoMo and LongMemEval results with AMA",
 "summary": "Run the research pipeline on LoCoMo and LongMemEval_s to check the reported scores.",
 "who": "A researcher or student who needs to verify or extend the AMA results for an ACL 2026 paper comparison.",
 "problem": "Published memory benchmark scores are hard to trust without rerunning the original pipeline on the same datasets.",
 "outcome": "You run the AMA evaluation entry points and produce scores you can compare against the README's reference numbers.",
 "steps": [
  "`git clone https://github.com/Sherlockwz/AMA.git`",
  "`cd AMA`",
  "`corepack enable pnpm`",
  "`./scripts/setup.sh`",
  "Follow the reproduction guide at `docs/reproduction.md` to obtain the LoCoMo and LongMemEval_s datasets and place them where the guide says.",
  "Run the evaluation entry points under `python/AdaptiveMemory/Core`, as described in the reproduction guide."
 ],
 "agent_prompt": "In the AMA repo at <path-to-AMA>, follow docs/reproduction.md to set up the LoCoMo dataset and run one evaluation entry point from python/AdaptiveMemory/Core. Expected output: an overall LLM Score for the LoCoMo run. Verify that the dataset is placed where the guide specifies before comparing your score to the README's 0.774 reference.",
 "agents": [
  "any"
 ],
 "difficulty": "Advanced",
 "time": "120 minutes",
 "basis": "readme",
 "caveat": "The benchmark datasets are not redistributed in the repo, so you must obtain them yourself before any evaluation can run.",
 "category": "Software and DevOps",
 "tags": [
  "Software and DevOps"
 ],
 "lang": "python",
 "requirements": [
  "Python 3.10 or newer",
  "Node.js 22.22.3 or newer (24.15 or newer also supported)",
  "pnpm 11 or newer",
  "An OpenAI-compatible chat-completions endpoint and embedding endpoint, with an API key",
  "Git, to clone the repository"
 ],
 "risk": "sensitive",
 "seen_in_the_wild": false,
 "mentioned_publicly": false,
 "risks": [
  {
   "tag": "credentials",
   "notice": "Full benchmark runs make many LLM and embedding calls, so the API keys in your environment will incur usage costs."
  }
 ],
 "best": false,
 "score": 0.7,
 "evidence": [],
 "url": "https://dedupedai.com/use-cases/sherlockwz-ama/reproduce-am-paper-benchmarks/",
 "md": "https://dedupedai.com/use-cases/sherlockwz-ama/reproduce-am-paper-benchmarks/index.md",
 "disclaimer": "Deduped lists open-source repositories that belong to other people. We do not own, maintain, audit, or vouch for any of them, and a repo appearing here is not an endorsement or a safety check. The use cases, steps, and prompts are written by AI and may be incomplete, outdated, or just wrong. Where a use case is marked \"Seen in the wild\", we link to public posts where someone described something similar, but we have not verified those posts and the use case itself is still our AI's own write-up. Use all of this at your own risk. Only run code, commands, or agents on systems and accounts you are authorized to use, and read the code and every command before you run it, especially anything that touches credentials, files, or the network. Nothing here is professional, security, legal, or financial advice. All trademarks and project names belong to their respective owners, and their use here does not imply affiliation. Content is provided as is, without warranties of any kind. Found a mistake, a safety problem, or something you want removed? Email hello@dedupedai.com and we will look at it promptly.",
 "api": "https://dedupedai.com/api/v1/usecases/sherlockwz-ama--reproduce-am-paper-benchmarks.json",
 "api_version": "v1"
}