{
 "schema_version": 1,
 "generated": "2026-10-20",
 "id": "yeonwoosung-llm-architecture-from-scratch--study-mixture-of-experts-routing",
 "repo": "YeonwooSung/llm-architecture-from-scratch",
 "repo_url": "https://github.com/YeonwooSung/llm-architecture-from-scratch",
 "repo_slug": "yeonwoosung-llm-architecture-from-scratch",
 "title": "Study Mixture of Experts layers in two models",
 "page_title": "How to study Mixture of Experts layers in two models with LLM Architecture From Scratch",
 "summary": "Read the MoE implementations in the DeepSeek v3 and Llama 4 Maverick folders to see how experts are used.",
 "who": "A researcher or engineer who is considering a Mixture of Experts design and needs reference code from models that the README lists as using it.",
 "problem": "Mixture of Experts layers are hard to understand from papers alone, and the routing details differ between model families.",
 "outcome": "A written note on how expert layers are structured in the two listed models, with the code locations for each part.",
 "steps": [
  "Find the rows marked with a check in the Mixture of Experts column, including DeepSeek v3 and Llama 4 Maverick.",
  "Open the deepseekv3 and llama4_maverick folders linked from the table.",
  "Locate the expert layer and the code that picks which experts run for each token.",
  "Write down the number of experts and the routing logic as each folder implements them.",
  "Compare the two notes and list where the designs match and where they differ."
 ],
 "agent_prompt": "In the repo YeonwooSung/llm-architecture-from-scratch, open the deepseekv3 and llama4_maverick folders and describe how each one implements its Mixture of Experts layer. Output a comparison table of expert structure and routing. Verify each claim against the code in the folder before reporting.",
 "agents": [
  "any"
 ],
 "difficulty": "Advanced",
 "time": "1 hour 30 minutes",
 "basis": "readme",
 "caveat": "The README marks only some models with Mixture of Experts, and the in-progress models listed at the bottom are not yet in the table.",
 "category": "Research",
 "tags": [
  "Research"
 ],
 "lang": "python",
 "requirements": [
  "Python, as listed for the repo",
  "PyTorch, as stated in the repo description",
  "Basic familiarity with transformer attention and normalization layers"
 ],
 "risk": "none",
 "seen_in_the_wild": false,
 "mentioned_publicly": false,
 "risks": [],
 "best": false,
 "score": 0.8,
 "evidence": [],
 "url": "https://dedupedai.com/use-cases/yeonwoosung-llm-architecture-from-scratch/study-mixture-of-experts-routing/",
 "md": "https://dedupedai.com/use-cases/yeonwoosung-llm-architecture-from-scratch/study-mixture-of-experts-routing/index.md",
 "disclaimer": "Deduped lists open-source repositories that belong to other people. We do not own, maintain, audit, or vouch for any of them, and a repo appearing here is not an endorsement or a safety check. The use cases, steps, and prompts are written by AI and may be incomplete, outdated, or just wrong. Where a use case is marked \"Seen in the wild\", we link to public posts where someone described something similar, but we have not verified those posts and the use case itself is still our AI's own write-up. Use all of this at your own risk. Only run code, commands, or agents on systems and accounts you are authorized to use, and read the code and every command before you run it, especially anything that touches credentials, files, or the network. Nothing here is professional, security, legal, or financial advice. All trademarks and project names belong to their respective owners, and their use here does not imply affiliation. Content is provided as is, without warranties of any kind. Found a mistake, a safety problem, or something you want removed? Email hello@dedupedai.com and we will look at it promptly.",
 "api": "https://dedupedai.com/api/v1/usecases/yeonwoosung-llm-architecture-from-scratch--study-mixture-of-experts-routing.json",
 "api_version": "v1"
}