{
 "schema_version": 1,
 "generated": "2026-10-26",
 "repo": "lyogavin/airllm",
 "name": "airllm",
 "owner": "lyogavin",
 "slug": "lyogavin-airllm",
 "github": "https://github.com/lyogavin/airllm",
 "what": "AirLLM cuts inference memory so very large language models run on a single small GPU without quantization, distillation or pruning, by streaming model layers instead of loading the whole model. The README reports 70B models on a 4GB card, DeepSeek-V3 (671B) on about 12GB and Kimi K3 on under 4GB, all through one AutoModel API, plus optional 4-bit or 8-bit compression for speed and new training support for huge models on small VRAM.",
 "why": "41k stars, picked up by one AI source we track",
 "language": "Jupyter Notebook",
 "requirements": [
  "Python and pip",
  "A CUDA GPU (a few GB of VRAM) or a Mac with mlx and torch for Llama-style models",
  "Lots of disk for split layer shards; some models need specific transformers versions",
  "Optional: bitsandbytes for compression"
 ],
 "audience_note": "Developers and researchers with a modest GPU who want to experiment with very large open models without renting big cloud machines.",
 "risk": "none",
 "section": "earlier",
 "weekly_rank": null,
 "category": "Software and DevOps",
 "url": "https://dedupedai.com/use-cases/lyogavin-airllm/",
 "use_case_count": 3,
 "use_cases": [
  {
   "id": "lyogavin-airllm--run-a-70b-model-on-a-4gb-gpu",
   "title": "Run a 70B-class model on a single 4GB GPU",
   "best": true,
   "url": "https://dedupedai.com/use-cases/lyogavin-airllm/run-a-70b-model-on-a-4gb-gpu/",
   "json": "https://dedupedai.com/use-cases/u/lyogavin-airllm/run-a-70b-model-on-a-4gb-gpu.json"
  },
  {
   "id": "lyogavin-airllm--speed-up-airllm-with-compression",
   "title": "Speed up large-model inference with 4-bit block compression",
   "best": false,
   "url": "https://dedupedai.com/use-cases/lyogavin-airllm/speed-up-airllm-with-compression/",
   "json": "https://dedupedai.com/use-cases/u/lyogavin-airllm/speed-up-airllm-with-compression.json"
  },
  {
   "id": "lyogavin-airllm--run-llama-style-models-on-a-mac",
   "title": "Run large Llama-style models on a Mac with AirLLM",
   "best": false,
   "url": "https://dedupedai.com/use-cases/lyogavin-airllm/run-llama-style-models-on-a-mac/",
   "json": "https://dedupedai.com/use-cases/u/lyogavin-airllm/run-llama-style-models-on-a-mac.json"
  }
 ],
 "disclaimer": "Third-party repos we don't own or vouch for. AI-generated use cases can be wrong: review everything and run at your own risk."
}