{
 "schema_version": 1,
 "generated": "2026-10-26",
 "url": "https://dedupedai.com/best-ai-repos-for/web-scraping-for-llms/",
 "task": "web scraping for LLMs and RAG",
 "question": "What is the best open-source repo for scraping websites into LLM-ready data?",
 "answer": "The best open-source AI repos for web scraping for LLMs and RAG right now are Firecrawl, Crawl4AI and Browser Use. Firecrawl is the most starred option and returns clean markdown or JSON through an API; Crawl4AI is fully self-hosted; browser-use helps when a site needs clicks and logins before the data appears.",
 "md": "https://dedupedai.com/best-ai-repos-for/web-scraping-for-llms/index.md",
 "repos": [
  {
   "repo": "firecrawl/firecrawl",
   "title": "Firecrawl",
   "url": "https://dedupedai.com/use-cases/firecrawl-firecrawl/",
   "github": "https://github.com/firecrawl/firecrawl",
   "what": "Firecrawl is an API that searches, scrapes, crawls and interacts with the web and returns clean markdown, structured JSON or screenshots for AI agents. It handles JavaScript-heavy pages, proxies and rate limits, parses PDFs and DOCX, can click and type before extracting, and plugs into agents as a skill or an MCP server. It is open source and also offered as a hosted service with an API key.",
   "who": "Developers building AI agents, RAG pipelines or research tools that need clean page content instead of raw HTML.",
   "stars": 195071,
   "language": "TypeScript",
   "requirements": [
    "A Firecrawl API key from the hosted service, or a self-hosted instance",
    "Python, Node.js, or curl; or an MCP client such as Claude Code or Cursor"
   ],
   "uc_count": 3,
   "use_cases": [
    {
     "title": "Turn any web page into clean markdown for an LLM",
     "url": "https://dedupedai.com/use-cases/firecrawl-firecrawl/turn-any-url-into-clean-markdown/"
    },
    {
     "title": "Give Claude Code or Cursor live web search and scraping",
     "url": "https://dedupedai.com/use-cases/firecrawl-firecrawl/give-your-coding-agent-live-web-access/"
    },
    {
     "title": "Crawl a whole docs site into a knowledge base",
     "url": "https://dedupedai.com/use-cases/firecrawl-firecrawl/crawl-a-docs-site-into-a-knowledge-base/"
    }
   ],
   "best": {
    "title": "Turn any web page into clean markdown for an LLM",
    "url": "https://dedupedai.com/use-cases/firecrawl-firecrawl/turn-any-url-into-clean-markdown/",
    "difficulty": "Beginner",
    "time": "10 minutes",
    "agents": [],
    "steps": [
     "Get an API key at Firecrawl.dev",
     "Python: from firecrawl import Firecrawl, create app = Firecrawl(api_key=\"fc-YOUR_API_KEY\"), then result = app.scrape('firecrawl.dev')",
     "Or from the CLI: firecrawl https://firecrawl.dev --only-main-content",
     "Ask for other formats such as structured JSON or screenshots when you need them"
    ],
    "md": "https://dedupedai.com/use-cases/firecrawl-firecrawl/turn-any-url-into-clean-markdown/index.md"
   }
  },
  {
   "repo": "unclecode/crawl4ai",
   "title": "Crawl4AI",
   "url": "https://dedupedai.com/use-cases/unclecode-crawl4ai/",
   "github": "https://github.com/unclecode/crawl4ai",
   "what": "Crawl4AI is an open-source web crawler and scraper that turns websites into clean, LLM-ready Markdown for RAG, agents and data pipelines, self-hosted or via a hosted API and MCP server.",
   "who": "Developers building RAG pipelines, agents or datasets from web content.",
   "stars": null,
   "language": "Python",
   "requirements": [
    "Python with pip",
    "Playwright Chromium (installed by crawl4ai-setup)"
   ],
   "uc_count": 3,
   "use_cases": [
    {
     "title": "Turn any web page into clean Markdown for RAG",
     "url": "https://dedupedai.com/use-cases/unclecode-crawl4ai/turn-website-into-markdown-for-rag/"
    },
    {
     "title": "Give Claude Code a web scraping tool over MCP",
     "url": "https://dedupedai.com/use-cases/unclecode-crawl4ai/give-claude-code-web-scraping-tool/"
    },
    {
     "title": "Scrape pages with one curl call, hosted or on your own server",
     "url": "https://dedupedai.com/use-cases/unclecode-crawl4ai/scrape-page-via-hosted-api/"
    }
   ],
   "best": {
    "title": "Turn any web page into clean Markdown for RAG",
    "url": "https://dedupedai.com/use-cases/unclecode-crawl4ai/turn-website-into-markdown-for-rag/",
    "difficulty": "Beginner",
    "time": "15 minutes",
    "agents": [
     "claude-code",
     "codex",
     "cursor"
    ],
    "steps": [
     "Install with pip install -U crawl4ai",
     "Run crawl4ai-setup once to install the browser, then check with crawl4ai-doctor",
     "From the shell, crwl https://news.ycombinator.com -o markdown prints a page as Markdown",
     "In Python, open AsyncWebCrawler() as crawler, await crawler.arun(url=...) and save result.markdown",
     "Use the Fit Markdown filters (PruningContentFilterLXML, or BM25ContentFilter for a query) to drop boilerplate",
     "Crawl a docs site with crwl https://docs.crawl4ai.com --deep-crawl bfs --max-pages 10"
    ],
    "md": "https://dedupedai.com/use-cases/unclecode-crawl4ai/turn-website-into-markdown-for-rag/index.md"
   }
  },
  {
   "repo": "browser-use/browser-use",
   "title": "Browser Use",
   "url": "https://dedupedai.com/use-cases/browser-use-browser-use/",
   "github": "https://github.com/browser-use/browser-use",
   "what": "browser-use is an open-source Python library for AI browser agents that navigate websites with any LLM, plus an optional cloud with stealth browsers and a hosted agent API.",
   "who": "Developers automating web tasks such as form filling, lookups and bookings with AI agents.",
   "stars": null,
   "language": "Python",
   "requirements": [
    "Python 3.12 with uv",
    "An LLM API key such as OpenAI, or a Browser Use key"
   ],
   "uc_count": 3,
   "use_cases": [
    {
     "title": "Automate a web task with a Python browser agent",
     "url": "https://dedupedai.com/use-cases/browser-use-browser-use/automate-web-task-with-python-agent/"
    },
    {
     "title": "Give your coding agent a browser skill",
     "url": "https://dedupedai.com/use-cases/browser-use-browser-use/give-coding-agent-browser-skill/"
    },
    {
     "title": "Run browser agents on cloud stealth browsers",
     "url": "https://dedupedai.com/use-cases/browser-use-browser-use/run-agent-on-cloud-browser/"
    }
   ],
   "best": {
    "title": "Automate a web task with a Python browser agent",
    "url": "https://dedupedai.com/use-cases/browser-use-browser-use/automate-web-task-with-python-agent/",
    "difficulty": "Intermediate",
    "time": "20 minutes",
    "agents": [
     "claude-code",
     "codex",
     "cursor"
    ],
    "steps": [
     "With uv installed, start a project with uv init --python 3.12 if needed",
     "Add the library: uv add browser-use",
     "Put OPENAI_API_KEY in .env (or BROWSER_USE_API_KEY to use the BU2 model)",
     "In agent.py create Agent(task=\"...\", llm=ChatOpenAI(...)), call await agent.run() and print history.final_result()",
     "Run it with uv run agent.py",
     "Swap in ChatBrowserUse(model='bu-2-0'), the model tuned for browser automation, if you have a Browser Use key"
    ],
    "md": "https://dedupedai.com/use-cases/browser-use-browser-use/automate-web-task-with-python-agent/index.md"
   }
  }
 ]
}