{
 "schema_version": 1,
 "generated": "2026-10-20",
 "id": "salikhussain71-code-pakgov-rag-project--reproduce-retrieval-benchmark-results",
 "repo": "salikhussain71-code/PAKGOV-RAG-project",
 "repo_url": "https://github.com/salikhussain71-code/PAKGOV-RAG-project",
 "repo_slug": "salikhussain71-code-pakgov-rag-project",
 "title": "Check the retrieval benchmark numbers yourself",
 "page_title": "How to check the retrieval benchmark numbers yourself with PAKGOV RAG Project",
 "summary": "Regenerate the results table from committed evaluation files and compare BM25, dense, and hybrid retrieval.",
 "who": "A researcher or reviewer who needs to verify the R@1, R@5, and MRR figures before citing them in a paper or report.",
 "problem": "Benchmark numbers are only useful if they can be checked, and comparing retrievers on different question sets gives misleading rankings.",
 "outcome": "A regenerated FINAL_RESULTS.md table, with the metrics checked against the committed raw outputs in data/interim/ and evaluation/retrieval/.",
 "steps": [
  "Clone https://github.com/salikhussain71-code/PAKGOV-RAG-project and open the project folder.",
  "Install the Python packages the evaluation script imports, such as `pip install streamlit sentence-transformers faiss-cpu rank_bm25`, if the script fails with a missing module. (inferred, check the README)",
  "Run `python app/eval/final_table.py`.",
  "Open evaluation/retrieval/FINAL_RESULTS.md and compare it with hybrid_rrf_metrics_50.json and dense_metrics_50.json.",
  "Read evaluation/retrieval/failure_analysis_50.md and note the questions where the gold source was not in the top 5."
 ],
 "agent_prompt": "Clone salikhussain71-code/PAKGOV-RAG-project, run python app/eval/final_table.py, and write the regenerated results to evaluation/retrieval/FINAL_RESULTS.md. Report the R@5 and MRR values for BM25, Dense, and Hybrid RRF (k=60). Verify that the values match the committed metrics JSON files in evaluation/retrieval/ and flag any difference.",
 "agents": [
  "any"
 ],
 "difficulty": "Intermediate",
 "time": "30 minutes",
 "basis": "readme",
 "caveat": "The README warns that the 50-question set is small, so a 10-point gap at R@5 equals 5 questions and the ranking should be treated as indicative.",
 "category": "Research",
 "tags": [
  "Research"
 ],
 "lang": "python",
 "requirements": [
  "Python installed (the README lists Python 3.14)",
  "Ability to clone the GitHub repository and run commands in a terminal",
  "Internet access for the first download of the sentence-transformers embedding model (inferred from the model being loaded by name)",
  "Constitution of Pakistan PDFs or the committed data files for the demo and evaluation"
 ],
 "risk": "none",
 "seen_in_the_wild": false,
 "mentioned_publicly": false,
 "risks": [],
 "best": false,
 "score": 0.9,
 "evidence": [],
 "url": "https://dedupedai.com/use-cases/salikhussain71-code-pakgov-rag-project/reproduce-retrieval-benchmark-results/",
 "md": "https://dedupedai.com/use-cases/salikhussain71-code-pakgov-rag-project/reproduce-retrieval-benchmark-results/index.md",
 "disclaimer": "Deduped lists open-source repositories that belong to other people. We do not own, maintain, audit, or vouch for any of them, and a repo appearing here is not an endorsement or a safety check. The use cases, steps, and prompts are written by AI and may be incomplete, outdated, or just wrong. Where a use case is marked \"Seen in the wild\", we link to public posts where someone described something similar, but we have not verified those posts and the use case itself is still our AI's own write-up. Use all of this at your own risk. Only run code, commands, or agents on systems and accounts you are authorized to use, and read the code and every command before you run it, especially anything that touches credentials, files, or the network. Nothing here is professional, security, legal, or financial advice. All trademarks and project names belong to their respective owners, and their use here does not imply affiliation. Content is provided as is, without warranties of any kind. Found a mistake, a safety problem, or something you want removed? Email hello@dedupedai.com and we will look at it promptly.",
 "api": "https://dedupedai.com/api/v1/usecases/salikhussain71-code-pakgov-rag-project--reproduce-retrieval-benchmark-results.json",
 "api_version": "v1"
}