{
 "schema_version": 1,
 "generated": "2026-10-20",
 "id": "baruquerodrigues-llm-legislativo--reproduce-byte-nll-evaluation",
 "repo": "BaruqueRodrigues/llm-legislativo",
 "repo_url": "https://github.com/BaruqueRodrigues/llm-legislativo",
 "repo_slug": "baruquerodrigues-llm-legislativo",
 "title": "Reproduce the byte_nll_v1 evaluation numbers",
 "page_title": "How to reproduce the byte_nll_v1 evaluation numbers with LLM Legislativo",
 "summary": "Rerun the byte-level NLL protocol on the test split and compare against the reported results.",
 "who": "A researcher checking whether the published perplexity and nats per byte figures hold on the same test split.",
 "problem": "Published metrics are only useful if the evaluation protocol can be run again and matched.",
 "outcome": "Your own byte_nll_v1 numbers for the test split, set next to the README values of 0,7297 nats per byte and 1,0528 bits per byte.",
 "steps": [
  "Read the byte_nll_v1 protocol description in the README section 7 and the code in src/legislm/evaluation/.",
  "Clone the repo with `git clone https://github.com/BaruqueRodrigues/llm-legislativo` and install with `pip install -r requirements-cpu.lock`. (inferred, check the README)",
  "Get the model and the test split from the Hugging Face links in the README.",
  "Run the evaluation module from src/legislm/evaluation/ on the test split and save the output. (inferred, check the README)"
 ],
 "agent_prompt": "In the repo https://github.com/BaruqueRodrigues/llm-legislativo, run the byte_nll_v1 evaluation in src/legislm/evaluation/ on the test split and report nats per byte for the whole set and for the Câmara and Senado subsets. Compare with the README values 0,7297 and 0,8817 and 0,7049. Verify the document and byte counts match 71.069 documents and 482 million bytes.",
 "agents": [
  "any"
 ],
 "difficulty": "Advanced",
 "time": "2 hours",
 "basis": "readme",
 "caveat": "The README does not give the evaluation command, so the entry point and the input paths must be read from the code, and the run may need a GPU for a reasonable time.",
 "category": "Research",
 "tags": [
  "Research"
 ],
 "lang": "python",
 "requirements": [
  "A Python environment (the README install section is cut off in the source, so exact setup steps are unconfirmed)",
  "Internet access to Hugging Face Hub and GitHub",
  "Enough disk space for the Parquet release if you download corpus files (the raw TAR archives total over 360 GiB)",
  "Basic familiarity with PyTorch and Hugging Face datasets for the model and corpus steps"
 ],
 "risk": "dual-use",
 "seen_in_the_wild": false,
 "mentioned_publicly": false,
 "risks": [],
 "best": false,
 "score": 0.7,
 "evidence": [],
 "url": "https://dedupedai.com/use-cases/baruquerodrigues-llm-legislativo/reproduce-byte-nll-evaluation/",
 "md": "https://dedupedai.com/use-cases/baruquerodrigues-llm-legislativo/reproduce-byte-nll-evaluation/index.md",
 "disclaimer": "Deduped lists open-source repositories that belong to other people. We do not own, maintain, audit, or vouch for any of them, and a repo appearing here is not an endorsement or a safety check. The use cases, steps, and prompts are written by AI and may be incomplete, outdated, or just wrong. Where a use case is marked \"Seen in the wild\", we link to public posts where someone described something similar, but we have not verified those posts and the use case itself is still our AI's own write-up. Use all of this at your own risk. Only run code, commands, or agents on systems and accounts you are authorized to use, and read the code and every command before you run it, especially anything that touches credentials, files, or the network. Nothing here is professional, security, legal, or financial advice. All trademarks and project names belong to their respective owners, and their use here does not imply affiliation. Content is provided as is, without warranties of any kind. Found a mistake, a safety problem, or something you want removed? Email hello@dedupedai.com and we will look at it promptly.",
 "api": "https://dedupedai.com/api/v1/usecases/baruquerodrigues-llm-legislativo--reproduce-byte-nll-evaluation.json",
 "api_version": "v1"
}