{
 "schema_version": 1,
 "generated": "2026-10-26",
 "id": "tencent-ui-mate--test-gui-agent-on-one-screenshot",
 "repo": "Tencent/UI-Mate",
 "repo_url": "https://github.com/Tencent/UI-Mate",
 "repo_slug": "tencent-ui-mate",
 "title": "Test a GUI agent on one screenshot locally",
 "page_title": "How to test a GUI agent on one screenshot locally with UI Mate",
 "summary": "Serve UI Mate with vLLM and get a predicted next action for a screenshot you provide.",
 "who": "A developer or QA engineer who wants to see how a computer-use agent reads a screen before wiring it into a larger workflow.",
 "problem": "Before running an agent on real work, you need to know whether it can read a specific screen and pick a sensible first move.",
 "outcome": "You get a predicted action for your own screenshot and instruction, with the screenshot and prediction written to outputs/.",
 "steps": [
  "Run `pip install openai pillow` in the repository environment.",
  "Run `vllm serve /path/to/UI-Mate-27B --trust-remote-code --served-model-name UI_Mate --port 8000 --tensor-parallel-size 2 --gpu-memory-utilization 0.85 --mm-encoder-tp-mode data --chat-template-content-format openai --limit-mm-per-prompt '{\"image\":6,\"video\":0}'`. (inferred, check the README)",
  "Run `curl -s http://127.0.0.1:8000/v1/models` and confirm the served model name.",
  "Run `python examples/run_agent.py --image /path/to/screen.png --instruction \"<your instruction>\" --base-url http://127.0.0.1:8000/v1`. (inferred, check the README)",
  "Open the files written to `outputs/` and check the predicted position and action."
 ],
 "agent_prompt": "Using the Tencent/UI-Mate repository, serve UI-Mate-27B with vLLM on port 8000, then run examples/run_agent.py with the screenshot at <screenshot path> and the instruction <instruction>. Report the predicted action and where the screenshot and prediction were saved in outputs/. Verify that the model name returned by /v1/models is UI_Mate before running.",
 "agents": [
  "any"
 ],
 "difficulty": "Intermediate",
 "time": "1 hour",
 "basis": "readme",
 "caveat": "The README example depends on a multi-GPU vLLM setup, so a single small machine may not run the 27B checkpoint without changes to these flags.",
 "category": "Work and operations",
 "tags": [
  "Work and operations"
 ],
 "lang": "javascript",
 "requirements": [
  "A GPU server able to serve the UI-Mate-27B checkpoint with vLLM (the README example uses two-way tensor parallelism)",
  "Python with pip, so you can install openai and pillow",
  "A clone of the Tencent/UI-Mate repository and the model weights from Hugging Face",
  "A sandboxed or test desktop, since the agent controls real keyboard and mouse input"
 ],
 "risk": "dual-use",
 "seen_in_the_wild": false,
 "mentioned_publicly": false,
 "risks": [],
 "best": true,
 "score": 1.0,
 "evidence": [],
 "url": "https://dedupedai.com/use-cases/tencent-ui-mate/test-gui-agent-on-one-screenshot/",
 "md": "https://dedupedai.com/use-cases/tencent-ui-mate/test-gui-agent-on-one-screenshot/index.md",
 "disclaimer": "Deduped lists open-source repositories that belong to other people. We do not own, maintain, audit, or vouch for any of them, and a repo appearing here is not an endorsement or a safety check. The use cases, steps, and prompts are written by AI and may be incomplete, outdated, or just wrong. Where a use case is marked \"Seen in the wild\", we link to public posts where someone described something similar, but we have not verified those posts and the use case itself is still our AI's own write-up. Use all of this at your own risk. Only run code, commands, or agents on systems and accounts you are authorized to use, and read the code and every command before you run it, especially anything that touches credentials, files, or the network. Nothing here is professional, security, legal, or financial advice. All trademarks and project names belong to their respective owners, and their use here does not imply affiliation. Content is provided as is, without warranties of any kind. Found a mistake, a safety problem, or something you want removed? Email hello@dedupedai.com and we will look at it promptly.",
 "api": "https://dedupedai.com/api/v1/usecases/tencent-ui-mate--test-gui-agent-on-one-screenshot.json",
 "api_version": "v1"
}