{
 "schema_version": 1,
 "generated": "2026-10-24",
 "id": "verlintas-betteraichat--drive-an-app-by-tapping-text-on-screen",
 "repo": "Verlintas/BetterAIChat",
 "repo_url": "https://github.com/Verlintas/BetterAIChat",
 "repo_slug": "verlintas-betteraichat",
 "title": "Drive an app by tapping its on-screen text",
 "page_title": "How to drive an app by tapping its on-screen text with Better AI Chat",
 "summary": "Let the AI find and tap buttons by their text in any app, after you grant Accessibility access.",
 "who": "A user with a repetitive multi-step task in a phone app, such as filling a form, who wants the AI to perform the taps.",
 "problem": "Tapping through the same screens by hand is slow, and tapping by fixed coordinates breaks when the layout shifts.",
 "outcome": "The AI completes a short sequence of taps and text entries in one app, and you confirm the result with a screenshot.",
 "steps": [
  "In Settings, turn on the Accessibility and Usage access toggles.",
  "Open the target app and start a chat in Build mode.",
  "Ask the AI to find the target button by its text. It uses ua_find_text or screen_ocr to inspect the screen, then ua_tap_text to tap it.",
  "For text entry or swipes, let the AI use ua_type or ua_swipe. Approve each confirmation prompt in Build mode.",
  "Ask the AI to run take_screenshot or screen_ocr to verify the final screen."
 ],
 "agent_prompt": "Using the repo Verlintas/BetterAIChat, with Accessibility and Usage access enabled, ask the Agent to open <target app> and tap the button labeled <button text>. Expected output: a screenshot or OCR result showing the next screen. Verify the screen changed as described and that each tap was confirmed.",
 "agents": [
  "any"
 ],
 "difficulty": "Intermediate",
 "time": "1 hour",
 "basis": "readme",
 "caveat": "The README says this path needs no root and no Shizuku, but the AI can only tap what the Accessibility service reports on screen, so results depend on each app's layout.",
 "category": "Personal productivity",
 "tags": [
  "Personal productivity"
 ],
 "lang": "kotlin",
 "requirements": [
  "An Android phone that can install an APK from outside the Play Store",
  "A browser on the phone or computer to download the APK from GitHub Releases",
  "An API key from a supported provider (OpenAI-compatible, Anthropic Claude, or Google Gemini)",
  "For building from source: JDK 17 and Android SDK with compileSdk 36",
  "Optional: Shizuku for the run_shell and app management tools, and Accessibility and Usage access toggles for full UI automation"
 ],
 "risk": "sensitive",
 "seen_in_the_wild": false,
 "mentioned_publicly": false,
 "risks": [
  {
   "tag": "reverse-engineering",
   "notice": "The AI taps and types inside other apps, so it may act on a screen you did not expect, and you should keep Build mode confirmations on for any app that handles money or accounts."
  }
 ],
 "best": false,
 "score": 0.7,
 "evidence": [],
 "url": "https://dedupedai.com/use-cases/verlintas-betteraichat/drive-an-app-by-tapping-text-on-screen/",
 "md": "https://dedupedai.com/use-cases/verlintas-betteraichat/drive-an-app-by-tapping-text-on-screen/index.md",
 "disclaimer": "Deduped lists open-source repositories that belong to other people. We do not own, maintain, audit, or vouch for any of them, and a repo appearing here is not an endorsement or a safety check. The use cases, steps, and prompts are written by AI and may be incomplete, outdated, or just wrong. Where a use case is marked \"Seen in the wild\", we link to public posts where someone described something similar, but we have not verified those posts and the use case itself is still our AI's own write-up. Use all of this at your own risk. Only run code, commands, or agents on systems and accounts you are authorized to use, and read the code and every command before you run it, especially anything that touches credentials, files, or the network. Nothing here is professional, security, legal, or financial advice. All trademarks and project names belong to their respective owners, and their use here does not imply affiliation. Content is provided as is, without warranties of any kind. Found a mistake, a safety problem, or something you want removed? Email hello@dedupedai.com and we will look at it promptly.",
 "api": "https://dedupedai.com/api/v1/usecases/verlintas-betteraichat--drive-an-app-by-tapping-text-on-screen.json",
 "api_version": "v1"
}