CtrlK
BlogDocsLog inGet started
Tessl Logo

tessl/pypi-uiautomator2

A simple, easy-to-use, and stable Android automation library

Overview
Eval results
Files

screen-input.mddocs/

Screen and Input Operations

Screen capture, device orientation control, hardware key simulation, and text input functionality for comprehensive device interaction.

Capabilities

Screen Capture

Capture device screen content and UI hierarchy information.

class Device:
    def screenshot(self, filename: Optional[str] = None, format="pillow", display_id: Optional[int] = None):
        """
        Take device screenshot.

        Parameters:
        - filename: Save path (if provided, returns None)
        - format: Output format ("pillow" or "opencv")
        - display_id: Specific display ID for multi-screen devices

        Returns:
        PIL Image or None if filename specified
        """

    def dump_hierarchy(self, compressed=False, pretty=False, max_depth: Optional[int] = None) -> str:
        """
        Get UI hierarchy XML representation.

        Parameters:
        - compressed: Return compressed XML
        - pretty: Pretty-print XML with indentation
        - max_depth: Maximum hierarchy depth to capture

        Returns:
        XML string of UI hierarchy
        """

Usage examples:

d = u2.connect()

# Take screenshot and save
d.screenshot("screenshot.png")

# Get screenshot as PIL Image
img = d.screenshot()
img.save("screenshot.jpg")
img.show()  # Display image

# Get screenshot in OpenCV format
import cv2
cv_img = d.screenshot(format="opencv")
cv2.imwrite("screenshot_cv.png", cv_img)

# Multi-display support
d.screenshot("display1.png", display_id=1)

# Get UI hierarchy
hierarchy = d.dump_hierarchy(pretty=True)
with open("hierarchy.xml", "w", encoding="utf-8") as f:
    f.write(hierarchy)

# Compressed hierarchy
compact_xml = d.dump_hierarchy(compressed=True, max_depth=30)

Hardware Key Input

Simulate hardware keys and buttons.

class Device:
    def press(self, key: Union[int, str], meta=None):
        """
        Press hardware key.

        Parameters:
        - key: Key name or key code
        - meta: Meta key modifier (optional)

        Supported key names:
        home, back, left, right, up, down, center, menu, search,
        enter, delete, del, recent, volume_up, volume_down,
        volume_mute, camera, power
        """

    def long_press(self, key: Union[int, str]):
        """
        Long press hardware key.

        Parameters:
        - key: Key name or key code
        """

    def keyevent(self, v):
        """
        Send keyevent via shell command.

        Parameters:
        - v: Keyevent name (case insensitive)
        """

Usage examples:

d = u2.connect()

# Basic key presses
d.press("home")      # Home button
d.press("back")      # Back button
d.press("menu")      # Menu button
d.press("power")     # Power button

# Navigation keys
d.press("up")        # Up arrow
d.press("down")      # Down arrow
d.press("left")      # Left arrow
d.press("right")     # Right arrow
d.press("center")    # Center/OK button

# Volume controls
d.press("volume_up")
d.press("volume_down")
d.press("volume_mute")

# Text input keys
d.press("enter")     # Enter key
d.press("delete")    # Delete/backspace

# Long press operations
d.long_press("power")  # Power menu
d.long_press("home")   # Recent apps

# Key codes
d.press(4)           # Back key (KEYCODE_BACK)
d.press(82)          # Menu key (KEYCODE_MENU)

# Keyevent shell commands
d.keyevent("POWER")
d.keyevent("HOME")

Text Input and Clipboard

Text input operations and clipboard management.

class Device:
    def send_keys(self, text: str, clear: bool = False):
        """
        Send text to focused input field.

        Parameters:
        - text: Text to input
        - clear: Clear existing text before input
        """

    def clear_text(self):
        """Clear text in focused input field"""

    @property
    def clipboard(self) -> Optional[str]:
        """Get clipboard content"""

    @clipboard.setter
    def clipboard(self, text: str):
        """Set clipboard content"""

    def set_clipboard(self, text, label=None):
        """
        Set clipboard content with optional label.

        Parameters:
        - text: Text content to copy
        - label: User-visible label for clip data
        """

Usage examples:

d = u2.connect()

# Text input to focused field
d.send_keys("Hello World")
d.send_keys("Replace text", clear=True)

# Clear input field
d.clear_text()

# Clipboard operations
d.clipboard = "Text to copy"
current_clipboard = d.clipboard
print(f"Clipboard: {current_clipboard}")

# Set clipboard with label
d.set_clipboard("Important data", label="Test Data")

# Complex text input workflow
username_field = d(resourceId="com.example:id/username")
username_field.click()
d.clear_text()
d.send_keys("john_doe")

password_field = d(resourceId="com.example:id/password")
password_field.click()
d.send_keys("secret123")
d.press("enter")

System UI Operations

Control system UI elements and notifications.

class Device:
    def open_notification(self):
        """Open notification panel"""

    def open_quick_settings(self):
        """Open quick settings panel"""

    def open_url(self, url: str):
        """
        Open URL in default browser.

        Parameters:
        - url: URL to open
        """

    @property
    def last_toast(self) -> Optional[str]:
        """Get last toast message"""

    def clear_toast(self):
        """Clear last toast message"""

Usage examples:

d = u2.connect()

# System UI access
d.open_notification()      # Pull down notification panel
d.open_quick_settings()    # Open quick settings

# Open URLs
d.open_url("https://www.google.com")
d.open_url("market://details?id=com.example.app")  # Play Store

# Toast monitoring
toast_msg = d.last_toast
if toast_msg:
    print(f"Toast: {toast_msg}")
d.clear_toast()

Advanced Input Method Support

Enhanced text input using input method editor (IME).

class InputMethodMixIn:
    def is_input_ime_installed(self) -> bool:
        """Check if UIAutomator2 input method is installed"""

    def install_input_ime(self):
        """Install UIAutomator2 input method"""

    def set_input_ime(self, enable: bool = True):
        """Enable/disable UIAutomator2 input method"""

Usage examples:

d = u2.connect()

# Check and install input method
if not d.is_input_ime_installed():
    d.install_input_ime()
    d.set_input_ime(True)

# Enhanced text input (automatically used when available)
d.send_keys("Text with special chars: @#$%^&*()")
d.send_keys("Unicode text: 你好世界 🌍")

# Input method provides better text input reliability
search_field = d(resourceId="search_box")
search_field.click()
d.send_keys("Complex search query with symbols & emojis 😀", clear=True)

Touch Gesture Builder

Low-level touch event construction for complex gestures.

class Device:
    @property
    def touch(self) -> TouchActions:
        """Touch gesture builder for complex multi-touch operations"""

class TouchActions:
    def down(self, x, y) -> TouchActions:
        """Touch down at coordinates"""

    def move(self, x, y) -> TouchActions:
        """Move touch to coordinates"""

    def up(self, x, y) -> TouchActions:
        """Release touch at coordinates"""

    def sleep(self, seconds: float) -> TouchActions:
        """Add delay in gesture sequence"""

Usage examples:

d = u2.connect()

# Custom touch gestures
d.touch.down(100, 100).sleep(0.1).move(200, 200).sleep(0.5).up(200, 200)

# Multi-step gesture
d.touch.down(100, 100).move(150, 150).move(200, 200).move(250, 250).up(250, 250)

# Touch and hold with movement
d.touch.down(300, 300).sleep(1.0).move(400, 400).sleep(0.5).up(400, 400)

# Precise timing control
d.touch.down(100, 100).sleep(0.05).move(101, 101).sleep(0.05).move(102, 102).up(102, 102)

Coordinate System Utilities

Coordinate conversion and screen dimension utilities.

class Device:
    @property
    def pos_rel2abs(self):
        """
        Function to convert relative (0-1) coordinates to absolute pixels.

        Returns:
        Function that takes (x, y) relative coordinates and returns absolute coordinates
        """

    def window_size(self) -> Tuple[int, int]:
        """
        Get device screen dimensions.

        Returns:
        Tuple of (width, height) in pixels
        """

Usage examples:

d = u2.connect()

# Get screen dimensions
width, height = d.window_size()
print(f"Screen: {width}x{height}")

# Convert relative to absolute coordinates
convert = d.pos_rel2abs
center_x, center_y = convert(0.5, 0.5)  # Screen center
top_left_x, top_left_y = convert(0.1, 0.1)  # Near top-left
bottom_right_x, bottom_right_y = convert(0.9, 0.9)  # Near bottom-right

# Use converted coordinates
d.click(center_x, center_y)  # Click center
d.swipe(top_left_x, top_left_y, bottom_right_x, bottom_right_y)  # Diagonal swipe

Install with Tessl CLI

npx tessl i tessl/pypi-uiautomator2

docs

app-management.md

device-management.md

image-processing.md

index.md

screen-input.md

ui-interaction.md

watchers-automation.md

xpath-selection.md

tile.json