0
# Screen and Input Operations
1

2
Screen capture, device orientation control, hardware key simulation, and text input functionality for comprehensive device interaction.
3

4
## Capabilities
5

6
### Screen Capture
7

8
Capture device screen content and UI hierarchy information.
9

10
```python { .api }
11
class Device:
12
    def screenshot(self, filename: Optional[str] = None, format="pillow", display_id: Optional[int] = None):
13
        """
14
        Take device screenshot.
15

16
        Parameters:
17
        - filename: Save path (if provided, returns None)
18
        - format: Output format ("pillow" or "opencv")
19
        - display_id: Specific display ID for multi-screen devices
20

21
        Returns:
22
        PIL Image or None if filename specified
23
        """
24

25
    def dump_hierarchy(self, compressed=False, pretty=False, max_depth: Optional[int] = None) -> str:
26
        """
27
        Get UI hierarchy XML representation.
28

29
        Parameters:
30
        - compressed: Return compressed XML
31
        - pretty: Pretty-print XML with indentation
32
        - max_depth: Maximum hierarchy depth to capture
33

34
        Returns:
35
        XML string of UI hierarchy
36
        """
37
```
38

39
Usage examples:
40

41
```python
42
d = u2.connect()
43

44
# Take screenshot and save
45
d.screenshot("screenshot.png")
46

47
# Get screenshot as PIL Image
48
img = d.screenshot()
49
img.save("screenshot.jpg")
50
img.show()  # Display image
51

52
# Get screenshot in OpenCV format
53
import cv2
54
cv_img = d.screenshot(format="opencv")
55
cv2.imwrite("screenshot_cv.png", cv_img)
56

57
# Multi-display support
58
d.screenshot("display1.png", display_id=1)
59

60
# Get UI hierarchy
61
hierarchy = d.dump_hierarchy(pretty=True)
62
with open("hierarchy.xml", "w", encoding="utf-8") as f:
63
    f.write(hierarchy)
64

65
# Compressed hierarchy
66
compact_xml = d.dump_hierarchy(compressed=True, max_depth=30)
67
```
68

69
### Hardware Key Input
70

71
Simulate hardware keys and buttons.
72

73
```python { .api }
74
class Device:
75
    def press(self, key: Union[int, str], meta=None):
76
        """
77
        Press hardware key.
78

79
        Parameters:
80
        - key: Key name or key code
81
        - meta: Meta key modifier (optional)
82

83
        Supported key names:
84
        home, back, left, right, up, down, center, menu, search,
85
        enter, delete, del, recent, volume_up, volume_down,
86
        volume_mute, camera, power
87
        """
88

89
    def long_press(self, key: Union[int, str]):
90
        """
91
        Long press hardware key.
92

93
        Parameters:
94
        - key: Key name or key code
95
        """
96

97
    def keyevent(self, v):
98
        """
99
        Send keyevent via shell command.
100

101
        Parameters:
102
        - v: Keyevent name (case insensitive)
103
        """
104
```
105

106
Usage examples:
107

108
```python
109
d = u2.connect()
110

111
# Basic key presses
112
d.press("home")      # Home button
113
d.press("back")      # Back button
114
d.press("menu")      # Menu button
115
d.press("power")     # Power button
116

117
# Navigation keys
118
d.press("up")        # Up arrow
119
d.press("down")      # Down arrow
120
d.press("left")      # Left arrow
121
d.press("right")     # Right arrow
122
d.press("center")    # Center/OK button
123

124
# Volume controls
125
d.press("volume_up")
126
d.press("volume_down")
127
d.press("volume_mute")
128

129
# Text input keys
130
d.press("enter")     # Enter key
131
d.press("delete")    # Delete/backspace
132

133
# Long press operations
134
d.long_press("power")  # Power menu
135
d.long_press("home")   # Recent apps
136

137
# Key codes
138
d.press(4)           # Back key (KEYCODE_BACK)
139
d.press(82)          # Menu key (KEYCODE_MENU)
140

141
# Keyevent shell commands
142
d.keyevent("POWER")
143
d.keyevent("HOME")
144
```
145

146
### Text Input and Clipboard
147

148
Text input operations and clipboard management.
149

150
```python { .api }
151
class Device:
152
    def send_keys(self, text: str, clear: bool = False):
153
        """
154
        Send text to focused input field.
155

156
        Parameters:
157
        - text: Text to input
158
        - clear: Clear existing text before input
159
        """
160

161
    def clear_text(self):
162
        """Clear text in focused input field"""
163

164
    @property
165
    def clipboard(self) -> Optional[str]:
166
        """Get clipboard content"""
167

168
    @clipboard.setter
169
    def clipboard(self, text: str):
170
        """Set clipboard content"""
171

172
    def set_clipboard(self, text, label=None):
173
        """
174
        Set clipboard content with optional label.
175

176
        Parameters:
177
        - text: Text content to copy
178
        - label: User-visible label for clip data
179
        """
180
```
181

182
Usage examples:
183

184
```python
185
d = u2.connect()
186

187
# Text input to focused field
188
d.send_keys("Hello World")
189
d.send_keys("Replace text", clear=True)
190

191
# Clear input field
192
d.clear_text()
193

194
# Clipboard operations
195
d.clipboard = "Text to copy"
196
current_clipboard = d.clipboard
197
print(f"Clipboard: {current_clipboard}")
198

199
# Set clipboard with label
200
d.set_clipboard("Important data", label="Test Data")
201

202
# Complex text input workflow
203
username_field = d(resourceId="com.example:id/username")
204
username_field.click()
205
d.clear_text()
206
d.send_keys("john_doe")
207

208
password_field = d(resourceId="com.example:id/password")
209
password_field.click()
210
d.send_keys("secret123")
211
d.press("enter")
212
```
213

214
### System UI Operations
215

216
Control system UI elements and notifications.
217

218
```python { .api }
219
class Device:
220
    def open_notification(self):
221
        """Open notification panel"""
222

223
    def open_quick_settings(self):
224
        """Open quick settings panel"""
225

226
    def open_url(self, url: str):
227
        """
228
        Open URL in default browser.
229

230
        Parameters:
231
        - url: URL to open
232
        """
233

234
    @property
235
    def last_toast(self) -> Optional[str]:
236
        """Get last toast message"""
237

238
    def clear_toast(self):
239
        """Clear last toast message"""
240
```
241

242
Usage examples:
243

244
```python
245
d = u2.connect()
246

247
# System UI access
248
d.open_notification()      # Pull down notification panel
249
d.open_quick_settings()    # Open quick settings
250

251
# Open URLs
252
d.open_url("https://www.google.com")
253
d.open_url("market://details?id=com.example.app")  # Play Store
254

255
# Toast monitoring
256
toast_msg = d.last_toast
257
if toast_msg:
258
    print(f"Toast: {toast_msg}")
259
d.clear_toast()
260
```
261

262
### Advanced Input Method Support
263

264
Enhanced text input using input method editor (IME).
265

266
```python { .api }
267
class InputMethodMixIn:
268
    def is_input_ime_installed(self) -> bool:
269
        """Check if UIAutomator2 input method is installed"""
270

271
    def install_input_ime(self):
272
        """Install UIAutomator2 input method"""
273

274
    def set_input_ime(self, enable: bool = True):
275
        """Enable/disable UIAutomator2 input method"""
276
```
277

278
Usage examples:
279

280
```python
281
d = u2.connect()
282

283
# Check and install input method
284
if not d.is_input_ime_installed():
285
    d.install_input_ime()
286
    d.set_input_ime(True)
287

288
# Enhanced text input (automatically used when available)
289
d.send_keys("Text with special chars: @#$%^&*()")
290
d.send_keys("Unicode text: 你好世界 🌍")
291

292
# Input method provides better text input reliability
293
search_field = d(resourceId="search_box")
294
search_field.click()
295
d.send_keys("Complex search query with symbols & emojis 😀", clear=True)
296
```
297

298
### Touch Gesture Builder
299

300
Low-level touch event construction for complex gestures.
301

302
```python { .api }
303
class Device:
304
    @property
305
    def touch(self) -> TouchActions:
306
        """Touch gesture builder for complex multi-touch operations"""
307

308
class TouchActions:
309
    def down(self, x, y) -> TouchActions:
310
        """Touch down at coordinates"""
311

312
    def move(self, x, y) -> TouchActions:
313
        """Move touch to coordinates"""
314

315
    def up(self, x, y) -> TouchActions:
316
        """Release touch at coordinates"""
317

318
    def sleep(self, seconds: float) -> TouchActions:
319
        """Add delay in gesture sequence"""
320
```
321

322
Usage examples:
323

324
```python
325
d = u2.connect()
326

327
# Custom touch gestures
328
d.touch.down(100, 100).sleep(0.1).move(200, 200).sleep(0.5).up(200, 200)
329

330
# Multi-step gesture
331
d.touch.down(100, 100).move(150, 150).move(200, 200).move(250, 250).up(250, 250)
332

333
# Touch and hold with movement
334
d.touch.down(300, 300).sleep(1.0).move(400, 400).sleep(0.5).up(400, 400)
335

336
# Precise timing control
337
d.touch.down(100, 100).sleep(0.05).move(101, 101).sleep(0.05).move(102, 102).up(102, 102)
338
```
339

340
### Coordinate System Utilities
341

342
Coordinate conversion and screen dimension utilities.
343

344
```python { .api }
345
class Device:
346
    @property
347
    def pos_rel2abs(self):
348
        """
349
        Function to convert relative (0-1) coordinates to absolute pixels.
350

351
        Returns:
352
        Function that takes (x, y) relative coordinates and returns absolute coordinates
353
        """
354

355
    def window_size(self) -> Tuple[int, int]:
356
        """
357
        Get device screen dimensions.
358

359
        Returns:
360
        Tuple of (width, height) in pixels
361
        """
362
```
363

364
Usage examples:
365

366
```python
367
d = u2.connect()
368

369
# Get screen dimensions
370
width, height = d.window_size()
371
print(f"Screen: {width}x{height}")
372

373
# Convert relative to absolute coordinates
374
convert = d.pos_rel2abs
375
center_x, center_y = convert(0.5, 0.5)  # Screen center
376
top_left_x, top_left_y = convert(0.1, 0.1)  # Near top-left
377
bottom_right_x, bottom_right_y = convert(0.9, 0.9)  # Near bottom-right
378

379
# Use converted coordinates
380
d.click(center_x, center_y)  # Click center
381
d.swipe(top_left_x, top_left_y, bottom_right_x, bottom_right_y)  # Diagonal swipe
382
```