Python UIAutomation for Windows - comprehensive library for automating Windows applications using Microsoft's UIAutomation framework
Bitmap operations for capturing screenshots of controls or screen areas, pixel color analysis, and image-based verification in automation scripts. This capability enables visual testing and verification workflows.
The main class for image capture and manipulation operations.
class Bitmap:
"""Class for image capture and manipulation."""
@staticmethod
def FromControl(control: Control) -> Bitmap:
"""
Capture a control as a bitmap image.
Args:
control: Control to capture
Returns:
Bitmap: Bitmap image of the control
"""
@staticmethod
def FromFile(filename: str) -> Bitmap:
"""
Load a bitmap from an image file.
Args:
filename: Path to image file
Returns:
Bitmap: Loaded bitmap image
"""
def ToFile(self, filename: str) -> None:
"""
Save the bitmap to an image file.
Args:
filename: Output file path (supports .bmp, .png, .jpg formats)
"""
def GetPixelColor(self, x: int, y: int) -> int:
"""
Get the color of a pixel at the specified coordinates.
Args:
x: X coordinate within the bitmap
y: Y coordinate within the bitmap
Returns:
int: Color value as an integer (0xAARRGGBB format)
"""
def SetPixelColor(self, x: int, y: int, color: int) -> None:
"""
Set the color of a pixel at the specified coordinates.
Args:
x: X coordinate within the bitmap
y: Y coordinate within the bitmap
color: Color value as an integer (0xAARRGGBB format)
"""
@property
def Width(self) -> int:
"""Width of the bitmap in pixels."""
@property
def Height(self) -> int:
"""Height of the bitmap in pixels."""Methods available on Control objects for capturing screenshots.
class Control:
def CaptureToImage(self, filename: str) -> None:
"""
Capture this control to an image file.
Args:
filename: Output file path for the screenshot
"""Pixel color information from the Win32API.
# Available via Win32API class
from uiautomation import Win32API
def GetPixelColor(x: int, y: int) -> int:
"""
Get the color of a pixel at screen coordinates.
Args:
x: Screen X coordinate
y: Screen Y coordinate
Returns:
int: Color value as an integer (0xAARRGGBB format)
"""import uiautomation
# Capture specific control (main way to capture images)
window = uiautomation.WindowControl(Name='Calculator')
if window.Exists():
window.CaptureToImage("calculator_screenshot.png")
# Capture a specific control
button = uiautomation.ButtonControl(Name='Submit')
if button.Exists():
button.CaptureToImage("button_screenshot.png")# Capture control to file
window = uiautomation.WindowControl(Name='Calculator')
if window.Exists():
window.CaptureToImage("calculator_window.png")
# Load bitmap from file
loaded_bitmap = uiautomation.Bitmap.FromFile("reference_image.png")# Get pixel color from screen
color = uiautomation.GetPixelColor(300, 200)
print(f"Pixel color: 0x{color:06X}")
# Convert to RGB
red, green, blue = uiautomation.ColorToRGB(color)
print(f"RGB: ({red}, {green}, {blue})")
# Check if pixel is a specific color
if color == uiautomation.Color.RED:
print("Pixel is red")
# Get pixel color from bitmap
bitmap = uiautomation.Bitmap.FromFile("image.png")
pixel_color = bitmap.GetPixelColor(50, 75)def verify_button_color(button_name, expected_color):
"""Verify a button has the expected background color."""
button = uiautomation.ButtonControl(Name=button_name)
if not button.Exists():
return False
# Capture button to temporary file
button.CaptureToImage("temp_button.png")
bitmap = uiautomation.Bitmap.FromFile("temp_button.png")
# Check center pixel color (assuming solid color button)
center_x = bitmap.Width // 2
center_y = bitmap.Height // 2
actual_color = bitmap.GetPixelColor(center_x, center_y)
# Compare colors (with tolerance for slight variations)
return color_similar(actual_color, expected_color, tolerance=10)
def color_similar(color1, color2, tolerance=5):
"""Check if two colors are similar within tolerance."""
r1, g1, b1 = uiautomation.ColorToRGB(color1)
r2, g2, b2 = uiautomation.ColorToRGB(color2)
return (abs(r1 - r2) <= tolerance and
abs(g1 - g2) <= tolerance and
abs(b1 - b2) <= tolerance)
# Use the verification function
if verify_button_color('OK', uiautomation.Color.GREEN):
print("OK button has correct green color")def find_control_by_image(reference_image_path, tolerance=0.95):
"""Find control by matching against reference image."""
# Note: Screen capture would need to be implemented using Win32API
# This is a conceptual example
current_screen = uiautomation.Bitmap.FromFile("current_screen.png")
reference = uiautomation.Bitmap.FromFile(reference_image_path)
# Simple template matching (this is a conceptual example)
# Real implementation would use image processing algorithms
for y in range(current_screen.Height - reference.Height):
for x in range(current_screen.Width - reference.Width):
if image_matches_at_position(current_screen, reference, x, y, tolerance):
return uiautomation.Point(x + reference.Width // 2,
y + reference.Height // 2)
return None
def image_matches_at_position(source, template, x, y, tolerance):
"""Check if template matches source at given position."""
# Simplified matching logic - real implementation would be more sophisticated
matching_pixels = 0
total_pixels = template.Width * template.Height
for ty in range(template.Height):
for tx in range(template.Width):
src_color = source.GetPixelColor(x + tx, y + ty)
tpl_color = template.GetPixelColor(tx, ty)
if color_similar(src_color, tpl_color, tolerance=10):
matching_pixels += 1
return (matching_pixels / total_pixels) >= tolerancedef compare_screenshots(image1_path, image2_path, difference_output=None):
"""Compare two screenshots and optionally generate difference image."""
img1 = uiautomation.Bitmap.FromFile(image1_path)
img2 = uiautomation.Bitmap.FromFile(image2_path)
if img1.Width != img2.Width or img1.Height != img2.Height:
print("Images have different dimensions")
return False
differences = 0
total_pixels = img1.Width * img1.Height
# Create difference image if requested
diff_bitmap = None
if difference_output:
diff_bitmap = uiautomation.Bitmap.FromFile(image1_path) # Copy of first image
for y in range(img1.Height):
for x in range(img1.Width):
color1 = img1.GetPixelColor(x, y)
color2 = img2.GetPixelColor(x, y)
if color1 != color2:
differences += 1
if diff_bitmap:
# Highlight differences in red
diff_bitmap.SetPixelColor(x, y, uiautomation.Color.RED)
if diff_bitmap and difference_output:
diff_bitmap.ToFile(difference_output)
similarity = 1.0 - (differences / total_pixels)
print(f"Images are {similarity * 100:.2f}% similar")
return similarity > 0.95 # 95% similarity threshold
# Compare before and after screenshots
compare_screenshots("before.png", "after.png", "differences.png")def visual_regression_test(test_name, control_selector):
"""Perform visual regression testing on a control."""
# Define reference image path
reference_path = f"reference_images/{test_name}.png"
current_path = f"current_images/{test_name}.png"
# Find and capture control
control = uiautomation.FindControl(**control_selector)
if not control.Exists():
print(f"Control not found for test: {test_name}")
return False
# Capture current state
control.CaptureToImage(current_path)
# Compare with reference if it exists
try:
if compare_screenshots(reference_path, current_path):
print(f"✓ Visual test passed: {test_name}")
return True
else:
print(f"✗ Visual test failed: {test_name}")
return False
except FileNotFoundError:
print(f"Reference image not found for {test_name}, creating baseline")
# Copy current as reference for future tests
import shutil
shutil.copy(current_path, reference_path)
return True
# Run visual tests
visual_regression_test("login_button", {"Name": "Login", "ControlType": uiautomation.ControlType.ButtonControl})
visual_regression_test("main_window", {"Name": "Application", "ControlType": uiautomation.ControlType.WindowControl})Install with Tessl CLI
npx tessl i tessl/pypi-uiautomation