CtrlK
BlogDocsLog inGet started
Tessl Logo

tessl/pypi-uiautomation

Python UIAutomation for Windows - comprehensive library for automating Windows applications using Microsoft's UIAutomation framework

Overview
Eval results
Files

screen-capture.mddocs/

Screen Capture

Bitmap operations for capturing screenshots of controls or screen areas, pixel color analysis, and image-based verification in automation scripts. This capability enables visual testing and verification workflows.

Capabilities

Bitmap Class

The main class for image capture and manipulation operations.

class Bitmap:
    """Class for image capture and manipulation."""
    
    @staticmethod
    def FromControl(control: Control) -> Bitmap:
        """
        Capture a control as a bitmap image.
        
        Args:
            control: Control to capture
            
        Returns:
            Bitmap: Bitmap image of the control
        """
    
    @staticmethod
    def FromFile(filename: str) -> Bitmap:
        """
        Load a bitmap from an image file.
        
        Args:
            filename: Path to image file
            
        Returns:
            Bitmap: Loaded bitmap image
        """
    
    def ToFile(self, filename: str) -> None:
        """
        Save the bitmap to an image file.
        
        Args:
            filename: Output file path (supports .bmp, .png, .jpg formats)
        """
    
    def GetPixelColor(self, x: int, y: int) -> int:
        """
        Get the color of a pixel at the specified coordinates.
        
        Args:
            x: X coordinate within the bitmap
            y: Y coordinate within the bitmap
            
        Returns:
            int: Color value as an integer (0xAARRGGBB format)
        """
    
    def SetPixelColor(self, x: int, y: int, color: int) -> None:
        """
        Set the color of a pixel at the specified coordinates.
        
        Args:
            x: X coordinate within the bitmap
            y: Y coordinate within the bitmap
            color: Color value as an integer (0xAARRGGBB format)
        """
    
    @property
    def Width(self) -> int:
        """Width of the bitmap in pixels."""
    
    @property
    def Height(self) -> int:
        """Height of the bitmap in pixels."""

Control Screen Capture Methods

Methods available on Control objects for capturing screenshots.

class Control:
    def CaptureToImage(self, filename: str) -> None:
        """
        Capture this control to an image file.
        
        Args:
            filename: Output file path for the screenshot
        """

System-Level Screen Capture

Pixel color information from the Win32API.

# Available via Win32API class
from uiautomation import Win32API

def GetPixelColor(x: int, y: int) -> int:
    """
    Get the color of a pixel at screen coordinates.
    
    Args:
        x: Screen X coordinate
        y: Screen Y coordinate
        
    Returns:
        int: Color value as an integer (0xAARRGGBB format)
    """

Usage Examples

Basic Screen Capture

import uiautomation

# Capture specific control (main way to capture images)
window = uiautomation.WindowControl(Name='Calculator')
if window.Exists():
    window.CaptureToImage("calculator_screenshot.png")

# Capture a specific control
button = uiautomation.ButtonControl(Name='Submit')
if button.Exists():
    button.CaptureToImage("button_screenshot.png")

Working with Bitmap Objects

# Capture control to file
window = uiautomation.WindowControl(Name='Calculator')
if window.Exists():
    window.CaptureToImage("calculator_window.png")

# Load bitmap from file
loaded_bitmap = uiautomation.Bitmap.FromFile("reference_image.png")

Pixel Color Analysis

# Get pixel color from screen
color = uiautomation.GetPixelColor(300, 200)
print(f"Pixel color: 0x{color:06X}")

# Convert to RGB
red, green, blue = uiautomation.ColorToRGB(color)
print(f"RGB: ({red}, {green}, {blue})")

# Check if pixel is a specific color
if color == uiautomation.Color.RED:
    print("Pixel is red")

# Get pixel color from bitmap
bitmap = uiautomation.Bitmap.FromFile("image.png")
pixel_color = bitmap.GetPixelColor(50, 75)

Visual Verification

def verify_button_color(button_name, expected_color):
    """Verify a button has the expected background color."""
    button = uiautomation.ButtonControl(Name=button_name)
    if not button.Exists():
        return False
    
    # Capture button to temporary file
    button.CaptureToImage("temp_button.png")
    bitmap = uiautomation.Bitmap.FromFile("temp_button.png")
    
    # Check center pixel color (assuming solid color button)
    center_x = bitmap.Width // 2
    center_y = bitmap.Height // 2
    actual_color = bitmap.GetPixelColor(center_x, center_y)
    
    # Compare colors (with tolerance for slight variations)
    return color_similar(actual_color, expected_color, tolerance=10)

def color_similar(color1, color2, tolerance=5):
    """Check if two colors are similar within tolerance."""
    r1, g1, b1 = uiautomation.ColorToRGB(color1)
    r2, g2, b2 = uiautomation.ColorToRGB(color2)
    
    return (abs(r1 - r2) <= tolerance and 
            abs(g1 - g2) <= tolerance and 
            abs(b1 - b2) <= tolerance)

# Use the verification function
if verify_button_color('OK', uiautomation.Color.GREEN):
    print("OK button has correct green color")

Image-Based Control Location

def find_control_by_image(reference_image_path, tolerance=0.95):
    """Find control by matching against reference image."""
    # Note: Screen capture would need to be implemented using Win32API
    # This is a conceptual example
    current_screen = uiautomation.Bitmap.FromFile("current_screen.png")
    reference = uiautomation.Bitmap.FromFile(reference_image_path)
    
    # Simple template matching (this is a conceptual example)
    # Real implementation would use image processing algorithms
    for y in range(current_screen.Height - reference.Height):
        for x in range(current_screen.Width - reference.Width):
            if image_matches_at_position(current_screen, reference, x, y, tolerance):
                return uiautomation.Point(x + reference.Width // 2, 
                                        y + reference.Height // 2)
    return None

def image_matches_at_position(source, template, x, y, tolerance):
    """Check if template matches source at given position."""
    # Simplified matching logic - real implementation would be more sophisticated
    matching_pixels = 0
    total_pixels = template.Width * template.Height
    
    for ty in range(template.Height):
        for tx in range(template.Width):
            src_color = source.GetPixelColor(x + tx, y + ty)
            tpl_color = template.GetPixelColor(tx, ty)
            if color_similar(src_color, tpl_color, tolerance=10):
                matching_pixels += 1
    
    return (matching_pixels / total_pixels) >= tolerance

Screenshot Comparison

def compare_screenshots(image1_path, image2_path, difference_output=None):
    """Compare two screenshots and optionally generate difference image."""
    img1 = uiautomation.Bitmap.FromFile(image1_path)
    img2 = uiautomation.Bitmap.FromFile(image2_path)
    
    if img1.Width != img2.Width or img1.Height != img2.Height:
        print("Images have different dimensions")
        return False
    
    differences = 0
    total_pixels = img1.Width * img1.Height
    
    # Create difference image if requested
    diff_bitmap = None
    if difference_output:
        diff_bitmap = uiautomation.Bitmap.FromFile(image1_path)  # Copy of first image
    
    for y in range(img1.Height):
        for x in range(img1.Width):
            color1 = img1.GetPixelColor(x, y)
            color2 = img2.GetPixelColor(x, y)
            
            if color1 != color2:
                differences += 1
                if diff_bitmap:
                    # Highlight differences in red
                    diff_bitmap.SetPixelColor(x, y, uiautomation.Color.RED)
    
    if diff_bitmap and difference_output:
        diff_bitmap.ToFile(difference_output)
    
    similarity = 1.0 - (differences / total_pixels)
    print(f"Images are {similarity * 100:.2f}% similar")
    return similarity > 0.95  # 95% similarity threshold

# Compare before and after screenshots
compare_screenshots("before.png", "after.png", "differences.png")

Automated Visual Testing

def visual_regression_test(test_name, control_selector):
    """Perform visual regression testing on a control."""
    # Define reference image path
    reference_path = f"reference_images/{test_name}.png"
    current_path = f"current_images/{test_name}.png"
    
    # Find and capture control
    control = uiautomation.FindControl(**control_selector)
    if not control.Exists():
        print(f"Control not found for test: {test_name}")
        return False
    
    # Capture current state
    control.CaptureToImage(current_path)
    
    # Compare with reference if it exists
    try:
        if compare_screenshots(reference_path, current_path):
            print(f"✓ Visual test passed: {test_name}")
            return True
        else:
            print(f"✗ Visual test failed: {test_name}")
            return False
    except FileNotFoundError:
        print(f"Reference image not found for {test_name}, creating baseline")
        # Copy current as reference for future tests
        import shutil
        shutil.copy(current_path, reference_path)
        return True

# Run visual tests
visual_regression_test("login_button", {"Name": "Login", "ControlType": uiautomation.ControlType.ButtonControl})
visual_regression_test("main_window", {"Name": "Application", "ControlType": uiautomation.ControlType.WindowControl})

Install with Tessl CLI

npx tessl i tessl/pypi-uiautomation

docs

automation-patterns.md

control-management.md

control-types.md

index.md

input-simulation.md

logging-debugging.md

screen-capture.md

windows-api.md

tile.json