or run

npx @tessl/cli init
Log in

Version

Tile

Overview

Evals

Files

docs

advanced-interactions.mdbidi-protocol.mdbrowser-automation.mdbrowser-options.mddriver-management.mdelement-location.mderror-handling.mdindex.mdwait-conditions.md
tile.json

advanced-interactions.mddocs/

Advanced Interactions

Complex user interactions including action chains, drag-and-drop operations, keyboard shortcuts, file uploads, and form handling for sophisticated browser automation.

Capabilities

Actions Class

Builder for complex user interaction sequences combining mouse movements, clicks, and keyboard input.

/**
 * Action sequence builder for complex interactions
 */
class Actions {
  constructor(driver: WebDriver, options?: object);
  
  /** Click element or current mouse position */
  click(element?: WebElement): Actions;
  
  /** Double-click element or current mouse position */
  doubleClick(element?: WebElement): Actions;
  
  /** Right-click element or current mouse position */
  contextClick(element?: WebElement): Actions;
  
  /** Press and hold mouse button */
  press(button?: string): Actions;
  
  /** Release mouse button */
  release(button?: string): Actions;
  
  /** Move mouse to element or coordinates */
  move(options: {x?: number, y?: number, duration?: number, origin?: string | WebElement}): Actions;
  
  /** Drag and drop from source to target */
  dragAndDrop(source: WebElement, target: WebElement): Actions;
  
  /** Drag element by offset */
  dragAndDropBy(element: WebElement, x: number, y: number): Actions;
  
  /** Press key down */
  keyDown(key: string, element?: WebElement): Actions;
  
  /** Release key */
  keyUp(key: string, element?: WebElement): Actions;
  
  /** Send keystrokes */
  sendKeys(...keys: (string | number)[]): Actions;
  
  /** Pause for specified duration */
  pause(duration: number, ...devices: string[]): Actions;
  
  /** Access keyboard device */
  keyboard(): Actions;
  
  /** Access mouse device */
  mouse(): Actions;
  
  /** Access wheel device */
  wheel(): Actions;
  
  /** Scroll by delta amounts */
  scroll(x: number, y: number, deltaX: number, deltaY: number, origin?: string | WebElement, duration?: number): Actions;
  
  /** Clear all actions */
  clear(): Actions;
  
  /** Execute the action sequence */
  perform(): Promise<void>;
}

Usage Examples:

const { Builder, By, Key } = require('selenium-webdriver');

let driver = await new Builder().forBrowser('chrome').build();
let actions = driver.actions();

// Simple click sequence
await actions
  .move({origin: await driver.findElement(By.id('button1'))})
  .click()
  .pause(1000)
  .move({origin: await driver.findElement(By.id('button2'))}) 
  .click()
  .perform();

// Drag and drop
let source = await driver.findElement(By.id('draggable'));
let target = await driver.findElement(By.id('droppable'));

await actions
  .dragAndDrop(source, target)
  .perform();

// Complex mouse movements
await actions
  .move({x: 100, y: 100})
  .press()
  .move({x: 200, y: 200, duration: 2000})
  .release()
  .perform();

// Keyboard shortcuts
await actions
  .keyDown(Key.CONTROL)
  .sendKeys('a')  // Ctrl+A
  .keyUp(Key.CONTROL)
  .sendKeys('Hello World')
  .perform();

// Right-click context menu
let element = await driver.findElement(By.id('target'));
await actions
  .contextClick(element)
  .perform();

// Select menu item after right-click
await actions
  .contextClick(element)
  .pause(500)
  .move({x: 50, y: 20, origin: element})
  .click()
  .perform();

Select Class

Helper for HTML select element interactions with support for single and multiple selections.

/**
 * HTML select element helper
 */
class Select {
  constructor(element: WebElement);
  
  /** Select option by index (0-based) */
  selectByIndex(index: number): Promise<void>;
  
  /** Select option by value attribute */
  selectByValue(value: string): Promise<void>;
  
  /** Select option by visible text */
  selectByVisibleText(text: string): Promise<void>;
  
  /** Deselect all options (multi-select only) */
  deselectAll(): Promise<void>;
  
  /** Deselect option by index */
  deselectByIndex(index: number): Promise<void>;
  
  /** Deselect option by value */
  deselectByValue(value: string): Promise<void>;
  
  /** Deselect option by visible text */
  deselectByVisibleText(text: string): Promise<void>;
  
  /** Get all option elements */
  getOptions(): Promise<WebElement[]>;
  
  /** Get all selected option elements */
  getAllSelectedOptions(): Promise<WebElement[]>;
  
  /** Get first selected option element */
  getFirstSelectedOption(): Promise<WebElement>;
  
  /** Check if multiple selections are allowed */
  isMultiple(): Promise<boolean>;
}

Usage Examples:

const { Select } = require('selenium-webdriver');

// Single select dropdown
let countrySelect = await driver.findElement(By.id('country'));
let select = new Select(countrySelect);

// Different selection methods
await select.selectByVisibleText('United States');
await select.selectByValue('US');
await select.selectByIndex(0);

// Multi-select listbox
let skillsSelect = await driver.findElement(By.id('skills'));
let multiSelect = new Select(skillsSelect);

if (await multiSelect.isMultiple()) {
  await multiSelect.selectByVisibleText('JavaScript');
  await multiSelect.selectByVisibleText('Python');
  await multiSelect.selectByVisibleText('Java');
  
  // Get selected options
  let selectedOptions = await multiSelect.getAllSelectedOptions();
  for (let option of selectedOptions) {
    console.log('Selected:', await option.getText());
  }
  
  // Deselect specific option
  await multiSelect.deselectByVisibleText('Java');
  
  // Clear all selections
  await multiSelect.deselectAll();
}

// Get all available options
let allOptions = await select.getOptions();
for (let option of allOptions) {
  let text = await option.getText();
  let value = await option.getAttribute('value');
  console.log(`Option: ${text} (value: ${value})`);
}

File Upload Handling

File upload operations with support for local and remote file detection.

/**
 * File detector for handling file uploads
 */
class FileDetector {
  /** Detect if value represents a file path */
  detect(value: string): string | null;
}

class LocalFileDetector extends FileDetector {
  /** Detect local file paths */
  detect(value: string): string | null;
}

class RemoteFileDetector extends FileDetector {
  /** Detect remote file paths */
  detect(value: string): string | null;
}

/**
 * WebDriver file upload methods
 */
interface WebDriver {
  /** Set file detector for upload handling */
  setFileDetector(detector: FileDetector): void;
}

Usage Examples:

const { LocalFileDetector } = require('selenium-webdriver/io');
const path = require('path');

// Set up file detector
driver.setFileDetector(new LocalFileDetector());

// Single file upload
let fileInput = await driver.findElement(By.css('input[type="file"]'));
let filePath = path.resolve(__dirname, 'test-file.pdf');
await fileInput.sendKeys(filePath);

// Multiple file upload
let multiFileInput = await driver.findElement(By.css('input[type="file"][multiple]'));
let file1 = path.resolve(__dirname, 'file1.jpg');
let file2 = path.resolve(__dirname, 'file2.png');
await multiFileInput.sendKeys(file1 + '\n' + file2);

// Wait for upload completion
await driver.wait(until.elementLocated(By.className('upload-success')), 10000);

// Remote file upload (for remote WebDriver)
const { RemoteFileDetector } = require('selenium-webdriver/io');
driver.setFileDetector(new RemoteFileDetector());

let remoteFilePath = '/path/to/remote/file.txt';
await fileInput.sendKeys(remoteFilePath);

Form Handling

Comprehensive form interaction patterns for various input types.

Usage Examples:

// Text input handling
let textField = await driver.findElement(By.name('username'));
await textField.clear();
await textField.sendKeys('testuser');

// Password field
let passwordField = await driver.findElement(By.name('password'));
await passwordField.sendKeys('securePassword123');

// Checkbox handling
let checkbox = await driver.findElement(By.id('terms'));
let isChecked = await checkbox.isSelected();
if (!isChecked) {
  await checkbox.click();
}

// Radio button selection
let radioButtons = await driver.findElements(By.name('gender'));
for (let radio of radioButtons) {
  let value = await radio.getAttribute('value');
  if (value === 'male') {
    await radio.click();
    break;
  }
}

// Textarea handling
let textarea = await driver.findElement(By.name('comments'));
await textarea.clear();
await textarea.sendKeys('This is a multi-line\ncomment with line breaks.');

// Date input
let dateInput = await driver.findElement(By.name('birthdate'));
await dateInput.sendKeys('01/15/1990');

// Number input
let numberInput = await driver.findElement(By.name('age'));
await numberInput.clear();
await numberInput.sendKeys('25');

// Email input validation
let emailInput = await driver.findElement(By.name('email'));
await emailInput.sendKeys('invalid-email');

let submitButton = await driver.findElement(By.css('button[type="submit"]'));
await submitButton.click();

// Check for validation errors
let emailError = await driver.findElements(By.css('.email-error'));
if (emailError.length > 0) {
  console.log('Email validation failed');
  await emailInput.clear();
  await emailInput.sendKeys('valid@example.com');
}

Keyboard Interactions

Advanced keyboard input handling including special keys and key combinations.

/**
 * Special keyboard keys
 */
const Key = {
  RETURN: '\uE006',
  ENTER: '\uE007',
  TAB: '\uE004',
  ESCAPE: '\uE00C',
  SPACE: '\uE00D',
  BACKSPACE: '\uE003',
  DELETE: '\uE017',
  
  // Arrow keys
  ARROW_UP: '\uE013',
  ARROW_DOWN: '\uE015',
  ARROW_LEFT: '\uE012',
  ARROW_RIGHT: '\uE014',
  
  // Function keys
  F1: '\uE031',
  F2: '\uE032',
  F12: '\uE03C',
  
  // Modifier keys
  SHIFT: '\uE008',
  CONTROL: '\uE009',
  ALT: '\uE00A',
  META: '\uE03D',
  
  // Other keys
  HOME: '\uE011',
  END: '\uE010',
  PAGE_UP: '\uE00E',
  PAGE_DOWN: '\uE00F',
  INSERT: '\uE016'
};

Usage Examples:

const { Key } = require('selenium-webdriver');

// Basic key combinations
let textField = await driver.findElement(By.name('content'));
await textField.sendKeys('Hello World');

// Select all and replace
await textField.sendKeys(Key.CONTROL, 'a'); // Ctrl+A
await textField.sendKeys('New content');

// Navigate with arrow keys
await textField.sendKeys(Key.HOME); // Go to beginning
await textField.sendKeys(Key.ARROW_RIGHT, Key.ARROW_RIGHT); // Move 2 positions right
await textField.sendKeys('XX'); // Insert text

// Copy and paste
await textField.sendKeys(Key.CONTROL, 'a'); // Select all
await textField.sendKeys(Key.CONTROL, 'c'); // Copy

let anotherField = await driver.findElement(By.name('destination'));
await anotherField.sendKeys(Key.CONTROL, 'v'); // Paste

// Function keys
await driver.sendKeys(Key.F5); // Refresh page
await driver.sendKeys(Key.F12); // Open developer tools

// Complex key sequences with Actions
let actions = driver.actions();
await actions
  .keyDown(Key.SHIFT)
  .sendKeys(Key.ARROW_RIGHT, Key.ARROW_RIGHT, Key.ARROW_RIGHT) // Select 3 characters
  .keyUp(Key.SHIFT)
  .sendKeys(Key.DELETE) // Delete selection
  .perform();

// Tab navigation
await driver.sendKeys(Key.TAB); // Move to next field
await driver.sendKeys(Key.SHIFT, Key.TAB); // Move to previous field

Mouse Operations

Detailed mouse interaction patterns for complex UI elements.

Usage Examples:

// Hover operations
let menuItem = await driver.findElement(By.id('menu-item'));
await driver.actions()
  .move({origin: menuItem})
  .perform();

// Wait for submenu to appear
let submenu = await driver.wait(
  until.elementIsVisible(By.id('submenu')), 
  3000
);

// Click submenu item
let submenuItem = await submenu.findElement(By.linkText('Settings'));
await submenuItem.click();

// Double-click operations
let editableText = await driver.findElement(By.className('editable'));
await driver.actions()
  .doubleClick(editableText)
  .perform();

// Drag and drop with offset
let slider = await driver.findElement(By.className('slider-handle'));
await driver.actions()
  .dragAndDropBy(slider, 100, 0) // Move 100px right
  .perform();

// Precise mouse movements
let canvas = await driver.findElement(By.tagName('canvas'));
await driver.actions()
  .move({origin: canvas, x: 50, y: 50})
  .press()
  .move({x: 150, y: 150, duration: 1000})
  .move({x: 50, y: 150, duration: 500})
  .move({x: 50, y: 50, duration: 500})
  .release()
  .perform();

// Mouse wheel scrolling (via JavaScript)
await driver.executeScript('arguments[0].scrollTop += 500', canvas);

// Right-click and menu selection
let contextTarget = await driver.findElement(By.id('context-target'));
await driver.actions()
  .contextClick(contextTarget)
  .perform();

// Wait for context menu
let contextMenu = await driver.wait(
  until.elementIsVisible(By.className('context-menu')), 
  2000
);

// Select menu option
let deleteOption = await contextMenu.findElement(By.xpath('.//li[text()="Delete"]'));
await deleteOption.click();

Touch and Mobile Interactions

Touch-based interactions for mobile testing scenarios.

Usage Examples:

// Touch tap (mobile)
let mobileButton = await driver.findElement(By.id('mobile-btn'));
await driver.actions()
  .move({origin: mobileButton})
  .press()
  .pause(100)
  .release()
  .perform();

// Swipe gestures
let swipeArea = await driver.findElement(By.className('swipe-container'));
await driver.actions()
  .move({origin: swipeArea, x: -100, y: 0})
  .press()
  .move({x: 100, y: 0, duration: 500})
  .release()
  .perform();

// Pinch zoom (two-finger gesture simulation)
let zoomArea = await driver.findElement(By.id('zoom-area'));
await driver.actions()
  .move({origin: zoomArea, x: -50, y: 0})
  .press()
  .move({origin: zoomArea, x: 50, y: 0})
  .press()
  .pause(100)
  .move({x: -75, y: 0, duration: 1000})
  .move({x: 75, y: 0, duration: 1000})
  .release()
  .release()
  .perform();

// Long press
let longPressTarget = await driver.findElement(By.className('long-press'));
await driver.actions()
  .move({origin: longPressTarget})
  .press()
  .pause(2000) // 2 second press
  .release()
  .perform();