Skip to main content

​
Overview

Microsoft UI Automation (UIA) is a modern accessibility API that provides programmatic access to UI elements. NVDA uses UIA to support modern Windows applications, Windows Terminal, Microsoft Office, and other UIA-aware applications.

​
Architecture

​
Core Components

The NVDA UIA implementation consists of several key modules:
  • UIAHandler: Core UIA initialization and management
  • UIAHandler.remote: Remote operations for better performance
  • UIAHandler.browseMode: Browse mode for UIA documents
  • UIAHandler.customAnnotations: Custom annotation support
  • UIAHandler.customProps: Custom property definitions

​
UIA Element Hierarchy

from comInterfaces import UIAutomationClient as UIA

# Get root element
rootElement = UIAHandler.handler.rootElement

# Navigate the tree
focusElement = UIAHandler.handler.clientObject.GetFocusedElement()
parent = focusElement.GetParent()
firstChild = focusElement.FindFirst(UIA.TreeScope_Children, condition)

# Access properties
name = element.CurrentName
controlType = element.CurrentControlType
value = element.GetCurrentPropertyValue(UIA.UIA_ValueValuePropertyId)

​
Working with UIA in NVDA

​
Checking if UIA Should Be Used

from UIAHandler import (
    shouldUseUIAInMSWord,
    isUIAWindow,
    goodUIAWindowClassNames,
    badUIAWindowClassNames,
)

def isUIAWindow(windowHandle):
    """Check if a window should use UIA."""
    # Get window class name
    className = winUser.getClassName(windowHandle)
    
    # Check good/bad lists
    if className in goodUIAWindowClassNames:
        return True
    if className in badUIAWindowClassNames:
        return False
    
    # Check if window exposes UIA
    try:
        element = UIAHandler.handler.clientObject.ElementFromHandle(windowHandle)
        return element is not None
    except COMError:
        return False

def shouldUseUIAInMSWord(appModule):
    """Determine if UIA should be used in Microsoft Word."""
    allow = config.conf["UIA"]["allowInMSWord"]
    
    if allow == AllowUiaInMSWord.ALWAYS:
        return True
    
    # Check if old object model is available
    canUseOlderApproach = bool(appModule.helperLocalBindingHandle)
    if not canUseOlderApproach:
        return True  # Must use UIA
    
    # Check Windows version and Office version
    if winVersion.getWinVer() < winVersion.WIN11:
        return False
    
    if allow == AllowUiaInMSWord.WHERE_SUITABLE:
        officeVersion = tuple(int(x) for x in appModule.productVersion.split(".")[:3])
        return officeVersion >= (16, 0, 15000)
    
    return False

​
Control Type Mapping

from UIAHandler import UIAControlTypesToNVDARoles
import controlTypes

# Map UIA control types to NVDA roles
UIAControlTypesToNVDARoles = {
    UIA.UIA_ButtonControlTypeId: controlTypes.Role.BUTTON,
    UIA.UIA_CheckBoxControlTypeId: controlTypes.Role.CHECKBOX,
    UIA.UIA_ComboBoxControlTypeId: controlTypes.Role.COMBOBOX,
    UIA.UIA_EditControlTypeId: controlTypes.Role.EDITABLETEXT,
    UIA.UIA_HyperlinkControlTypeId: controlTypes.Role.LINK,
    UIA.UIA_ImageControlTypeId: controlTypes.Role.GRAPHIC,
    UIA.UIA_ListControlTypeId: controlTypes.Role.LIST,
    UIA.UIA_ListItemControlTypeId: controlTypes.Role.LISTITEM,
    UIA.UIA_MenuControlTypeId: controlTypes.Role.POPUPMENU,
    UIA.UIA_MenuItemControlTypeId: controlTypes.Role.MENUITEM,
    UIA.UIA_TreeControlTypeId: controlTypes.Role.TREEVIEW,
    UIA.UIA_TreeItemControlTypeId: controlTypes.Role.TREEVIEWITEM,
    # ... more mappings
}

def getRole(uiaElement):
    """Get NVDA role for a UIA element."""
    controlType = uiaElement.CurrentControlType
    return UIAControlTypesToNVDARoles.get(controlType, controlTypes.Role.UNKNOWN)

​
UIA Events

​
Event Registration

import UIAHandler
from comtypes import COMObject
from comInterfaces.UIAutomationClient import IUIAutomationEventHandler

class UIAEventHandler(COMObject):
    """Handler for UIA events."""
    
    _com_interfaces_ = [IUIAutomationEventHandler]
    
    def __init__(self, eventId, element, scope=UIA.TreeScope_Element):
        self.eventId = eventId
        super().__init__()
        
        # Register for event
        UIAHandler.handler.clientObject.AddAutomationEventHandler(
            eventId,
            element,
            scope,
            None,  # cache request
            self,
        )
    
    def HandleAutomationEvent(self, sender, eventId):
        """Called when the event fires."""
        try:
            # Process event
            obj = UIAHandler.getNVDAObjectFromUIAElement(sender)
            if obj:
                eventHandler.queueEvent("UIA_event", obj)
        except Exception:
            log.error("Error handling UIA event", exc_info=True)

# Common event IDs
focusChangedEventId = UIA.UIA_AutomationFocusChangedEventId
propertyChangedEventId = UIA.UIA_AutomationPropertyChangedEventId
liveRegionChangedEventId = UIA.UIA_LiveRegionChangedEventId

​
Event Mapping

UIAEventIdsToNVDAEventNames = {
    UIA.UIA_LiveRegionChangedEventId: "liveRegionChange",
    UIA.UIA_SelectionItem_ElementSelectedEventId: "UIA_elementSelected",
    UIA.UIA_MenuOpenedEventId: "gainFocus",
    UIA.UIA_ToolTipOpenedEventId: "UIA_toolTipOpened",
    UIA.UIA_Window_WindowOpenedEventId: "UIA_window_windowOpen",
    UIA.UIA_SystemAlertEventId: "UIA_systemAlert",
    UIA.UIA_Text_TextSelectionChangedEventId: "caret",
}

UIAPropertyIdsToNVDAEventNames = {
    UIA.UIA_NamePropertyId: "nameChange",
    UIA.UIA_ValueValuePropertyId: "valueChange",
    UIA.UIA_ExpandCollapseExpandCollapseStatePropertyId: "stateChange",
    UIA.UIA_ToggleToggleStatePropertyId: "stateChange",
    UIA.UIA_IsEnabledPropertyId: "stateChange",
}

​
Text Patterns

​
Using TextPattern

from comInterfaces.UIAutomationClient import IUIAutomationTextPattern

class UIATextInfo(textInfos.TextInfo):
    """TextInfo implementation using UIA TextPattern."""
    
    def __init__(self, obj, position, _rangeObj=None):
        super().__init__(obj, position)
        self._rangeObj = _rangeObj
        
        if not self._rangeObj:
            # Get text pattern from element
            pattern = obj.UIAElement.GetCurrentPattern(UIA.UIA_TextPatternId)
            self._pattern = pattern.QueryInterface(IUIAutomationTextPattern)
            
            # Create range based on position
            if position == textInfos.POSITION_FIRST:
                self._rangeObj = self._pattern.DocumentRange
                self._rangeObj.MoveEndpointByRange(
                    UIA.TextPatternRangeEndpoint_End,
                    self._rangeObj,
                    UIA.TextPatternRangeEndpoint_Start,
                )
            elif position == textInfos.POSITION_LAST:
                self._rangeObj = self._pattern.DocumentRange
                self._rangeObj.MoveEndpointByRange(
                    UIA.TextPatternRangeEndpoint_Start,
                    self._rangeObj,
                    UIA.TextPatternRangeEndpoint_End,
                )
            elif position == textInfos.POSITION_CARET:
                selection = self._pattern.GetSelection()
                if selection.length > 0:
                    self._rangeObj = selection.GetElement(0)
                else:
                    self._rangeObj = self._pattern.DocumentRange
    
    def _get_text(self):
        """Get text in this range."""
        return self._rangeObj.GetText(-1)  # -1 = all text
    
    def move(self, unit, direction, endPoint=None):
        """Move by the specified unit."""
        if endPoint == "start":
            endpoint = UIA.TextPatternRangeEndpoint_Start
        elif endPoint == "end":
            endpoint = UIA.TextPatternRangeEndpoint_End
        else:
            # Collapse to start, move start, then expand
            self.collapse()
            endpoint = UIA.TextPatternRangeEndpoint_Start
        
        # Map NVDA unit to UIA unit
        uiaUnit = NVDAUnitsToUIAUnits.get(unit, UIA.TextUnit_Character)
        
        # Perform the move
        moved = self._rangeObj.MoveEndpointByUnit(
            endpoint,
            uiaUnit,
            direction,
        )
        
        return moved

# Unit mapping
NVDAUnitsToUIAUnits = {
    textInfos.UNIT_CHARACTER: UIA.TextUnit_Character,
    textInfos.UNIT_WORD: UIA.TextUnit_Word,
    textInfos.UNIT_LINE: UIA.TextUnit_Line,
    textInfos.UNIT_PARAGRAPH: UIA.TextUnit_Paragraph,
    textInfos.UNIT_PAGE: UIA.TextUnit_Page,
    textInfos.UNIT_STORY: UIA.TextUnit_Document,
}

​
Formatting Information

def getFormatFieldAtRange(self, textRange):
    """Get formatting for a text range."""
    formatField = textInfos.FormatField()
    
    # Font information
    try:
        fontName = textRange.GetAttributeValue(UIA.UIA_FontNameAttributeId)
        if fontName != UIA.UIA_NotSupported:
            formatField["font-name"] = fontName
        
        fontSize = textRange.GetAttributeValue(UIA.UIA_FontSizeAttributeId)
        if fontSize != UIA.UIA_NotSupported:
            formatField["font-size"] = f"{fontSize}pt"
        
        bold = textRange.GetAttributeValue(UIA.UIA_IsBoldAttributeId)
        if bold:
            formatField["bold"] = True
        
        italic = textRange.GetAttributeValue(UIA.UIA_IsItalicAttributeId)
        if italic:
            formatField["italic"] = True
    
    except COMError:
        pass
    
    return formatField

​
Remote Operations

Remote operations allow batching multiple UIA calls for better performance: 
from UIAHandler import remote

class RemoteOperation:
    """Execute multiple UIA operations in a batch."""
    
    def __init__(self):
        self.operations = []
    
    def getText(self, element):
        """Queue getting text from an element."""
        op = remote.createRemoteOperation()
        op.addCall("GetText", element)
        self.operations.append(op)
        return len(self.operations) - 1
    
    def execute(self):
        """Execute all queued operations."""
        results = []
        for op in self.operations:
            try:
                result = remote.executeRemoteOperation(op)
                results.append(result)
            except Exception:
                results.append(None)
        return results

​
Custom Annotations

from UIAHandler import customAnnotations

def getCustomAnnotations(element):
    """Get custom annotations from a UIA element."""
    try:
        # Get custom annotation pattern
        pattern = element.GetCurrentPattern(UIA.UIA_CustomAnnotationPatternId)
        if not pattern:
            return []
        
        annotations = []
        annotationObjects = pattern.CurrentAnnotationObjects
        
        for annotObj in annotationObjects:
            annotType = annotObj.CurrentAnnotationTypeId
            annotName = annotObj.CurrentAnnotationTypeName
            
            annotations.append({
                "type": annotType,
                "name": annotName,
            })
        
        return annotations
    
    except COMError:
        return []

​
Best Practices

Performance

Use remote operations to batch UIA calls and reduce COM overhead

Error Handling

Always catch COMError when accessing UIA properties - elements may become invalid

Caching

Use UIA cache requests to pre-fetch commonly accessed properties

Event Filtering

Only register for events you actually need - UIA can generate many events
UIA elements can become stale if the underlying UI changes. Always be prepared to handle exceptions.

​
Common Patterns

​
Finding Elements

def findDescendant(element, condition):
    """Find a descendant element matching a condition."""
    return element.FindFirst(UIA.TreeScope_Descendants, condition)

def createPropertyCondition(propertyId, value):
    """Create a condition for a property value."""
    return UIAHandler.handler.clientObject.CreatePropertyCondition(
        propertyId,
        value,
    )

# Example: Find a button by name
buttonCondition = UIAHandler.handler.clientObject.CreateAndCondition(
    createPropertyCondition(UIA.UIA_ControlTypePropertyId, UIA.UIA_ButtonControlTypeId),
    createPropertyCondition(UIA.UIA_NamePropertyId, "OK"),
)
button = findDescendant(rootElement, buttonCondition)

​
Walking the Tree

def walkTree(element, callback, maxDepth=10, currentDepth=0):
    """Walk UIA tree calling callback for each element."""
    if currentDepth >= maxDepth:
        return
    
    # Process current element
    if callback(element) is False:
        return  # Stop if callback returns False
    
    # Get children
    try:
        walker = UIAHandler.handler.clientObject.RawViewWalker
        child = walker.GetFirstChildElement(element)
        
        while child:
            walkTree(child, callback, maxDepth, currentDepth + 1)
            child = walker.GetNextSiblingElement(child)
    
    except COMError:
        pass

​
Reference

​
Key Modules

  • UIAHandler: Core UIA implementation
  • UIAHandler.types: Type definitions and constants
  • UIAHandler.utils: Utility functions
  • UIAHandler.remote: Remote operations
  • comInterfaces.UIAutomationClient: COM interfaces

​
Important Constants

# Control types
UIA.UIA_ButtonControlTypeId
UIA.UIA_EditControlTypeId
UIA.UIA_DocumentControlTypeId

# Property IDs
UIA.UIA_NamePropertyId
UIA.UIA_ValueValuePropertyId
UIA.UIA_IsEnabledPropertyId

# Pattern IDs
UIA.UIA_TextPatternId
UIA.UIA_ValuePatternId
UIA.UIA_SelectionPatternId

# Tree scope
UIA.TreeScope_Element
UIA.TreeScope_Children
UIA.TreeScope_Descendants

​
Configuration

# Check UIA settings
config.conf["UIA"]["enabled"]
config.conf["UIA"]["allowInMSWord"]
config.conf["UIA"]["allowInChromium"]

Developer Guide

See the NVDA Developer Guide for complete plugin development information

Microsoft UIA Documentation

Official Microsoft documentation for UI Automation

Build docs developers (and LLMs) love

Get started for freeTalk to us