Preliminary support for UI Automation in Windows Console

source/NVDAObjects/UIA/__init__.py

@@ -858,6 +858,14 @@ def findOverlayClasses(self,clsList):
 		except ValueError:
 			pass
 
+		# Support Windows Console's UIA interface
+		if (
+			self.windowClassName == "ConsoleWindowClass"
+			and self.UIAElement.cachedAutomationId == "Text Area"
+			and config.conf['UIA']['consoleUIA']
+		):
+			from .winConsoleUIA import winConsoleUIA
+			clsList.append(winConsoleUIA)
 		# Add editableText support if UIA supports a text pattern
 		if self.TextInfo==UIATextInfo:
 			clsList.append(EditableTextWithoutAutoSelectDetection)

source/NVDAObjects/UIA/winConsoleUIA.py

@@ -0,0 +1,62 @@
+# NVDAObjects/UIA/winConsoleUIA.py
+# A part of NonVisual Desktop Access (NVDA)
+# This file is covered by the GNU General Public License.
+# See the file COPYING for more details.
+# Copyright (C) 2019 Bill Dengler
+
+import time
+import textInfos
+import UIAHandler
+
+from scriptHandler import script
+from winVersion import winVersion
+from . import UIATextInfo
+from ..behaviors import Terminal
+
+
+class consoleUIATextInfo(UIATextInfo):
+	_expandCollapseBeforeReview = False
+
+	def __init__(self, obj, position, _rangeObj=None):
+		super(consoleUIATextInfo, self).__init__(obj, position, _rangeObj)
+		if position == textInfos.POSITION_CARET:
+			if winVersion.build >= 18362:  # Windows 10 version 1903
+				# The UIA implementation in 1903 causes the caret to be
+				# off-by-one, so move it one position to the right
+				# to compensate.
+				self._rangeObj.MoveEndpointByUnit(
+					UIAHandler.TextPatternRangeEndpoint_Start,
+					UIAHandler.NVDAUnitsToUIAUnits['character'],
+					1
+				)
+
+
+class winConsoleUIA(Terminal):
+	_TextInfo = consoleUIATextInfo
+	_isTyping = False
+	_lastCharTime = 0
+	_TYPING_TIMEOUT = 1
+
+	def _reportNewText(self, line):
+		# Additional typed character filtering beyond that in LiveText
+		if self._isTyping and time.time() - self._lastCharTime <= self._TYPING_TIMEOUT:
+			return
+		super(winConsoleUIA, self)._reportNewText(line)
+
+	def event_typedCharacter(self, ch):
+		if not ch.isspace():
+			self._isTyping = True
+		self._lastCharTime = time.time()
+		super(winConsoleUIA, self).event_typedCharacter(ch)
+
+	@script(gestures=["kb:enter", "kb:numpadEnter", "kb:tab"])
+	def script_clear_isTyping(self, gesture):
+		gesture.send()
+		self._isTyping = False
+
+	def _getTextLines(self):
+		# Filter out extraneous empty lines from UIA
+		# Todo: do this (also) somewhere else so they aren't in document review either
+		ptr = self.UIATextPattern.GetVisibleRanges()
+		res = [ptr.GetElement(i).GetText(-1) for i in range(ptr.length)]
+		return res

source/NVDAObjects/window/__init__.py

@@ -12,6 +12,7 @@
 from logHandler import log
 import controlTypes
 import api
+import config
 import displayModel
 import eventHandler
 from NVDAObjects import NVDAObject
@@ -120,7 +121,7 @@ def findOverlayClasses(self,clsList):
 			from .scintilla import Scintilla as newCls
 		elif windowClassName in ("AkelEditW", "AkelEditA"):
 			from .akelEdit import AkelEdit as newCls
-		elif windowClassName=="ConsoleWindowClass":
+		elif windowClassName=="ConsoleWindowClass" and not config.conf['UIA']['consoleUIA']:
 			from .winConsole import WinConsole as newCls
 		elif windowClassName=="EXCEL7":
 			from .excel import Excel7Window as newCls

source/_UIAHandler.py

@@ -59,13 +59,15 @@
 	"SysListView32",
 	"EXCEL7",
 	"Button",
-	# #7497: Windows 10 Fall Creators Update has an incomplete UIA implementation for console windows, therefore for now we should ignore it.
-	# It does not implement caret/selection, and probably has no new text events.
-	"ConsoleWindowClass",
 	# #8944: The Foxit UIA implementation is incomplete and should not be used for now.
 	"FoxitDocWnd",
 ]
 
+# #7497: Windows 10 Fall Creators Update has an incomplete UIA implementation for console windows, therefore for now we should ignore it.
+# It does not implement caret/selection, and probably has no new text events.
+if not config.conf['UIA']['consoleUIA']:
+	badUIAWindowClassNames.append("ConsoleWindowClass")
+
 # #8405: used to detect UIA dialogs prior to Windows 10 RS5.
 UIADialogClassNames=[
 	"#32770",
@@ -140,7 +142,7 @@
 
 UIAEventIdsToNVDAEventNames={
 	UIA_LiveRegionChangedEventId:"liveRegionChange",
-	#UIA_Text_TextChangedEventId:"textChanged",
+	UIA_Text_TextChangedEventId:"textChange",
 	UIA_SelectionItem_ElementSelectedEventId:"UIA_elementSelected",
 	UIA_MenuOpenedEventId:"gainFocus",
 	UIA_SelectionItem_ElementAddedToSelectionEventId:"stateChange",

source/config/configSpec.py

@@ -187,6 +187,7 @@
 [UIA]
 	enabled = boolean(default=true)
 	useInMSWordWhenAvailable = boolean(default=false)
+	consoleUIA = boolean(default=false)
 
 [update]
 	autoCheck = boolean(default=true)

source/globalCommands.py

@@ -983,8 +983,9 @@ def script_review_top(self,gesture):
 
 	def script_review_previousLine(self,gesture):
 		info=api.getReviewPosition().copy()
-		info.expand(textInfos.UNIT_LINE)
-		info.collapse()
+		if info._expandCollapseBeforeReview:
+			info.expand(textInfos.UNIT_LINE)
+			info.collapse()
 		res=info.move(textInfos.UNIT_LINE,-1)
 		if res==0:
 			# Translators: a message reported when review cursor is at the top line of the current navigator object.
@@ -1014,8 +1015,9 @@ def script_review_currentLine(self,gesture):
 
 	def script_review_nextLine(self,gesture):
 		info=api.getReviewPosition().copy()
-		info.expand(textInfos.UNIT_LINE)
-		info.collapse()
+		if info._expandCollapseBeforeReview:
+			info.expand(textInfos.UNIT_LINE)
+			info.collapse()
 		res=info.move(textInfos.UNIT_LINE,1)
 		if res==0:
 			# Translators: a message reported when review cursor is at the bottom line of the current navigator object.
@@ -1041,8 +1043,9 @@ def script_review_bottom(self,gesture):
 
 	def script_review_previousWord(self,gesture):
 		info=api.getReviewPosition().copy()
-		info.expand(textInfos.UNIT_WORD)
-		info.collapse()
+		if info._expandCollapseBeforeReview:
+			info.expand(textInfos.UNIT_WORD)
+			info.collapse()
 		res=info.move(textInfos.UNIT_WORD,-1)
 		if res==0:
 			# Translators: a message reported when review cursor is at the top line of the current navigator object.
@@ -1071,8 +1074,9 @@ def script_review_currentWord(self,gesture):
 
 	def script_review_nextWord(self,gesture):
 		info=api.getReviewPosition().copy()
-		info.expand(textInfos.UNIT_WORD)
-		info.collapse()
+		if info._expandCollapseBeforeReview:
+			info.expand(textInfos.UNIT_WORD)
+			info.collapse()
 		res=info.move(textInfos.UNIT_WORD,1)
 		if res==0:
 			# Translators: a message reported when review cursor is at the bottom line of the current navigator object.
@@ -1087,8 +1091,9 @@ def script_review_nextWord(self,gesture):
 
 	def script_review_startOfLine(self,gesture):
 		info=api.getReviewPosition().copy()
-		info.expand(textInfos.UNIT_LINE)
-		info.collapse()
+		if info._expandCollapseBeforeReview:
+			info.expand(textInfos.UNIT_LINE)
+			info.collapse()
 		api.setReviewPosition(info)
 		info.expand(textInfos.UNIT_CHARACTER)
 		ui.reviewMessage(_("Left"))
@@ -1101,8 +1106,9 @@ def script_review_previousCharacter(self,gesture):
 		lineInfo=api.getReviewPosition().copy()
 		lineInfo.expand(textInfos.UNIT_LINE)
 		charInfo=api.getReviewPosition().copy()
-		charInfo.expand(textInfos.UNIT_CHARACTER)
-		charInfo.collapse()
+		if charInfo._expandCollapseBeforeReview:
+			charInfo.expand(textInfos.UNIT_CHARACTER)
+			charInfo.collapse()
 		res=charInfo.move(textInfos.UNIT_CHARACTER,-1)
 		if res==0 or charInfo.compareEndPoints(lineInfo,"startToStart")<0:
 			# Translators: a message reported when review cursor is at the leftmost character of the current navigator object's text.
@@ -1158,8 +1164,9 @@ def script_review_nextCharacter(self,gesture):
 		lineInfo=api.getReviewPosition().copy()
 		lineInfo.expand(textInfos.UNIT_LINE)
 		charInfo=api.getReviewPosition().copy()
-		charInfo.expand(textInfos.UNIT_CHARACTER)
-		charInfo.collapse()
+		if charInfo._expandCollapseBeforeReview:
+			charInfo.expand(textInfos.UNIT_CHARACTER)
+			charInfo.collapse()
 		res=charInfo.move(textInfos.UNIT_CHARACTER,1)
 		if res==0 or charInfo.compareEndPoints(lineInfo,"endToEnd")>=0:
 			# Translators: a message reported when review cursor is at the rightmost character of the current navigator object's text.

source/gui/settingsDialogs.py

@@ -2050,6 +2050,13 @@ def __init__(self, parent):
 		self.UIAInMSWordCheckBox.SetValue(config.conf["UIA"]["useInMSWordWhenAvailable"])
 		self.UIAInMSWordCheckBox.defaultValue = self._getDefaultValue(["UIA", "useInMSWordWhenAvailable"])
 
+		# Translators: This is the label for a checkbox in the
+		#  Advanced settings panel.
+		label = _("Use UI Automation to access the Windows Console (restart NVDA for changes to take effect)")
+		self.ConsoleUIACheckBox=UIAGroup.addItem(wx.CheckBox(self, label=label))
+		self.ConsoleUIACheckBox.SetValue(config.conf["UIA"]["consoleUIA"])
+		self.ConsoleUIACheckBox.defaultValue = self._getDefaultValue(["UIA", "consoleUIA"])
+
 		# Translators: This is the label for a group of advanced options in the
 		#  Advanced settings panel
 		label = _("Browse mode")
@@ -2152,6 +2159,7 @@ def onSave(self):
 		log.debug("Saving advanced config")
 		config.conf["development"]["enableScratchpadDir"]=self.scratchpadCheckBox.IsChecked()
 		config.conf["UIA"]["useInMSWordWhenAvailable"]=self.UIAInMSWordCheckBox.IsChecked()
+		config.conf["UIA"]["consoleUIA"]=self.ConsoleUIACheckBox.IsChecked()
 		config.conf["virtualBuffers"]["autoFocusFocusableElements"] = self.autoFocusFocusableElementsCheckBox.IsChecked()
 		config.conf["editableText"]["caretMoveTimeoutMs"]=self.caretMoveTimeoutSpinControl.GetValue()
 		for index,key in enumerate(self.logCategories):

source/keyboardHandler.py

@@ -197,6 +197,7 @@ def internal_keyDownEvent(vkCode,scanCode,extended,injected):
 		# #6017: handle typed characters in Win10 RS2 and above where we can't detect typed characters in-process 
 		# This code must be in the 'finally' block as code above returns in several places yet we still want to execute this particular code.
 		focus=api.getFocusObject()
+		from NVDAObjects.UIA.winConsoleUIA import winConsoleUIA
 		if (
 			# This is only possible in Windows 10 RS2 and above
 			winVersion.winVersion.build>=14986
@@ -205,10 +206,13 @@ def internal_keyDownEvent(vkCode,scanCode,extended,injected):
 			# and not if this gesture is a modifier key
 			and not isNVDAModifierKey(vkCode,extended) and not vkCode in KeyboardInputGesture.NORMAL_MODIFIER_KEYS
 			and ( # Either of
-				# We couldn't inject in-process, and its not a console window (console windows have their own specific typed character support)
+				# We couldn't inject in-process, and its not a legacy console window.
+				# console windows have their own specific typed character support.
 				(not focus.appModule.helperLocalBindingHandle and focus.windowClassName!='ConsoleWindowClass')
 				# or the focus is within a UWP app, where WM_CHAR never gets sent 
 				or focus.windowClassName.startswith('Windows.UI.Core')
+				#Or this is a UIA console window, where WM_CHAR messages are doubled
+				or isinstance(focus, winConsoleUIA)
 			)
 		):
 			keyStates=(ctypes.c_byte*256)()

source/textInfos/__init__.py

@@ -264,6 +264,10 @@ class TextInfo(baseObject.AutoPropertyObject):
 	@type bookmark: L{Bookmark}
 	"""
 
+	#: whether this textInfo should be expanded then collapsed around its enclosing unit before review.
+	#: This can be problematic for some implementations.
+	_expandCollapseBeforeReview = True
+
 	def __init__(self,obj,position):
 		"""Constructor.
 		Subclasses must extend this, calling the superclass method first.

source/winConsoleHandler.py

@@ -46,7 +46,6 @@
 COMMON_LVB_UNDERSCORE=0x8000
 
 
-
 @wincon.PHANDLER_ROUTINE
 def _consoleCtrlHandler(event):
 	if event in (wincon.CTRL_C_EVENT,wincon.CTRL_BREAK_EVENT):

user_docs/en/userGuide.t2t

@@ -1670,6 +1670,9 @@ This includes in Microsoft Word itself, and also the Microsoft Outlook message v
 However, There may be some information which is either not exposed, or exposed incorrectly in some versions of Microsoft Office, which means this UI automation support cannot always be relied upon.
 We still do not recommend that  the majority of users turn this on by default, though we do welcome users of Office 2016/365 to test this feature and provide feedback.
 
+==== Use UI automation to access the Windows Console====[AdvancedSettingsConsoleUIA]
+When this option is enabled, NVDA will use a new, work in progress version of its support for Windows Console which takes advantage of [accessibility improvements made by Microsoft https://devblogs.microsoft.com/commandline/whats-new-in-windows-console-in-windows-10-fall-creators-update/]. This feature is highly experimental and is still incomplete, so its use is not yet recommended. However, once completed, it is anticipated that this new support will become the default, improving NVDA's performance and stability in Windows command consoles.
+
 ==== Automatically set system focus to focusable elements in Browse Mode ====[BrowseModeSettingsAutoFocusFocusableElements]
 Key: NVDA+8