Support for custom braille tables (#16208)

Fixes #3304.
Fixes #9863.
Supersedes PR #9864, #10172.
Addresses #505 (comment)

Summary of the issue:
In NVDA, there is no easy and reliable way for an add-on to provide a new braille table. For an experienced users wishing to do so there are two options:

Alter manually an existing table in louis/tables.
The new table still has its original name in the settings GUI.
This change is lost upon NVDA updates.

Set the absolute path to the table file in the configuration in lieu of the usual file name.
The settings GUI shows an empty entry for this one.
Forces to manually alter nvda.ini.
Forces to copy in the same directory the whole dependency chain of the new table plus braille-patterns.cti. This is because liblouis default table resolver only looks for tables in a single directory, See Make it easier for add-ons to supply custom braille tables #5489 (comment)

Description of how this pull request fixes the issue:
Add a new brailleTables optional directory in both the user scratchpad directory and the add-on directory structure.

Support reading tables metadata from an optional brailleTables section of the add-on manifest or from a manifest.ini file with the same format found in the root of the scratchpad directory, allowing a user to provide a display name and set output/input/contracted capabilities with no code.

Implement a custom liblouis table resolver that resolves tables based on what is registered in the brailleTables module:

When liblouis calls the resolver without a base file specified, the table is looked up from the brailleTables module and either resolved from the add-ons brailleTables directory or the built-in tables directory
When liblouis calls the resolver with a base file specified (e.g. when processing includes in tables), the table is looked up from the folder of the base table and/or the built-in tables directory
Enforce the existing fallback mechanism to ensure there still is braille output if the configured table cannot be found e.g. because an add-on or the scratchpad directory was disabled. This now applies both to the main configuration and individual profiles and also covers braille input.

Note that if an add-on author wants a table to be listed in the GUI, he/she should always define the table in the manifest. Contrary to earlier incarnations of this pr, replacing a table in an add-on (i.e. when it has the same filename as a built-in table) without defining it in the manifest is no longer possible. Therefore it is also not possible to replace unlisted tables that are included by listed tables. For example, if you want to replace spaces.uti as included in nl-comp8.utb, you weel need to both define and bundle a replacement of nl-comp8.utb and spaces.uti in your add-on.

.gitignore

@@ -41,6 +41,7 @@ uninstaller/UAC.nsh
 *.pyo
 *.dmp
 tests/unit/nvda.ini
+tests/unit/brailleTables
 tests/system/settingsCache/*
 !tests/system/settingsCache/2020.4/*.txt
 source/locale/en/LC_MESSAGES/nvda.po

contributors.txt

@@ -233,3 +233,5 @@ Larry Wang
 Doug Lee
 Doc Mehta
 Andre Louis
+Accessolutions
+Julien Cochuyt

nvdaHelper/liblouis/sconscript

@@ -24,13 +24,15 @@ Import([
 	"thirdPartyEnv",
 	"sourceDir",
 ])
-sourceDir:Base = sourceDir
+sourceDir: Base = sourceDir
 thirdPartyEnv: Environment = thirdPartyEnv
 env: Environment = typing.cast(Environment, thirdPartyEnv.Clone())
 
 louisRootDir = env.Dir("#include/liblouis")
 louisSourceDir = louisRootDir.Dir("liblouis")
+louisTableDir = louisRootDir.Dir("tables")
 outDir = sourceDir.Dir("louis")
+unitTestTablesDir = env.Dir("#tests/unit/brailleTables")
 signExec=env['signExec'] if env['certFile'] else None
 
 RE_AC_INIT = re.compile(r"^AC_INIT\(\[(?P<package>.*)\], \[(?P<version>.*)\], \[(?P<bugReport>.*)\], \[(?P<tarName>.*)\], \[(?P<url>.*)\]\)")
@@ -96,7 +98,18 @@ env.Install(sourceDir, louisLib)
 louisPython = env.Substfile(outDir.File("__init__.py"), louisRootDir.File("python/louis/__init__.py.in"),
 	SUBST_DICT={"###LIBLOUIS_SONAME###": louisLib[0].name})
 
-env.Install(outDir.Dir("tables"),
-	[f for f in env.Glob("%s/tables/*" % louisRootDir)
+env.Install(
+	outDir.Dir("tables"),
+	[
+		f for f in env.Glob(f"{louisTableDir}/*")
 		if f.name not in ("Makefile", "Makefile.am", "Makefile.in", "README", "maketablelist.sh")
-	])
+	]
+)
+# Custom tables unit test
+testTable = env.InstallAs(unitTestTablesDir.File("test.utb"), louisTableDir.File("en-us-comp8-ext.utb"))
+env.Depends(testTable, env.Install(unitTestTablesDir, [
+	louisTableDir.File("latinLetterDef8Dots.uti"),
+	louisTableDir.File("en-us-comp8-ext.utb")
+]))
+# Ensure the braille tables for tests are installed with scons source
+env.Alias("source", testTable)

projectDocs/dev/developerGuide/developerGuide.md

@@ -755,9 +755,9 @@ Then the handler needs to be registered - preferably in the constructor of your
     addonHandler.isCLIParamKnown.register(processArgs)
 ## Packaging Code as NVDA Add-ons {#Addons}
 
-To make it easy for users to share and install plugins and drivers, they can be packaged in to a single NVDA add-on package which the user can then install into a copy of NVDA via the Add-on Store found under Tools in the NVDA menu.
+To make it easy for users to share and install plugins, drivers and braille translation tables, they can be packaged in to a single NVDA add-on package which the user can then install into a copy of NVDA via the Add-on Store found under Tools in the NVDA menu.
 Add-on packages are only supported in NVDA 2012.2 and later.
-An add-on package is simply a standard zip archive with the file extension of "`nvda-addon`" which contains a manifest file, optional install/uninstall code and one or more directories containing plugins and/or drivers.
+An add-on package is simply a standard zip archive with the file extension of "`nvda-addon`" which contains a manifest file, optional install/uninstall code and one or more directories containing plugins, drivers and/or and braille translation tables.
 
 ### Non-ASCII File Names in Zip Archives {#toc34}
 
@@ -819,6 +819,9 @@ The lastTestedNVDAVersion field in particular is used to ensure that users can b
 It allows the add-on author to make an assurance that the add-on will not cause instability, or break the users system.
 When this is not provided, or is less than the current version of NVDA (ignoring minor point updates e.g. 2018.3.1) then the user will be warned not to install the add-on.
 
+The manifest can also specify information regarding the additional braille translation tables provided by the add-on.
+Please refer to the [braille translation tables section](#BrailleTables) later on in this document.
+
 #### An Example Manifest File {#toc37}
     --- start ---
     name = "myTestAddon"
@@ -839,6 +842,7 @@ The following plugins and drivers can be included in an add-on:
 * Braille display drivers: Place them in a brailleDisplayDrivers directory in the archive.
 * Global plugins: Place them in a globalPlugins directory in the archive.
 * Synthesizer drivers: Place them in a synthDrivers directory in the archive.
+* [Braille translation tables](#BrailleTables): Place them in a brailleTables directory in the archive.
 
 ### Optional install / Uninstall code {#toc39}
 
@@ -889,6 +893,49 @@ This will open the file named in the docFileName parameter of the manifest.
 NVDA will search for this file in the appropriate language directories.
 For example, if docFileName is set to readme.html and the user is using English, NVDA will open doc\en\readme.html.
 
+### Braille translation tables {#BrailleTables}
+
+Although NVDA ships with more than a hundred braille translation tables provided by [the liblouis project](https://liblouis.io/) aimed at fitting most needs, it also supports the addition of custom tables.
+Custom tables must be placed in the brailleTables directory of an add-on or subdirectory of the scratchpad directory.
+These tables can either replace standard tables shipped with NVDA or be completely new ones.
+
+When adding a table, some information must be provided such as its display name in the Preferences dialog, whether it supports input and/or output and whether it is for contracted braille.
+When an add-on ships with tables, this information is included in its manifest in the optional brailleTables section.
+For example:
+```
+--- start ---
+[brailleTables]
+[[fr-bfu-tabmod-comp8.utb]]
+displayName = French (unified) 8 dot computer braille - Addition
+contracted = False
+output = True
+input = True
+
+[[no-no-8dot.utb]]
+displayName = Norwegian 8 dot computer braille - Replacement
+contracted = False
+output = True
+input = True
+--- end ---
+```
+
+In the above example, `fr-bfu-tabmod-comp8.utb` is a new table,  while `no-no-8dot.utb` replaces a table that is already included in NVDA.
+Both tables need to be shipped in the brailleTables directory of the add-on.
+It is also possible to include a table in the manifest that is shipped with NVDA but otherwise unavailable for selection in the Preferences dialog.
+In that case, the table does not need to be shipped in the add-on's brailleTables directory.
+
+Providing a custom table, whether it has the same file name as a standard table or a different name, thus requires you to define the table in the add-on's manifest.
+The only exception to this rule applies to tables that are included within other tables.
+While they don't have to be included in the manifest of the add-on, they can only be included from other tables that are part of the same add-on.
+
+Custom tables can also be placed in the brailleTables subdirectory of the scratchpad directory.
+In this case, the table metadata can be placed in a `manifest.ini` file in the root of the scratchpad in the exact same format as the example above.
+Basically, this means that, whether using an add-on or the scratchpad, the requirements and implementation steps are equal.
+Note that a `manifest.ini` file in the scratchpad is only parsed for braille table metadata.
+Other add-on metadata in the file is ignored.
+
+Please refer to the [liblouis documentation](https://liblouis.io/documentation/) for detailed information regarding the braille translation tables format.
+
 ## NVDA Python Console {#PythonConsole}
 
 The NVDA Python console emulates the interactive Python interpreter from within NVDA.

source/addonHandler/__init__.py

@@ -1,6 +1,6 @@
 # A part of NonVisual Desktop Access (NVDA)
 # Copyright (C) 2012-2024 Rui Batista, NV Access Limited, Noelia Ruiz Martínez,
-# Joseph Lee, Babbage B.V., Arnold Loubriat, Łukasz Golonka, Leonard de Ruijter
+# Joseph Lee, Babbage B.V., Arnold Loubriat, Łukasz Golonka, Leonard de Ruijter, Julien Cochuyt
 # This file is covered by the GNU General Public License.
 # See the file COPYING for more details.
 
@@ -998,6 +998,15 @@ class AddonManifest(ConfigObj):
 # Name of default documentation file for the add-on.
 docFileName = string(default=None)
 
+# Custom braille tables
+[brailleTables]
+	# The key is the table file name (not the full path)
+	[[__many__]]
+		displayName = string()
+		contracted = boolean(default=false)
+		input = boolean(default=true)
+		output = boolean(default=true)
+
 # NOTE: apiVersion:
 # EG: 2019.1.0 or 0.0.0
 # Must have 3 integers separated by dots.
@@ -1014,7 +1023,7 @@ def __init__(self, input, translatedInput=None):
 		@param translatedInput: translated manifest input
 		@type translatedInput: file-like object
 		"""
-		super(AddonManifest, self).__init__(input, configspec=self.configspec, encoding='utf-8', default_encoding='utf-8')
+		super().__init__(input, configspec=self.configspec, encoding='utf-8', default_encoding='utf-8')
 		self._errors = None
 		val = Validator({"apiVersion":validate_apiVersionString})
 		result = self.validate(val, copy=True, preserve_errors=True)
@@ -1032,6 +1041,10 @@ def __init__(self, input, translatedInput=None):
 				val=self._translatedConfig.get(k)
 				if val:
 					self[k]=val
+			for fileName, tableConfig in self._translatedConfig.get("brailleTables", {}).items():
+				value = tableConfig.get("displayName")
+				if value:
+					self["brailleTables"][fileName]["displayName"] = value
 
 	@property
 	def errors(self):

source/braille.py

@@ -2,7 +2,7 @@
 # This file is covered by the GNU General Public License.
 # See the file COPYING for more details.
 # Copyright (C) 2008-2024 NV Access Limited, Joseph Lee, Babbage B.V., Davy Kager, Bram Duvigneau,
-# Leonard de Ruijter, Burman's Computer and Education Ltd.
+# Leonard de Ruijter, Burman's Computer and Education Ltd., Julien Cochuyt
 
 import itertools
 import os
@@ -70,6 +70,8 @@
 	from NVDAObjects import NVDAObject
 	from speech.types import SpeechSequence
 
+FALLBACK_TABLE = config.conf.getConfigValidation(("braille", "translationTable")).default
+"""Table to use if the output table configuration is invalid."""
 
 roleLabels: typing.Dict[controlTypes.Role, str] = {
 	# Translators: Displayed in braille for an object which is a
@@ -494,8 +496,7 @@ def update(self):
 		if config.conf["braille"]["expandAtCursor"] and self.cursorPos is not None:
 			mode |= louis.compbrlAtCursor
 		self.brailleCells, self.brailleToRawPos, self.rawToBraillePos, self.brailleCursorPos = louisHelper.translate(
-			[os.path.join(brailleTables.TABLES_DIR, config.conf["braille"]["translationTable"]),
-				"braille-patterns.cti"],
+			[handler.table.fileName, "braille-patterns.cti"],
 			self.rawText,
 			typeform=self.rawTextTypeforms,
 			mode=mode,
@@ -2036,6 +2037,7 @@ class BrailleHandler(baseObject.AutoPropertyObject):
 
 	def __init__(self):
 		louisHelper.initialize()
+		self._table: brailleTables.BrailleTable = brailleTables.getTable(FALLBACK_TABLE)
 		self.display: Optional[BrailleDisplayDriver] = None
 		self._displaySize: int = 0
 		"""
@@ -2106,6 +2108,18 @@ def terminate(self):
 			self.ackTimerHandle = None
 		louisHelper.terminate()
 
+	table: brailleTables.BrailleTable
+	"""Type definition for auto prop '_get_table/_set_table'"""
+
+	def _get_table(self) -> brailleTables.BrailleTable:
+		"""The translation table to use for braille output.
+		"""
+		return self._table
+
+	def _set_table(self, table: brailleTables.BrailleTable):
+		self._table = table
+		config.conf["braille"]["translationTable"] = table.fileName
+
 	# The list containing the regions that will be shown in braille when the speak function is called
 	# and the braille mode is set to speech output
 	_showSpeechInBrailleRegions: list[TextRegion] = []
@@ -2674,12 +2688,20 @@ def initialDisplay(self):
 
 	def handlePostConfigProfileSwitch(self):
 		display = config.conf["braille"]["display"]
-		# Do not choose a new display if:
-		if not (
-			# The display in the new profile is equal to the last requested display name
-			display == self._lastRequestedDisplayName
-			# or the new profile uses auto detection, which supports detection of the currently active display.
-			or (display == AUTO_DISPLAY_NAME and bdDetect.driverIsEnabledForAutoDetection(self.display.name))
+		# #7459: the syncBraille has been dropped in favor of the native hims driver.
+		# Migrate to renamed drivers as smoothly as possible.
+		newDriverName = RENAMED_DRIVERS.get(display)
+		if newDriverName:
+			display = config.conf["braille"]["display"] = newDriverName
+		if (
+			self.display is None
+			# Do not choose a new display if:
+			or not (
+				# The display in the new profile is equal to the last requested display name
+				display == self._lastRequestedDisplayName
+				# or the new profile uses auto detection, which supports detection of the currently active display.
+				or (display == AUTO_DISPLAY_NAME and bdDetect.driverIsEnabledForAutoDetection(self.display.name))
+			)
 		):
 			self.setDisplayByName(display)
 		elif (
@@ -2693,6 +2715,20 @@ def handlePostConfigProfileSwitch(self):
 			self._detector._limitToDevices = bdDetect.getBrailleDisplayDriversEnabledForDetection()
 
 		self._tether = config.conf["braille"]["tetherTo"]
+		tableName = config.conf["braille"]["translationTable"]
+		# #6140: Migrate to new table names as smoothly as possible.
+		newTableName = brailleTables.RENAMED_TABLES.get(tableName)
+		if newTableName:
+			tableName = config.conf["braille"]["translationTable"] = newTableName
+		if tableName != self._table.fileName:
+			try:
+				self._table = brailleTables.getTable(tableName)
+			except LookupError:
+				log.error(
+					f"Invalid translation table ({tableName}), "
+					f"falling back to default ({FALLBACK_TABLE})."
+				)
+				self._table = brailleTables.getTable(FALLBACK_TABLE)
 
 	def handleDisplayUnavailable(self):
 		"""Called when the braille display becomes unavailable.
@@ -2801,19 +2837,9 @@ def initialize():
 	log.info("Using liblouis version %s" % louis.version())
 	import serial
 	log.info("Using pySerial version %s"%serial.VERSION)
-	# #6140: Migrate to new table names as smoothly as possible.
-	oldTableName = config.conf["braille"]["translationTable"]
-	newTableName = brailleTables.RENAMED_TABLES.get(oldTableName)
-	if newTableName:
-		config.conf["braille"]["translationTable"] = newTableName
 	handler = BrailleHandler()
-	# #7459: the syncBraille has been dropped in favor of the native hims driver.
-	# Migrate to renamed drivers as smoothly as possible.
-	oldDriverName = config.conf["braille"]["display"]
-	newDriverName = RENAMED_DRIVERS.get(oldDriverName)
-	if newDriverName:
-		config.conf["braille"]["display"] = newDriverName
-	handler.setDisplayByName(config.conf["braille"]["display"])
+	handler.handlePostConfigProfileSwitch()
+	config.post_configProfileSwitch.register(handler.handlePostConfigProfileSwitch)
 
 def pumpAll():
 	"""Runs tasks at the end of each core cycle. For now just region updates, e.g. for caret movement."""