feat(developer): support normalization=disabled 🙀

#10554

common/web/types/src/kmx/kmx-plus.ts

@@ -75,6 +75,11 @@ export class Meta extends Section {
   indicator: StrsItem;
   version: StrsItem; // semver version string, defaults to "0"
   settings: KeyboardSettings;
+
+  /** convenience for checking settings */
+  get normalizionDisabled() {
+    return this.settings & KeyboardSettings.normalizationDisabled;
+  }
 };
 
 // 'strs'
@@ -205,11 +210,15 @@ export class Strs extends Section {
     }
     // nfd
     if (opts?.nfd) {
-      if (opts?.markers) {
-        s = MarkerParser.nfd_markers(s, false);
-      } else {
-        s = s.normalize("NFD");
-      }
+      if (!sections.meta) {
+        throw Error(`Internal Error: need 'meta' section to check normalization mode.`);
+      } else if (!sections.meta.normalizionDisabled) {
+        if (opts?.markers) {
+          s = MarkerParser.nfd_markers(s, false);
+        } else {
+          s = s.normalize("NFD");
+        }
+      } // else: disabled, do nothing
     }
     return s;
   }

developer/src/kmc-ldml/src/compiler/compiler.ts

@@ -25,6 +25,8 @@ export const SECTION_COMPILERS = [
 
   // First the former 'global' sections
   StrsCompiler,
+  // meta depends on strs, but potentially needed by anything else using strs
+  MetaCompiler,
   ListCompiler,
   ElemCompiler,
   UsetCompiler,
@@ -36,7 +38,6 @@ export const SECTION_COMPILERS = [
   KeysCompiler,
   LayrCompiler,
   LocaCompiler,
-  MetaCompiler,
   TranCompiler,
 ];
 

developer/src/kmc-ldml/src/compiler/keys.ts

@@ -1,4 +1,4 @@
-import { constants } from '@keymanapp/ldml-keyboard-constants';
+import { SectionIdent, constants } from '@keymanapp/ldml-keyboard-constants';
 import { LDMLKeyboard, KMXPlus, Constants, MarkerParser } from '@keymanapp/common-types';
 import { CompilerMessages } from './messages.js';
 import { SectionCompiler } from "./section-compiler.js";
@@ -46,6 +46,17 @@ export class KeysCompiler extends SectionCompiler {
     return constants.section.keys;
   }
 
+  public get dependencies(): Set<SectionIdent> {
+    const defaults = new Set(<SectionIdent[]>[
+      constants.section.elem,
+      constants.section.list,
+      constants.section.meta,
+      constants.section.strs,
+      constants.section.vars,
+    ]);
+    return defaults;
+  }
+
   /**
    *
    * @returns just the non-touch layers.

developer/src/kmc-ldml/src/compiler/meta.ts

@@ -1,4 +1,4 @@
-import { constants } from "@keymanapp/ldml-keyboard-constants";
+import { SectionIdent, constants } from "@keymanapp/ldml-keyboard-constants";
 import { KMXPlus } from '@keymanapp/common-types';
 
 import { CompilerMessages } from "./messages.js";
@@ -46,6 +46,11 @@ export class MetaCompiler extends SectionCompiler {
     return true;
   }
 
+  public get dependencies(): Set<SectionIdent> {
+    const strsOnly = new Set(<SectionIdent[]>[constants.section.strs]);
+    return strsOnly;
+  }
+
   public compile(sections: DependencySections): Meta {
     let result = new Meta();
     result.author        = sections.strs.allocString(this.keyboard3.info?.author);

developer/src/kmc-ldml/src/compiler/tran.ts

@@ -185,11 +185,12 @@ export abstract class TransformCompiler<T extends TransformCompilerType, TranBas
   }
   public get dependencies(): Set<SectionIdent> {
     const defaults = new Set(<SectionIdent[]>[
-      constants.section.strs,
-      constants.section.list,
       constants.section.elem,
-      constants.section.vars,
+      constants.section.list,
+      constants.section.meta,
+      constants.section.strs,
       constants.section.uset,
+      constants.section.vars,
     ]);
     defaults.delete(this.id);
     return defaults;

developer/src/kmc-ldml/test/helpers/index.ts

@@ -256,7 +256,7 @@ export function testCompilationCases(compiler: SectionCompilerNew, cases : Compi
         assert.includeDeepMembers(callbacks.messages, testcase.warnings, 'expected warnings to be included');
       } else if (!expectFailure) {
         // no warnings, so expect zero messages
-        assert.strictEqual(callbacks.messages.length, 0, 'expected zero messages');
+        assert.sameDeepMembers(callbacks.messages, [], 'expected zero messages but got ' + callbacks.messages);
       }
 
       // run the user-supplied callback if any

developer/src/kmc-ldml/test/test-keys.ts

@@ -5,7 +5,10 @@ import { assertCodePoints, compilerTestCallbacks, loadSectionFixture, testCompil
 import { KMXPlus, Constants, MarkerParser } from '@keymanapp/common-types';
 import { CompilerMessages } from '../src/compiler/messages.js';
 import { constants } from '@keymanapp/ldml-keyboard-constants';
+import { MetaCompiler } from '../src/compiler/meta.js';
+const keysDependencies = [ ...BASIC_DEPENDENCIES, MetaCompiler ];
 import Keys = KMXPlus.Keys;
+import { BASIC_DEPENDENCIES } from '../src/compiler/empty-compiler.js';
 const K = Constants.USVirtualKeyCodes;
 
 describe('keys', function () {
@@ -79,7 +82,7 @@ describe('keys', function () {
       callback(sect) {
         const keys = <Keys> sect;
         assert.ok(keys);
-        assert.equal(compilerTestCallbacks.messages.length, 0);
+
         assert.equal(keys.keys.length, 13 + KeysCompiler.reserved_count); // includes flick and gesture keys
 
         const [w] = keys.keys.filter(({ id }) => id.value === 'w');
@@ -116,6 +119,9 @@ describe('keys', function () {
         assertCodePoints(aacute.to.value, 'á');
 
       },
+      warnings: [
+        CompilerMessages.Hint_NormalizationDisabled()
+      ],
     },
     {
       subpath: 'sections/keys/escaped.xml',
@@ -193,14 +199,14 @@ describe('keys', function () {
         assert.equal(flickw.flicks[0].keyId.value, 'dd');
       },
     },
-  ]);
+  ], keysDependencies);
 });
 
 describe('keys.kmap', function () {
   this.slow(500); // 0.5 sec -- json schema validation takes a while
 
   it('should compile minimal kmap data', async function() {
-    let keys = await loadSectionFixture(KeysCompiler, 'sections/keys/minimal.xml', compilerTestCallbacks) as Keys;
+    let keys = await loadSectionFixture(KeysCompiler, 'sections/keys/minimal.xml', compilerTestCallbacks, keysDependencies) as Keys;
     assert.isNotNull(keys);
     assert.equal(compilerTestCallbacks.messages.length, 0);
     // skip reserved (gap) keys
@@ -385,39 +391,39 @@ describe('keys.kmap', function () {
         CompilerMessages.Error_InvalidScanCode({ form: "zzz", codes: ['ff'] }),
       ],
     },
-  ]);
+  ], keysDependencies);
 
   it('should reject layouts with too many hardware rows', async function() {
-    let keys = await loadSectionFixture(KeysCompiler, 'sections/keys/invalid-hardware-too-many-rows.xml', compilerTestCallbacks) as Keys;
+    let keys = await loadSectionFixture(KeysCompiler, 'sections/keys/invalid-hardware-too-many-rows.xml', compilerTestCallbacks, keysDependencies) as Keys;
     assert.isNull(keys);
     assert.equal(compilerTestCallbacks.messages.length, 1);
 
     assert.deepEqual(compilerTestCallbacks.messages[0], CompilerMessages.Error_HardwareLayerHasTooManyRows());
   });
 
   it('should reject layouts with too many hardware keys', async function() {
-    let keys = await loadSectionFixture(KeysCompiler, 'sections/keys/invalid-hardware-too-many-keys.xml', compilerTestCallbacks) as Keys;
+    let keys = await loadSectionFixture(KeysCompiler, 'sections/keys/invalid-hardware-too-many-keys.xml', compilerTestCallbacks, keysDependencies) as Keys;
     assert.isNull(keys);
     assert.equal(compilerTestCallbacks.messages.length, 1);
 
     assert.deepEqual(compilerTestCallbacks.messages[0], CompilerMessages.Error_RowOnHardwareLayerHasTooManyKeys({row: 1, hardware: 'us', modifiers: 'none'}));
   });
 
   it('should reject layouts with undefined keys', async function() {
-    let keys = await loadSectionFixture(KeysCompiler, 'sections/keys/invalid-undefined-key.xml', compilerTestCallbacks) as Keys;
+    let keys = await loadSectionFixture(KeysCompiler, 'sections/keys/invalid-undefined-key.xml', compilerTestCallbacks, keysDependencies) as Keys;
     assert.isNull(keys);
     assert.equal(compilerTestCallbacks.messages.length, 1);
 
     assert.deepEqual(compilerTestCallbacks.messages[0], CompilerMessages.Error_KeyNotFoundInKeyBag({col: 1, form: 'hardware', keyId: 'foo', layer: 'base', row: 1}));
   });
   it('should reject layouts with invalid keys', async function() {
-    let keys = await loadSectionFixture(KeysCompiler, 'sections/keys/invalid-key-missing-attrs.xml', compilerTestCallbacks) as Keys;
+    let keys = await loadSectionFixture(KeysCompiler, 'sections/keys/invalid-key-missing-attrs.xml', compilerTestCallbacks, keysDependencies) as Keys;
     assert.isNull(keys);
     assert.equal(compilerTestCallbacks.messages.length, 1);
     assert.deepEqual(compilerTestCallbacks.messages[0], CompilerMessages.Error_KeyMissingToGapOrSwitch({keyId: 'Q'}));
   });
   it('should accept layouts with gap/switch keys', async function() {
-    let keys = await loadSectionFixture(KeysCompiler, 'sections/keys/gap-switch.xml', compilerTestCallbacks) as Keys;
+    let keys = await loadSectionFixture(KeysCompiler, 'sections/keys/gap-switch.xml', compilerTestCallbacks, keysDependencies) as Keys;
     assert.isNotNull(keys);
     assert.equal(compilerTestCallbacks.messages.length, 0);
     assert.equal(keys.keys.length, 4 + KeysCompiler.reserved_count);

developer/src/kmc-ldml/test/test-tran.ts

@@ -9,7 +9,8 @@ import { KMXPlus, MarkerParser } from '@keymanapp/common-types';
 import Tran = KMXPlus.Tran;// for tests…
 import Bksp = KMXPlus.Bksp;// for tests…
 import { constants } from '@keymanapp/ldml-keyboard-constants';
-const tranDependencies = [ ...BASIC_DEPENDENCIES, UsetCompiler ];
+import { MetaCompiler } from './compiler/meta.js';
+const tranDependencies = [ ...BASIC_DEPENDENCIES, UsetCompiler, MetaCompiler ];
 const bkspDependencies = tranDependencies;
 
 describe('tran', function () {