Hotfix Unicode CLDR database (#4404) (#4406)

* Hotfix Unicode CLDR database (#4404) * Hotfix Unicode CLDR database * Add entry * Add test * Update PR number
microsoft · Sep 6, 2022 · 991a74e · 991a74e
1 parent 1b842df
commit 991a74e
Show file tree

Hide file tree

Showing 7 changed files with 204 additions and 2 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -22,6 +22,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+### Fixed
+
+-  QFE: Fixes [#4403](https://github.com/microsoft/BotFramework-WebChat/issues/4403). Patched Unicode CLDR database which caused file upload in Polish to appear blank, by [@compulim](https://github.com/compulim), in PR [#4406](https://github.com/microsoft/BotFramework-WebChat/pull/4406)
+
 ## [4.14.1] - 2021-09-07
 
 ### Fixed

diff --git a/...file-upload-polish-js-upload-a-file-in-polish-should-render-properly-1-snap.png b/...file-upload-polish-js-upload-a-file-in-polish-should-render-properly-1-snap.png
diff --git a/__tests__/html/localization.fileUpload.polish.html b/__tests__/html/localization.fileUpload.polish.html
@@ -0,0 +1,55 @@
+<!DOCTYPE html>
+<html lang="en-US">
+  <head>
+    <link href="/assets/index.css" rel="stylesheet" type="text/css" />
+    <script crossorigin="anonymous" src="/test-harness.js"></script>
+    <script crossorigin="anonymous" src="/test-page-object.js"></script>
+    <script crossorigin="anonymous" src="/__dist__/webchat-es5.js"></script>
+  </head>
+  <body>
+    <div id="webchat"></div>
+    <script>
+      run(async function () {
+        const directLine = testHelpers.createDirectLineWithTranscript([
+          {
+            type: 'message',
+            id: 'A0000001',
+            timestamp: '2022-09-06T20:49:30.3628667Z',
+            localTimestamp: '2022-09-06T13:49:29.166-07:00',
+            localTimezone: 'America/Los_Angeles',
+            channelId: 'directline',
+            from: {
+              role: 'user'
+            },
+            locale: 'en-US',
+            attachments: [
+              {
+                contentType: 'application/zip',
+                contentUrl:
+                  'https://docs.botframework.com/static/devportal/client/images/bot-framework-default-placeholder.png',
+                name: 'some-file.zip'
+              }
+            ],
+            channelData: {
+              attachmentSizes: [1024]
+            }
+          }
+        ]);
+
+        const store = testHelpers.createStore();
+
+        WebChat.renderWebChat(
+          {
+            directLine,
+            locale: 'pl-PL',
+            store
+          },
+          document.getElementById('webchat')
+        );
+
+        await pageConditions.uiConnected();
+        await host.snapshot();
+      });
+    </script>
+  </body>
+</html>
diff --git a/__tests__/html/localization.fileUpload.polish.js b/__tests__/html/localization.fileUpload.polish.js
@@ -0,0 +1,6 @@
+/** @jest-environment ./packages/test/harness/src/host/jest/WebDriverEnvironment.js */
+
+describe('upload a file in Polish', () => {
+  test('should render properly', () =>
+    runHTML('localization.fileUpload.polish.html'));
+});
diff --git a/packages/api/package.json b/packages/api/package.json
@@ -30,7 +30,7 @@
     "prestart": "npm run build:babel",
     "start": "concurrently --kill-others --names \"babel,globalize,tsc\" \"npm run start:babel\" \"npm run start:globalize\" \"npm run start:typescript\"",
     "start:babel": "npm run build:babel -- --skip-initial-build --watch",
-    "start:globalize": "node-dev --respawn scripts/createPrecompiledGlobalize.js",
+    "start:globalize": "node-dev --respawn scripts/createPrecompiledGlobalize.mjs",
     "start:typescript": "npm run build:typescript -- --watch"
   },
   "devDependencies": {

diff --git a/packages/support/cldr-data/package.json b/packages/support/cldr-data/package.json
@@ -35,11 +35,12 @@
   "files": [
     "src/index.js",
     "src/install.js",
+    "src/patch.mjs",
     "urls.json"
   ],
   "scripts": {
     "eslint": "npm run precommit",
-    "install": "node ./src/install.js",
+    "install": "node ./src/install.js && node ./src/patch.mjs",
     "precommit": "npm run precommit:eslint -- src",
     "precommit:eslint": "eslint",
     "prettier": "prettier --check src/**/*.{js,ts,tsx}"

diff --git a/packages/support/cldr-data/src/patch.mjs b/packages/support/cldr-data/src/patch.mjs
@@ -0,0 +1,136 @@
+import { fileURLToPath } from 'url';
+import { resolve } from 'path';
+import fs from 'fs/promises';
+
+// There is an issue in the Unicode CLDR database (v36):
+//
+// - Polish has 4 different plural types: "one", "few", "many", "other"
+// - However, some units, say "short/digital-kilobyte", only have "other" defined
+// - When we localize 1024 (number) into kilobytes, it use the "one" type
+// - Since "short/digital-kilobyte/one" is not defined in the database, `globalize` throw exception
+//
+// In all of our supported languages, we also observed the same issue in Portuguese as well.
+//
+// As a hotfix, we are patching the Unicode CLDR database for all `[long/short/narrow]/digital-*` rules to make sure it include all plurals needed for that language.
+//
+// For a long term fix, we should move forward to a newer version of CLDR database, which is outlined in https://github.com/rxaviers/cldr-data-npm/issues/78.
+
+let FORBIDDEN_PROPERTY_NAMES;
+
+function getForbiddenPropertyNames() {
+  return (
+    FORBIDDEN_PROPERTY_NAMES ||
+    (FORBIDDEN_PROPERTY_NAMES = Object.freeze(
+      Array.from(
+        new Set([
+          // As-of writing, `Object.prototype` includes:
+          //   __defineGetter__
+          //   __defineSetter__
+          //   __lookupGetter__
+          //   __lookupSetter
+          //   __proto__
+          //   constructor
+          //   hasOwnProperty
+          //   isPrototypeOf
+          //   propertyIsEnumerable
+          //   toLocaleString
+          //   toString
+          //   valueOf
+          ...Object.getOwnPropertyNames(Object.prototype),
+
+          'prototype'
+        ])
+      )
+    ))
+  );
+}
+
+function isForbiddenPropertyName(propertyName) {
+  return getForbiddenPropertyNames().includes(propertyName);
+}
+
+function toDist(filename) {
+  if (filename.includes('..')) {
+    throw new Error('Filename cannot contains "..".');
+  }
+
+  return resolve(fileURLToPath(import.meta.url), '../../dist/', filename);
+}
+
+// ESLint "wrap-iife" rule is conflicting with Prettier.
+// eslint-disable-next-line wrap-iife
+(async function () {
+  const plurals = JSON.parse(await fs.readFile(toDist('supplemental/plurals.json'), 'utf8'));
+
+  const languagePlurals = new Map();
+
+  Object.entries(plurals.supplemental['plurals-type-cardinal']).forEach(([language, pluralsTypeCardinal]) => {
+    const plurals = ['other'];
+
+    languagePlurals.set(language, plurals);
+
+    if (!(`pluralRule-count-other` in pluralsTypeCardinal)) {
+      throw new Error(`Language ${language} does not have plural type "other".`);
+    }
+
+    ['zero', 'one', 'two', 'few', 'many'].forEach(pluralType => {
+      `pluralRule-count-${pluralType}` in pluralsTypeCardinal && plurals.push(pluralType);
+    });
+  });
+
+  const patchedLanguages = [];
+
+  await Promise.all(
+    Array.from(languagePlurals.entries()).map(async ([language, supportedPluralTypes]) => {
+      if (!/^[\w-]+$/u.test(language) && isForbiddenPropertyName(language)) {
+        throw new Error(`Invalid language code "${language}".`);
+      }
+
+      let units;
+
+      try {
+        units = JSON.parse(await fs.readFile(toDist(`main/${language}/units.json`), 'utf8'));
+      } catch (err) {
+        if (err.code === 'ENOENT') {
+          return;
+        }
+
+        throw err;
+      }
+
+      let numFound = 0;
+
+      ['long', 'short', 'narrow'].forEach(form => {
+        Object.entries(units.main[language].units[form]).forEach(([unitName, entry]) => {
+          if (!unitName.startsWith('digital-')) {
+            return;
+          }
+
+          if ('unitPattern-count-other' in entry) {
+            const { 'unitPattern-count-other': other } = entry;
+
+            supportedPluralTypes.forEach(pluralType => {
+              const name = `unitPattern-count-${pluralType}`;
+
+              if (!(name in entry)) {
+                entry[name] = other;
+                numFound++;
+              }
+            });
+          }
+        });
+      });
+
+      if (numFound) {
+        patchedLanguages.push(`${language} (${numFound} issues)`);
+
+        // eslint-disable-next-line no-magic-numbers
+        await fs.writeFile(toDist(`main/${language}/units.json`), JSON.stringify(units, null, 2));
+      }
+    })
+  );
+
+  // We are display output in CLI.
+  // eslint-disable-next-line no-console
+  console.log(`Patched ${patchedLanguages.length} languages: ${patchedLanguages.join(', ')}.`);
+})();