From 991a74ee0c1d7242545c16b60fab98716d5ddd72 Mon Sep 17 00:00:00 2001 From: William Wong Date: Tue, 6 Sep 2022 16:58:25 -0700 Subject: [PATCH] Hotfix Unicode CLDR database (#4404) (#4406) * Hotfix Unicode CLDR database (#4404) * Hotfix Unicode CLDR database * Add entry * Add test * Update PR number --- CHANGELOG.md | 4 + ...n-polish-should-render-properly-1-snap.png | Bin 0 -> 10569 bytes .../html/localization.fileUpload.polish.html | 55 +++++++ .../html/localization.fileUpload.polish.js | 6 + packages/api/package.json | 2 +- packages/support/cldr-data/package.json | 3 +- packages/support/cldr-data/src/patch.mjs | 136 ++++++++++++++++++ 7 files changed, 204 insertions(+), 2 deletions(-) create mode 100644 __tests__/__image_snapshots__/html/localization-file-upload-polish-js-upload-a-file-in-polish-should-render-properly-1-snap.png create mode 100644 __tests__/html/localization.fileUpload.polish.html create mode 100644 __tests__/html/localization.fileUpload.polish.js create mode 100644 packages/support/cldr-data/src/patch.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fd2c880fc..8705bd8c37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +### Fixed + +- QFE: Fixes [#4403](https://github.com/microsoft/BotFramework-WebChat/issues/4403). Patched Unicode CLDR database which caused file upload in Polish to appear blank, by [@compulim](https://github.com/compulim), in PR [#4406](https://github.com/microsoft/BotFramework-WebChat/pull/4406) + ## [4.14.1] - 2021-09-07 ### Fixed diff --git a/__tests__/__image_snapshots__/html/localization-file-upload-polish-js-upload-a-file-in-polish-should-render-properly-1-snap.png b/__tests__/__image_snapshots__/html/localization-file-upload-polish-js-upload-a-file-in-polish-should-render-properly-1-snap.png new file mode 100644 index 0000000000000000000000000000000000000000..0b3497e20b3d6479159fda2faa5ddc0eddc1f2d2 GIT binary patch literal 10569 zcmeHtc|4T+`}eeIr;em#X(6FF6;)Jh_h5*}F?<7Yc>ii@AE) z2!-0@fI@A4u!9HQc`MXUfxk9+8R=X?6*dTup-@Lrn9CPU{1T^_{(&ZJ`o`SywLjjG z_PvXGvz7Eh^V301?F`q`k9L&zUD+1bpJ|il$#oc7Uo@X!NEdwNv|8o05A_@MJ$?7= z56gXL1umbuwXgolX3~onF)!}^bFNzB&bPm|?$zr(QALi_5ydwoDvzi>tQww0pSKrw zE$cdidZ64;ghrt>xjW=hhqO~ReK@swFY3yE2h^WWHMcx4+$w~6e!K~F>%c=ERQkUk z`L{0rg5h71{HqNA%Hh8@@vk@jPX~e?5$m^a-x^9tNaRwfGe71~AC&95x~y*BzU`@^ z>f=1=Udfz}$iH#(W`1w)trZT3+=ANFTT?sane*@SB=*oSb5 zo%NUg-2UH*KV`FAh1XBZH;DLZ-*9oM)hY#oGsB#ev9YdG_}-p6+O}r9(M=I5J4!cR z-qahG{@J`pwL_%9txZdBd9?HyVM5SuKpeB%RXF1#I;8REq4lYjDo=(2`R&@KH{qJW z1MV*L@gq#yaS06o!O_Sn>E;l<>ZXRZ7@UK*lf%qD zbkLNhb>g9u7*B@ccxtAtLvt(@|E48#bf3nWw}~A3oXdmvbYW6PcE!wu2K8Fo^p-I8 zbmZUm z&3^Q2xId@PAk4x0l5atmbw0PJGgGz)%Up?8cWLLF>@WCHQ&VcWG|06eI4yLL1qjqsKqe$*Bo{M@Ie>OL6TBO53F6b2{oy}_aoVB#G z*;Ffi(EX}nGE0x-eLK+C`j8l>{vA0t$EhuDd!9IEq2um2>C2yJ52mKXWgc8h@74Nh zSqrzIcedO*T$cznGER%RB_ZN!CC6W8G9Usg<226h=Gy|SwZ#Ve?cQG) zvq^SaXFqpHJ;XlWWnyIV;{Gc8e0{K-ao4=`;E?wyDdW#=-g7Mx#F$(+Zzq#LhrF8b z$s@TZbdJ2V@cvSCI;W;=Zr75VO0U>HrswNO@tiaoHr&;5qQ_JJ>J#S@re5@?q1a+O z+febG_q2uuV+*f-+dejxD4^2W+y2+fcCT-7CqrfOM*q&e$)eXkdTNPafM)NUm=*ba zr2!wdJk~z$qkGMq(XKc?F(1z_s`x-G7W=w?)>CD%`TbeWXYOmx#5c9IkxJWa1K(h@ zPd$oBxb0@zeqr0`pH%jAWsn!weX8u?0^5}<)IZYd^V-UY-JHcwT#>}Qy}jPa<<3biuEwVa{fZW=>vQ*uRSa5~x*aMYD{;AWBd@^Ced60l znY65$rUBMqrH8m}rG%`=x9CE%wyF2k+B`zTs$vc`mpmgUlG7w96o=)9pMeTi=WTlm z2VWKLS{c4t{;m{%SK4V@9CM5%ERdZ2K(=Ro&U3WN!-M8^%{hUa{)jjP4>nA_;WQ(=+>N5V{@OZg zU$7fHa*tZhnu1b+|ffq~gGU)b-*ER!+k<**2*`BAp#!wz0)DI^L~fbEO|~50n9pC3;>5@HM-MTVC>FSIt2*r{+ zM#k4Y=E!BglAhb@Uu&xD+9kh1zedEr(cP$}HWkRZv}zj!whcQh-rdFxm?(T^$DW-> zTdt0E)S^ua_S%ZJ{N6Se&z|>bBs3N@GG47Dh%3+?m`k-6m_CML?A?6V&6@W&8r%+@ zDCeqNx5;D4ZWt+lWgr4cwo1-RXqfRg(pg6Z-B20Y!2w3l2|Bq4178%dPP0z9QM9Da zT9+)c8kgVO_5HHq$;(SFL)7spZ)IC$Wwj_x+Hm0{TO*TbtEy({w(9)XF!g)e=ep%n zUsyWn{4}REs_gs^k7Qm0rqWgB=D@{?GT*$EG&A%gq5}R#v1qkViha^5Nr*_Ymd0?D zicE$S*aUy$Dx02{ESy!x-pF=RWuud?h6WdtUz-~_zFMPhqHmp^wI{r#sBDkAOXHRY z*1Tohg!5Lc^pq6M=H})kv}p0Hwb;u|v!A_>pBZ9(|3CYP@*AYvVe71nBJ@_2{eGSQ zT2rVfH0|BHmoSLeL{!_Tw$-gPIh2x>%9$p&s+Fwc$B!4|7}B9(Vae?-k%!M;VbX{JF=hKtTydD~ z{iv&_*J|y{#9JXz1K_RZw!{*~J@gSFAro6$>f^^eQ_M>9>eV5+ zm{+f|rHr!@$KP!FuqS#Z#HW%;W4BwGT3Z*CICbRe#kI5CJttpc?%YW`C}(pzIJj_@ zr?>m|xu%8T`qmn;;Cus(HIrkhYug-Ot}Sj zrPjDGX{JmX-dIypQ!;WNs^K{{q5o&8OMp~s1dYupx7v}^wCdK#EpLJ{zC$0>|b-uL=v+*Ais83QYW7s_k z35j|6G9zHRLg&1F!{qF5RIjsARP21XG|HW44*8S!=zk&e|8s%H=+g`c2&fzK#1G#< z^(GE`Rt!!t=knBif8%`MtkP0ahD72ThL0B&^y599E^zSRNlKAj%{LKVc~tL@JG}pK zXO}x5=D6vB-DFpp!9;t3v5gIRe0==jXo z#bBP01LraI$r>9bk9%~rwL^J@l~NvV*-kEXA@f3mF~2H(sWyZ$32UEU_h`fMPHiPl z=1Bh;xu%xo?y2`1$xJCOu#%z0hxWECd;R)#&gyF5=Doe$Z|>I%g$pYi+0{g_Pfs+b zhuyD_XN=R(nVFfEH18Z;U0q;tOk!fbOkw#Y8ylP4+}xHl?Pn&C>Rp+%95+@Kx#rA` zH<~3SC8T&+1w&KQ%zzoXc~DRg`NU{}RT-|M(0bSI-NuH7Q9wV1pjE%jtSntpsq?S2 z@FR-3si_w~e*8$A`*wsu4_G#9JRu{a2T&(73?+~f;q?%RK=CLPELq35rgApF-nReO>eBPKt|< zKH4pNU0*-EkiKMTYHt3&f2K~Pk)orKV~mgku+{5qs=$E*;*{1LtuS84;UBMav$LB9 zA~gzk@}C?n?Q(5hU*od-pUL4CN1G`_%Rexe;FP7MX0BycP#E^xcYs+`~&`Bs4kpx ze%>=QG!%o|b<6A4#_EVh9(2sq)YMZYX?|g0LG8lPqnhxNlrcVz+qQM9-}LT1dz?#> z)6$Ngu`0b*WZ$?XY#~gA3C<7=4fPe4SlufVhV2L?*;6J}p)YV>eV z+ux?{;^j57vrEU}aB2_Ho`Zjn{L=*(QBY7IPl>7r{#;KEA1 zErF3>fhItt*wYqiot>7Uj~`2)I&~FhaL(_6NC^e~@IcNOMM&P^g}9WgEVirEB^qc1 zyz!a*+4pO`|LZeQ%C_lZ9i2U*YBx4G9o2BLV;Jevr`uTTOJzQhu1Q$RNa%5ObsM-> zZ9_wBoA+h9Lvxy$7fWQnu<+@!t||ZSX=WuJ@WtQB1CEc?Z6u@djp${Esm>C@@N0YT z#Zf~zl0)x@hk)!tP-h8@A}g>cR@ORKV|^}?lmRGl`11FDQpOX0NnCT9HevE@%>m#8 zU=C4tIMMYhR~}!wbZK#ABspM7*|l2A!lp8MG+DLkPq>OX`RrCa9)^>oEaDyq09}`OvISm7&U%U+Q#1h;K!&91MFP#S8D|=bvUz{#w{BX#YfWi&72z+Ze>tvchxd zmuT%yyQf=F&--`$N5TKU+uC{AQ>T==>`Ur zDc{)O4-j`#8MFM+AJ%?q+{q#WaJuVsWR+;<=WFD)jRu$YV-e` z;BZ+RLAnEK)-A2A3T{2`je8p2-K$${Gd4(iygx}DG-$V=EWAQz&d<;Lbo{FDqKF30 zUYzQ#SlAQEt8oEt*3@dCQ2-esY3@dHsU{^ev*d7~g~e+yJ9nDTToPw_h6=6)CoQog z$HXwkgRKChW*h5ks_!H73Te7qc=jljJS3l-o0~&Suwm&>m9MOgbze}K339-+VVNtl zv)dNb6hF~EHeET>{^FQc!>1i{c92%m9zEKO1kHt!Mve5h>O;OB@?gM8{r&yej$!N# zxJgOS8ewR7So-YQ<{~v!RV#3b+V9_^x~5`3_q`AoQSrViARr)lU=+e;;*v*les=aL zZEfwOsoUO9!Lh*XrpuSX8|q)5*V(jblfApi@3*=bOmml8bpeD$L^+|wNJOoTkr5CP zDTq3uBP}7Jotm0TA+jiEn(VDWJBJ1a(gBkODNasKCBYj(^@+;HP*;=Nw^I>CRtU8s zNj6ol)Vp~Dmyiq{urwJ}H)I4+vk>CcHXa^bTzvnb^Y@>BixA0!%vxYkbVb#7+Qif} z=Fy`^@_w_oto?^#phYd`8`aVwg3;=gjE#-)t@?^37VAIlBF8@MmTUX@)=mNw#wXTp z7m3b77!8q<%tpKg@ss(vx#p(=GV;OeffysWuo=~+03I4#O-5Y&vY@qhlq_xZc%vN> z1n++T1B|h*vC#mcBkp_zheIIFe%y|PJYF$%WOE340r%;zouK;C($X!Dw(a&FN%F-( z&h4xYu7*@A54jKmztgXG_oQWI&6cN`cqYCh$9JZO>=F!u*jet84l$-JQ%?|9m5#M9 zrFQMwMG}Wrj;}4V%%O5ZM~)x`$3&mZfxu15fU8j^>AljD#GmfvqWe&R%Ws~xdnYiC`Z9;j47);Uj4&%vpwf{YAt?aP->sj7DI zk3s(Zr{2?-MxW{}EqbEVlctZD|0 zw7AsehCraK+w&Rpw%B*Z2Fcvx?F9)Va&2G-rtooqpEV>fT>}FMYL#konxtayF(bt7 z>p9wmRwPcRBQX(NaWN`XnR=8Xw!XjWOtm5GCGd6`G9z9m@iL3C^20g%Dr zCvMPYN)0EGBYqqBiU;9jw6$Huspv+8hr=Kd$6-%&ob0K=x-Ei?pSbe;#pk~35M{1} z3uQhzP9H*{b}S>m3jkSZ4Dz1K5+hKa45J)4p3{d1o8r*{rS?MNfMD2gL!^zSQmN{{ zd$(Bo^@R#uaQ^90Lyn8149k^&|Nh+&hs(Vw8wbxLt{`%ohlht?bJsFn1)mE^ydY?e zBX#`vB|tG5ILn6PV@S~HmoYJ!b+VP$psP=xKW~E}b8uv2d0~FE;oedqD{W5 zJFtI$YI(U5=&ZhN@LCB_-NeQQ(pM(*M&{f(BSS;O_zN!aNcINdgfYkwW=zCe9`8Av zW(h-Cl8Seo}( z8Z=!7mcRfPNo}~=kA3dhh*I!ZZ08le4)@?Jjt=mfTrwRIAZ zQ`~JPtNPrTU@^!WQ?hwfsyUHJ7v>FvKA?jSo+;~k&oWHEg7njk&W=T(1Y-a7%NM@} zr@^72oT{pxg+n=Gw&=1R!h}aK<-=HuT76V(YDf ztu*?G=VWPu2%2ExT2{$u=Dh8~t0Zw&8iNZFsRR5FHrAFI1y+jI9$y87)C0>$Y_)6F z4{x&Xe7!2v;U(QfQd=mhsHgzvm=pHcwtc&dn%YfxU0YXY?Ck8Euu-JAef($F`uh6R zw2ZQ{1wdJau$Ya^76LM64&O-*I^)=+^Ni7~;U? zTc-m&&YwGXfS*6Tsp&d?tVLVv@g74kE6{?IIb-NvBz+CV*%oGGWblTgj~%-R$<9Pm z^TEid`h^Q-;B<&;!al51L~kLW+Nx^mG>pm+^2k!*!((Avw)6V+u$!8iQYI&z4OT&i zhP+iufC*-;!hj}#gt4FBZ%lnUOvq9yDzy41_3z)SVf0&N!5JZ=&;DQogT^3gGVGIy zSs~l`kgNjGfJij=oCV+igbLa;`Hv*okDqy=v@Bn<FKj)#xRBU_zmoZY^R@-lT(1bqbwXTP&IHA#u>N<*&LVSD0Eas z#S$Du7mIaW_5i7xUs`HM7H%M}Id9%vcWTc!1m{36OhM{GEUletOMUa^PuMUxwr1%U zkgcG=S0aSREGYJkWF&kN2sGJ>!9SFfB!jUlBt6+?D>r&(q&nv+n4OWMV~$z=ja1P4 z9CNA-RHk`?T3skZeQyNie<|+5ssKF;p2HFt3-p!rPY!WIcxRUm5}0G2f#acJ!hh&c_7T_ei%_y=1ObRK zd^_T10?R2xrs==?pK&5Le)BOs>&{rdyZdB)BzV1ML46cVW@4dHW7&CPJdYX+78-lT zDho0TqU)1C-_k&Ix|qH?ARE`<(K)l9`Pe-HGyYiGHL{w8QEPJjUd8Sd2-7&oAfSav z6=2>|)7glYG=L!91j`uM+Z=#6r#fA(1x`(oztCG*oYW#G-Sm9(%PEKAhuPn+Q3cHR<`PE)v#fBsm;Oe_X*>0sW zbYe7C@--|et&;i%bA3=jqwy5Q29uXoDw&!!U%og%{zQx(w%wHCVtLH`kL3II_FmQw zC=CZ=3KpGi=sNZ?O9>XHi?A9GP}gn^{2~nF@5UNy!xRSJ>FSO3qNPDpZ`kC^&1Z(T!L?bbIL literal 0 HcmV?d00001 diff --git a/__tests__/html/localization.fileUpload.polish.html b/__tests__/html/localization.fileUpload.polish.html new file mode 100644 index 0000000000..d09862cf40 --- /dev/null +++ b/__tests__/html/localization.fileUpload.polish.html @@ -0,0 +1,55 @@ + + + + + + + + + +
+ + + diff --git a/__tests__/html/localization.fileUpload.polish.js b/__tests__/html/localization.fileUpload.polish.js new file mode 100644 index 0000000000..77f8c7735e --- /dev/null +++ b/__tests__/html/localization.fileUpload.polish.js @@ -0,0 +1,6 @@ +/** @jest-environment ./packages/test/harness/src/host/jest/WebDriverEnvironment.js */ + +describe('upload a file in Polish', () => { + test('should render properly', () => + runHTML('localization.fileUpload.polish.html')); +}); diff --git a/packages/api/package.json b/packages/api/package.json index 1f91c9d975..07b4f23325 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -30,7 +30,7 @@ "prestart": "npm run build:babel", "start": "concurrently --kill-others --names \"babel,globalize,tsc\" \"npm run start:babel\" \"npm run start:globalize\" \"npm run start:typescript\"", "start:babel": "npm run build:babel -- --skip-initial-build --watch", - "start:globalize": "node-dev --respawn scripts/createPrecompiledGlobalize.js", + "start:globalize": "node-dev --respawn scripts/createPrecompiledGlobalize.mjs", "start:typescript": "npm run build:typescript -- --watch" }, "devDependencies": { diff --git a/packages/support/cldr-data/package.json b/packages/support/cldr-data/package.json index 80950579aa..133435a7bf 100644 --- a/packages/support/cldr-data/package.json +++ b/packages/support/cldr-data/package.json @@ -35,11 +35,12 @@ "files": [ "src/index.js", "src/install.js", + "src/patch.mjs", "urls.json" ], "scripts": { "eslint": "npm run precommit", - "install": "node ./src/install.js", + "install": "node ./src/install.js && node ./src/patch.mjs", "precommit": "npm run precommit:eslint -- src", "precommit:eslint": "eslint", "prettier": "prettier --check src/**/*.{js,ts,tsx}" diff --git a/packages/support/cldr-data/src/patch.mjs b/packages/support/cldr-data/src/patch.mjs new file mode 100644 index 0000000000..f16462cd71 --- /dev/null +++ b/packages/support/cldr-data/src/patch.mjs @@ -0,0 +1,136 @@ +import { fileURLToPath } from 'url'; +import { resolve } from 'path'; +import fs from 'fs/promises'; + +// There is an issue in the Unicode CLDR database (v36): +// +// - Polish has 4 different plural types: "one", "few", "many", "other" +// - However, some units, say "short/digital-kilobyte", only have "other" defined +// - When we localize 1024 (number) into kilobytes, it use the "one" type +// - Since "short/digital-kilobyte/one" is not defined in the database, `globalize` throw exception +// +// In all of our supported languages, we also observed the same issue in Portuguese as well. +// +// As a hotfix, we are patching the Unicode CLDR database for all `[long/short/narrow]/digital-*` rules to make sure it include all plurals needed for that language. +// +// For a long term fix, we should move forward to a newer version of CLDR database, which is outlined in https://github.com/rxaviers/cldr-data-npm/issues/78. + +let FORBIDDEN_PROPERTY_NAMES; + +function getForbiddenPropertyNames() { + return ( + FORBIDDEN_PROPERTY_NAMES || + (FORBIDDEN_PROPERTY_NAMES = Object.freeze( + Array.from( + new Set([ + // As-of writing, `Object.prototype` includes: + // __defineGetter__ + // __defineSetter__ + // __lookupGetter__ + // __lookupSetter + // __proto__ + // constructor + // hasOwnProperty + // isPrototypeOf + // propertyIsEnumerable + // toLocaleString + // toString + // valueOf + ...Object.getOwnPropertyNames(Object.prototype), + + 'prototype' + ]) + ) + )) + ); +} + +function isForbiddenPropertyName(propertyName) { + return getForbiddenPropertyNames().includes(propertyName); +} + +function toDist(filename) { + if (filename.includes('..')) { + throw new Error('Filename cannot contains "..".'); + } + + return resolve(fileURLToPath(import.meta.url), '../../dist/', filename); +} + +// ESLint "wrap-iife" rule is conflicting with Prettier. +// eslint-disable-next-line wrap-iife +(async function () { + const plurals = JSON.parse(await fs.readFile(toDist('supplemental/plurals.json'), 'utf8')); + + const languagePlurals = new Map(); + + Object.entries(plurals.supplemental['plurals-type-cardinal']).forEach(([language, pluralsTypeCardinal]) => { + const plurals = ['other']; + + languagePlurals.set(language, plurals); + + if (!(`pluralRule-count-other` in pluralsTypeCardinal)) { + throw new Error(`Language ${language} does not have plural type "other".`); + } + + ['zero', 'one', 'two', 'few', 'many'].forEach(pluralType => { + `pluralRule-count-${pluralType}` in pluralsTypeCardinal && plurals.push(pluralType); + }); + }); + + const patchedLanguages = []; + + await Promise.all( + Array.from(languagePlurals.entries()).map(async ([language, supportedPluralTypes]) => { + if (!/^[\w-]+$/u.test(language) && isForbiddenPropertyName(language)) { + throw new Error(`Invalid language code "${language}".`); + } + + let units; + + try { + units = JSON.parse(await fs.readFile(toDist(`main/${language}/units.json`), 'utf8')); + } catch (err) { + if (err.code === 'ENOENT') { + return; + } + + throw err; + } + + let numFound = 0; + + ['long', 'short', 'narrow'].forEach(form => { + Object.entries(units.main[language].units[form]).forEach(([unitName, entry]) => { + if (!unitName.startsWith('digital-')) { + return; + } + + if ('unitPattern-count-other' in entry) { + const { 'unitPattern-count-other': other } = entry; + + supportedPluralTypes.forEach(pluralType => { + const name = `unitPattern-count-${pluralType}`; + + if (!(name in entry)) { + entry[name] = other; + numFound++; + } + }); + } + }); + }); + + if (numFound) { + patchedLanguages.push(`${language} (${numFound} issues)`); + + // eslint-disable-next-line no-magic-numbers + await fs.writeFile(toDist(`main/${language}/units.json`), JSON.stringify(units, null, 2)); + } + }) + ); + + // We are display output in CLI. + // eslint-disable-next-line no-console + console.log(`Patched ${patchedLanguages.length} languages: ${patchedLanguages.join(', ')}.`); +})();