From 2c9bc81253da3936fdb343b76f02283c462578d7 Mon Sep 17 00:00:00 2001 From: tienifr Date: Mon, 8 Apr 2024 15:36:35 +0700 Subject: [PATCH 1/5] add support for latin characters in emoji suggestion --- src/CONST.ts | 4 ++-- src/libs/EmojiTrie.ts | 6 +++++- src/libs/GetStyledTextArray.ts | 5 ++++- src/libs/StringUtils.ts | 11 ++++++++++- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/CONST.ts b/src/CONST.ts index f9229d5185b4..d70d4c5d72d9 100755 --- a/src/CONST.ts +++ b/src/CONST.ts @@ -1680,8 +1680,8 @@ const CONST = { // Extract attachment's source from the data's html string ATTACHMENT_DATA: /(data-expensify-source|data-name)="([^"]+)"/g, - EMOJI_NAME: /:[\w+-]+:/g, - EMOJI_SUGGESTIONS: /:[a-zA-Z0-9_+-]{1,40}$/, + EMOJI_NAME: /:[\p{L}0-9_+-]+:/gu, + EMOJI_SUGGESTIONS: /:[\p{L}0-9_+-]{1,40}$/u, AFTER_FIRST_LINE_BREAK: /\n.*/g, LINE_BREAK: /\r|\n/g, CODE_2FA: /^\d{6}$/, diff --git a/src/libs/EmojiTrie.ts b/src/libs/EmojiTrie.ts index d0f0b0dcfab6..4b4994125ef2 100644 --- a/src/libs/EmojiTrie.ts +++ b/src/libs/EmojiTrie.ts @@ -2,6 +2,7 @@ import emojis, {localeEmojis} from '@assets/emojis'; import type {Emoji, HeaderEmoji, PickerEmoji} from '@assets/emojis/types'; import CONST from '@src/CONST'; import Timing from './actions/Timing'; +import StringUtils from './StringUtils'; import Trie from './Trie'; type EmojiMetaData = { @@ -33,7 +34,10 @@ function addKeywordsToTrie(trie: Trie, keywords: string[], item: keywords.forEach((keyword) => { const keywordNode = trie.search(keyword); if (!keywordNode) { - trie.add(keyword, {suggestions: [{code: item.code, types: item.types, name}]}); + const keywordWithoutAccents = StringUtils.normalizeAccents(keyword); + if (keywordWithoutAccents !== keyword) { + trie.add(keywordWithoutAccents, {suggestions: [{code: item.code, types: item.types, name}]}); + } } else { const suggestion = {code: item.code, types: item.types, name}; const suggestions = shouldPrependKeyword ? [suggestion, ...(keywordNode.metaData.suggestions ?? [])] : [...(keywordNode.metaData.suggestions ?? []), suggestion]; diff --git a/src/libs/GetStyledTextArray.ts b/src/libs/GetStyledTextArray.ts index 9bc6ccc33b4c..ffae31dc861b 100644 --- a/src/libs/GetStyledTextArray.ts +++ b/src/libs/GetStyledTextArray.ts @@ -1,4 +1,5 @@ import Str from 'expensify-common/lib/str'; +import StringUtils from './StringUtils'; type StyledText = { text: string; @@ -8,7 +9,9 @@ type StyledText = { const getStyledTextArray = (name: string, prefix: string): StyledText[] => { const texts = []; const prefixLowercase = prefix.toLowerCase(); - const prefixLocation = name.toLowerCase().search(Str.escapeForRegExp(prefixLowercase)); + const prefixLocation = StringUtils.normalizeAccents(name) + .toLowerCase() + .search(Str.escapeForRegExp(StringUtils.normalizeAccents(prefixLowercase))); if (prefixLocation === 0 && prefix.length === name.length) { texts.push({text: name, isColored: true}); diff --git a/src/libs/StringUtils.ts b/src/libs/StringUtils.ts index 2fb918c7a233..c9fa40c4da8a 100644 --- a/src/libs/StringUtils.ts +++ b/src/libs/StringUtils.ts @@ -63,6 +63,15 @@ function removeInvisibleCharacters(value: string): string { return result.trim(); } +/** + * Remove accents/diacritics + * @param text - The input string + * @returns The string with all accents/diacritics removed + */ +function normalizeAccents(text: string) { + return text.normalize('NFD').replace(/[\u0300-\u036f]/g, ''); +} + /** * Replace all CRLF with LF * @param value - The input string @@ -72,4 +81,4 @@ function normalizeCRLF(value?: string): string | undefined { return value?.replace(/\r\n/g, '\n'); } -export default {sanitizeString, isEmptyString, removeInvisibleCharacters, normalizeCRLF}; +export default {sanitizeString, isEmptyString, removeInvisibleCharacters, normalizeAccents, normalizeCRLF}; From 723d36568be6b1c1d8604caa75f9e577e3fb4c27 Mon Sep 17 00:00:00 2001 From: tienifr Date: Mon, 8 Apr 2024 17:33:26 +0700 Subject: [PATCH 2/5] add original keyword to trie --- src/libs/EmojiTrie.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/libs/EmojiTrie.ts b/src/libs/EmojiTrie.ts index 4b4994125ef2..d51881e0e5d5 100644 --- a/src/libs/EmojiTrie.ts +++ b/src/libs/EmojiTrie.ts @@ -35,9 +35,8 @@ function addKeywordsToTrie(trie: Trie, keywords: string[], item: const keywordNode = trie.search(keyword); if (!keywordNode) { const keywordWithoutAccents = StringUtils.normalizeAccents(keyword); - if (keywordWithoutAccents !== keyword) { - trie.add(keywordWithoutAccents, {suggestions: [{code: item.code, types: item.types, name}]}); - } + const keywordToAdd = keywordWithoutAccents !== keyword ? keywordWithoutAccents : keyword; + trie.add(keywordToAdd, {suggestions: [{code: item.code, types: item.types, name}]}); } else { const suggestion = {code: item.code, types: item.types, name}; const suggestions = shouldPrependKeyword ? [suggestion, ...(keywordNode.metaData.suggestions ?? [])] : [...(keywordNode.metaData.suggestions ?? []), suggestion]; From af77cbf86ff775b5db0992b43df8a6ee5e161199 Mon Sep 17 00:00:00 2001 From: tienifr Date: Mon, 8 Apr 2024 17:34:12 +0700 Subject: [PATCH 3/5] add original keyword to trie --- src/libs/EmojiTrie.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libs/EmojiTrie.ts b/src/libs/EmojiTrie.ts index d51881e0e5d5..c0fd4434c48f 100644 --- a/src/libs/EmojiTrie.ts +++ b/src/libs/EmojiTrie.ts @@ -35,8 +35,10 @@ function addKeywordsToTrie(trie: Trie, keywords: string[], item: const keywordNode = trie.search(keyword); if (!keywordNode) { const keywordWithoutAccents = StringUtils.normalizeAccents(keyword); - const keywordToAdd = keywordWithoutAccents !== keyword ? keywordWithoutAccents : keyword; - trie.add(keywordToAdd, {suggestions: [{code: item.code, types: item.types, name}]}); + if (keywordWithoutAccents !== keyword) { + trie.add(keywordWithoutAccents, {suggestions: [{code: item.code, types: item.types, name}]}); + } + trie.add(keyword, {suggestions: [{code: item.code, types: item.types, name}]}); } else { const suggestion = {code: item.code, types: item.types, name}; const suggestions = shouldPrependKeyword ? [suggestion, ...(keywordNode.metaData.suggestions ?? [])] : [...(keywordNode.metaData.suggestions ?? []), suggestion]; From 166ecec82acf5ed580bfbb8aefb9f32812bd151c Mon Sep 17 00:00:00 2001 From: tienifr Date: Wed, 22 May 2024 19:30:04 +0700 Subject: [PATCH 4/5] add and update normalized emoji in trie --- src/libs/EmojiTrie.ts | 31 +++++++++++++++++++++++-------- src/libs/StringUtils.ts | 5 ----- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/src/libs/EmojiTrie.ts b/src/libs/EmojiTrie.ts index c8ed46d1c8d6..a8155d3f2fd6 100644 --- a/src/libs/EmojiTrie.ts +++ b/src/libs/EmojiTrie.ts @@ -34,19 +34,25 @@ type EmojiTrie = { function addKeywordsToTrie(trie: Trie, keywords: string[], item: Emoji, name: string, shouldPrependKeyword = false) { keywords.forEach((keyword) => { const keywordNode = trie.search(keyword); + const normalizedKeyword = StringUtils.normalizeAccents(keyword); + if (!keywordNode) { - const keywordWithoutAccents = StringUtils.normalizeAccents(keyword); - if (keywordWithoutAccents !== keyword) { - trie.add(keywordWithoutAccents, {suggestions: [{code: item.code, types: item.types, name}]}); + const metadata = {suggestions: [{code: item.code, types: item.types, name}]}; + if (normalizedKeyword !== keyword) { + trie.add(normalizedKeyword, metadata); } - trie.add(keyword, {suggestions: [{code: item.code, types: item.types, name}]}); + trie.add(keyword, metadata); } else { const suggestion = {code: item.code, types: item.types, name}; const suggestions = shouldPrependKeyword ? [suggestion, ...(keywordNode.metaData.suggestions ?? [])] : [...(keywordNode.metaData.suggestions ?? []), suggestion]; - trie.update(keyword, { + const newMetadata = { ...keywordNode.metaData, suggestions, - }); + }; + if (normalizedKeyword !== keyword) { + trie.update(normalizedKeyword, newMetadata); + } + trie.update(keyword, newMetadata); } }); } @@ -73,12 +79,21 @@ function createTrie(lang: SupportedLanguage = CONST.LOCALES.DEFAULT): Trie { const englishName = item.name; const localeName = langEmojis?.[item.code]?.name ?? englishName; + const normalizedName = StringUtils.normalizeAccents(localeName); const node = trie.search(localeName); if (!node) { - trie.add(localeName, {code: item.code, types: item.types, name: localeName, suggestions: []}); + const metadata = {code: item.code, types: item.types, name: localeName, suggestions: []}; + if (normalizedName !== localeName) { + trie.add(normalizedName, metadata); + } + trie.add(localeName, metadata); } else { - trie.update(localeName, {code: item.code, types: item.types, name: localeName, suggestions: node.metaData.suggestions}); + const newMetadata = {code: item.code, types: item.types, name: localeName, suggestions: node.metaData.suggestions}; + if (normalizedName !== localeName) { + trie.update(normalizedName, newMetadata); + } + trie.update(localeName, newMetadata); } const nameParts = getNameParts(localeName).slice(1); // We remove the first part because we already index the full name. diff --git a/src/libs/StringUtils.ts b/src/libs/StringUtils.ts index 30a2954f33b6..6fab57accc3f 100644 --- a/src/libs/StringUtils.ts +++ b/src/libs/StringUtils.ts @@ -63,11 +63,6 @@ function removeInvisibleCharacters(value: string): string { return result.trim(); } -/** - * Remove accents/diacritics - * @param text - The input string - * @returns The string with all accents/diacritics removed - */ function normalizeAccents(text: string) { return text.normalize('NFD').replace(/[\u0300-\u036f]/g, ''); } From 08ac2fafe6fef08d07e21ad91c5fe16c2c12ac05 Mon Sep 17 00:00:00 2001 From: tienifr Date: Fri, 24 May 2024 13:23:13 +0700 Subject: [PATCH 5/5] add back comment --- src/libs/StringUtils.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libs/StringUtils.ts b/src/libs/StringUtils.ts index 6fab57accc3f..30a2954f33b6 100644 --- a/src/libs/StringUtils.ts +++ b/src/libs/StringUtils.ts @@ -63,6 +63,11 @@ function removeInvisibleCharacters(value: string): string { return result.trim(); } +/** + * Remove accents/diacritics + * @param text - The input string + * @returns The string with all accents/diacritics removed + */ function normalizeAccents(text: string) { return text.normalize('NFD').replace(/[\u0300-\u036f]/g, ''); }