perf: improve the accuracy of language recognition

raycast · Jul 17, 2022 · 294dbfa · 294dbfa
1 parent 7d41811
commit 294dbfa
Show file tree

Hide file tree

Showing 9 changed files with 107 additions and 94 deletions.
diff --git a/extensions/easydict/README.md b/extensions/easydict/README.md
@@ -177,7 +177,7 @@ Name case are insensitive, use comma to separate. Example: `deepl,apple,baidu,te
 
 </details>
 
-### Apple Translate
+### 🍎 Apple Translate
 
 `Easydict` support MacOS system translate, for more information, please see [How to use macOS Apple System Translation in Easydict?](https://github.com/tisfeng/Raycast-Easydict/wiki/%E5%A6%82%E4%BD%95%E5%9C%A8-Easydict-%E4%B8%AD%E4%BD%BF%E7%94%A8-macOS-%E8%8B%B9%E6%9E%9C%E7%B3%BB%E7%BB%9F%E7%BF%BB%E8%AF%91%EF%BC%9F)
 

diff --git a/extensions/easydict/SECURITY.md b/extensions/easydict/SECURITY.md
@@ -2,18 +2,8 @@
 
 ## Supported Versions
 
-Use this section to tell people about which versions of your project are
-currently being supported with security updates.
-
-| Version | Supported          |
-| ------- | ------------------ |
-| 1.1.x   | :white_check_mark: |
-| 1.0     | :x:                |
+Latest commits in master branch.
 
 ## Reporting a Vulnerability
 
-Use this section to tell people how to report a vulnerability.
-
-Tell them where to go, how often they can expect to get an update on a
-reported vulnerability, what to expect if the vulnerability is accepted or
-declined, etc.
+If you believe you have found a security vulnerability in `Easydict`, please open an issue to let me know, thanks.
diff --git a/extensions/easydict/src/consts.ts b/extensions/easydict/src/consts.ts
@@ -2,7 +2,7 @@
  * @author: tisfeng
  * @createTime: 2022-06-24 22:36
  * @lastEditor: tisfeng
- * @lastEditTime: 2022-07-16 16:18
+ * @lastEditTime: 2022-07-17 11:54
  * @fileName: consts.ts
  *
  * Copyright (c) 2022 by tisfeng, All Rights Reserved.
@@ -85,7 +85,7 @@ export const languageItemList: LanguageItem[] = [
   {
     youdaoLanguageId: "zh-CHS",
     appleLanguageId: "zh_CN",
-    appleChineseLanguageTitle: "中文",
+    appleDetectChineseLanguageTitle: "中文",
     deepLSourceLanguageId: "ZH",
     francLanguageId: "cmn",
     aliyunLanguageId: "zh",
@@ -99,7 +99,7 @@ export const languageItemList: LanguageItem[] = [
   {
     youdaoLanguageId: "zh-CHT",
     appleLanguageId: "zh-TW",
-    appleChineseLanguageTitle: "中文",
+    appleDetectChineseLanguageTitle: "中文",
     francLanguageId: "cmn",
     aliyunLanguageId: "zh-tw",
     tencentLanguageId: "zh-TW",
@@ -111,7 +111,7 @@ export const languageItemList: LanguageItem[] = [
   {
     youdaoLanguageId: "en",
     appleLanguageId: "en_US",
-    appleChineseLanguageTitle: "英语",
+    appleDetectChineseLanguageTitle: "英语",
     deepLSourceLanguageId: "EN",
     deepLTargetLanguageId: "EN-US", // "EN-GB" "EN-US"
     francLanguageId: "eng",
@@ -128,7 +128,7 @@ export const languageItemList: LanguageItem[] = [
   {
     youdaoLanguageId: "ja",
     appleLanguageId: "ja_JP",
-    appleChineseLanguageTitle: "日语",
+    appleDetectChineseLanguageTitle: "日语",
     deepLSourceLanguageId: "JA",
     francLanguageId: "jpn",
     aliyunLanguageId: "ja",
@@ -143,7 +143,7 @@ export const languageItemList: LanguageItem[] = [
   {
     youdaoLanguageId: "ko",
     appleLanguageId: "ko_KR",
-    appleChineseLanguageTitle: "韩语",
+    appleDetectChineseLanguageTitle: "韩语",
     francLanguageId: "kor",
     aliyunLanguageId: "ko",
     tencentDetectLanguageId: "kr",
@@ -156,7 +156,7 @@ export const languageItemList: LanguageItem[] = [
   {
     youdaoLanguageId: "fr",
     appleLanguageId: "fr_FR",
-    appleChineseLanguageTitle: "法语",
+    appleDetectChineseLanguageTitle: "法语",
     deepLSourceLanguageId: "FR",
     francLanguageId: "fra",
     aliyunLanguageId: "fr",
@@ -170,7 +170,7 @@ export const languageItemList: LanguageItem[] = [
   {
     youdaoLanguageId: "es",
     appleLanguageId: "es_ES",
-    appleChineseLanguageTitle: "西班牙语",
+    appleDetectChineseLanguageTitle: "西班牙语",
     deepLSourceLanguageId: "ES",
     francLanguageId: "spa",
     aliyunLanguageId: "es",
@@ -183,7 +183,7 @@ export const languageItemList: LanguageItem[] = [
   {
     youdaoLanguageId: "pt",
     appleLanguageId: "pt_BR",
-    appleChineseLanguageTitle: "葡萄牙语",
+    appleDetectChineseLanguageTitle: "葡萄牙语",
     deepLSourceLanguageId: "PT",
     deepLTargetLanguageId: "PT-PT", // "PT-PT" "PT-BR"
     francLanguageId: "por",
@@ -196,7 +196,7 @@ export const languageItemList: LanguageItem[] = [
   {
     youdaoLanguageId: "it",
     appleLanguageId: "it_IT",
-    appleChineseLanguageTitle: "意大利语",
+    appleDetectChineseLanguageTitle: "意大利语",
     deepLSourceLanguageId: "IT",
     francLanguageId: "ita",
     aliyunLanguageId: "it",
@@ -208,7 +208,7 @@ export const languageItemList: LanguageItem[] = [
   {
     youdaoLanguageId: "de",
     appleLanguageId: "de_DE",
-    appleChineseLanguageTitle: "德语",
+    appleDetectChineseLanguageTitle: "德语",
     deepLSourceLanguageId: "DE",
     francLanguageId: "deu",
     aliyunLanguageId: "de",
@@ -221,7 +221,7 @@ export const languageItemList: LanguageItem[] = [
   {
     youdaoLanguageId: "ru",
     appleLanguageId: "ru_RU",
-    appleChineseLanguageTitle: "俄语",
+    appleDetectChineseLanguageTitle: "俄语",
     deepLSourceLanguageId: "RU",
     francLanguageId: "rus",
     aliyunLanguageId: "ru",
@@ -233,7 +233,7 @@ export const languageItemList: LanguageItem[] = [
   {
     youdaoLanguageId: "ar",
     appleLanguageId: "ar_AE",
-    appleChineseLanguageTitle: "阿拉伯语",
+    appleDetectChineseLanguageTitle: "阿拉伯语",
     francLanguageId: "arb",
     aliyunLanguageId: "ar",
     tencentLanguageId: "ar",
@@ -243,6 +243,7 @@ export const languageItemList: LanguageItem[] = [
   },
   {
     youdaoLanguageId: "sv",
+    appleDetectChineseLanguageTitle: "瑞典语",
     deepLSourceLanguageId: "SV",
     francLanguageId: "swe",
     aliyunLanguageId: "sv",
@@ -252,6 +253,7 @@ export const languageItemList: LanguageItem[] = [
   },
   {
     youdaoLanguageId: "ro",
+    appleDetectChineseLanguageTitle: "罗马尼亚语",
     deepLSourceLanguageId: "RO",
     francLanguageId: "ron",
     aliyunLanguageId: "ro",
@@ -261,6 +263,7 @@ export const languageItemList: LanguageItem[] = [
   },
   {
     youdaoLanguageId: "th",
+    appleDetectChineseLanguageTitle: "泰语",
     francLanguageId: "tha",
     aliyunLanguageId: "th",
     tencentLanguageId: "th",
@@ -270,6 +273,7 @@ export const languageItemList: LanguageItem[] = [
   },
   {
     youdaoLanguageId: "sk",
+    appleDetectChineseLanguageTitle: "斯洛伐克语",
     deepLSourceLanguageId: "SK",
     francLanguageId: "slk",
     aliyunLanguageId: "sk",
@@ -279,6 +283,7 @@ export const languageItemList: LanguageItem[] = [
   },
   {
     youdaoLanguageId: "nl",
+    appleDetectChineseLanguageTitle: "荷兰语",
     deepLSourceLanguageId: "NL",
     francLanguageId: "nld",
     aliyunLanguageId: "nl",
@@ -288,6 +293,7 @@ export const languageItemList: LanguageItem[] = [
   },
   {
     youdaoLanguageId: "hu",
+    appleDetectChineseLanguageTitle: "匈牙利语",
     deepLSourceLanguageId: "HU",
     francLanguageId: "hun",
     aliyunLanguageId: "hu",
@@ -297,6 +303,7 @@ export const languageItemList: LanguageItem[] = [
   },
   {
     youdaoLanguageId: "el",
+    appleDetectChineseLanguageTitle: "希腊语",
     deepLSourceLanguageId: "EL",
     francLanguageId: "ell",
     aliyunLanguageId: "el",
@@ -306,6 +313,7 @@ export const languageItemList: LanguageItem[] = [
   },
   {
     youdaoLanguageId: "da",
+    appleDetectChineseLanguageTitle: "丹麦语",
     deepLSourceLanguageId: "DA",
     francLanguageId: "dan",
     aliyunLanguageId: "da",
@@ -315,6 +323,7 @@ export const languageItemList: LanguageItem[] = [
   },
   {
     youdaoLanguageId: "fi",
+    appleDetectChineseLanguageTitle: "芬兰语",
     deepLSourceLanguageId: "FI",
     francLanguageId: "fin",
     aliyunLanguageId: "fi",
@@ -324,6 +333,7 @@ export const languageItemList: LanguageItem[] = [
   },
   {
     youdaoLanguageId: "pl",
+    appleDetectChineseLanguageTitle: "波兰语",
     deepLSourceLanguageId: "PL",
     francLanguageId: "pol",
     aliyunLanguageId: "pl",
@@ -333,6 +343,7 @@ export const languageItemList: LanguageItem[] = [
   },
   {
     youdaoLanguageId: "cs",
+    appleDetectChineseLanguageTitle: "捷克语",
     deepLSourceLanguageId: "CS",
     francLanguageId: "ces",
     aliyunLanguageId: "cs",

diff --git a/extensions/easydict/src/detectLanguage.ts b/extensions/easydict/src/detectLanguage.ts
@@ -2,7 +2,7 @@
  * @author: tisfeng
  * @createTime: 2022-06-24 17:07
  * @lastEditor: tisfeng
- * @lastEditTime: 2022-07-16 17:25
+ * @lastEditTime: 2022-07-17 18:29
  * @fileName: detectLanguage.ts
  *
  * Copyright (c) 2022 by tisfeng, All Rights Reserved.
@@ -52,7 +52,7 @@ let delayLocalDetectLanguageTimer: NodeJS.Timeout;
 /**
  * Record all API detected language, if has detected two identical language id, use it.
  */
-const detectedLanguageTypeResultList: LanguageDetectTypeResult[] = [];
+const detectedAPILanguageTypeResultList: LanguageDetectTypeResult[] = [];
 
 const defaultConfirmedConfidence = 0.8;
 
@@ -66,7 +66,7 @@ export function detectLanguage(
   callback: (detectedLanguageResult: LanguageDetectTypeResult) => void
 ): void {
   console.log(`start detectLanguage`);
-  let localDetectResult = getLocalTextLanguageDetectResult(text, defaultConfirmedConfidence);
+  const localDetectResult = getLocalTextLanguageDetectResult(text, defaultConfirmedConfidence);
   if (localDetectResult.confirmed) {
     console.log("use local detect confirmed:", localDetectResult.type, localDetectResult.youdaoLanguageId);
     // Todo: may be do not need to clear timeout, when API detect success, callback once again.
@@ -78,10 +78,8 @@ export function detectLanguage(
   // Start a delay timer to detect local language, use it only if API detect over time.
   clearTimeout(delayLocalDetectLanguageTimer);
   delayLocalDetectLanguageTimer = setTimeout(() => {
-    console.log(`API detect over time, use local detect language`);
     isDetectedLanguage = true;
-    localDetectResult = getLocalTextLanguageDetectResult(text, defaultConfirmedConfidence);
-    console.log(`use local detect language --->: ${localDetectResult}`);
+    console.log(`API detect over time, use local detect language --->: ${localDetectResult}`);
     callback(localDetectResult);
   }, delayDetectLanguageTime);
 
@@ -122,6 +120,7 @@ function raceDetectTextLanguage(
   callback?: (detectTypeResult: LanguageDetectTypeResult) => void
 ) {
   console.log(`start raceDetectTextLanguage: ${[...detectLanguageActionMap.keys()]}`);
+  // console.log("race local detect language: ", localLanguageDetectTypeResult);
   isDetectedLanguage = false;
   const detectLanguageActionList = detectLanguageActionMap.values();
   Promise.race(detectLanguageActionList)
@@ -205,7 +204,7 @@ function handleDetectedLanguageTypeResult(
   }
 
   // Second, iterate detectedLanguageTypeList, check if has detected two identical language id, if true, use it.
-  for (const languageTypeReuslt of detectedLanguageTypeResultList as LanguageDetectTypeResult[]) {
+  for (const languageTypeReuslt of detectedAPILanguageTypeResultList as LanguageDetectTypeResult[]) {
     const detectedYoudaoLanguageId = detectedlanguageTypeResult.youdaoLanguageId;
     if (
       languageTypeReuslt.youdaoLanguageId === detectedYoudaoLanguageId &&
@@ -219,20 +218,23 @@ function handleDetectedLanguageTypeResult(
   }
 
   // If this API detected language is not confirmed, record it in the detectedLanguageTypeList.
-  detectedLanguageTypeResultList.push(detectedlanguageTypeResult);
+  detectedAPILanguageTypeResultList.push(detectedlanguageTypeResult);
 
   /**
-   * Finally, iterate detectedLanguageTypeList, to compare with the local detect language list, if true, use it.
+   * Finally, iterate API detectedLanguageTypeList, to compare with the local detect language list, if true, use it.
    * If matched, mark it as confirmed, else use it directly, but not confirmed.
    */
   if (detectLanguageActionMap.size === 0) {
     console.log(`try compare API detected language list with local deteced list`);
-    console.log(`---> API detected language list: ${JSON.stringify(detectedLanguageTypeResultList, null, 4)}`);
-
-    const detectedLanguageArray = localLanguageDetectTypeResult.detectedLanguageArray;
-    if (detectedLanguageArray) {
-      for (const [languageId, confidence] of detectedLanguageArray) {
-        for (const languageTypeReuslt of detectedLanguageTypeResultList) {
+    console.log(`---> API detected language list: ${JSON.stringify(detectedAPILanguageTypeResultList, null, 4)}`);
+
+    const detectedLocalLanguageArray = localLanguageDetectTypeResult.detectedLanguageArray;
+    // console.log(`---> local detected language list: ${JSON.stringify(detectedLocalLanguageArray, null, 4)}`);
+    if (detectedLocalLanguageArray?.length) {
+      for (const [languageId, confidence] of detectedLocalLanguageArray) {
+        // console.log(`---> local detected language: ${languageId}, confidence: ${confidence}`);
+        for (const languageTypeReuslt of detectedAPILanguageTypeResultList) {
+          // console.log(`---> API detected language: ${JSON.stringify(languageTypeReuslt, null, 4)}`);
           if (confidence > 0 && languageTypeReuslt.youdaoLanguageId === languageId && isValidLanguageId(languageId)) {
             languageTypeReuslt.confirmed = true;
             console.warn(`---> local detect identical language: ${JSON.stringify(languageTypeReuslt, null, 4)}`);
@@ -304,7 +306,7 @@ function getFinalLanguageDetectResult(
 function getLocalTextLanguageDetectResult(
   text: string,
   confirmedConfidence: number,
-  lowConfidence = 0.1
+  lowConfidence = 0.2
 ): LanguageDetectTypeResult {
   console.log(`start local detect language, confirmed confidence (>${confirmedConfidence})`);
 
@@ -326,6 +328,7 @@ function getLocalTextLanguageDetectResult(
           type: francDetectResult.type,
           youdaoLanguageId: languageId,
           confirmed: false,
+          detectedLanguageArray: francDetectResult.detectedLanguageArray,
         };
         return lowConfidenceDetectTypeResult;
       }
@@ -347,13 +350,12 @@ function getLocalTextLanguageDetectResult(
   }
 
   // finally, use "auto" as fallback.
+  console.log(`final use auto`);
   const finalAutoLanguageTypeResult: LanguageDetectTypeResult = {
     type: LanguageDetectType.Simple,
     youdaoLanguageId: "auto",
     confirmed: false,
   };
-
-  console.log(`final use auto`);
   return finalAutoLanguageTypeResult;
 }
 

diff --git a/extensions/easydict/src/easydict.tsx b/extensions/easydict/src/easydict.tsx
@@ -2,7 +2,7 @@
  * @author: tisfeng
  * @createTime: 2022-06-23 14:19
  * @lastEditor: tisfeng
- * @lastEditTime: 2022-07-16 17:56
+ * @lastEditTime: 2022-07-17 18:36
  * @fileName: easydict.tsx
  *
  * Copyright (c) 2022 by tisfeng, All Rights Reserved.
@@ -216,7 +216,8 @@ export default function () {
 
       const youdaoResult = youdaoTranslateTypeResult.result as YoudaoTranslateResult;
       console.log(`youdao translate result: ${JSON.stringify(youdaoResult, null, 2)}`);
-      console.warn(`---> Youdao cost time: ${Date.now() - startTime} ms`);
+      // From the input text query, to the end of Youdao translation request.
+      console.warn(`---> Entire request cost time: ${Date.now() - startTime} ms`);
       const youdaoErrorCode = youdaoResult.errorCode;
       youdaoTranslateTypeResult.errorInfo = getYoudaoErrorInfo(youdaoErrorCode);