From f0eafc1c599305a8c269599e068709a408d545e5 Mon Sep 17 00:00:00 2001 From: canisminor1990 Date: Wed, 22 Nov 2023 22:47:55 +0800 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20feat:=20I18n=20support=20concurrenc?= =?UTF-8?q?y=20query?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/lobe-i18n/README.md | 1 + packages/lobe-i18n/README.zh-CN.md | 21 +++--- packages/lobe-i18n/package.json | 1 + packages/lobe-i18n/src/core/I18n.ts | 73 +++++++++++--------- packages/lobe-i18n/src/store/initialState.ts | 11 ++- packages/lobe-i18n/src/store/selectors.ts | 6 +- packages/lobe-i18n/src/types/config.ts | 4 ++ 7 files changed, 67 insertions(+), 50 deletions(-) diff --git a/packages/lobe-i18n/README.md b/packages/lobe-i18n/README.md index 838706b..7283692 100644 --- a/packages/lobe-i18n/README.md +++ b/packages/lobe-i18n/README.md @@ -150,6 +150,7 @@ This project provides some additional configuration items set with environment v | reference | No | `string` | - | Provide some context for more accurate translations | | splitToken | No | `number` | - | Split the localized JSON file by tokens, automatically calculated by default | | temperature | No | `number` | `0` | Sampling temperature to use | +| concurrency | No | `number` | `5` | Number of concurrently pending promises returned | | experimental | No | `experimental` | | Experimental features, see below | | markdown | No | `markdown` | | See `markdown` configuration below | diff --git a/packages/lobe-i18n/README.zh-CN.md b/packages/lobe-i18n/README.zh-CN.md index f715d95..7af8f8c 100644 --- a/packages/lobe-i18n/README.zh-CN.md +++ b/packages/lobe-i18n/README.zh-CN.md @@ -149,6 +149,7 @@ $ lobe-i18n md | reference | 否 | `string` | - | 提供一些上下文以获得更准确的翻译 | | splitToken | 否 | `number` | - | 按令牌分割本地化 JSON 文件，默认自动计算 | | temperature | 否 | `number` | `0` | 使用的采样温度 | +| concurrency | 否 | `number` | `5` | 同时并发的队列请求数量 | | experimental | 否 | `experimental` | | 实验性功能，见下文 | | markdown | 否 | `markdown` | | 见 `markdown` 配置说明 | @@ -285,16 +286,16 @@ $ lobe-i18n ## 📝 Markdown 配置 -| 属性名称 | 必填 | 类型 | 默认值 | 描述 | -| ---------------- | ---- | ------------------------- | ---------------------------- | ----------------------------------------- | -| entry | 是 | `string[]` | - | 入口文件或文件夹，支持 `glob` | -| entryLocale | 否 | `string` | 同父级 | 作为翻译参考的语言 | -| entryExtension | 否 | `string` | `.md` | 入口文件扩展名 | -| exclude | 否 | `string[]` | - | 需要过滤的文件，支持 `glob` | -| outputLocales | 否 | `string[]` | 同父级 | 需要进行翻译的所有语言 | -| outputExtensions | 否 | `function` | `(locale) => '.{locale}.md'` | 输出文件的扩展名生成 | -| mode | 否 | `string``mdast``function` | `string` | 翻译的模式选择，解释见下文 | -| translateCode | 否 | `boolean` | `false` | 在 `mdast` 下是否翻译代码块，其他模式无效 | +| 属性名称 | 必填 | 类型 | 默认值 | 描述 | +| ---------------- | ---- | --------------------------- | ---------------------------- | ----------------------------------------- | +| entry | 是 | `string[]` | - | 入口文件或文件夹，支持 `glob` | +| entryLocale | 否 | `string` | 同父级 | 作为翻译参考的语言 | +| entryExtension | 否 | `string` | `.md` | 入口文件扩展名 | +| exclude | 否 | `string[]` | - | 需要过滤的文件，支持 `glob` | +| outputLocales | 否 | `string[]` | 同父级 | 需要进行翻译的所有语言 | +| outputExtensions | 否 | `function` | `(locale) => '.{locale}.md'` | 输出文件的扩展名生成 | +| mode | 否 | `string`,`mdast`,`function` | `string` | 翻译的模式选择，解释见下文 | +| translateCode | 否 | `boolean` | `false` | 在 `mdast` 下是否翻译代码块，其他模式无效 | #### `outputExtensions` diff --git a/packages/lobe-i18n/package.json b/packages/lobe-i18n/package.json index be30520..7964a42 100644 --- a/packages/lobe-i18n/package.json +++ b/packages/lobe-i18n/package.json @@ -69,6 +69,7 @@ "langchain": "latest", "lodash-es": "^4", "multimatch": "^7", + "p-map": "^6", "pangu": "^4", "react": "^18", "remark-frontmatter": "^4", diff --git a/packages/lobe-i18n/src/core/I18n.ts b/packages/lobe-i18n/src/core/I18n.ts index e856ae8..0538e16 100644 --- a/packages/lobe-i18n/src/core/I18n.ts +++ b/packages/lobe-i18n/src/core/I18n.ts @@ -1,4 +1,5 @@ import { merge } from 'lodash-es'; +import pMap from 'p-map'; import { TranslateMarkdown } from '@/core/TranslateMarkdown'; import { LocaleObj } from '@/types'; @@ -81,8 +82,6 @@ export class I18n { this.maxStep = splitString.length; this.step = 0; - const translatedSplitString: string[] = []; - onProgress?.({ isLoading: true, maxStep: this.maxStep, @@ -90,21 +89,25 @@ export class I18n { step: 0, }); - for (const text of splitString) { - onProgress?.({ - isLoading: this.step < this.maxStep, - maxStep: this.maxStep, - progress: this.step < this.maxStep ? Math.floor((this.step / this.maxStep) * 100) : 100, - step: this.step, - }); - const result = await this.translateLocaleService.runByString({ - from, - text, - to, - }); - if (result) translatedSplitString.push(result); - if (this.step < this.maxStep) this.step++; - } + const translatedSplitString: string[] = await pMap( + splitString, + async (text) => { + onProgress?.({ + isLoading: this.step < this.maxStep, + maxStep: this.maxStep, + progress: this.step < this.maxStep ? Math.floor((this.step / this.maxStep) * 100) : 100, + step: this.step, + }); + const result = await this.translateLocaleService.runByString({ + from, + text, + to, + }); + if (this.step < this.maxStep) this.step++; + return result; + }, + { concurrency: this.config?.concurrency }, + ); onProgress?.({ isLoading: false, @@ -150,8 +153,6 @@ export class I18n { this.maxStep = splitJson.length; this.step = 0; - const translatedSplitJson: LocaleObj[] = []; - onProgress?.({ isLoading: true, maxStep: this.maxStep, @@ -159,21 +160,25 @@ export class I18n { step: 0, }); - for (const json of splitJson) { - onProgress?.({ - isLoading: this.step < this.maxStep, - maxStep: this.maxStep, - progress: this.step < this.maxStep ? Math.floor((this.step / this.maxStep) * 100) : 100, - step: this.step, - }); - const result = await this.translateLocaleService.runByJson({ - from, - json, - to, - }); - if (result) translatedSplitJson.push(result); - if (this.step < this.maxStep) this.step++; - } + const translatedSplitJson: LocaleObj[] = await pMap( + splitJson, + async (json) => { + onProgress?.({ + isLoading: this.step < this.maxStep, + maxStep: this.maxStep, + progress: this.step < this.maxStep ? Math.floor((this.step / this.maxStep) * 100) : 100, + step: this.step, + }); + const result = await this.translateLocaleService.runByJson({ + from, + json, + to, + }); + if (this.step < this.maxStep) this.step++; + return result; + }, + { concurrency: this.config?.concurrency }, + ); onProgress?.({ isLoading: false, diff --git a/packages/lobe-i18n/src/store/initialState.ts b/packages/lobe-i18n/src/store/initialState.ts index 0fb7318..969d39e 100644 --- a/packages/lobe-i18n/src/store/initialState.ts +++ b/packages/lobe-i18n/src/store/initialState.ts @@ -1,6 +1,15 @@ -import { I18nConfig } from '@/types/config'; +import { I18nConfig, MarkdownModeType } from '@/types/config'; import { LanguageModel } from '@/types/models'; +import { getDefaultExtension } from '@/utils/getDefaultExtension'; export const DEFAULT_CONFIG: Partial = { + concurrency: 5, + markdown: { + entry: [], + entryExtension: '.md', + mode: MarkdownModeType.STRING, + outputExtensions: getDefaultExtension, + }, modelName: LanguageModel.GPT3_5, + temperature: 0, }; diff --git a/packages/lobe-i18n/src/store/selectors.ts b/packages/lobe-i18n/src/store/selectors.ts index 4e67553..3d0b323 100644 --- a/packages/lobe-i18n/src/store/selectors.ts +++ b/packages/lobe-i18n/src/store/selectors.ts @@ -4,9 +4,8 @@ import dotenv from 'dotenv'; import { merge } from 'lodash-es'; import { DEFAULT_CONFIG } from '@/store/initialState'; -import { Config, ConfigKeys, I18nConfig, MarkdownConfig, MarkdownModeType } from '@/types/config'; +import { Config, ConfigKeys, I18nConfig, MarkdownConfig } from '@/types/config'; import { checkOptionKeys } from '@/utils/checkOptionKeys'; -import { getDefaultExtension } from '@/utils/getDefaultExtension'; import { config, explorer, schema } from './config'; @@ -42,10 +41,7 @@ const getMarkdownConfigFile = (): MarkdownConfig => { } const markdownConfig = merge(config?.markdown || {}, { - entryExtension: config?.markdown?.entryExtension || '.md', entryLocale: config?.markdown?.entryLocale || config.entryLocale, - mode: config?.markdown?.mode || MarkdownModeType.STRING, - outputExtensions: config?.markdown?.outputExtensions || getDefaultExtension, outputLocales: config?.markdown?.outputLocales || config.outputLocales, }); diff --git a/packages/lobe-i18n/src/types/config.ts b/packages/lobe-i18n/src/types/config.ts index aa00fc3..b56f7c4 100644 --- a/packages/lobe-i18n/src/types/config.ts +++ b/packages/lobe-i18n/src/types/config.ts @@ -1,6 +1,10 @@ import { LanguageModel } from './models'; export interface I18nConfigLocale { + /** + * @description Number of concurrently pending promises returned + */ + concurrency?: number; /** * @description The entry file or folder */