From ec6d942d83eb9fa8df38f713fce6638daaf6daac Mon Sep 17 00:00:00 2001 From: balibabu Date: Wed, 23 Oct 2024 15:45:03 +0800 Subject: [PATCH] feat: Added auto_keywords and auto_questions fields to the parsing configuration page #2687 (#2987) ### What problem does this PR solve? feat: Added auto_keywords and auto_questions fields to the parsing configuration page #2687 ### Type of change - [x] New Feature (non-breaking change which adds functionality) --- web/src/components/auto-keywords-item.tsx | 36 +++++++++++++++++++ .../components/chunk-method-modal/hooks.ts | 12 ++++++- .../components/chunk-method-modal/index.tsx | 11 +++++- web/src/locales/en.ts | 4 +++ web/src/locales/zh-traditional.ts | 4 +++ web/src/locales/zh.ts | 4 +++ .../knowledge-setting/configuration.tsx | 13 +++++++ 7 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 web/src/components/auto-keywords-item.tsx diff --git a/web/src/components/auto-keywords-item.tsx b/web/src/components/auto-keywords-item.tsx new file mode 100644 index 00000000000..c703d050bf0 --- /dev/null +++ b/web/src/components/auto-keywords-item.tsx @@ -0,0 +1,36 @@ +import { useTranslate } from '@/hooks/common-hooks'; +import { Form, InputNumber } from 'antd'; + +const style = { + width: '100%', +}; + +export const AutoKeywordsItem = () => { + const { t } = useTranslate('knowledgeDetails'); + + return ( + + + + ); +}; + +export const AutoQuestionsItem = () => { + const { t } = useTranslate('knowledgeDetails'); + + return ( + + + + ); +}; diff --git a/web/src/components/chunk-method-modal/hooks.ts b/web/src/components/chunk-method-modal/hooks.ts index c1ae1b8edde..28d1611269b 100644 --- a/web/src/components/chunk-method-modal/hooks.ts +++ b/web/src/components/chunk-method-modal/hooks.ts @@ -1,7 +1,7 @@ import { useHandleChunkMethodSelectChange } from '@/hooks/logic-hooks'; import { useSelectParserList } from '@/hooks/user-setting-hooks'; import { FormInstance } from 'antd'; -import { useEffect, useMemo, useState } from 'react'; +import { useCallback, useEffect, useMemo, useState } from 'react'; const ParserListMap = new Map([ [ @@ -118,3 +118,13 @@ export const useFetchParserListOnMount = ( return { parserList: nextParserList, handleChange, selectedTag }; }; + +const hideAutoKeywords = ['qa', 'table', 'resume', 'knowledge_graph']; + +export const useShowAutoKeywords = () => { + const showAutoKeywords = useCallback((selectedTag: string) => { + return hideAutoKeywords.every((x) => selectedTag !== x); + }, []); + + return showAutoKeywords; +}; diff --git a/web/src/components/chunk-method-modal/index.tsx b/web/src/components/chunk-method-modal/index.tsx index cea88b905e9..6548f8d754b 100644 --- a/web/src/components/chunk-method-modal/index.tsx +++ b/web/src/components/chunk-method-modal/index.tsx @@ -17,11 +17,12 @@ import { } from 'antd'; import omit from 'lodash/omit'; import React, { useEffect, useMemo } from 'react'; -import { useFetchParserListOnMount } from './hooks'; +import { useFetchParserListOnMount, useShowAutoKeywords } from './hooks'; import { useTranslate } from '@/hooks/common-hooks'; import { IParserConfig } from '@/interfaces/database/document'; import { IChangeParserConfigRequestBody } from '@/interfaces/request/document'; +import { AutoKeywordsItem, AutoQuestionsItem } from '../auto-keywords-item'; import Delimiter from '../delimiter'; import EntityTypesItem from '../entity-types-item'; import ExcelToHtml from '../excel-to-html'; @@ -108,6 +109,8 @@ const ChunkMethodModal: React.FC = ({ const showExcelToHtml = selectedTag === 'naive' && documentExtension === 'xlsx'; + const showAutoKeywords = useShowAutoKeywords(); + const afterClose = () => { form.resetFields(); }; @@ -283,6 +286,12 @@ const ChunkMethodModal: React.FC = ({ )} + {showAutoKeywords(selectedTag) && ( + <> + + + + )} {showExcelToHtml && } {showRaptorParseConfiguration(selectedTag) && ( diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts index 0b63dd333a2..5d8cfcf3782 100644 --- a/web/src/locales/en.ts +++ b/web/src/locales/en.ts @@ -154,6 +154,10 @@ export default { delimiter: `Delimiter`, html4excel: 'Excel to HTML', html4excelTip: `Excel will be parsed into HTML table or not. If it's FALSE, every row in Excel will be formed as a chunk.`, + autoKeywords: 'Auto keywords', + autoKeywordsTip: `Extract N keywords for every chunk to boost their rank score while querying such keywords. Extra tokens will be comsumed for LLM that you set in 'System model settings'. You can check the result in the chunk list.`, + autoQuestions: 'Auto questions', + autoQuestionsTip: `Extract N questions for every chunk to boost their rank score while querying such questions. Extra tokens will be comsumed for LLM that you set in 'System model settings'. You can check the result in the chunk list. This function will not destroy the entire chunking process if errors occur except adding empty result to the original chunk.`, }, knowledgeConfiguration: { titleDescription: diff --git a/web/src/locales/zh-traditional.ts b/web/src/locales/zh-traditional.ts index 5edf21d0f0b..4157c298e09 100644 --- a/web/src/locales/zh-traditional.ts +++ b/web/src/locales/zh-traditional.ts @@ -150,6 +150,10 @@ export default { delimiter: `分段標識符`, html4excel: '表格轉HTML', html4excelTip: `Excel 是否會被解析為 HTML 表格。如果為 FALSE，Excel 中的每一行都會形成一個區塊。`, + autoKeywords: '自動關鍵字', + autoKeywordsTip: `在查詢此類關鍵字時，為每個區塊提取 N 個關鍵字以提高其排名分數。在「系統模型設定」中設定的 LLM 將消耗額外的 token。您可以在區塊清單中查看結果。 `, + autoQuestions: '自動問題', + autoQuestionsTip: `在查詢此類問題時，為每個區塊提取 N 個問題以提高其排名分數。在「系統模型設定」中設定的 LLM 將消耗額外的 token。您可以在區塊清單中查看結果。如果發生錯誤，此功能不會破壞整個分塊過程，除了將空結果新增至原始區塊。 `, }, knowledgeConfiguration: { titleDescription: '在這裡更新您的知識庫詳細信息，尤其是解析方法。', diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.ts index 36e39e126b4..693929d9fef 100644 --- a/web/src/locales/zh.ts +++ b/web/src/locales/zh.ts @@ -151,6 +151,10 @@ export default { delimiter: `分段标识符`, html4excel: '表格转HTML', html4excelTip: `Excel 是否将被解析为 HTML 表。如果为 FALSE，Excel 中的每一行都将形成一个块。`, + autoKeywords: '自动关键词', + autoKeywordsTip: `在查询此类关键词时，为每个块提取 N 个关键词以提高其排名得分。在“系统模型设置”中设置的 LLM 将消耗额外的 token。您可以在块列表中查看结果。`, + autoQuestions: '自动问题', + autoQuestionsTip: `在查询此类问题时，为每个块提取 N 个问题以提高其排名得分。在“系统模型设置”中设置的 LLM 将消耗额外的 token。您可以在块列表中查看结果。如果发生错误，此功能不会破坏整个分块过程，除了将空结果添加到原始块。`, }, knowledgeConfiguration: { titleDescription: '在这里更新您的知识库详细信息，尤其是解析方法。', diff --git a/web/src/pages/add-knowledge/components/knowledge-setting/configuration.tsx b/web/src/pages/add-knowledge/components/knowledge-setting/configuration.tsx index 64b1cbb820c..8800b718420 100644 --- a/web/src/pages/add-knowledge/components/knowledge-setting/configuration.tsx +++ b/web/src/pages/add-knowledge/components/knowledge-setting/configuration.tsx @@ -1,3 +1,8 @@ +import { + AutoKeywordsItem, + AutoQuestionsItem, +} from '@/components/auto-keywords-item'; +import { useShowAutoKeywords } from '@/components/chunk-method-modal/hooks'; import Delimiter from '@/components/delimiter'; import EntityTypesItem from '@/components/entity-types-item'; import ExcelToHtml from '@/components/excel-to-html'; @@ -27,6 +32,7 @@ const ConfigurationForm = ({ form }: { form: FormInstance }) => { useFetchKnowledgeConfigurationOnMount(form); const { t } = useTranslate('knowledgeConfiguration'); const handleChunkMethodSelectChange = useHandleChunkMethodSelectChange(form); + const showAutoKeywords = useShowAutoKeywords(); return (
@@ -120,6 +126,12 @@ const ConfigurationForm = ({ form }: { form: FormInstance }) => { )} + {showAutoKeywords(parserId) && ( + <> + + + + )} {parserId === 'naive' && ( <> @@ -128,6 +140,7 @@ const ConfigurationForm = ({ form }: { form: FormInstance }) => { )} + {showRaptorParseConfiguration(parserId) && ( )}