diff --git a/apps/agentfabric/app.py b/apps/agentfabric/app.py index ffeb254c..8ca79470 100644 --- a/apps/agentfabric/app.py +++ b/apps/agentfabric/app.py @@ -6,6 +6,7 @@ import gradio as gr import json +import modelscope_gradio_components as mgr import yaml from apps.agentfabric.builder_core import (beauty_output, gen_response_and_process, @@ -15,10 +16,12 @@ is_valid_plugin_configuration, parse_configuration, save_avatar_image, save_builder_configuration, save_plugin_configuration) -from gradio_utils import ChatBot, format_cover_html, format_goto_publish_html +from gradio_utils import format_cover_html, format_goto_publish_html from i18n import I18n from modelscope_agent.schemas import Message from modelscope_agent.utils.logger import agent_logger as logger +from modelscope_gradio_components.components.Chatbot.llm_thinking_presets import \ + qwen from publish_util import (pop_user_info_from_config, prepare_agent_zip, reload_agent_zip) from user_core import init_user_chatbot_agent @@ -75,90 +78,6 @@ def check_uuid(uuid_str): return uuid_str -def process_configuration(uuid_str, bot_avatar, name, description, - instructions, model, agent_language, suggestions, - knowledge_files, capabilities_checkboxes, - openapi_schema, openapi_auth, openapi_auth_apikey, - openapi_auth_apikey_type, openapi_privacy_policy, - state): - uuid_str = check_uuid(uuid_str) - tool_cfg = state['tool_cfg'] - capabilities = state['capabilities'] - bot_avatar, bot_avatar_path = save_avatar_image(bot_avatar, uuid_str) - suggestions_filtered = [row for row in suggestions if row[0]] - if len(suggestions_filtered) == 0: - suggestions_filtered == [['']] - user_dir = get_user_dir(uuid_str) - if knowledge_files is not None: - new_knowledge_files = [ - os.path.join(user_dir, os.path.basename((f.name))) - for f in knowledge_files - ] - for src_file, dst_file in zip(knowledge_files, new_knowledge_files): - if not os.path.exists(dst_file): - shutil.copy(src_file.name, dst_file) - else: - new_knowledge_files = [] - - builder_cfg = { - 'name': name, - 'avatar': bot_avatar, - 'description': description, - 'instruction': instructions, - 'prompt_recommend': [row[0] for row in suggestions_filtered], - 'knowledge': new_knowledge_files, - 'tools': { - capability: dict([(key, value if key != 'use' else - (capability in capabilities_checkboxes)) - for key, value in tool_cfg[capability].items()]) - for capability in map(lambda item: item[1], capabilities) - }, - 'model': model, - 'language': agent_language, - } - - try: - try: - schema_dict = json.loads(openapi_schema) - except json.decoder.JSONDecodeError: - schema_dict = yaml.safe_load(openapi_schema) - except Exception as e: - raise gr.Error( - f'OpenAPI schema format error, should be one of json and yaml: {e}' - ) - - openapi_plugin_cfg = { - 'schema': schema_dict, - 'auth': { - 'type': openapi_auth, - 'apikey': openapi_auth_apikey, - 'apikey_type': openapi_auth_apikey_type - }, - 'privacy_policy': openapi_privacy_policy - } - if is_valid_plugin_configuration(openapi_plugin_cfg): - save_plugin_configuration(openapi_plugin_cfg, uuid_str) - except Exception as e: - logger.error( - uuid=uuid_str, - error=str(e), - content={'error_traceback': traceback.format_exc()}) - - save_builder_configuration(builder_cfg, uuid_str) - update_builder(uuid_str, state) - init_user(uuid_str, state) - return [ - gr.HTML.update( - visible=True, - value=format_cover_html(builder_cfg, bot_avatar_path)), - gr.Chatbot.update( - visible=False, - avatar_images=get_avatar_image(bot_avatar, uuid_str)), - gr.Dataset.update(samples=suggestions_filtered), - gr.DataFrame.update(value=suggestions_filtered) - ] - - # 创建 Gradio 界面 demo = gr.Blocks(css='assets/app.css') with demo: @@ -185,13 +104,23 @@ def process_configuration(uuid_str, bot_avatar, name, description, # "Create" 标签页的 Chatbot 组件 start_text = '欢迎使用agent创建助手。我可以帮助您创建一个定制agent。'\ '您希望您的agent主要用于什么领域或任务？比如，您可以说，我想做一个RPG游戏agent' - create_chatbot = gr.Chatbot( - show_label=False, value=[[None, start_text]]) - create_chat_input = gr.Textbox( + create_chatbot = mgr.Chatbot( + show_label=False, + value=[[None, start_text]], + flushing=False, + show_copy_button=True, + llm_thinking_presets=[ + qwen( + action_input_title='调用 ', + action_output_title='完成调用') + ]) + create_chat_input = mgr.MultimodalInput( label=i18n.get('message'), - placeholder=i18n.get('message_placeholder')) - create_send_button = gr.Button( - i18n.get('sendOnLoading'), interactive=False) + placeholder=i18n.get('message_placeholder'), + interactive=False, + upload_button_props=dict(visible=False), + submit_button_props=dict( + label=i18n.get('sendOnLoading'))) configure_tab = gr.Tab(i18n.get_whole('configure'), id=1) with configure_tab: @@ -200,8 +129,7 @@ def process_configuration(uuid_str, bot_avatar, name, description, with gr.Row(): bot_avatar_comp = gr.Image( label=i18n.get('form_avatar'), - placeholder='Chatbot avatar image', - source='upload', + sources=['upload'], interactive=True, type='filepath', scale=1, @@ -295,7 +223,7 @@ def process_configuration(uuid_str, bot_avatar, name, description, f"""
{i18n.get('preview')}
""") user_chat_bot_cover = gr.HTML(format_cover_html({}, None)) - user_chatbot = ChatBot( + user_chatbot = mgr.Chatbot( value=[[None, None]], elem_id='user_chatbot', elem_classes=['markdown-body'], @@ -303,22 +231,26 @@ def process_configuration(uuid_str, bot_avatar, name, description, height=650, latex_delimiters=[], show_label=False, - visible=False) - preview_chat_input = gr.Textbox( + visible=False, + show_copy_button=True, + llm_thinking_presets=[ + qwen( + action_input_title='调用 ', + action_output_title='完成调用') + ]) + preview_chat_input = mgr.MultimodalInput( + interactive=False, label=i18n.get('message'), - placeholder=i18n.get('message_placeholder')) + placeholder=i18n.get('message_placeholder'), + upload_button_props=dict( + label=i18n.get('upload_btn'), + file_types=['file', 'image', 'audio', 'video', 'text'], + file_count='multiple'), + submit_button_props=dict(label=i18n.get('sendOnLoading'))) user_chat_bot_suggest = gr.Dataset( label=i18n.get('prompt_suggestion'), components=[preview_chat_input], samples=[]) - # preview_send_button = gr.Button('Send') - with gr.Row(): - upload_button = gr.UploadButton( - i18n.get('upload_btn'), - file_types=['file', 'image', 'audio', 'video', 'text'], - file_count='multiple') - preview_send_button = gr.Button( - i18n.get('sendOnLoading'), interactive=False) user_chat_bot_suggest.select( lambda evt: evt[0], inputs=[user_chat_bot_suggest], @@ -355,8 +287,8 @@ def process_configuration(uuid_str, bot_avatar, name, description, # bot user_chat_bot_cover, user_chat_bot_suggest, - preview_send_button, - create_send_button, + preview_chat_input, + create_chat_input, ] # 初始化表单 @@ -380,7 +312,7 @@ def init_ui_config(uuid_str, _state, builder_cfg, model_cfg, tool_cfg): state: _state, bot_avatar_comp: - gr.Image.update(value=bot_avatar), + gr.Image(value=bot_avatar), name_input: builder_cfg.get('name', ''), description_input: @@ -388,7 +320,7 @@ def init_ui_config(uuid_str, _state, builder_cfg, model_cfg, tool_cfg): instructions_input: builder_cfg.get('instruction'), model_selector: - gr.Dropdown.update( + gr.Dropdown( value=builder_cfg.get('model', models[0]), choices=models), agent_language_selector: builder_cfg.get('language') or 'zh', @@ -398,7 +330,7 @@ def init_ui_config(uuid_str, _state, builder_cfg, model_cfg, tool_cfg): builder_cfg.get('knowledge', []) if len(builder_cfg['knowledge']) > 0 else None, capabilities_checkboxes: - gr.CheckboxGroup.update( + gr.CheckboxGroup( value=[ tool for tool in builder_cfg.get('tools', {}).keys() if builder_cfg.get('tools').get(tool).get('use', False) @@ -408,7 +340,9 @@ def init_ui_config(uuid_str, _state, builder_cfg, model_cfg, tool_cfg): user_chat_bot_cover: format_cover_html(builder_cfg, bot_avatar), user_chat_bot_suggest: - gr.Dataset.update(samples=[[item] for item in suggests]), + gr.Dataset( + components=[preview_chat_input], + samples=[[item] for item in suggests]), } # tab 切换的事件处理 @@ -443,15 +377,15 @@ def format_message_with_builder_cfg(_state, chatbot, builder_cfg, create_chatbot: chatbot, user_chat_bot_cover: - gr.HTML.update( + gr.HTML( visible=True, value=format_cover_html(builder_cfg, bot_avatar_path)), user_chatbot: - gr.Chatbot.update( + gr.update( visible=False, avatar_images=get_avatar_image(bot_avatar, uuid_str)), user_chat_bot_suggest: - gr.Dataset.update(samples=suggestion) + gr.Dataset(components=[preview_chat_input], samples=suggestion) } def create_send_message(chatbot, input, _state, uuid_str): @@ -460,15 +394,15 @@ def create_send_message(chatbot, input, _state, uuid_str): builder_agent = _state['builder_agent'] builder_memory = _state['builder_memory'] - chatbot.append((input, '')) + chatbot.append([{'text': input.text, 'files': input.files}, None]) yield { create_chatbot: chatbot, - create_chat_input: gr.Textbox.update(value=''), + create_chat_input: None, } response = '' for frame in gen_response_and_process( builder_agent, - query=input, + query=input.text, memory=builder_memory, print_info=True, uuid_str=uuid_str): @@ -495,12 +429,12 @@ def create_send_message(chatbot, input, _state, uuid_str): frame_text = content response = beauty_output(f'{response}{frame_text}', step_result) - chatbot[-1] = (input, response) + chatbot[-1][1] = response yield { create_chatbot: chatbot, } - create_send_button.click( + create_chat_input.submit( create_send_message, inputs=[create_chatbot, create_chat_input, state, uuid_str], outputs=[ @@ -508,6 +442,94 @@ def create_send_message(chatbot, input, _state, uuid_str): user_chat_bot_suggest, create_chat_input ]) + def process_configuration(uuid_str, bot_avatar, name, description, + instructions, model, agent_language, suggestions, + knowledge_files, capabilities_checkboxes, + openapi_schema, openapi_auth, + openapi_auth_apikey, openapi_auth_apikey_type, + openapi_privacy_policy, state): + uuid_str = check_uuid(uuid_str) + tool_cfg = state['tool_cfg'] + capabilities = state['capabilities'] + bot_avatar, bot_avatar_path = save_avatar_image(bot_avatar, uuid_str) + suggestions_filtered = [row for row in suggestions if row[0]] + if len(suggestions_filtered) == 0: + suggestions_filtered = [['']] + user_dir = get_user_dir(uuid_str) + if knowledge_files is not None: + new_knowledge_files = [ + os.path.join(user_dir, os.path.basename((f.name))) + for f in knowledge_files + ] + for src_file, dst_file in zip(knowledge_files, + new_knowledge_files): + if not os.path.exists(dst_file): + shutil.copy(src_file.name, dst_file) + else: + new_knowledge_files = [] + builder_cfg = { + 'name': name, + 'avatar': bot_avatar, + 'description': description, + 'instruction': instructions, + 'prompt_recommend': [row[0] for row in suggestions_filtered], + 'knowledge': new_knowledge_files, + 'tools': { + capability: + dict([(key, value if key != 'use' else + (capability in capabilities_checkboxes)) + for key, value in tool_cfg[capability].items()]) + for capability in map(lambda item: item[1], capabilities) + }, + 'model': model, + 'language': agent_language, + } + + try: + try: + schema_dict = json.loads(openapi_schema) + except json.decoder.JSONDecodeError: + schema_dict = yaml.safe_load(openapi_schema) + except Exception as e: + raise gr.Error( + f'OpenAPI schema format error, should be one of json and yaml: {e}' + ) + + openapi_plugin_cfg = { + 'schema': schema_dict, + 'auth': { + 'type': openapi_auth, + 'apikey': openapi_auth_apikey, + 'apikey_type': openapi_auth_apikey_type + }, + 'privacy_policy': openapi_privacy_policy + } + if is_valid_plugin_configuration(openapi_plugin_cfg): + save_plugin_configuration(openapi_plugin_cfg, uuid_str) + except Exception as e: + logger.error( + uuid=uuid_str, + error=str(e), + content={'error_traceback': traceback.format_exc()}) + + save_builder_configuration(builder_cfg, uuid_str) + update_builder(uuid_str, state) + init_user(uuid_str, state) + return { + user_chat_bot_cover: + gr.HTML( + visible=True, + value=format_cover_html(builder_cfg, bot_avatar_path)), + user_chatbot: + gr.update( + visible=False, + avatar_images=get_avatar_image(bot_avatar, uuid_str)), + suggestion_input: [item[:] for item in suggestions_filtered], + user_chat_bot_suggest: + gr.Dataset( + components=[preview_chat_input], samples=suggestions_filtered), + } + # 配置 "Configure" 标签页的提交按钮功能 configure_button.click( process_configuration, @@ -528,18 +550,24 @@ def preview_send_message(chatbot, input, _state, uuid_str): # 将发送的消息添加到聊天历史 # _uuid_str = check_uuid(uuid_str) user_agent = _state['user_agent'] + append_files = [] + for file in input.files: + file_name = os.path.basename(file.path) + # covert xxx.json to xxx_uuid_str.json + file_name = file_name.replace('.', f'_{uuid_str}.') + file_path = os.path.join(get_ci_dir(), file_name) + if not os.path.exists(file_path): + # make sure file path's directory exists + os.makedirs(os.path.dirname(file_path), exist_ok=True) + shutil.copy(file.path, file_path) + append_files.append(file_path) user_memory = _state['user_memory'] - if 'new_file_paths' in _state: - new_file_paths = _state['new_file_paths'] - else: - new_file_paths = [] - _state['new_file_paths'] = [] - chatbot.append((input, '')) + chatbot.append([{'text': input.text, 'files': input.files}, None]) yield { - user_chatbot: gr.Chatbot.update(visible=True, value=chatbot), - user_chat_bot_cover: gr.HTML.update(visible=False), - preview_chat_input: gr.Textbox.update(value='') + user_chatbot: mgr.Chatbot(visible=True, value=chatbot), + user_chat_bot_cover: gr.HTML(visible=False), + preview_chat_input: None } # get short term memory history @@ -547,27 +575,29 @@ def preview_send_message(chatbot, input, _state, uuid_str): # get long term memory knowledge, currently get one file uploaded_file = None - if len(new_file_paths) > 0: - uploaded_file = new_file_paths[0] - ref_doc = user_memory.run(query=input, url=uploaded_file, checked=True) + if len(append_files) > 0: + uploaded_file = append_files[0] + ref_doc = user_memory.run( + query=input.text, url=uploaded_file, checked=True) response = '' try: for frame in user_agent.run( - input, + input.text, history=history, ref_doc=ref_doc, - append_files=new_file_paths): + append_files=append_files): + # append_files=new_file_paths): # important! do not change this response += frame - chatbot[-1] = (input, response) + chatbot[-1][1] = response yield {user_chatbot: chatbot} if len(history) == 0: user_memory.update_history( Message(role='system', content=user_agent.system_prompt)) user_memory.update_history([ - Message(role='user', content=input), + Message(role='user', content=input.text), Message(role='assistant', content=response), ]) except Exception as e: @@ -578,51 +608,14 @@ def preview_send_message(chatbot, input, _state, uuid_str): msg = 'Too many people are calling, please try again later.' else: msg = str(e) - chatbot[-1] = (input, msg) + chatbot[-1][1] = msg yield {user_chatbot: chatbot} - preview_send_button.click( + preview_chat_input.submit( preview_send_message, inputs=[user_chatbot, preview_chat_input, state, uuid_str], outputs=[user_chatbot, user_chat_bot_cover, preview_chat_input]) - def upload_file(chatbot, upload_button, _state, uuid_str): - uuid_str = check_uuid(uuid_str) - new_file_paths = [] - if 'file_paths' in _state: - file_paths = _state['file_paths'] - else: - file_paths = [] - for file in upload_button: - file_name = os.path.basename(file.name) - # covert xxx.json to xxx_uuid_str.json - file_name = file_name.replace('.', f'_{uuid_str}.') - file_path = os.path.join(get_ci_dir(), file_name) - if not os.path.exists(file_path): - # make sure file path's directory exists - os.makedirs(os.path.dirname(file_path), exist_ok=True) - shutil.copy(file.name, file_path) - file_paths.append(file_path) - new_file_paths.append(file_path) - if file_name.endswith(('.jpeg', '.png', '.jpg')): - chatbot += [((file_path, ), None)] - - else: - chatbot.append((None, f'上传文件{file_name}，成功')) - yield { - user_chatbot: gr.Chatbot.update(visible=True, value=chatbot), - user_chat_bot_cover: gr.HTML.update(visible=False), - preview_chat_input: gr.Textbox.update(value='') - } - - _state['file_paths'] = file_paths - _state['new_file_paths'] = new_file_paths - - upload_button.upload( - upload_file, - inputs=[user_chatbot, upload_button, state, uuid_str], - outputs=[user_chatbot, user_chat_bot_cover, preview_chat_input]) - # configuration for publish def publish_agent(name, uuid_str, state): uuid_str = check_uuid(uuid_str) @@ -690,24 +683,26 @@ def change_lang(language): preview_header: gr.HTML( f"""
{i18n.get('preview')}
"""), - preview_send_button: - gr.Button.update(value=i18n.get('send')), + preview_chat_input: + mgr.MultimodalInput( + label=i18n.get('message'), + placeholder=i18n.get('message_placeholder'), + submit_button_props=dict(label=i18n.get('send')), + upload_button_props=dict( + label=i18n.get('upload_btn'), + file_types=['file', 'image', 'audio', 'video', 'text'], + file_count='multiple')), create_chat_input: - gr.Textbox( + mgr.MultimodalInput( label=i18n.get('message'), - placeholder=i18n.get('message_placeholder')), - create_send_button: - gr.Button.update(value=i18n.get('send')), + placeholder=i18n.get('message_placeholder'), + submit_button_props=dict(label=i18n.get('send'))), user_chat_bot_suggest: - gr.Dataset(label=i18n.get('prompt_suggestion')), - preview_chat_input: - gr.Textbox( - label=i18n.get('message'), - placeholder=i18n.get('message_placeholder')), + gr.Dataset( + components=[preview_chat_input], + label=i18n.get('prompt_suggestion')), publish_accordion: gr.Accordion(label=i18n.get('publish')), - upload_button: - gr.UploadButton(i18n.get('upload_btn')), header: gr.Markdown(i18n.get('header')), publish_alert_md: @@ -723,9 +718,8 @@ def change_lang(language): inputs=[language], outputs=configure_updated_outputs + [ configure_button, create_chat_input, open_api_accordion, - preview_header, preview_chat_input, publish_accordion, - upload_button, header, publish_alert_md, build_hint_md, - publish_hint_md + preview_header, preview_chat_input, publish_accordion, header, + publish_alert_md, build_hint_md, publish_hint_md ]) def init_all(uuid_str, _state): @@ -740,14 +734,18 @@ def init_all(uuid_str, _state): yield { state: _state, - preview_send_button: - gr.Button.update(value=i18n.get('send'), interactive=True), - create_send_button: - gr.Button.update(value=i18n.get('send'), interactive=True), + preview_chat_input: + mgr.MultimodalInput( + submit_button_props=dict(label=i18n.get('send')), + interactive=True), + create_chat_input: + mgr.MultimodalInput( + submit_button_props=dict(label=i18n.get('send')), + interactive=True), } demo.load( init_all, inputs=[uuid_str, state], outputs=configure_updated_outputs) -demo.queue(concurrency_count=10) -demo.launch(show_error=True) +demo.queue() +demo.launch(show_error=True, max_threads=10) diff --git a/apps/agentfabric/appBot.py b/apps/agentfabric/appBot.py index 93e28d16..b6eeb74c 100644 --- a/apps/agentfabric/appBot.py +++ b/apps/agentfabric/appBot.py @@ -4,9 +4,13 @@ import traceback import gradio as gr +import modelscope_gradio_components as mgr from config_utils import get_avatar_image, get_ci_dir, parse_configuration -from gradio_utils import ChatBot, format_cover_html +from gradio_utils import format_cover_html +from modelscope_agent.schemas import Message from modelscope_agent.utils.logger import agent_logger as logger +from modelscope_gradio_components.components.Chatbot.llm_thinking_presets import \ + qwen from user_core import init_user_chatbot_agent uuid_str = 'local_user' @@ -33,9 +37,10 @@ def check_uuid(uuid_str): def init_user(state): try: seed = state.get('session_seed', random.randint(0, 1000000000)) - user_agent = init_user_chatbot_agent(uuid_str) + user_agent, user_memory = init_user_chatbot_agent(uuid_str) user_agent.seed = seed state['user_agent'] = user_agent + state['user_memory'] = user_memory except Exception as e: logger.error( uuid=uuid_str, @@ -57,27 +62,28 @@ def init_user(state): with gr.Column(scale=4): with gr.Column(): # Preview - user_chatbot = ChatBot( + user_chatbot = mgr.Chatbot( value=[[None, '尝试问我一点什么吧～']], elem_id='user_chatbot', elem_classes=['markdown-body'], avatar_images=avatar_pairs, height=600, latex_delimiters=[], - show_label=False) + show_label=False, + show_copy_button=True, + llm_thinking_presets=[ + qwen( + action_input_title='调用 ', + action_output_title='完成调用') + ]) with gr.Row(): - with gr.Column(scale=12): - preview_chat_input = gr.Textbox( - show_label=False, - container=False, - placeholder='跟我聊聊吧～') - with gr.Column(min_width=70, scale=1): - upload_button = gr.UploadButton( - '上传', - file_types=['file', 'image', 'audio', 'video', 'text'], - file_count='multiple') - with gr.Column(min_width=70, scale=1): - preview_send_button = gr.Button('发送', variant='primary') + user_chatbot_input = mgr.MultimodalInput( + interactive=True, + placeholder='跟我聊聊吧～', + upload_button_props=dict( + file_count='multiple', + file_types=['file', 'image', 'audio', 'video', + 'text'])) with gr.Column(scale=1): user_chat_bot_cover = gr.HTML( @@ -85,87 +91,65 @@ def init_user(state): user_chat_bot_suggest = gr.Examples( label='Prompt Suggestions', examples=suggests, - inputs=[preview_chat_input]) + inputs=[user_chatbot_input]) - def upload_file(chatbot, upload_button, _state): + def send_message(chatbot, input, _state): + # 将发送的消息添加到聊天历史 + if 'user_agent' not in _state: + init_user(_state) + # 将发送的消息添加到聊天历史 _uuid_str = check_uuid(uuid_str) - new_file_paths = [] - if 'file_paths' in _state: - file_paths = _state['file_paths'] - else: - file_paths = [] - for file in upload_button: - file_name = os.path.basename(file.name) + user_agent = _state['user_agent'] + user_memory = _state['user_memory'] + append_files = [] + for file in input.files: + file_name = os.path.basename(file.path) # covert xxx.json to xxx_uuid_str.json file_name = file_name.replace('.', f'_{_uuid_str}.') file_path = os.path.join(get_ci_dir(), file_name) if not os.path.exists(file_path): # make sure file path's directory exists os.makedirs(os.path.dirname(file_path), exist_ok=True) - shutil.copy(file.name, file_path) - file_paths.append(file_path) - new_file_paths.append(file_path) - if file_name.endswith(('.jpeg', '.png', '.jpg')): - chatbot += [((file_path, ), None)] - - else: - chatbot.append((None, f'上传文件{file_name}，成功')) + shutil.copy(file.path, file_path) + append_files.append(file_path) + chatbot.append([{'text': input.text, 'files': input.files}, None]) yield { - user_chatbot: gr.Chatbot.update(visible=True, value=chatbot), - preview_chat_input: gr.Textbox.update(value='') + user_chatbot: chatbot, + user_chatbot_input: None, } - _state['file_paths'] = file_paths - _state['new_file_paths'] = new_file_paths - - upload_button.upload( - upload_file, - inputs=[user_chatbot, upload_button, state], - outputs=[user_chatbot, preview_chat_input]) + # get short term memory history + history = user_memory.get_history() - def send_message(chatbot, input, _state): - # 将发送的消息添加到聊天历史 - if 'user_agent' not in _state: - init_user(_state) - - user_agent = _state['user_agent'] - if 'new_file_paths' in _state: - new_file_paths = _state['new_file_paths'] - else: - new_file_paths = [] - _state['new_file_paths'] = [] - chatbot.append((input, '')) - yield { - user_chatbot: chatbot, - preview_chat_input: gr.Textbox.update(value=''), - } + # get long term memory knowledge, currently get one file + uploaded_file = None + if len(append_files) > 0: + uploaded_file = append_files[0] + ref_doc = user_memory.run( + query=input.text, url=uploaded_file, checked=True) response = '' try: - for frame in user_agent.stream_run( - input, - print_info=True, - remote=False, - append_files=new_file_paths): - # is_final = frame.get("frame_is_final") - llm_result = frame.get('llm_text', '') - exec_result = frame.get('exec_result', '') - # llm_result = llm_result.split("<|user|>")[0].strip() - if len(exec_result) != 0: - # action_exec_result - if isinstance(exec_result, dict): - exec_result = str(exec_result['result']) - frame_text = f'{exec_result}' - else: - # llm result - frame_text = llm_result + for frame in user_agent.run( + input.text, + history=history, + ref_doc=ref_doc, + append_files=append_files): # important! do not change this - response += frame_text - chatbot[-1] = (input, response) + response += frame + chatbot[-1][1] = response yield { user_chatbot: chatbot, } + if len(history) == 0: + user_memory.update_history( + Message(role='system', content=user_agent.system_prompt)) + + user_memory.update_history([ + Message(role='user', content=input.text), + Message(role='assistant', content=response), + ]) except Exception as e: if 'dashscope.common.error.AuthenticationError' in str(e): msg = 'DASHSCOPE_API_KEY should be set via environment variable. You can acquire this in ' \ @@ -174,15 +158,15 @@ def send_message(chatbot, input, _state): msg = 'Too many people are calling, please try again later.' else: msg = str(e) - chatbot[-1] = (input, msg) + chatbot[-1][1] = msg yield {user_chatbot: chatbot} - preview_send_button.click( - send_message, - inputs=[user_chatbot, preview_chat_input, state], - outputs=[user_chatbot, preview_chat_input]) + gr.on([user_chatbot_input.submit], + fn=send_message, + inputs=[user_chatbot, user_chatbot_input, state], + outputs=[user_chatbot, user_chatbot_input]) demo.load(init_user, inputs=[state], outputs=[state]) -demo.queue(concurrency_count=10) -demo.launch(show_error=True) +demo.queue() +demo.launch(show_error=True, max_threads=10) diff --git a/apps/agentfabric/gradio_utils.py b/apps/agentfabric/gradio_utils.py index 00b0db70..ddc47554 100644 --- a/apps/agentfabric/gradio_utils.py +++ b/apps/agentfabric/gradio_utils.py @@ -7,8 +7,6 @@ from urllib import parse import json -import markdown -from gradio.components import Chatbot as ChatBotBase ALREADY_CONVERTED_MARK = '' @@ -78,7 +76,7 @@ def format_cover_html(configuration, bot_avatar_path): return f"""
- +
{configuration.get("name", "")}
{configuration.get("description", "")}
@@ -105,328 +103,26 @@ def format_goto_publish_html(label, zip_url, agent_user_params, disable=False): """ -class ChatBot(ChatBotBase): - - def normalize_markdown(self, bot_message): - lines = bot_message.split('\n') - normalized_lines = [] - inside_list = False - - for i, line in enumerate(lines): - if re.match(r'^(\d+\.|-|\*|\+)\s', line.strip()): - if not inside_list and i > 0 and lines[i - 1].strip() != '': - normalized_lines.append('') - inside_list = True - normalized_lines.append(line) - elif inside_list and line.strip() == '': - if i < len(lines) - 1 and not re.match(r'^(\d+\.|-|\*|\+)\s', - lines[i + 1].strip()): - normalized_lines.append(line) - continue - else: - inside_list = False - normalized_lines.append(line) - - return '\n'.join(normalized_lines) - - def convert_markdown(self, bot_message): - if bot_message.count('```') % 2 != 0: - bot_message += '\n```' - - bot_message = self.normalize_markdown(bot_message) - - result = markdown.markdown( +def postprocess_messages( + message_pairs: list[list[str | tuple[str] | tuple[str, str] | None] + | tuple] +) -> list[list[str | dict | None]]: + if message_pairs is None: + return [] + processed_messages = [] + for message_pair in message_pairs: + assert isinstance( + message_pair, (tuple, list) + ), f'Expected a list of lists or list of tuples. Received: {message_pair}' + assert ( + len(message_pair) == 2 + ), f'Expected a list of lists of length 2 or list of tuples of length 2. Received: {message_pair}' + + user_message, bot_message = message_pair + + # TODO: 是否需要进行任何其他的后处理 + processed_messages.append([ + user_message, bot_message, - extensions=[ - 'toc', 'extra', 'tables', 'codehilite', - 'markdown_cjk_spacing.cjk_spacing', 'pymdownx.magiclink' - ], - extension_configs={ - 'markdown_katex': { - 'no_inline_svg': True, # fix for WeasyPrint - 'insert_fonts_css': True, - }, - 'codehilite': { - 'linenums': False, - 'guess_lang': True - }, - 'mdx_truly_sane_lists': { - 'nested_indent': 2, - 'truly_sane': True, - } - }) - result = ''.join(result) - return result - - @staticmethod - def prompt_parse(message): - output = '' - if 'Thought' in message: - if 'Action' in message or 'Action Input:' in message: - re_pattern_thought = re.compile( - pattern=r'([\s\S]+)Thought:([\s\S]+)Action:') - - res = re_pattern_thought.search(message) - - if res is None: - re_pattern_thought_only = re.compile( - pattern=r'Thought:([\s\S]+)Action:') - res = re_pattern_thought_only.search(message) - llm_result = '' - else: - llm_result = res.group(1).strip() - action_thought_result = res.group(2).strip() - - re_pattern_action = re.compile( - pattern= - r'Action:([\s\S]+)Action Input:([\s\S]+)<\|startofexec\|>') - res = re_pattern_action.search(message) - if res is None: - action, action_parameters = parse_action_response(message) - else: - action = res.group(1).strip() - action_parameters = res.group(2) - action_result = json.dumps({ - 'api_name': action, - 'parameters': action_parameters - }) - output += f'{llm_result}\n{action_thought_result}\n<|startofthink|>\n{action_result}\n<|endofthink|>\n' - if '<|startofexec|>' in message: - re_pattern3 = re.compile( - pattern=r'<\|startofexec\|>([\s\S]+)<\|endofexec\|>') - res3 = re_pattern3.search(message) - observation = res3.group(1).strip() - output += f'\n<|startofexec|>\n{observation}\n<|endofexec|>\n' - if 'Final Answer' in message: - re_pattern2 = re.compile( - pattern=r'Thought:([\s\S]+)Final Answer:([\s\S]+)') - res2 = re_pattern2.search(message) - # final_thought_result = res2.group(1).strip() - final_answer_result = res2.group(2).strip() - output += f'{final_answer_result}\n' - - if output == '': - return message - print(output) - return output - else: - return message - - def convert_bot_message(self, bot_message): - - bot_message = ChatBot.prompt_parse(bot_message) - # print('processed bot message----------') - # print(bot_message) - # print('processed bot message done') - start_pos = 0 - result = '' - find_json_pattern = re.compile(r'{[\s\S]+}') - START_OF_THINK_TAG, END_OF_THINK_TAG = '<|startofthink|>', '<|endofthink|>' - START_OF_EXEC_TAG, END_OF_EXEC_TAG = '<|startofexec|>', '<|endofexec|>' - while start_pos < len(bot_message): - try: - start_of_think_pos = bot_message.index(START_OF_THINK_TAG, - start_pos) - end_of_think_pos = bot_message.index(END_OF_THINK_TAG, - start_pos) - if start_pos < start_of_think_pos: - result += self.convert_markdown( - bot_message[start_pos:start_of_think_pos]) - think_content = bot_message[start_of_think_pos - + len(START_OF_THINK_TAG - ):end_of_think_pos].strip() - json_content = find_json_pattern.search(think_content) - think_content = json_content.group( - ) if json_content else think_content - try: - think_node = json.loads(think_content) - plugin_name = think_node.get( - 'plugin_name', - think_node.get('plugin', - think_node.get('api_name', 'unknown'))) - summary = f'选择插件【{plugin_name}】，调用处理中...' - del think_node['url'] - # think_node.pop('url', None) - - detail = f'```json\n\n{json.dumps(think_node, indent=3, ensure_ascii=False)}\n\n```' - except Exception: - summary = '思考中...' - detail = think_content - # traceback.print_exc() - # detail += traceback.format_exc() - result += '
' + summary + '' + self.convert_markdown( - detail) + '
' - # print(f'detail:{detail}') - start_pos = end_of_think_pos + len(END_OF_THINK_TAG) - except Exception: - # result += traceback.format_exc() - break - # continue - - try: - start_of_exec_pos = bot_message.index(START_OF_EXEC_TAG, - start_pos) - end_of_exec_pos = bot_message.index(END_OF_EXEC_TAG, start_pos) - # print(start_of_exec_pos) - # print(end_of_exec_pos) - # print(bot_message[start_of_exec_pos:end_of_exec_pos]) - # print('------------------------') - if start_pos < start_of_exec_pos: - result += self.convert_markdown( - bot_message[start_pos:start_of_think_pos]) - exec_content = bot_message[start_of_exec_pos - + len(START_OF_EXEC_TAG - ):end_of_exec_pos].strip() - try: - summary = '完成插件调用.' - detail = f'```json\n\n{exec_content}\n\n```' - except Exception: - pass - - result += '
' + summary + '' + self.convert_markdown( - detail) + '
' - - start_pos = end_of_exec_pos + len(END_OF_EXEC_TAG) - except Exception: - # result += traceback.format_exc() - continue - if start_pos < len(bot_message): - result += self.convert_markdown(bot_message[start_pos:]) - result += ALREADY_CONVERTED_MARK - return result - - def convert_bot_message_for_qwen(self, bot_message): - - start_pos = 0 - result = '' - find_json_pattern = re.compile(r'{[\s\S]+}') - ACTION = 'Action:' - ACTION_INPUT = 'Action Input' - OBSERVATION = 'Observation' - RESULT_START = '' - RESULT_END = '' - while start_pos < len(bot_message): - try: - action_pos = bot_message.index(ACTION, start_pos) - action_input_pos = bot_message.index(ACTION_INPUT, start_pos) - result += self.convert_markdown( - bot_message[start_pos:action_pos]) - # Action: image_gen - # Action Input - # {"text": "金庸武侠 世界", "resolution": "1280x720"} - # Observation: ![IMAGEGEN](https://dashscope-result-sh.oss-cn-shanghai.aliyuncs.com/1d/e9/20231116/723609ee/d046d2d9-0c95-420b-9467-f0e831f5e2b7-1.png?Expires=1700227460&OSSAccessKeyId=LTAI5tQZd8AEcZX6KZV4G8qL&Signature=R0PlEazQF9uBD%2Fh9tkzOkJMGyg8%3D) # noqa E501 - action_name = bot_message[action_pos - + len(ACTION - ):action_input_pos].strip() - # action_start action_end 使用 Action Input 到 Observation 之间 - action_input_end = bot_message[action_input_pos:].index( - OBSERVATION) - 1 - action_input = bot_message[action_input_pos:action_input_pos - + action_input_end].strip() - is_json = find_json_pattern.search(action_input) - if is_json: - action_input = is_json.group() - else: - action_input = re.sub(r'^Action Input[:]?[\s]*', '', - action_input) - - summary = f'调用工具 {action_name}' - if is_json: - detail = f'```json\n\n{json.dumps(json.loads(action_input), indent=4, ensure_ascii=False)}\n\n```' - else: - detail = action_input - result += '
' + summary + '' + self.convert_markdown( - detail) + '
' - start_pos = action_input_pos + action_input_end + 1 - try: - observation_pos = bot_message.index(OBSERVATION, start_pos) - idx = observation_pos + len(OBSERVATION) - obs_message = bot_message[idx:] - observation_start_id = obs_message.index( - RESULT_START) + len(RESULT_START) - observation_end_idx = obs_message.index(RESULT_END) - summary = '完成调用' - exec_content = obs_message[ - observation_start_id:observation_end_idx] - detail = f'```\n\n{exec_content}\n\n```' - start_pos = idx + observation_end_idx + len(RESULT_END) - except Exception: - summary = '执行中...' - detail = '' - exec_content = None - - result += '
' + summary + '' + self.convert_markdown( - detail) + '
' - if exec_content is not None and '[IMAGEGEN]' in exec_content: - # convert local file to base64 - re_pattern = re.compile(pattern=r'!\[[^\]]+\]\(([^)]+)\)') - res = re_pattern.search(exec_content) - if res: - image_path = res.group(1).strip() - if os.path.isfile(image_path): - exec_content = convert_url( - exec_content, - covert_image_to_base64(image_path)) - result += self.convert_markdown(f'{exec_content}') - - except Exception: - # import traceback; traceback.print_exc() - result += self.convert_markdown(bot_message[start_pos:]) - start_pos = len(bot_message[start_pos:]) - break - - result += ALREADY_CONVERTED_MARK - return result - - def postprocess( - self, - message_pairs: list[list[str | tuple[str] | tuple[str, str] | None] - | tuple], - ) -> list[list[str | dict | None]]: - """ - Parameters: - message_pairs: List of lists representing the message and response pairs. - Each message and response should be a string, which may be in Markdown format. - It can also be a tuple whose first element is a string or pathlib. - Path filepath or URL to an image/video/audio, and second (optional) element is the alt text, - in which case the media file is displayed. It can also be None, in which case that message is not displayed. - Returns: - List of lists representing the message and response. Each message and response will be a string of HTML, - or a dictionary with media information. Or None if the message is not to be displayed. - """ - if message_pairs is None: - return [] - processed_messages = [] - for message_pair in message_pairs: - assert isinstance( - message_pair, (tuple, list) - ), f'Expected a list of lists or list of tuples. Received: {message_pair}' - assert ( - len(message_pair) == 2 - ), f'Expected a list of lists of length 2 or list of tuples of length 2. Received: {message_pair}' - if isinstance(message_pair[0], tuple) or isinstance( - message_pair[1], tuple): - processed_messages.append([ - self._postprocess_chat_messages(message_pair[0]), - self._postprocess_chat_messages(message_pair[1]), - ]) - else: - # 处理不是元组的情况 - user_message, bot_message = message_pair - - if user_message and not user_message.endswith( - ALREADY_CONVERTED_MARK): - convert_md = self.convert_markdown( - html.escape(user_message)) - user_message = f'{convert_md}' + ALREADY_CONVERTED_MARK - if bot_message and not bot_message.endswith( - ALREADY_CONVERTED_MARK): - # bot_message = self.convert_bot_message(bot_message) - bot_message = self.convert_bot_message_for_qwen( - bot_message) - processed_messages.append([ - user_message, - bot_message, - ]) - - return processed_messages + ]) + return processed_messages diff --git a/apps/agentfabric/requirements.txt b/apps/agentfabric/requirements.txt index 03b5ba90..b43cc31f 100644 --- a/apps/agentfabric/requirements.txt +++ b/apps/agentfabric/requirements.txt @@ -1,6 +1,7 @@ dashscope faiss-cpu -gradio==3.47.1 +gradio==4.8 +https://modelscope-demo-widget.oss-cn-zhangjiakou.aliyuncs.com/modelscope_gradio_components-0.0.1-py3-none-any.whl langchain markdown-cjk-spacing mdx_truly_sane_lists diff --git a/apps/agentfabric/user_core.py b/apps/agentfabric/user_core.py index b8f790b6..de6b0cae 100644 --- a/apps/agentfabric/user_core.py +++ b/apps/agentfabric/user_core.py @@ -30,8 +30,11 @@ def init_user_chatbot_agent(uuid_str='', session='default'): function_list = parse_tool_cfg(tool_cfg) function_list = add_openapi_plugin_to_additional_tool( plugin_cfg, function_list) + llm_config = {'model': builder_cfg.model, 'model_server': 'dashscope'} - agent = RolePlay(function_list=function_list, llm=llm_config) + instruction = builder_cfg.instruction + agent = RolePlay( + function_list=function_list, llm=llm_config, instruction=instruction) # build memory storage_path = get_user_dir(uuid_str) diff --git a/apps/msgpt/gradio_chatbot.py b/apps/msgpt/gradio_chatbot.py index 7316ad00..f6d54cdd 100644 --- a/apps/msgpt/gradio_chatbot.py +++ b/apps/msgpt/gradio_chatbot.py @@ -198,7 +198,7 @@ def process_exec_result(self, exec_result: str): return final_result match_audio = re.search( - r'