From 88033b3c6964d95a7809d8512e704112ef4a712f Mon Sep 17 00:00:00 2001 From: skyline2006 Date: Mon, 29 Jan 2024 23:37:14 +0800 Subject: [PATCH 1/5] feat: openapi --- modelscope_agent/llm/__init__.py | 4 +-- modelscope_agent/llm/base.py | 4 +-- modelscope_agent/llm/openai.py | 54 +++++++++++++++++++++++++++----- 3 files changed, 51 insertions(+), 11 deletions(-) diff --git a/modelscope_agent/llm/__init__.py b/modelscope_agent/llm/__init__.py index 84b3974de..39b717293 100644 --- a/modelscope_agent/llm/__init__.py +++ b/modelscope_agent/llm/__init__.py @@ -3,7 +3,7 @@ from .base import LLM_REGISTRY, BaseChatModel from .dashscope import DashScopeLLM, QwenChatAtDS from .modelscope import ModelScopeChatGLM, ModelScopeLLM -from .openai import OpenAi +from .openai import OpenAi, OpenAPILocal from .zhipu import GLM4, ZhipuLLM @@ -25,5 +25,5 @@ def get_chat_model(model: str, model_server: str, **kwargs) -> BaseChatModel: __all__ = [ 'LLM_REGISTRY', 'BaseChatModel', 'OpenAi', 'DashScopeLLM', 'QwenChatAtDS', - 'ModelScopeLLM', 'ModelScopeChatGLM', 'ZhipuLLM', 'GLM4' + 'ModelScopeLLM', 'ModelScopeChatGLM', 'ZhipuLLM', 'GLM4', 'OpenAPILocal' ] diff --git a/modelscope_agent/llm/base.py b/modelscope_agent/llm/base.py index 5702ca93c..86f2755da 100644 --- a/modelscope_agent/llm/base.py +++ b/modelscope_agent/llm/base.py @@ -74,9 +74,9 @@ def chat(self, assert len(messages) > 0, 'messages list must not be empty' if stream: - return self._chat_stream(messages, stop=stop, **kwargs) + return self._chat_stream(messages=messages, stop=stop, prompt=prompt, **kwargs) else: - return self._chat_no_stream(messages, stop=stop, **kwargs) + return self._chat_no_stream(messages=messages, stop=stop, prompt=prompt, **kwargs) @retry(max_retries=3, delay_seconds=0.5) def chat_with_functions(self, diff --git a/modelscope_agent/llm/openai.py b/modelscope_agent/llm/openai.py index 530949360..4d6a65017 100644 --- a/modelscope_agent/llm/openai.py +++ b/modelscope_agent/llm/openai.py @@ -1,7 +1,7 @@ import os from typing import Dict, Iterator, List, Optional -import openai +from openai import OpenAI from modelscope_agent.llm.base import BaseChatModel, register_llm @@ -11,16 +11,17 @@ class OpenAi(BaseChatModel): def __init__(self, model: str, model_server: str, **kwargs): super().__init__(model, model_server) - openai.api_base = kwargs.get('api_base', + api_base = kwargs.get('api_base', 'https://api.openai.com/v1').strip() - openai.api_key = kwargs.get( + api_key = kwargs.get( 'api_key', os.getenv('OPENAI_API_KEY', default='EMPTY')).strip() + self.client = OpenAI(api_key=api_key, base_url=api_base) def _chat_stream(self, messages: List[Dict], stop: Optional[List[str]] = None, **kwargs) -> Iterator[str]: - response = openai.ChatCompletion.create( + response = self.client.completions.create( model=self.model, messages=messages, stop=stop, @@ -35,7 +36,7 @@ def _chat_no_stream(self, messages: List[Dict], stop: Optional[List[str]] = None, **kwargs) -> str: - response = openai.ChatCompletion.create( + response = self.client.completions.create( model=self.model, messages=messages, stop=stop, @@ -49,13 +50,52 @@ def chat_with_functions(self, functions: Optional[List[Dict]] = None, **kwargs) -> Dict: if functions: - response = openai.ChatCompletion.create( + response = self.client.completions.create( model=self.model, messages=messages, functions=functions, **kwargs) else: - response = openai.ChatCompletion.create( + response = self.client.completions.create( model=self.model, messages=messages, **kwargs) # TODO: error handling return response.choices[0].message + + +@register_llm('openapi') +class OpenAPILocal(BaseChatModel): + def __init__(self, model: str, model_server: str, **kwargs): + super().__init__(model, model_server) + openai_api_key = "EMPTY" + openai_api_base = "http://localhost:8000/v1" + self.client = OpenAI( + api_key=openai_api_key, + base_url=openai_api_base, + ) + + def _chat_stream(self, + prompt: str, + **kwargs) -> Iterator[str]: + response = self.client.completions.create( + model=self.model, + prompt=prompt, + stream=True, + ) + # TODO: error handling + for chunk in response: + if hasattr(chunk.choices[0], 'text'): + yield chunk.choices[0].text + + def _chat_no_stream(self, + prompt: str, + **kwargs) -> str: + response = self.client.completions.create( + model=self.model, + prompt=prompt, + stream=False, + ) + # TODO: error handling + return response.choices[0].message.content + + def support_function_calling(self) -> bool: + return False From f0ca639bb347684f81537d5154d89c45f0242bf0 Mon Sep 17 00:00:00 2001 From: skyline2006 Date: Tue, 30 Jan 2024 10:12:46 +0800 Subject: [PATCH 2/5] fix pre-commit --- modelscope_agent/llm/base.py | 6 ++++-- modelscope_agent/llm/openai.py | 29 +++++++++++++---------------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/modelscope_agent/llm/base.py b/modelscope_agent/llm/base.py index 86f2755da..f54c01e0c 100644 --- a/modelscope_agent/llm/base.py +++ b/modelscope_agent/llm/base.py @@ -74,9 +74,11 @@ def chat(self, assert len(messages) > 0, 'messages list must not be empty' if stream: - return self._chat_stream(messages=messages, stop=stop, prompt=prompt, **kwargs) + return self._chat_stream( + messages=messages, stop=stop, prompt=prompt, **kwargs) else: - return self._chat_no_stream(messages=messages, stop=stop, prompt=prompt, **kwargs) + return self._chat_no_stream( + messages=messages, stop=stop, prompt=prompt, **kwargs) @retry(max_retries=3, delay_seconds=0.5) def chat_with_functions(self, diff --git a/modelscope_agent/llm/openai.py b/modelscope_agent/llm/openai.py index 4d6a65017..6f38d73a8 100644 --- a/modelscope_agent/llm/openai.py +++ b/modelscope_agent/llm/openai.py @@ -1,8 +1,8 @@ import os from typing import Dict, Iterator, List, Optional -from openai import OpenAI from modelscope_agent.llm.base import BaseChatModel, register_llm +from openai import OpenAI @register_llm('openai') @@ -11,10 +11,10 @@ class OpenAi(BaseChatModel): def __init__(self, model: str, model_server: str, **kwargs): super().__init__(model, model_server) - api_base = kwargs.get('api_base', - 'https://api.openai.com/v1').strip() - api_key = kwargs.get( - 'api_key', os.getenv('OPENAI_API_KEY', default='EMPTY')).strip() + api_base = kwargs.get('api_base', 'https://api.openai.com/v1').strip() + api_key = kwargs.get('api_key', + os.getenv('OPENAI_API_KEY', + default='EMPTY')).strip() self.client = OpenAI(api_key=api_key, base_url=api_base) def _chat_stream(self, @@ -64,36 +64,33 @@ def chat_with_functions(self, @register_llm('openapi') class OpenAPILocal(BaseChatModel): + def __init__(self, model: str, model_server: str, **kwargs): super().__init__(model, model_server) - openai_api_key = "EMPTY" - openai_api_base = "http://localhost:8000/v1" + openai_api_key = 'EMPTY' + openai_api_base = 'http://localhost:8000/v1' self.client = OpenAI( api_key=openai_api_key, base_url=openai_api_base, - ) + ) - def _chat_stream(self, - prompt: str, - **kwargs) -> Iterator[str]: + def _chat_stream(self, prompt: str, **kwargs) -> Iterator[str]: response = self.client.completions.create( model=self.model, prompt=prompt, stream=True, - ) + ) # TODO: error handling for chunk in response: if hasattr(chunk.choices[0], 'text'): yield chunk.choices[0].text - def _chat_no_stream(self, - prompt: str, - **kwargs) -> str: + def _chat_no_stream(self, prompt: str, **kwargs) -> str: response = self.client.completions.create( model=self.model, prompt=prompt, stream=False, - ) + ) # TODO: error handling return response.choices[0].message.content From 782b034befa69a0c7082669f92d527cac3ad158d Mon Sep 17 00:00:00 2001 From: suluyan Date: Thu, 1 Feb 2024 13:47:25 +0800 Subject: [PATCH 3/5] update openai llm --- modelscope_agent/llm/openai.py | 100 ++++++++++++++++++++------------- 1 file changed, 60 insertions(+), 40 deletions(-) diff --git a/modelscope_agent/llm/openai.py b/modelscope_agent/llm/openai.py index 6f38d73a8..8b770aa36 100644 --- a/modelscope_agent/llm/openai.py +++ b/modelscope_agent/llm/openai.py @@ -1,6 +1,7 @@ import os -from typing import Dict, Iterator, List, Optional +from typing import Dict, Iterator, List, Optional, Union +from modelscope_agent.utils.retry import retry from modelscope_agent.llm.base import BaseChatModel, register_llm from openai import OpenAI @@ -8,24 +9,28 @@ @register_llm('openai') class OpenAi(BaseChatModel): - def __init__(self, model: str, model_server: str, **kwargs): + def __init__(self, model: str, model_server: str, is_chat: bool = True, is_function_call: Optional[bool] = None, support_stream: Optional[bool] = None, **kwargs): super().__init__(model, model_server) + print(kwargs, model, model_server) api_base = kwargs.get('api_base', 'https://api.openai.com/v1').strip() api_key = kwargs.get('api_key', os.getenv('OPENAI_API_KEY', default='EMPTY')).strip() self.client = OpenAI(api_key=api_key, base_url=api_base) + self.is_function_call = is_function_call + self.is_chat = is_chat + self.support_stream = support_stream def _chat_stream(self, messages: List[Dict], stop: Optional[List[str]] = None, **kwargs) -> Iterator[str]: - response = self.client.completions.create( + response = self.client.chat.completions.create( model=self.model, messages=messages, stop=stop, - stream=True, + stream=True , **kwargs) # TODO: error handling for chunk in response: @@ -36,7 +41,7 @@ def _chat_no_stream(self, messages: List[Dict], stop: Optional[List[str]] = None, **kwargs) -> str: - response = self.client.completions.create( + response = self.client.chat.completions.create( model=self.model, messages=messages, stop=stop, @@ -45,6 +50,56 @@ def _chat_no_stream(self, # TODO: error handling return response.choices[0].message.content + def support_function_calling(self): + if self.is_function_call == None: + return super().support_function_calling() + else: + return self.is_function_call + + def support_raw_prompt(self) -> bool: + if self.is_chat == None: + return super().support_raw_prompt() + else: + return self.is_chat + + @retry(max_retries=3, delay_seconds=0.5) + def chat(self, + prompt: Optional[str] = None, + messages: Optional[List[Dict]] = None, + stop: Optional[List[str]] = None, + stream: bool = False, + **kwargs) -> Union[str, Iterator[str]]: + if isinstance(self.support_stream, bool): + stream = self.support_stream + if self.support_raw_prompt(): + return self.chat_with_raw_prompt(prompt=prompt, stream=stream, stop=stop, **kwargs) + if not messages and prompt and isinstance(prompt, str): + messages = [{'role': 'user', 'content': prompt}] + return super().chat(messages=messages, stop=stop, stream=stream, **kwargs) + + def chat_with_raw_prompt(self, + prompt: str, + stream: bool=True, + **kwargs) -> str: + from prompts import prompts + prompt = prompts[self.model] + max_tokens = kwargs.get('max_tokens', 2000) + response = self.client.completions.create( + model=self.model, + prompt=prompt, + stream=stream, + max_tokens=max_tokens + ) + print(response) + # TODO: error handling + if stream: + for chunk in response: + if hasattr(chunk.choices[0], 'text'): + yield chunk.choices[0].text + else: + return response.choices[0].text + + def chat_with_functions(self, messages: List[Dict], functions: Optional[List[Dict]] = None, @@ -61,38 +116,3 @@ def chat_with_functions(self, # TODO: error handling return response.choices[0].message - -@register_llm('openapi') -class OpenAPILocal(BaseChatModel): - - def __init__(self, model: str, model_server: str, **kwargs): - super().__init__(model, model_server) - openai_api_key = 'EMPTY' - openai_api_base = 'http://localhost:8000/v1' - self.client = OpenAI( - api_key=openai_api_key, - base_url=openai_api_base, - ) - - def _chat_stream(self, prompt: str, **kwargs) -> Iterator[str]: - response = self.client.completions.create( - model=self.model, - prompt=prompt, - stream=True, - ) - # TODO: error handling - for chunk in response: - if hasattr(chunk.choices[0], 'text'): - yield chunk.choices[0].text - - def _chat_no_stream(self, prompt: str, **kwargs) -> str: - response = self.client.completions.create( - model=self.model, - prompt=prompt, - stream=False, - ) - # TODO: error handling - return response.choices[0].message.content - - def support_function_calling(self) -> bool: - return False From 233147002e88585a66437a1122d3f3a6ec8593f4 Mon Sep 17 00:00:00 2001 From: suluyan Date: Thu, 1 Feb 2024 13:47:25 +0800 Subject: [PATCH 4/5] fix bugs --- modelscope_agent/llm/__init__.py | 4 ++-- modelscope_agent/llm/openai.py | 33 ++++++++++++++++++-------------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/modelscope_agent/llm/__init__.py b/modelscope_agent/llm/__init__.py index 39b717293..84b3974de 100644 --- a/modelscope_agent/llm/__init__.py +++ b/modelscope_agent/llm/__init__.py @@ -3,7 +3,7 @@ from .base import LLM_REGISTRY, BaseChatModel from .dashscope import DashScopeLLM, QwenChatAtDS from .modelscope import ModelScopeChatGLM, ModelScopeLLM -from .openai import OpenAi, OpenAPILocal +from .openai import OpenAi from .zhipu import GLM4, ZhipuLLM @@ -25,5 +25,5 @@ def get_chat_model(model: str, model_server: str, **kwargs) -> BaseChatModel: __all__ = [ 'LLM_REGISTRY', 'BaseChatModel', 'OpenAi', 'DashScopeLLM', 'QwenChatAtDS', - 'ModelScopeLLM', 'ModelScopeChatGLM', 'ZhipuLLM', 'GLM4', 'OpenAPILocal' + 'ModelScopeLLM', 'ModelScopeChatGLM', 'ZhipuLLM', 'GLM4' ] diff --git a/modelscope_agent/llm/openai.py b/modelscope_agent/llm/openai.py index 8b770aa36..05369b380 100644 --- a/modelscope_agent/llm/openai.py +++ b/modelscope_agent/llm/openai.py @@ -1,18 +1,23 @@ import os from typing import Dict, Iterator, List, Optional, Union -from modelscope_agent.utils.retry import retry from modelscope_agent.llm.base import BaseChatModel, register_llm +from modelscope_agent.utils.retry import retry from openai import OpenAI @register_llm('openai') class OpenAi(BaseChatModel): - def __init__(self, model: str, model_server: str, is_chat: bool = True, is_function_call: Optional[bool] = None, support_stream: Optional[bool] = None, **kwargs): + def __init__(self, + model: str, + model_server: str, + is_chat: bool = True, + is_function_call: Optional[bool] = None, + support_stream: Optional[bool] = None, + **kwargs): super().__init__(model, model_server) - print(kwargs, model, model_server) api_base = kwargs.get('api_base', 'https://api.openai.com/v1').strip() api_key = kwargs.get('api_key', os.getenv('OPENAI_API_KEY', @@ -30,7 +35,7 @@ def _chat_stream(self, model=self.model, messages=messages, stop=stop, - stream=True , + stream=True, **kwargs) # TODO: error handling for chunk in response: @@ -60,7 +65,8 @@ def support_raw_prompt(self) -> bool: if self.is_chat == None: return super().support_raw_prompt() else: - return self.is_chat + # if not chat, then prompt + return not self.is_chat @retry(max_retries=3, delay_seconds=0.5) def chat(self, @@ -77,12 +83,15 @@ def chat(self, messages = [{'role': 'user', 'content': prompt}] return super().chat(messages=messages, stop=stop, stream=stream, **kwargs) + def _out_generator(self, response): + for chunk in response: + if hasattr(chunk.choices[0], 'text'): + yield chunk.choices[0].text + def chat_with_raw_prompt(self, prompt: str, - stream: bool=True, + stream: bool = True, **kwargs) -> str: - from prompts import prompts - prompt = prompts[self.model] max_tokens = kwargs.get('max_tokens', 2000) response = self.client.completions.create( model=self.model, @@ -90,16 +99,13 @@ def chat_with_raw_prompt(self, stream=stream, max_tokens=max_tokens ) - print(response) + # TODO: error handling if stream: - for chunk in response: - if hasattr(chunk.choices[0], 'text'): - yield chunk.choices[0].text + return self._out_generator(response) else: return response.choices[0].text - def chat_with_functions(self, messages: List[Dict], functions: Optional[List[Dict]] = None, @@ -115,4 +121,3 @@ def chat_with_functions(self, model=self.model, messages=messages, **kwargs) # TODO: error handling return response.choices[0].message - From ecae5169f49be4fe7b9d5143fb7d4a7fa409a109 Mon Sep 17 00:00:00 2001 From: skyline2006 Date: Sun, 4 Feb 2024 16:03:30 +0800 Subject: [PATCH 5/5] fix pre-commit --- modelscope_agent/llm/openai.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/modelscope_agent/llm/openai.py b/modelscope_agent/llm/openai.py index 05369b380..88413bc94 100644 --- a/modelscope_agent/llm/openai.py +++ b/modelscope_agent/llm/openai.py @@ -56,13 +56,13 @@ def _chat_no_stream(self, return response.choices[0].message.content def support_function_calling(self): - if self.is_function_call == None: + if self.is_function_call is None: return super().support_function_calling() else: return self.is_function_call def support_raw_prompt(self) -> bool: - if self.is_chat == None: + if self.is_chat is None: return super().support_raw_prompt() else: # if not chat, then prompt @@ -78,10 +78,12 @@ def chat(self, if isinstance(self.support_stream, bool): stream = self.support_stream if self.support_raw_prompt(): - return self.chat_with_raw_prompt(prompt=prompt, stream=stream, stop=stop, **kwargs) + return self.chat_with_raw_prompt( + prompt=prompt, stream=stream, stop=stop, **kwargs) if not messages and prompt and isinstance(prompt, str): messages = [{'role': 'user', 'content': prompt}] - return super().chat(messages=messages, stop=stop, stream=stream, **kwargs) + return super().chat( + messages=messages, stop=stop, stream=stream, **kwargs) def _out_generator(self, response): for chunk in response: @@ -97,8 +99,7 @@ def chat_with_raw_prompt(self, model=self.model, prompt=prompt, stream=stream, - max_tokens=max_tokens - ) + max_tokens=max_tokens) # TODO: error handling if stream: