From cd0f14b238dbf53dfded8a9452d828fc09b29ff0 Mon Sep 17 00:00:00 2001 From: andy-yang-1 Date: Fri, 1 Dec 2023 21:14:37 +0000 Subject: [PATCH 1/3] support openai>=1.0.0 --- fastchat/serve/openai_api_server.py | 1 + .../test_sentence_similarity.py | 2 +- tests/test_openai_api.py | 114 +++++++++++++----- 3 files changed, 87 insertions(+), 30 deletions(-) diff --git a/fastchat/serve/openai_api_server.py b/fastchat/serve/openai_api_server.py index 120bc42e1..a42c8f00c 100644 --- a/fastchat/serve/openai_api_server.py +++ b/fastchat/serve/openai_api_server.py @@ -375,6 +375,7 @@ async def show_available_models(): return ModelList(data=model_cards) +@app.post("/v1chat/completions", dependencies=[Depends(check_api_key)]) @app.post("/v1/chat/completions", dependencies=[Depends(check_api_key)]) async def create_chat_completion(request: ChatCompletionRequest): """Creates a completion for the chat message""" diff --git a/playground/test_embedding/test_sentence_similarity.py b/playground/test_embedding/test_sentence_similarity.py index 0b9a54081..d7a8f6e5f 100644 --- a/playground/test_embedding/test_sentence_similarity.py +++ b/playground/test_embedding/test_sentence_similarity.py @@ -7,7 +7,7 @@ from scipy.spatial.distance import cosine -def get_embedding_from_api(word, model="vicuna-7b-v1.1"): +def get_embedding_from_api(word, model="vicuna-7b-v1.5"): if "ada" in model: resp = openai.Embedding.create( model=model, diff --git a/tests/test_openai_api.py b/tests/test_openai_api.py index 064069833..08d23bc05 100644 --- a/tests/test_openai_api.py +++ b/tests/test_openai_api.py @@ -4,8 +4,14 @@ Launch: python3 launch_openai_api_test_server.py """ +from distutils.version import LooseVersion +import warnings import openai +try: + from openai import OpenAI, AsyncOpenAI +except ImportError: + warnings.warn("openai<1.0 is deprecated") from fastchat.utils import run_cmd @@ -14,20 +20,36 @@ def test_list_models(): - model_list = openai.Model.list() - names = [x["id"] for x in model_list["data"]] + if LooseVersion(openai.__version__) < LooseVersion("1.0"): + model_list = openai.Model.list() + else: + client = OpenAI(api_key=openai.api_key, base_url=openai.api_base) + model_list = client.models.list() + names = [x.id for x in model_list.data] return names def test_completion(model, logprob): prompt = "Once upon a time" - completion = openai.Completion.create( - model=model, - prompt=prompt, - logprobs=logprob, - max_tokens=64, - temperature=0, - ) + if LooseVersion(openai.__version__) < LooseVersion("1.0"): + completion = openai.Completion.create( + model=model, + prompt=prompt, + logprobs=logprob, + max_tokens=64, + temperature=0, + ) + else: + client = OpenAI(api_key=openai.api_key, base_url=openai.api_base) + # legacy + completion = client.completions.create( + model=model, + prompt=prompt, + logprobs=logprob, + max_tokens=64, + temperature=0, + ) + print(f"full text: {prompt + completion.choices[0].text}", flush=True) if completion.choices[0].logprobs is not None: print( @@ -38,42 +60,76 @@ def test_completion(model, logprob): def test_completion_stream(model): prompt = "Once upon a time" - res = openai.Completion.create( - model=model, - prompt=prompt, - max_tokens=64, - stream=True, - temperature=0, - ) + if LooseVersion(openai.__version__) < LooseVersion("1.0"): + res = openai.Completion.create( + model=model, + prompt=prompt, + max_tokens=64, + stream=True, + temperature=0, + ) + else: + client = OpenAI(api_key=openai.api_key, base_url=openai.api_base) + # legacy + res = client.completions.create( + model=model, + prompt=prompt, + max_tokens=64, + stream=True, + temperature=0, + ) print(prompt, end="") for chunk in res: - content = chunk["choices"][0]["text"] + content = chunk.choices[0].text print(content, end="", flush=True) print() def test_embedding(model): - embedding = openai.Embedding.create(model=model, input="Hello world!") - print(f"embedding len: {len(embedding['data'][0]['embedding'])}") - print(f"embedding value[:5]: {embedding['data'][0]['embedding'][:5]}") + if LooseVersion(openai.__version__) < LooseVersion("1.0"): + embedding = openai.Embedding.create(model=model, input="Hello world!") + else: + client = OpenAI(api_key=openai.api_key, base_url=openai.api_base) + embedding = client.embeddings.create(model=model, input="Hello world!") + print(f"embedding len: {len(embedding.data[0].embedding)}") + print(f"embedding value[:5]: {embedding.data[0].embedding[:5]}") def test_chat_completion(model): - completion = openai.ChatCompletion.create( - model=model, - messages=[{"role": "user", "content": "Hello! What is your name?"}], - temperature=0, - ) + if LooseVersion(openai.__version__) < LooseVersion("1.0"): + completion = openai.ChatCompletion.create( + model=model, + messages=[{"role": "user", "content": "Hello! What is your name?"}], + temperature=0, + ) + else: + client = OpenAI(api_key=openai.api_key, base_url=openai.api_base) + completion = client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": "Hello! What is your name?"}], + temperature=0, + ) print(completion.choices[0].message.content) def test_chat_completion_stream(model): messages = [{"role": "user", "content": "Hello! What is your name?"}] - res = openai.ChatCompletion.create( - model=model, messages=messages, stream=True, temperature=0 - ) + if LooseVersion(openai.__version__) < LooseVersion("1.0"): + res = openai.ChatCompletion.create( + model=model, messages=messages, stream=True, temperature=0 + ) + else: + client = OpenAI(api_key=openai.api_key, base_url=openai.api_base) + res = client.chat.completions.create( + model=model, messages=messages, stream=True, temperature=0 + ) for chunk in res: - content = chunk["choices"][0]["delta"].get("content", "") + try: + content = chunk.choices[0].delta.content + if content is None: + content = "" + except Exception as e: + content = chunk.choices[0].delta.get("content", "") print(content, end="", flush=True) print() From 4def076562a2424252b4ab4c2b2dc172a106babd Mon Sep 17 00:00:00 2001 From: andy-yang-1 Date: Fri, 1 Dec 2023 21:21:46 +0000 Subject: [PATCH 2/3] update doc --- docs/openai_api.md | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/docs/openai_api.md b/docs/openai_api.md index f3c0fba93..aa86d5b39 100644 --- a/docs/openai_api.md +++ b/docs/openai_api.md @@ -39,24 +39,30 @@ pip install --upgrade openai Then, interact with model vicuna: ```python -import openai +from openai import OpenAI # to get proper authentication, make sure to use a valid key that's listed in # the --api-keys flag. if no flag value is provided, the `api_key` will be ignored. -openai.api_key = "EMPTY" -openai.api_base = "http://localhost:8000/v1" +client = OpenAI(api_key="EMPTY", base_url="http://localhost:8000/v1", default_headers={"x-foo": "true"}) model = "vicuna-7b-v1.5" prompt = "Once upon a time" -# create a completion -completion = openai.Completion.create(model=model, prompt=prompt, max_tokens=64) +# create a completion (legacy) +completion = client.completions.create( + model=model, + prompt=prompt +) # print the completion print(prompt + completion.choices[0].text) # create a chat completion -completion = openai.ChatCompletion.create( - model=model, - messages=[{"role": "user", "content": "Hello! What is your name?"}] +completion = client.chat.completions.create( + model="vicuna-7b-v1.5", + response_format={ "type": "json_object" }, + messages=[ + {"role": "system", "content": "You are a helpful assistant designed to output JSON."}, + {"role": "user", "content": "Who won the world series in 2020?"} + ] ) # print the completion print(completion.choices[0].message.content) From fdac4492d5c71f2a7ba914605e3f7b18fe323807 Mon Sep 17 00:00:00 2001 From: andy-yang-1 Date: Fri, 1 Dec 2023 22:34:46 +0000 Subject: [PATCH 3/3] fix style --- tests/test_openai_api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_openai_api.py b/tests/test_openai_api.py index 08d23bc05..82e95adcc 100644 --- a/tests/test_openai_api.py +++ b/tests/test_openai_api.py @@ -8,6 +8,7 @@ import warnings import openai + try: from openai import OpenAI, AsyncOpenAI except ImportError: