From 62ad7167333eea0f98f7136445da8f8c73b87b85 Mon Sep 17 00:00:00 2001 From: Rodo Date: Sun, 3 Mar 2024 20:47:46 +1000 Subject: [PATCH 01/14] test: discord_bot/llm.py --- src/__init__.py | 0 src/discord_bot/__init__.py | 0 src/discord_bot/bot.py | 4 ++-- src/discord_bot/llm.py | 16 ++++++++------- tests/__init__.py | 0 tests/test_demo.py | 2 -- tests/test_discord_bot_llm.py | 37 +++++++++++++++++++++++++++++++++++ 7 files changed, 48 insertions(+), 11 deletions(-) create mode 100644 src/__init__.py create mode 100644 src/discord_bot/__init__.py create mode 100644 tests/__init__.py delete mode 100644 tests/test_demo.py create mode 100644 tests/test_discord_bot_llm.py diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/discord_bot/__init__.py b/src/discord_bot/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/discord_bot/bot.py b/src/discord_bot/bot.py index 6cc5b7b..2f7d938 100644 --- a/src/discord_bot/bot.py +++ b/src/discord_bot/bot.py @@ -1,7 +1,7 @@ import os import interactions import dotenv -import llm +import src.discord_bot.llm dotenv.load_dotenv() @@ -41,7 +41,7 @@ async def ask_model(ctx: interactions.SlashContext, model: str = "", prompt: str await ctx.defer() - response = await llm.answer_question(model, prompt, AI_SERVER_URL) + response = await src.discord_bot.llm.answer_question(model, prompt, AI_SERVER_URL) await ctx.send(response) diff --git a/src/discord_bot/llm.py b/src/discord_bot/llm.py index 6c2e5de..2244bec 100644 --- a/src/discord_bot/llm.py +++ b/src/discord_bot/llm.py @@ -8,12 +8,14 @@ async def answer_question(model: str, question: str, server_url: str) -> str: - client = openai.AsyncOpenAI(base_url=server_url, api_key="FAKE") - response = await client.chat.completions.create( - model=model, - messages=[{"role": "user", "content": question}], - ) - - out = response.choices[0].message.content or "No response from the model" + try: + client = openai.AsyncOpenAI(base_url=server_url, api_key="FAKE") + response = await client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": question}], + ) + out = response.choices[0].message.content or "No response from the model" + except Exception as e: + out = f"Error: {e}" return out diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_demo.py b/tests/test_demo.py deleted file mode 100644 index b4840fb..0000000 --- a/tests/test_demo.py +++ /dev/null @@ -1,2 +0,0 @@ -def test_demo() -> None: - assert 1 == 1 diff --git a/tests/test_discord_bot_llm.py b/tests/test_discord_bot_llm.py new file mode 100644 index 0000000..2a44130 --- /dev/null +++ b/tests/test_discord_bot_llm.py @@ -0,0 +1,37 @@ +import os +import dotenv +import pytest +import src.discord_bot.llm + +AI_SERVER_URL = os.getenv("AI_SERVER_URL") or "http://localhost:8000" +dotenv.load_dotenv() + + +@pytest.mark.asyncio +async def test_answer_question__LLM_should_response() -> None: + model = "gpt-3.5-turbo" + prompt = "reply me with exactly 123hi and 123hi only, no capitalised letters" + + response = await src.discord_bot.llm.answer_question(model, prompt, AI_SERVER_URL) + + assert response == "123hi" + + +@pytest.mark.asyncio +async def test_answer_question__invalid_server_url() -> None: + model = "gpt-3.5-turbo" + prompt = "Hello, world!" + + response = await src.discord_bot.llm.answer_question(model, prompt, "http://fakeurl.com") + + assert response.startswith("Error") + + +@pytest.mark.asyncio +async def test_answer_question__invalid_model() -> None: + model = "not-a-gpt" + prompt = "Hello, world!" + + response = await src.discord_bot.llm.answer_question(model, prompt, AI_SERVER_URL) + + assert response.startswith("Error") From 6a9008f0da847a95a4d0c746847baafa2bdb624e Mon Sep 17 00:00:00 2001 From: Sam Huynh Date: Sun, 3 Mar 2024 23:42:14 +1100 Subject: [PATCH 02/14] chore: delete init files --- src/__init__.py | 0 src/discord_bot/__init__.py | 0 tests/__init__.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 src/__init__.py delete mode 100644 src/discord_bot/__init__.py delete mode 100644 tests/__init__.py diff --git a/src/__init__.py b/src/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/discord_bot/__init__.py b/src/discord_bot/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29..0000000 From bffec00ee1a149b82fbd0c184723db64f87632ac Mon Sep 17 00:00:00 2001 From: Sam Huynh Date: Sun, 3 Mar 2024 23:43:29 +1100 Subject: [PATCH 03/14] chore: add discord_bot path to src and update import path in test --- poetry.lock | 65 +++++++++++++++++++++-------------- pyproject.toml | 1 + tests/test_discord_bot_llm.py | 8 ++--- 3 files changed, 45 insertions(+), 29 deletions(-) diff --git a/poetry.lock b/poetry.lock index e4ab6d8..70efa58 100644 --- a/poetry.lock +++ b/poetry.lock @@ -168,6 +168,17 @@ tornado = ["tornado (>=4.3)"] twisted = ["twisted"] zookeeper = ["kazoo"] +[[package]] +name = "async-timeout" +version = "4.0.3" +description = "Timeout context manager for asyncio programs" +optional = false +python-versions = ">=3.7" +files = [ + {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, + {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, +] + [[package]] name = "attrs" version = "23.2.0" @@ -431,13 +442,13 @@ wmi = ["wmi (>=1.5.1)"] [[package]] name = "email-validator" -version = "2.1.0.post1" +version = "2.1.1" description = "A robust email address syntax and deliverability validation library." optional = false python-versions = ">=3.8" files = [ - {file = "email_validator-2.1.0.post1-py3-none-any.whl", hash = "sha256:c973053efbeddfef924dc0bd93f6e77a1ea7ee0fce935aea7103c7a3d6d2d637"}, - {file = "email_validator-2.1.0.post1.tar.gz", hash = "sha256:a4b0bd1cf55f073b924258d19321b1f3aa74b4b5a71a42c305575dba920e1a44"}, + {file = "email_validator-2.1.1-py3-none-any.whl", hash = "sha256:97d882d174e2a65732fb43bfce81a3a834cbc1bde8bf419e30ef5ea976370a05"}, + {file = "email_validator-2.1.1.tar.gz", hash = "sha256:200a70680ba08904be6d1eef729205cc0d687634399a5924d842533efb824b84"}, ] [package.dependencies] @@ -710,13 +721,13 @@ socks = ["socksio (==1.*)"] [[package]] name = "huggingface-hub" -version = "0.20.3" +version = "0.21.3" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.8.0" files = [ - {file = "huggingface_hub-0.20.3-py3-none-any.whl", hash = "sha256:d988ae4f00d3e307b0c80c6a05ca6dbb7edba8bba3079f74cda7d9c2e562a7b6"}, - {file = "huggingface_hub-0.20.3.tar.gz", hash = "sha256:94e7f8e074475fbc67d6a71957b678e1b4a74ff1b64a644fd6cbb83da962d05d"}, + {file = "huggingface_hub-0.21.3-py3-none-any.whl", hash = "sha256:b183144336fdf2810a8c109822e0bb6ef1fd61c65da6fb60e8c3f658b7144016"}, + {file = "huggingface_hub-0.21.3.tar.gz", hash = "sha256:26a15b604e4fc7bad37c467b76456543ec849386cbca9cd7e1e135f53e500423"}, ] [package.dependencies] @@ -733,11 +744,12 @@ all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "gradio", "jedi", cli = ["InquirerPy (==0.3.4)"] dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "gradio", "jedi", "mypy (==1.5.1)", "numpy", "pydantic (>1.1,<2.0)", "pydantic (>1.1,<3.0)", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.1.3)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] +hf-transfer = ["hf-transfer (>=0.1.4)"] inference = ["aiohttp", "pydantic (>1.1,<2.0)", "pydantic (>1.1,<3.0)"] quality = ["mypy (==1.5.1)", "ruff (>=0.1.3)"] tensorflow = ["graphviz", "pydot", "tensorflow"] testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "gradio", "jedi", "numpy", "pydantic (>1.1,<2.0)", "pydantic (>1.1,<3.0)", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] -torch = ["torch"] +torch = ["safetensors", "torch"] typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] [[package]] @@ -814,13 +826,13 @@ i18n = ["Babel (>=2.7)"] [[package]] name = "litellm" -version = "1.27.1" +version = "1.28.11" description = "Library to easily interface with LLM API providers" optional = false python-versions = ">=3.8, !=2.7.*, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*, !=3.7.*" files = [ - {file = "litellm-1.27.1-py3-none-any.whl", hash = "sha256:39fb6b5c342684e404f0c7ca0e463e5ef5387a86118bce7173ac494c4f1082b2"}, - {file = "litellm-1.27.1.tar.gz", hash = "sha256:a425d910783bef5330a34beef26d25829337d92330fc40dc297e72bd0b77d04a"}, + {file = "litellm-1.28.11-py3-none-any.whl", hash = "sha256:0f8ca98ce8391d2f44c188ccfe54309d023fd77b0c2e897c68431c08040282cd"}, + {file = "litellm-1.28.11.tar.gz", hash = "sha256:34fe319e06d1791b8e9b6b31e34af140d596014420258f986038a54470d7447c"}, ] [package.dependencies] @@ -1106,13 +1118,13 @@ signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] [[package]] name = "openai" -version = "1.12.0" +version = "1.13.3" description = "The official Python library for the openai API" optional = false python-versions = ">=3.7.1" files = [ - {file = "openai-1.12.0-py3-none-any.whl", hash = "sha256:a54002c814e05222e413664f651b5916714e4700d041d5cf5724d3ae1a3e3481"}, - {file = "openai-1.12.0.tar.gz", hash = "sha256:99c5d257d09ea6533d689d1cc77caa0ac679fa21efef8893d8b0832a86877f1b"}, + {file = "openai-1.13.3-py3-none-any.whl", hash = "sha256:5769b62abd02f350a8dd1a3a242d8972c947860654466171d60fb0972ae0a41c"}, + {file = "openai-1.13.3.tar.gz", hash = "sha256:ff6c6b3bc7327e715e4b3592a923a5a1c7519ff5dd764a83d69f633d49e77a7b"}, ] [package.dependencies] @@ -1247,13 +1259,13 @@ virtualenv = ">=20.10.0" [[package]] name = "pydantic" -version = "2.6.2" +version = "2.6.3" description = "Data validation using Python type hints" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic-2.6.2-py3-none-any.whl", hash = "sha256:37a5432e54b12fecaa1049c5195f3d860a10e01bdfd24f1840ef14bd0d3aeab3"}, - {file = "pydantic-2.6.2.tar.gz", hash = "sha256:a09be1c3d28f3abe37f8a78af58284b236a92ce520105ddc91a6d29ea1176ba7"}, + {file = "pydantic-2.6.3-py3-none-any.whl", hash = "sha256:72c6034df47f46ccdf81869fddb81aade68056003900a8724a4f160700016a2a"}, + {file = "pydantic-2.6.3.tar.gz", hash = "sha256:e07805c4c7f5c6826e33a1d4c9d47950d7eaf34868e2690f8594d2e30241f11f"}, ] [package.dependencies] @@ -1494,15 +1506,18 @@ files = [ [[package]] name = "redis" -version = "5.0.1" +version = "5.0.2" description = "Python client for Redis database and key-value store" optional = false python-versions = ">=3.7" files = [ - {file = "redis-5.0.1-py3-none-any.whl", hash = "sha256:ed4802971884ae19d640775ba3b03aa2e7bd5e8fb8dfaed2decce4d0fc48391f"}, - {file = "redis-5.0.1.tar.gz", hash = "sha256:0dab495cd5753069d3bc650a0dde8a8f9edde16fc5691b689a566eda58100d0f"}, + {file = "redis-5.0.2-py3-none-any.whl", hash = "sha256:4caa8e1fcb6f3c0ef28dba99535101d80934b7d4cd541bbb47f4a3826ee472d1"}, + {file = "redis-5.0.2.tar.gz", hash = "sha256:3f82cc80d350e93042c8e6e7a5d0596e4dd68715babffba79492733e1f367037"}, ] +[package.dependencies] +async-timeout = ">=4.0.3" + [package.extras] hiredis = ["hiredis (>=1.0.0)"] ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)"] @@ -1700,13 +1715,13 @@ files = [ [[package]] name = "sniffio" -version = "1.3.0" +version = "1.3.1" description = "Sniff out which async library your code is running under" optional = false python-versions = ">=3.7" files = [ - {file = "sniffio-1.3.0-py3-none-any.whl", hash = "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"}, - {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, + {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"}, + {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, ] [[package]] @@ -1938,13 +1953,13 @@ telegram = ["requests"] [[package]] name = "typing-extensions" -version = "4.9.0" +version = "4.10.0" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, - {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, + {file = "typing_extensions-4.10.0-py3-none-any.whl", hash = "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475"}, + {file = "typing_extensions-4.10.0.tar.gz", hash = "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb"}, ] [[package]] diff --git a/pyproject.toml b/pyproject.toml index 1955808..95a3ba5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,6 +4,7 @@ version = "0.1.0" description = "" authors = ["VAIT"] readme = "README.md" +packages = [{include = "discord_bot", from = "src"}] [tool.poetry.dependencies] python = "^3.12" diff --git a/tests/test_discord_bot_llm.py b/tests/test_discord_bot_llm.py index 2a44130..c0539f8 100644 --- a/tests/test_discord_bot_llm.py +++ b/tests/test_discord_bot_llm.py @@ -1,7 +1,7 @@ import os import dotenv import pytest -import src.discord_bot.llm +from discord_bot.llm import answer_question AI_SERVER_URL = os.getenv("AI_SERVER_URL") or "http://localhost:8000" dotenv.load_dotenv() @@ -12,7 +12,7 @@ async def test_answer_question__LLM_should_response() -> None: model = "gpt-3.5-turbo" prompt = "reply me with exactly 123hi and 123hi only, no capitalised letters" - response = await src.discord_bot.llm.answer_question(model, prompt, AI_SERVER_URL) + response = await answer_question(model, prompt, AI_SERVER_URL) assert response == "123hi" @@ -22,7 +22,7 @@ async def test_answer_question__invalid_server_url() -> None: model = "gpt-3.5-turbo" prompt = "Hello, world!" - response = await src.discord_bot.llm.answer_question(model, prompt, "http://fakeurl.com") + response = await answer_question(model, prompt, "http://fakeurl.com") assert response.startswith("Error") @@ -32,6 +32,6 @@ async def test_answer_question__invalid_model() -> None: model = "not-a-gpt" prompt = "Hello, world!" - response = await src.discord_bot.llm.answer_question(model, prompt, AI_SERVER_URL) + response = await answer_question(model, prompt, AI_SERVER_URL) assert response.startswith("Error") From d86bbbc5acb5a41f655a502c76af37a1d65a8de8 Mon Sep 17 00:00:00 2001 From: Duy Hung Tran Date: Mon, 4 Mar 2024 10:46:03 +0700 Subject: [PATCH 04/14] test: litellm should handle concurrent request at the same time --- tests/test_discord_bot_llm.py | 40 +++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/tests/test_discord_bot_llm.py b/tests/test_discord_bot_llm.py index c0539f8..79506ef 100644 --- a/tests/test_discord_bot_llm.py +++ b/tests/test_discord_bot_llm.py @@ -1,3 +1,5 @@ +import asyncio +import time import os import dotenv import pytest @@ -9,17 +11,17 @@ @pytest.mark.asyncio async def test_answer_question__LLM_should_response() -> None: - model = "gpt-3.5-turbo" - prompt = "reply me with exactly 123hi and 123hi only, no capitalised letters" + model = "phi" + prompt = "Respond shortly: hello!" response = await answer_question(model, prompt, AI_SERVER_URL) - assert response == "123hi" + assert not response.startswith("Error") @pytest.mark.asyncio async def test_answer_question__invalid_server_url() -> None: - model = "gpt-3.5-turbo" + model = "phi" prompt = "Hello, world!" response = await answer_question(model, prompt, "http://fakeurl.com") @@ -35,3 +37,33 @@ async def test_answer_question__invalid_model() -> None: response = await answer_question(model, prompt, AI_SERVER_URL) assert response.startswith("Error") + + +@pytest.mark.asyncio +async def test_answer_concurrent_question__should_be_at_the_same_time(): + model = "phi" + prompt = "Respond shortly: hello" + n_models = 2 + + # Get the average time for generating a character in a single run + start = time.time() + out_single = await answer_question(model, prompt, AI_SERVER_URL) + average_single_time = (time.time() - start) / len(out_single) + + # Get the average time for generating a character when running n_models concurrently + start = time.time() + out_concurrent = await _concurrent_call(model, n_models, prompt, AI_SERVER_URL) + average_concurrent_time = (time.time() - start) / sum([len(x) for x in out_concurrent]) + + assert ( + average_concurrent_time < average_single_time * n_models + ), f"Running {n_models} separately should take more time than running them concurrently" + + +async def _concurrent_call(model, n_models, prompt, server_url): + asyncMethod = [] + for _ in range(n_models): + asyncMethod.append(answer_question(model, prompt, server_url)) + + out = await asyncio.gather(*asyncMethod) + return out From 24e4696c71ddd42efd9fab892be9c4e19c0d6084 Mon Sep 17 00:00:00 2001 From: Duy Hung Tran Date: Mon, 4 Mar 2024 16:13:50 +0700 Subject: [PATCH 05/14] feat: update github action to add setup llm proxy server for testing --- .github/actions/setup-ai/action.yml | 19 +++++++++++++++++++ .github/workflows/test-and-lint.yml | 4 ++++ 2 files changed, 23 insertions(+) create mode 100644 .github/actions/setup-ai/action.yml diff --git a/.github/actions/setup-ai/action.yml b/.github/actions/setup-ai/action.yml new file mode 100644 index 0000000..8e0c871 --- /dev/null +++ b/.github/actions/setup-ai/action.yml @@ -0,0 +1,19 @@ +name: setup-ai-server +description: "Setup AI server for testing" + +runs: + using: composite + steps: + - name: Install ollama + shell: bash + run: curl -fsSL https://ollama.com/install.sh | sh + + - name: Start ollama + if: steps.cache-ollama.outputs.cache-hit != 'true' + shell: bash + run: ollama pull tinydolphin + + - name: Start litellm + shell: bash + run: poetry run litellm --config llm_assistant/ollama/test_proxy_config.yaml & + \ No newline at end of file diff --git a/.github/workflows/test-and-lint.yml b/.github/workflows/test-and-lint.yml index 750735d..40c8b67 100644 --- a/.github/workflows/test-and-lint.yml +++ b/.github/workflows/test-and-lint.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - add-functional-tests-for-discord-bot pull_request: branches: - main @@ -39,6 +40,9 @@ jobs: - name: Set up Python uses: ./.github/actions/setup-python + - name: Set up AI server + uses: ./.github/actions/setup-ai + - name: Run pytest run: poetry run pytest From a4dfa4da78692e49243d4d15b80b8ca5a8d2383c Mon Sep 17 00:00:00 2001 From: Duy Hung Tran Date: Mon, 4 Mar 2024 16:16:21 +0700 Subject: [PATCH 06/14] feat: update github action to add setup llm proxy server for testing --- poetry.lock | 30 ++++++++++++++++++++++++------ pyproject.toml | 2 ++ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index 70efa58..43d6cbd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1387,23 +1387,41 @@ tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] [[package]] name = "pytest" -version = "8.0.2" +version = "8.1.0" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.8" files = [ - {file = "pytest-8.0.2-py3-none-any.whl", hash = "sha256:edfaaef32ce5172d5466b5127b42e0d6d35ebbe4453f0e3505d96afd93f6b096"}, - {file = "pytest-8.0.2.tar.gz", hash = "sha256:d4051d623a2e0b7e51960ba963193b09ce6daeb9759a451844a21e4ddedfc1bd"}, + {file = "pytest-8.1.0-py3-none-any.whl", hash = "sha256:ee32db7af8de4629a455806befa90559f307424c07b8413ccfc30bf5b221dd7e"}, + {file = "pytest-8.1.0.tar.gz", hash = "sha256:f8fa04ab8f98d185113ae60ea6d79c22f8143b14bc1caeced44a0ab844928323"}, ] [package.dependencies] colorama = {version = "*", markers = "sys_platform == \"win32\""} iniconfig = "*" packaging = "*" -pluggy = ">=1.3.0,<2.0" +pluggy = ">=1.4,<2.0" [package.extras] -testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] +testing = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "pytest-asyncio" +version = "0.23.5" +description = "Pytest support for asyncio" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-asyncio-0.23.5.tar.gz", hash = "sha256:3a048872a9c4ba14c3e90cc1aa20cbc2def7d01c7c8db3777ec281ba9c057675"}, + {file = "pytest_asyncio-0.23.5-py3-none-any.whl", hash = "sha256:4e7093259ba018d58ede7d5315131d21923a60f8a6e9ee266ce1589685c89eac"}, +] + +[package.dependencies] +pytest = ">=7.0.0,<9" + +[package.extras] +docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1.0)"] +testing = ["coverage (>=6.2)", "hypothesis (>=5.7.1)"] [[package]] name = "python-dotenv" @@ -2166,4 +2184,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.12" -content-hash = "d307d249e5bcc92ff009ddff03c51bd678a32015b142b8ff0cfa714b71b21aef" +content-hash = "a1ad6ecedfa5a85e5568b3c6d3e35f8eb0abe4d802165f154ac7ad7202fe83e2" diff --git a/pyproject.toml b/pyproject.toml index 95a3ba5..dd1e431 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,8 @@ pre-commit = "^3.6.1" ruff = "^0.2.1" pytest = "^8.0.0" mypy = "^1.8.0" +pytest-asyncio = "^0.23.5" + [tool.ruff] # Exclude a variety of commonly ignored directories. From 31ed1c24c36306b44f561186f96550180c3411a8 Mon Sep 17 00:00:00 2001 From: Duy Hung Tran Date: Mon, 4 Mar 2024 16:20:25 +0700 Subject: [PATCH 07/14] feat: update github action to add setup llm proxy server for testing --- .github/actions/setup-ai/action.yml | 2 +- src/litellm/proxy_config_for_testing.yaml | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 src/litellm/proxy_config_for_testing.yaml diff --git a/.github/actions/setup-ai/action.yml b/.github/actions/setup-ai/action.yml index 8e0c871..3b7b253 100644 --- a/.github/actions/setup-ai/action.yml +++ b/.github/actions/setup-ai/action.yml @@ -15,5 +15,5 @@ runs: - name: Start litellm shell: bash - run: poetry run litellm --config llm_assistant/ollama/test_proxy_config.yaml & + run: poetry run litellm --config src/litellm/proxy_config_for_testing.yaml & \ No newline at end of file diff --git a/src/litellm/proxy_config_for_testing.yaml b/src/litellm/proxy_config_for_testing.yaml new file mode 100644 index 0000000..c2d596a --- /dev/null +++ b/src/litellm/proxy_config_for_testing.yaml @@ -0,0 +1,17 @@ +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: ollama/phi + - model_name: gpt-4 + litellm_params: + model: ollama/phi + - model_name: phi + litellm_params: + model: ollama/phi + +litellm_params: + drop_params: True + +router_settings: + num_retries: 2 + timeout: 60 # seconds From 608a72d44da9ca43f04d668742789c61f602ba2a Mon Sep 17 00:00:00 2001 From: Duy Hung Tran Date: Mon, 4 Mar 2024 16:26:48 +0700 Subject: [PATCH 08/14] feat: update github action to add setup llm proxy server for testing --- src/litellm/proxy_config_for_testing.yaml | 10 ++-------- tests/test_discord_bot_llm.py | 14 ++------------ 2 files changed, 4 insertions(+), 20 deletions(-) diff --git a/src/litellm/proxy_config_for_testing.yaml b/src/litellm/proxy_config_for_testing.yaml index c2d596a..730b8aa 100644 --- a/src/litellm/proxy_config_for_testing.yaml +++ b/src/litellm/proxy_config_for_testing.yaml @@ -1,13 +1,7 @@ model_list: - - model_name: gpt-3.5-turbo + - model_name: tinydolphin litellm_params: - model: ollama/phi - - model_name: gpt-4 - litellm_params: - model: ollama/phi - - model_name: phi - litellm_params: - model: ollama/phi + model: ollama/tinydolphin litellm_params: drop_params: True diff --git a/tests/test_discord_bot_llm.py b/tests/test_discord_bot_llm.py index 79506ef..7b9b18a 100644 --- a/tests/test_discord_bot_llm.py +++ b/tests/test_discord_bot_llm.py @@ -11,7 +11,7 @@ @pytest.mark.asyncio async def test_answer_question__LLM_should_response() -> None: - model = "phi" + model = "tinydolphin" prompt = "Respond shortly: hello!" response = await answer_question(model, prompt, AI_SERVER_URL) @@ -19,16 +19,6 @@ async def test_answer_question__LLM_should_response() -> None: assert not response.startswith("Error") -@pytest.mark.asyncio -async def test_answer_question__invalid_server_url() -> None: - model = "phi" - prompt = "Hello, world!" - - response = await answer_question(model, prompt, "http://fakeurl.com") - - assert response.startswith("Error") - - @pytest.mark.asyncio async def test_answer_question__invalid_model() -> None: model = "not-a-gpt" @@ -41,7 +31,7 @@ async def test_answer_question__invalid_model() -> None: @pytest.mark.asyncio async def test_answer_concurrent_question__should_be_at_the_same_time(): - model = "phi" + model = "tinydolphin" prompt = "Respond shortly: hello" n_models = 2 From 0785e0f9d28c3c1bc086f9ca67a686c8bb1de035 Mon Sep 17 00:00:00 2001 From: Duy Hung Tran Date: Mon, 4 Mar 2024 16:53:40 +0700 Subject: [PATCH 09/14] fix: make mypy work --- .github/workflows/test-and-lint.yml | 2 +- pyproject.toml | 7 ++++++- src/discord_bot/bot.py | 4 ++-- tests/test_discord_bot_llm.py | 5 +++-- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-and-lint.yml b/.github/workflows/test-and-lint.yml index 40c8b67..02b0ff6 100644 --- a/.github/workflows/test-and-lint.yml +++ b/.github/workflows/test-and-lint.yml @@ -55,4 +55,4 @@ jobs: uses: ./.github/actions/setup-python - name: Typecheck with mypy - run: poetry run mypy . + run: poetry run mypy . --no-namespace-packages diff --git a/pyproject.toml b/pyproject.toml index dd1e431..c845dca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,6 +86,11 @@ show_error_codes = true warn_return_any = true warn_unused_ignores = true +# Prevent mypy from trigger the missing import error for our written packages +[[tool.mypy.overrides]] +module = ["discord_bot.*", "src.discord_bot.*"] +ignore_missing_imports = true + [build-system] requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" +build-backend = "poetry.core.masonry.api" \ No newline at end of file diff --git a/src/discord_bot/bot.py b/src/discord_bot/bot.py index 2f7d938..04fa42e 100644 --- a/src/discord_bot/bot.py +++ b/src/discord_bot/bot.py @@ -1,7 +1,7 @@ import os import interactions import dotenv -import src.discord_bot.llm +from src.discord_bot.llm import answer_question dotenv.load_dotenv() @@ -41,7 +41,7 @@ async def ask_model(ctx: interactions.SlashContext, model: str = "", prompt: str await ctx.defer() - response = await src.discord_bot.llm.answer_question(model, prompt, AI_SERVER_URL) + response = await answer_question(model, prompt, AI_SERVER_URL) await ctx.send(response) diff --git a/tests/test_discord_bot_llm.py b/tests/test_discord_bot_llm.py index 7b9b18a..b41235c 100644 --- a/tests/test_discord_bot_llm.py +++ b/tests/test_discord_bot_llm.py @@ -3,6 +3,7 @@ import os import dotenv import pytest +from typing import List from discord_bot.llm import answer_question AI_SERVER_URL = os.getenv("AI_SERVER_URL") or "http://localhost:8000" @@ -30,7 +31,7 @@ async def test_answer_question__invalid_model() -> None: @pytest.mark.asyncio -async def test_answer_concurrent_question__should_be_at_the_same_time(): +async def test_answer_concurrent_question__should_be_at_the_same_time() -> None: model = "tinydolphin" prompt = "Respond shortly: hello" n_models = 2 @@ -50,7 +51,7 @@ async def test_answer_concurrent_question__should_be_at_the_same_time(): ), f"Running {n_models} separately should take more time than running them concurrently" -async def _concurrent_call(model, n_models, prompt, server_url): +async def _concurrent_call(model: str, n_models: int, prompt: str, server_url: str) -> List[str]: asyncMethod = [] for _ in range(n_models): asyncMethod.append(answer_question(model, prompt, server_url)) From eb04fe41204563498911c0d46a2ed2781976264c Mon Sep 17 00:00:00 2001 From: Duy Hung Tran Date: Mon, 4 Mar 2024 16:55:47 +0700 Subject: [PATCH 10/14] chore: only trigger gh action for main branch --- .github/workflows/test-and-lint.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test-and-lint.yml b/.github/workflows/test-and-lint.yml index 02b0ff6..08ce5dc 100644 --- a/.github/workflows/test-and-lint.yml +++ b/.github/workflows/test-and-lint.yml @@ -4,7 +4,6 @@ on: push: branches: - main - - add-functional-tests-for-discord-bot pull_request: branches: - main From bf72f175c774b840b9ba57d92bb71649bc62ae63 Mon Sep 17 00:00:00 2001 From: Duy Hung Tran Date: Tue, 5 Mar 2024 16:22:49 +0700 Subject: [PATCH 11/14] chore: refactor testing code --- .github/workflows/test-and-lint.yml | 2 +- poetry.lock | 16 ++++++++-------- pyproject.toml | 2 +- src/discord_bot/llm.py | 6 +++--- tests/test_discord_bot_llm.py | 3 +-- 5 files changed, 14 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test-and-lint.yml b/.github/workflows/test-and-lint.yml index 08ce5dc..84bce65 100644 --- a/.github/workflows/test-and-lint.yml +++ b/.github/workflows/test-and-lint.yml @@ -43,7 +43,7 @@ jobs: uses: ./.github/actions/setup-ai - name: Run pytest - run: poetry run pytest + run: poetry run pytest tests/ typecheck: runs-on: ubuntu-latest diff --git a/poetry.lock b/poetry.lock index 43d6cbd..fe9806a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -826,13 +826,13 @@ i18n = ["Babel (>=2.7)"] [[package]] name = "litellm" -version = "1.28.11" +version = "1.29.1" description = "Library to easily interface with LLM API providers" optional = false python-versions = ">=3.8, !=2.7.*, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*, !=3.7.*" files = [ - {file = "litellm-1.28.11-py3-none-any.whl", hash = "sha256:0f8ca98ce8391d2f44c188ccfe54309d023fd77b0c2e897c68431c08040282cd"}, - {file = "litellm-1.28.11.tar.gz", hash = "sha256:34fe319e06d1791b8e9b6b31e34af140d596014420258f986038a54470d7447c"}, + {file = "litellm-1.29.1-py3-none-any.whl", hash = "sha256:0db15ad2a94cbb65022afc9ea5416f3830e10729163bea01a8f4cf7de6447f5f"}, + {file = "litellm-1.29.1.tar.gz", hash = "sha256:050aa5e34d65c76e9990d369f4fbd2ae96ad56e821b32875134454f46855c4a8"}, ] [package.dependencies] @@ -1387,23 +1387,23 @@ tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] [[package]] name = "pytest" -version = "8.1.0" +version = "8.0.2" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.8" files = [ - {file = "pytest-8.1.0-py3-none-any.whl", hash = "sha256:ee32db7af8de4629a455806befa90559f307424c07b8413ccfc30bf5b221dd7e"}, - {file = "pytest-8.1.0.tar.gz", hash = "sha256:f8fa04ab8f98d185113ae60ea6d79c22f8143b14bc1caeced44a0ab844928323"}, + {file = "pytest-8.0.2-py3-none-any.whl", hash = "sha256:edfaaef32ce5172d5466b5127b42e0d6d35ebbe4453f0e3505d96afd93f6b096"}, + {file = "pytest-8.0.2.tar.gz", hash = "sha256:d4051d623a2e0b7e51960ba963193b09ce6daeb9759a451844a21e4ddedfc1bd"}, ] [package.dependencies] colorama = {version = "*", markers = "sys_platform == \"win32\""} iniconfig = "*" packaging = "*" -pluggy = ">=1.4,<2.0" +pluggy = ">=1.3.0,<2.0" [package.extras] -testing = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] [[package]] name = "pytest-asyncio" diff --git a/pyproject.toml b/pyproject.toml index c845dca..3b154f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,4 +93,4 @@ ignore_missing_imports = true [build-system] requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" \ No newline at end of file +build-backend = "poetry.core.masonry.api" diff --git a/src/discord_bot/llm.py b/src/discord_bot/llm.py index 2244bec..db5d72a 100644 --- a/src/discord_bot/llm.py +++ b/src/discord_bot/llm.py @@ -15,7 +15,7 @@ async def answer_question(model: str, question: str, server_url: str) -> str: messages=[{"role": "user", "content": question}], ) out = response.choices[0].message.content or "No response from the model" - except Exception as e: - out = f"Error: {e}" - return out + return out + except Exception as e: + return f"Error: {e}" diff --git a/tests/test_discord_bot_llm.py b/tests/test_discord_bot_llm.py index b41235c..4bd3cd6 100644 --- a/tests/test_discord_bot_llm.py +++ b/tests/test_discord_bot_llm.py @@ -1,12 +1,11 @@ import asyncio import time -import os import dotenv import pytest from typing import List from discord_bot.llm import answer_question -AI_SERVER_URL = os.getenv("AI_SERVER_URL") or "http://localhost:8000" +AI_SERVER_URL = "http://localhost:8000" dotenv.load_dotenv() From fda4965ade8b9aba6ed6f26427415e17bd42ab2a Mon Sep 17 00:00:00 2001 From: Duy Hung Tran Date: Wed, 6 Mar 2024 09:27:41 +0700 Subject: [PATCH 12/14] chore: make mypy work correctly --- .github/workflows/test-and-lint.yml | 2 +- poetry.lock | 6 +++--- pyproject.toml | 5 ----- src/discord_bot/bot.py | 2 +- 4 files changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test-and-lint.yml b/.github/workflows/test-and-lint.yml index 84bce65..b45707f 100644 --- a/.github/workflows/test-and-lint.yml +++ b/.github/workflows/test-and-lint.yml @@ -54,4 +54,4 @@ jobs: uses: ./.github/actions/setup-python - name: Typecheck with mypy - run: poetry run mypy . --no-namespace-packages + run: poetry run mypy . diff --git a/poetry.lock b/poetry.lock index fe9806a..6f301e9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -826,13 +826,13 @@ i18n = ["Babel (>=2.7)"] [[package]] name = "litellm" -version = "1.29.1" +version = "1.29.3" description = "Library to easily interface with LLM API providers" optional = false python-versions = ">=3.8, !=2.7.*, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*, !=3.7.*" files = [ - {file = "litellm-1.29.1-py3-none-any.whl", hash = "sha256:0db15ad2a94cbb65022afc9ea5416f3830e10729163bea01a8f4cf7de6447f5f"}, - {file = "litellm-1.29.1.tar.gz", hash = "sha256:050aa5e34d65c76e9990d369f4fbd2ae96ad56e821b32875134454f46855c4a8"}, + {file = "litellm-1.29.3-py3-none-any.whl", hash = "sha256:73a79f50c22919a2b5e834123b676270cf99673f70247e9b5ce964bebb65b73a"}, + {file = "litellm-1.29.3.tar.gz", hash = "sha256:0b2d8ed698d24183b1f2520070bb1c28e62b554899731a8ce36351f4349622d4"}, ] [package.dependencies] diff --git a/pyproject.toml b/pyproject.toml index 3b154f4..dd1e431 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,11 +86,6 @@ show_error_codes = true warn_return_any = true warn_unused_ignores = true -# Prevent mypy from trigger the missing import error for our written packages -[[tool.mypy.overrides]] -module = ["discord_bot.*", "src.discord_bot.*"] -ignore_missing_imports = true - [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" diff --git a/src/discord_bot/bot.py b/src/discord_bot/bot.py index 04fa42e..a4488e5 100644 --- a/src/discord_bot/bot.py +++ b/src/discord_bot/bot.py @@ -1,7 +1,7 @@ import os import interactions import dotenv -from src.discord_bot.llm import answer_question +from discord_bot.llm import answer_question dotenv.load_dotenv() From e4f49c0093ad2fd89b553771bfcba26912d781b5 Mon Sep 17 00:00:00 2001 From: Duy Hung Tran Date: Wed, 6 Mar 2024 09:41:26 +0700 Subject: [PATCH 13/14] chore: make mypy work correctly --- pyproject.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index dd1e431..3b154f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,6 +86,11 @@ show_error_codes = true warn_return_any = true warn_unused_ignores = true +# Prevent mypy from trigger the missing import error for our written packages +[[tool.mypy.overrides]] +module = ["discord_bot.*", "src.discord_bot.*"] +ignore_missing_imports = true + [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" From 6dcd9cf073e20135f0c21aa8da7c1072383ac990 Mon Sep 17 00:00:00 2001 From: Sam Huynh Date: Wed, 6 Mar 2024 15:22:26 +1100 Subject: [PATCH 14/14] feat(pipelines): clean up setup ai action --- .github/actions/setup-ai/action.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/actions/setup-ai/action.yml b/.github/actions/setup-ai/action.yml index 3b7b253..9b6666c 100644 --- a/.github/actions/setup-ai/action.yml +++ b/.github/actions/setup-ai/action.yml @@ -6,14 +6,13 @@ runs: steps: - name: Install ollama shell: bash - run: curl -fsSL https://ollama.com/install.sh | sh - + run: curl -fsSL https://ollama.com/install.sh | sh + - name: Start ollama if: steps.cache-ollama.outputs.cache-hit != 'true' shell: bash run: ollama pull tinydolphin - + - name: Start litellm shell: bash run: poetry run litellm --config src/litellm/proxy_config_for_testing.yaml & - \ No newline at end of file