From 73a088ae2e54b9426e736cb39931fce1295fc010 Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Wed, 4 Sep 2024 01:15:04 -0700 Subject: [PATCH] agent builder tests --- .github/workflows/contrib-openai.yml | 417 ++++++++++--------- test/agentchat/contrib/test_agent_builder.py | 22 +- 2 files changed, 221 insertions(+), 218 deletions(-) diff --git a/.github/workflows/contrib-openai.yml b/.github/workflows/contrib-openai.yml index 7e8fb003317..9a99caec964 100644 --- a/.github/workflows/contrib-openai.yml +++ b/.github/workflows/contrib-openai.yml @@ -18,140 +18,141 @@ permissions: # contents: read # deployments: read jobs: - RetrieveChatTest: - strategy: - matrix: - os: [ubuntu-latest] - python-version: ["3.10"] - runs-on: ${{ matrix.os }} - environment: openai1 - services: - pgvector: - image: ankane/pgvector - env: - POSTGRES_DB: postgres - POSTGRES_USER: postgres - POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} - POSTGRES_HOST_AUTH_METHOD: trust - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - steps: - # checkout to pr branch - - name: Checkout - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install packages and dependencies - run: | - docker --version - python -m pip install --upgrade pip wheel - pip install -e . - python -c "import autogen" - pip install pytest-cov>=5 pytest-asyncio - - name: Install packages for test when needed - run: | - pip install docker - pip install -e .[retrievechat,retrievechat-qdrant,retrievechat-pgvector] - - name: Coverage - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} - AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} - OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} - run: | - pytest test/agentchat/contrib/retrievechat/ test/agentchat/contrib/retrievechat - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - file: ./coverage.xml - flags: unittests - AgentEvalTest: - strategy: - matrix: - os: [ubuntu-latest] - python-version: ["3.10"] - runs-on: ${{ matrix.os }} - environment: openai1 - steps: - # checkout to pr branch - - name: Checkout - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install packages and dependencies - run: | - docker --version - python -m pip install --upgrade pip wheel - pip install -e . - python -c "import autogen" - pip install pytest-cov>=5 pytest-asyncio - - name: Coverage - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} - AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} - OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} - run: | - pytest test/agentchat/contrib/agent_eval/test_agent_eval.py - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - file: ./coverage.xml - flags: unittests + # RetrieveChatTest: + # strategy: + # matrix: + # os: [ubuntu-latest] + # python-version: ["3.10"] + # runs-on: ${{ matrix.os }} + # environment: openai1 + # services: + # pgvector: + # image: ankane/pgvector + # env: + # POSTGRES_DB: postgres + # POSTGRES_USER: postgres + # POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} + # POSTGRES_HOST_AUTH_METHOD: trust + # options: >- + # --health-cmd pg_isready + # --health-interval 10s + # --health-timeout 5s + # --health-retries 5 + # ports: + # - 5432:5432 + # steps: + # # checkout to pr branch + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # ref: ${{ github.event.pull_request.head.sha }} + # - name: Set up Python ${{ matrix.python-version }} + # uses: actions/setup-python@v5 + # with: + # python-version: ${{ matrix.python-version }} + # - name: Install packages and dependencies + # run: | + # docker --version + # python -m pip install --upgrade pip wheel + # pip install -e . + # python -c "import autogen" + # pip install pytest-cov>=5 pytest-asyncio + # - name: Install packages for test when needed + # run: | + # pip install docker + # pip install -e .[retrievechat,retrievechat-qdrant,retrievechat-pgvector] + # - name: Coverage + # env: + # OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + # AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} + # AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} + # OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} + # run: | + # pytest test/agentchat/contrib/retrievechat/ test/agentchat/contrib/retrievechat + # - name: Upload coverage to Codecov + # uses: codecov/codecov-action@v3 + # with: + # file: ./coverage.xml + # flags: unittests - GPTAssistantAgent: - strategy: - matrix: - os: [ubuntu-latest] - python-version: ["3.11"] - runs-on: ${{ matrix.os }} - environment: openai1 - steps: - # checkout to pr branch - - name: Checkout - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install packages and dependencies - run: | - docker --version - python -m pip install --upgrade pip wheel - pip install -e . - python -c "import autogen" - pip install pytest-cov>=5 pytest-asyncio - - name: Install packages for test when needed - run: | - pip install docker - - name: Coverage - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} - AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} - OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} - run: | - pytest test/agentchat/contrib/test_gpt_assistant.py - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - file: ./coverage.xml - flags: unittests + # AgentEvalTest: + # strategy: + # matrix: + # os: [ubuntu-latest] + # python-version: ["3.10"] + # runs-on: ${{ matrix.os }} + # environment: openai1 + # steps: + # # checkout to pr branch + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # ref: ${{ github.event.pull_request.head.sha }} + # - name: Set up Python ${{ matrix.python-version }} + # uses: actions/setup-python@v5 + # with: + # python-version: ${{ matrix.python-version }} + # - name: Install packages and dependencies + # run: | + # docker --version + # python -m pip install --upgrade pip wheel + # pip install -e . + # python -c "import autogen" + # pip install pytest-cov>=5 pytest-asyncio + # - name: Coverage + # env: + # OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + # AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} + # AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} + # OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} + # run: | + # pytest test/agentchat/contrib/agent_eval/test_agent_eval.py + # - name: Upload coverage to Codecov + # uses: codecov/codecov-action@v3 + # with: + # file: ./coverage.xml + # flags: unittests + + # GPTAssistantAgent: + # strategy: + # matrix: + # os: [ubuntu-latest] + # python-version: ["3.11"] + # runs-on: ${{ matrix.os }} + # environment: openai1 + # steps: + # # checkout to pr branch + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # ref: ${{ github.event.pull_request.head.sha }} + # - name: Set up Python ${{ matrix.python-version }} + # uses: actions/setup-python@v5 + # with: + # python-version: ${{ matrix.python-version }} + # - name: Install packages and dependencies + # run: | + # docker --version + # python -m pip install --upgrade pip wheel + # pip install -e . + # python -c "import autogen" + # pip install pytest-cov>=5 pytest-asyncio + # - name: Install packages for test when needed + # run: | + # pip install docker + # - name: Coverage + # env: + # OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + # AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} + # AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} + # OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} + # run: | + # pytest test/agentchat/contrib/test_gpt_assistant.py + # - name: Upload coverage to Codecov + # uses: codecov/codecov-action@v3 + # with: + # file: ./coverage.xml + # flags: unittests TeachableAgent: strategy: matrix: @@ -229,44 +230,44 @@ jobs: with: file: ./coverage.xml flags: unittests - WebSurfer: - strategy: - matrix: - os: [ubuntu-latest] - python-version: ["3.11"] - runs-on: ${{ matrix.os }} - environment: openai1 - steps: - # checkout to pr branch - - name: Checkout - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install packages and dependencies - run: | - docker --version - python -m pip install --upgrade pip wheel - pip install -e .[websurfer] - python -c "import autogen" - pip install pytest-cov>=5 - - name: Coverage - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} - AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} - OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} - BING_API_KEY: ${{ secrets.BING_API_KEY }} - run: | - pytest test/agentchat/contrib/test_web_surfer.py - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - file: ./coverage.xml - flags: unittests + # WebSurfer: + # strategy: + # matrix: + # os: [ubuntu-latest] + # python-version: ["3.11"] + # runs-on: ${{ matrix.os }} + # environment: openai1 + # steps: + # # checkout to pr branch + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # ref: ${{ github.event.pull_request.head.sha }} + # - name: Set up Python ${{ matrix.python-version }} + # uses: actions/setup-python@v5 + # with: + # python-version: ${{ matrix.python-version }} + # - name: Install packages and dependencies + # run: | + # docker --version + # python -m pip install --upgrade pip wheel + # pip install -e .[websurfer] + # python -c "import autogen" + # pip install pytest-cov>=5 + # - name: Coverage + # env: + # OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + # AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} + # AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} + # OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} + # BING_API_KEY: ${{ secrets.BING_API_KEY }} + # run: | + # pytest test/agentchat/contrib/test_web_surfer.py + # - name: Upload coverage to Codecov + # uses: codecov/codecov-action@v3 + # with: + # file: ./coverage.xml + # flags: unittests ImageGen: strategy: @@ -303,40 +304,40 @@ jobs: file: ./coverage.xml flags: unittests - AgentOptimizer: - strategy: - matrix: - os: [ubuntu-latest] - python-version: ["3.11"] - runs-on: ${{ matrix.os }} - environment: openai1 - steps: - # checkout to pr branch - - name: Checkout - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install packages and dependencies - run: | - docker --version - python -m pip install --upgrade pip wheel - pip install -e . - python -c "import autogen" - pip install pytest-cov>=5 - - name: Coverage - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} - AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} - OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} - run: | - pytest test/agentchat/contrib/test_agent_optimizer.py - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - file: ./coverage.xml - flags: unittests + # AgentOptimizer: + # strategy: + # matrix: + # os: [ubuntu-latest] + # python-version: ["3.11"] + # runs-on: ${{ matrix.os }} + # environment: openai1 + # steps: + # # checkout to pr branch + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # ref: ${{ github.event.pull_request.head.sha }} + # - name: Set up Python ${{ matrix.python-version }} + # uses: actions/setup-python@v5 + # with: + # python-version: ${{ matrix.python-version }} + # - name: Install packages and dependencies + # run: | + # docker --version + # python -m pip install --upgrade pip wheel + # pip install -e . + # python -c "import autogen" + # pip install pytest-cov>=5 + # - name: Coverage + # env: + # OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + # AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} + # AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} + # OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} + # run: | + # pytest test/agentchat/contrib/test_agent_optimizer.py + # - name: Upload coverage to Codecov + # uses: codecov/codecov-action@v3 + # with: + # file: ./coverage.xml + # flags: unittests diff --git a/test/agentchat/contrib/test_agent_builder.py b/test/agentchat/contrib/test_agent_builder.py index b7f7ec88825..4612577c46e 100755 --- a/test/agentchat/contrib/test_agent_builder.py +++ b/test/agentchat/contrib/test_agent_builder.py @@ -51,8 +51,8 @@ def test_build(): builder = AgentBuilder( config_file_or_env=OAI_CONFIG_LIST, config_file_location=KEY_LOC, - builder_model=["gpt-4", "gpt-4-1106-preview"], - agent_model=["gpt-4", "gpt-4-1106-preview"], + builder_model_tags=["gpt-4"], + agent_model_tags=["gpt-4"], ) building_task = ( "Find a paper on arxiv by programming, and analyze its application in some domain. " @@ -83,8 +83,8 @@ def test_build_from_library(): builder = AgentBuilder( config_file_or_env=OAI_CONFIG_LIST, config_file_location=KEY_LOC, - builder_model=["gpt-4", "gpt-4-1106-preview"], - agent_model=["gpt-4", "gpt-4-1106-preview"], + builder_model_tags=["gpt-4"], + agent_model_tags=["gpt-4"], ) building_task = ( "Find a paper on arxiv by programming, and analyze its application in some domain. " @@ -136,8 +136,8 @@ def test_save(): builder = AgentBuilder( config_file_or_env=OAI_CONFIG_LIST, config_file_location=KEY_LOC, - builder_model=["gpt-4", "gpt-4-1106-preview"], - agent_model=["gpt-4", "gpt-4-1106-preview"], + builder_model_tags=["gpt-4"], + agent_model_tags=["gpt-4"], ) building_task = ( "Find a paper on arxiv by programming, and analyze its application in some domain. " @@ -173,8 +173,10 @@ def test_load(): builder = AgentBuilder( config_file_or_env=OAI_CONFIG_LIST, config_file_location=KEY_LOC, - builder_model=["gpt-4", "gpt-4-1106-preview"], - agent_model=["gpt-4", "gpt-4-1106-preview"], + # builder_model=["gpt-4", "gpt-4-1106-preview"], + # agent_model=["gpt-4", "gpt-4-1106-preview"], + builder_model_tags=["gpt-4"], + agent_model_tags=["gpt-4"], ) config_save_path = f"{here}/example_test_agent_builder_config.json" @@ -202,8 +204,8 @@ def test_clear_agent(): builder = AgentBuilder( config_file_or_env=OAI_CONFIG_LIST, config_file_location=KEY_LOC, - builder_model=["gpt-4", "gpt-4-1106-preview"], - agent_model=["gpt-4", "gpt-4-1106-preview"], + builder_model_tags=["gpt-4"], + agent_model_tags=["gpt-4"], ) config_save_path = f"{here}/example_test_agent_builder_config.json"