diff --git a/.github/workflows/e2e-agent-samples.yml b/.github/workflows/e2e-agent-samples.yml deleted file mode 100644 index 7a80f456..00000000 --- a/.github/workflows/e2e-agent-samples.yml +++ /dev/null @@ -1,666 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -# ============================================================================= -# E2E Tests: Agent Samples -# ============================================================================= -# Runs E2E integration tests for all agent samples -# Uses external PowerShell scripts for maintainability -# ============================================================================= - -name: E2E Agent Samples - -on: - # Run nightly at 6 AM UTC - schedule: - - cron: '0 6 * * *' - push: - branches: - - main - - 'feature/*' - paths: - - 'python/**' - - 'nodejs/**' - - 'dotnet/**' - - 'tests/e2e/**' - - '.github/workflows/e2e-agent-samples.yml' - - 'scripts/e2e/**' - pull_request: - branches: - - main - paths: - - 'python/**' - - 'nodejs/**' - - 'dotnet/**' - - 'tests/e2e/**' - - '.github/workflows/e2e-agent-samples.yml' - - 'scripts/e2e/**' - workflow_dispatch: - inputs: - sample: - description: 'Specific sample to test (leave empty for all)' - required: false - default: '' - debug: - description: 'Enable debug logging' - required: false - default: 'false' - -permissions: - contents: read - -env: - # MCP Authentication - Required for all samples - MCP_CLIENT_ID: ${{ secrets.MCP_CLIENT_ID }} - MCP_TENANT_ID: ${{ secrets.MCP_TENANT_ID }} - MCP_TEST_USERNAME: ${{ secrets.MCP_TEST_USERNAME }} - MCP_TEST_PASSWORD: ${{ secrets.MCP_TEST_PASSWORD }} - MCP_ENVIRONMENT: 'Development' - - # Common settings - SCRIPTS_PATH: scripts/e2e - - # E2E Integration Tests - located in this repository - E2E_TESTS_PATH: 'tests/e2e' - -jobs: - # =========================================================================== - # Python OpenAI Sample - # =========================================================================== - python-openai: - name: Python OpenAI Agent - runs-on: windows-latest - if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'python-openai' }} - - env: - SAMPLE_PATH: python/openai/sample-agent - AGENT_PORT: 3979 - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup .NET SDK - uses: actions/setup-dotnet@v4 - with: - dotnet-version: '8.0.x' - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: Install uv package manager - run: pip install uv - - - name: Install dependencies - working-directory: ${{ env.SAMPLE_PATH }} - run: uv sync - - - name: Acquire Bearer Token (ROPC) - id: token - shell: pwsh - run: | - $token = & "${{ env.SCRIPTS_PATH }}/Acquire-BearerToken.ps1" ` - -ClientId "${{ secrets.MCP_CLIENT_ID }}" ` - -TenantId "${{ secrets.MCP_TENANT_ID }}" ` - -Username "${{ secrets.MCP_TEST_USERNAME }}" ` - -Password "${{ secrets.MCP_TEST_PASSWORD }}" - echo "BEARER_TOKEN=$token" >> $env:GITHUB_OUTPUT - echo "::add-mask::$token" - - - name: Copy ToolingManifest.json - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Copy-ToolingManifest.ps1" -TargetPath "${{ env.SAMPLE_PATH }}" - - - name: Generate .env configuration - shell: pwsh - run: | - $configMappings = @{ - "AZURE_OPENAI_API_KEY" = "${{ secrets.PYTHON_OPENAI_AZURE_OPENAI_API_KEY }}" - "AZURE_OPENAI_ENDPOINT" = "${{ secrets.PYTHON_OPENAI_AZURE_OPENAI_ENDPOINT }}" - "AZURE_OPENAI_DEPLOYMENT" = "${{ secrets.PYTHON_OPENAI_AZURE_OPENAI_DEPLOYMENT }}" - "AZURE_OPENAI_API_VERSION" = "2024-12-01-preview" - "USE_AGENTIC_AUTH" = "false" - "MCP_PLATFORM_ENDPOINT" = "https://agent365.svc.cloud.microsoft" - "AGENT_ID" = "${{ secrets.PYTHON_OPENAI_AGENT_ID }}" - "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__AUTHTYPE" = "ClientSecret" - "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID" = "${{ secrets.PYTHON_OPENAI_AGENT_ID }}" - "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET" = "${{ secrets.PYTHON_OPENAI_CLIENT_SECRET }}" - "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID" = "${{ secrets.TENANT_ID }}" - "CONNECTIONSMAP__0__SERVICEURL" = "*" - "CONNECTIONSMAP__0__CONNECTION" = "SERVICE_CONNECTION" - "ENABLE_OBSERVABILITY" = "true" - } - & "${{ env.SCRIPTS_PATH }}/Generate-EnvConfig.ps1" ` - -OutputPath "${{ env.SAMPLE_PATH }}/.env" ` - -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` - -Port ${{ env.AGENT_PORT }} ` - -ConfigMappings $configMappings - - - name: Start Agent - id: start-agent - shell: pwsh - run: | - $agentPid = & "${{ env.SCRIPTS_PATH }}/Start-Agent.ps1" ` - -AgentPath "${{ env.SAMPLE_PATH }}" ` - -StartCommand "uv run python start_with_generic_host.py" ` - -Port ${{ env.AGENT_PORT }} ` - -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` - -Environment "Development" ` - -Runtime "python" - echo "AGENT_PID=$agentPid" >> $env:GITHUB_OUTPUT - - - name: Verify Agent Running - shell: pwsh - run: | - Write-Host "=== Verifying Agent ===" -ForegroundColor Cyan - - # Verify agent process is still running - $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" - Write-Host "Checking agent process (PID: $agentPid)..." -ForegroundColor Gray - - if ($agentPid) { - try { - $proc = Get-Process -Id $agentPid -ErrorAction Stop - Write-Host "Agent process (PID: $agentPid) is running: $($proc.ProcessName)" -ForegroundColor Green - } catch { - Write-Host "ERROR: Agent process (PID: $agentPid) is NOT running!" -ForegroundColor Red - - # Show logs - $logFile = "${{ env.SAMPLE_PATH }}/agent.log" - if (Test-Path $logFile) { - Write-Host "Agent logs:" -ForegroundColor Yellow - Get-Content $logFile - } - throw "Agent process has stopped" - } - } - - # Quick health check before running tests - $agentUrl = "http://localhost:${{ env.AGENT_PORT }}" - Write-Host "Verifying agent at $agentUrl..." -ForegroundColor Gray - $healthResponse = Invoke-WebRequest -Uri "$agentUrl/api/health" -Method GET -UseBasicParsing -ErrorAction SilentlyContinue - Write-Host "Health check: $($healthResponse.StatusCode)" -ForegroundColor Green - - - name: Restore E2E Test Dependencies - shell: pwsh - run: | - dotnet restore "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" - - - name: Run .NET E2E Tests - shell: pwsh - run: | - dotnet test "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" ` - --no-restore ` - --filter "Category=HTTP" ` - --logger "trx;LogFileName=test-results.trx" ` - --results-directory "${{ runner.temp }}/TestResults" - env: - AGENT_PORT: ${{ env.AGENT_PORT }} - AGENT_URL: http://localhost:${{ env.AGENT_PORT }} - TEST_RESULTS_DIR: ${{ runner.temp }}/TestConversations - - - name: Upload Test Results - if: always() - uses: actions/upload-artifact@v4 - with: - name: test-results-python-openai - path: ${{ runner.temp }}/TestResults/ - retention-days: 30 - - - name: Upload Test Conversations - if: always() - uses: actions/upload-artifact@v4 - with: - name: test-conversations-python-openai - path: ${{ runner.temp }}/TestConversations/ - retention-days: 30 - continue-on-error: true - - - name: Capture Agent Logs - if: always() - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Capture-AgentLogs.ps1" -AgentPath "${{ env.SAMPLE_PATH }}" - - - name: Cleanup Agent Process - if: always() - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Stop-AgentProcess.ps1" ` - -AgentPID "${{ steps.start-agent.outputs.AGENT_PID }}" ` - -Port ${{ env.AGENT_PORT }} - - # =========================================================================== - # Node.js OpenAI Sample - # =========================================================================== - nodejs-openai: - name: Node.js OpenAI Agent - runs-on: windows-latest - if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'nodejs-openai' }} - - env: - SAMPLE_PATH: nodejs/openai/sample-agent - AGENT_PORT: 3979 - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup .NET SDK - uses: actions/setup-dotnet@v4 - with: - dotnet-version: '8.0.x' - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '20' - - - name: Install dependencies - working-directory: ${{ env.SAMPLE_PATH }} - run: | - if (Test-Path "package-lock.json") { - npm ci - } else { - npm install - } - if (Test-Path "tsconfig.json") { - npm run build - } - shell: pwsh - - - name: Acquire Bearer Token (ROPC) - id: token - shell: pwsh - run: | - $token = & "${{ env.SCRIPTS_PATH }}/Acquire-BearerToken.ps1" ` - -ClientId "${{ secrets.MCP_CLIENT_ID }}" ` - -TenantId "${{ secrets.MCP_TENANT_ID }}" ` - -Username "${{ secrets.MCP_TEST_USERNAME }}" ` - -Password "${{ secrets.MCP_TEST_PASSWORD }}" - echo "BEARER_TOKEN=$token" >> $env:GITHUB_OUTPUT - echo "::add-mask::$token" - - - name: Copy ToolingManifest.json - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Copy-ToolingManifest.ps1" -TargetPath "${{ env.SAMPLE_PATH }}" - - - name: Generate .env configuration - shell: pwsh - run: | - $configMappings = @{ - "NODE_ENV" = "development" - "AZURE_OPENAI_API_KEY" = "${{ secrets.NODEJS_OPENAI_AZURE_OPENAI_API_KEY }}" - "AZURE_OPENAI_ENDPOINT" = "${{ secrets.NODEJS_OPENAI_AZURE_OPENAI_ENDPOINT }}" - "AZURE_OPENAI_DEPLOYMENT" = "${{ secrets.NODEJS_OPENAI_AZURE_OPENAI_DEPLOYMENT }}" - "AZURE_OPENAI_API_VERSION" = "2024-12-01-preview" - "connections__service_connection__settings__authType" = "ClientSecret" - "connections__service_connection__settings__clientId" = "${{ secrets.NODEJS_OPENAI_AGENT_ID }}" - "connections__service_connection__settings__clientSecret" = "${{ secrets.NODEJS_OPENAI_CLIENT_SECRET }}" - "connections__service_connection__settings__tenantId" = "${{ secrets.TENANT_ID }}" - "connectionsMap__0__serviceUrl" = "*" - "connectionsMap__0__connection" = "service_connection" - "agentic_scopes" = "ea9ffc3e-8a23-4a7d-836d-234d7c7565c1/.default" - } - & "${{ env.SCRIPTS_PATH }}/Generate-EnvConfig.ps1" ` - -OutputPath "${{ env.SAMPLE_PATH }}/.env" ` - -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` - -Port ${{ env.AGENT_PORT }} ` - -ConfigMappings $configMappings - - - name: Start Agent - id: start-agent - shell: pwsh - run: | - $agentPid = & "${{ env.SCRIPTS_PATH }}/Start-Agent.ps1" ` - -AgentPath "${{ env.SAMPLE_PATH }}" ` - -StartCommand "node dist/index.js" ` - -Port ${{ env.AGENT_PORT }} ` - -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` - -Environment "Development" ` - -Runtime "nodejs" - echo "AGENT_PID=$agentPid" >> $env:GITHUB_OUTPUT - - - name: Verify Agent Running - shell: pwsh - run: | - $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" - if ($agentPid) { - try { - $proc = Get-Process -Id $agentPid -ErrorAction Stop - Write-Host "Agent process (PID: $agentPid) is running: $($proc.ProcessName)" -ForegroundColor Green - } catch { - Write-Host "ERROR: Agent process (PID: $agentPid) is NOT running!" -ForegroundColor Red - throw "Agent process has stopped" - } - } - $agentUrl = "http://localhost:${{ env.AGENT_PORT }}" - $healthResponse = Invoke-WebRequest -Uri "$agentUrl/api/health" -Method GET -UseBasicParsing -ErrorAction SilentlyContinue - Write-Host "Health check: $($healthResponse.StatusCode)" -ForegroundColor Green - - - name: Restore E2E Test Dependencies - run: dotnet restore "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" - - - name: Run .NET E2E Tests - shell: pwsh - run: | - dotnet test "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" ` - --no-restore ` - --filter "Category=HTTP" ` - --logger "trx;LogFileName=test-results.trx" ` - --results-directory "${{ runner.temp }}/TestResults" - env: - AGENT_PORT: ${{ env.AGENT_PORT }} - AGENT_URL: http://localhost:${{ env.AGENT_PORT }} - TEST_RESULTS_DIR: ${{ runner.temp }}/TestConversations - - - name: Upload Test Results - if: always() - uses: actions/upload-artifact@v4 - with: - name: test-results-nodejs-openai - path: ${{ runner.temp }}/TestResults/ - retention-days: 30 - - - name: Capture Agent Logs - if: always() - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Capture-AgentLogs.ps1" -AgentPath "${{ env.SAMPLE_PATH }}" - - - name: Cleanup - if: always() - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Stop-AgentProcess.ps1" ` - -AgentPID "${{ steps.start-agent.outputs.AGENT_PID }}" ` - -Port ${{ env.AGENT_PORT }} - - # =========================================================================== - # .NET Semantic Kernel Sample - # =========================================================================== - dotnet-semantic-kernel: - name: .NET Semantic Kernel Agent - runs-on: windows-latest - if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'dotnet-semantic-kernel' }} - - env: - SAMPLE_PATH: dotnet/semantic-kernel/sample-agent - AGENT_PORT: 3978 - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup .NET - uses: actions/setup-dotnet@v4 - with: - dotnet-version: '8.0.x' - - - name: Restore dependencies - working-directory: ${{ env.SAMPLE_PATH }} - run: dotnet restore - - - name: Build - working-directory: ${{ env.SAMPLE_PATH }} - run: dotnet build --no-restore - - - name: Acquire Bearer Token (ROPC) - id: token - shell: pwsh - run: | - $token = & "${{ env.SCRIPTS_PATH }}/Acquire-BearerToken.ps1" ` - -ClientId "${{ secrets.MCP_CLIENT_ID }}" ` - -TenantId "${{ secrets.MCP_TENANT_ID }}" ` - -Username "${{ secrets.MCP_TEST_USERNAME }}" ` - -Password "${{ secrets.MCP_TEST_PASSWORD }}" - echo "BEARER_TOKEN=$token" >> $env:GITHUB_OUTPUT - echo "::add-mask::$token" - - - name: Copy ToolingManifest.json - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Copy-ToolingManifest.ps1" -TargetPath "${{ env.SAMPLE_PATH }}" - - - name: Generate appsettings.json - shell: pwsh - run: | - $configMappings = @{ - "AIServices:UseAzureOpenAI" = "true" - "AIServices:AzureOpenAI:Endpoint" = "${{ secrets.DOTNET_SK_AZURE_OPENAI_ENDPOINT }}" - "AIServices:AzureOpenAI:ApiKey" = "${{ secrets.DOTNET_SK_AZURE_OPENAI_API_KEY }}" - "AIServices:AzureOpenAI:DeploymentName" = "${{ secrets.DOTNET_SK_AZURE_OPENAI_DEPLOYMENT }}" - "TokenValidation:Enabled" = "false" - "TokenValidation:TenantId" = "${{ secrets.TENANT_ID }}" - "Connections:ServiceConnection:Settings:AuthType" = "ClientSecret" - "Connections:ServiceConnection:Settings:ClientId" = "${{ secrets.DOTNET_SK_CLIENT_ID }}" - "Connections:ServiceConnection:Settings:ClientSecret" = "${{ secrets.DOTNET_SK_CLIENT_SECRET }}" - "Connections:ServiceConnection:Settings:TenantId" = "${{ secrets.TENANT_ID }}" - } - & "${{ env.SCRIPTS_PATH }}/Generate-AppSettings.ps1" ` - -OutputPath "${{ env.SAMPLE_PATH }}/appsettings.json" ` - -ConfigMappings $configMappings - - - name: Start Agent - id: start-agent - shell: pwsh - run: | - $agentPid = & "${{ env.SCRIPTS_PATH }}/Start-Agent.ps1" ` - -AgentPath "${{ env.SAMPLE_PATH }}" ` - -StartCommand "dotnet run --no-build" ` - -Port ${{ env.AGENT_PORT }} ` - -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` - -Environment "Development" ` - -Runtime "dotnet" - echo "AGENT_PID=$agentPid" >> $env:GITHUB_OUTPUT - - - name: Verify Agent Running - shell: pwsh - run: | - $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" - if ($agentPid) { - try { - $proc = Get-Process -Id $agentPid -ErrorAction Stop - Write-Host "Agent process (PID: $agentPid) is running: $($proc.ProcessName)" -ForegroundColor Green - } catch { - Write-Host "ERROR: Agent process (PID: $agentPid) is NOT running!" -ForegroundColor Red - throw "Agent process has stopped" - } - } - $response = Invoke-WebRequest -Uri "http://localhost:${{ env.AGENT_PORT }}/api/health" -UseBasicParsing -ErrorAction SilentlyContinue - Write-Host "Health: $($response.StatusCode)" -ForegroundColor Green - - - name: Restore E2E Test Dependencies - run: dotnet restore "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" - - - name: Run .NET E2E Tests - shell: pwsh - run: | - dotnet test "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" ` - --no-restore ` - --filter "Category=HTTP" ` - --logger "trx;LogFileName=test-results.trx" ` - --results-directory "${{ runner.temp }}/TestResults" - env: - AGENT_PORT: ${{ env.AGENT_PORT }} - AGENT_URL: http://localhost:${{ env.AGENT_PORT }} - TEST_RESULTS_DIR: ${{ runner.temp }}/TestConversations - - - name: Upload Test Results - if: always() - uses: actions/upload-artifact@v4 - with: - name: test-results-dotnet-sk - path: ${{ runner.temp }}/TestResults/ - retention-days: 30 - - - name: Capture Logs - if: always() - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Capture-AgentLogs.ps1" -AgentPath "${{ env.SAMPLE_PATH }}" - - - name: Cleanup - if: always() - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Stop-AgentProcess.ps1" ` - -AgentPID "${{ steps.start-agent.outputs.AGENT_PID }}" ` - -Port ${{ env.AGENT_PORT }} - - # =========================================================================== - # .NET Agent Framework Sample - # =========================================================================== - dotnet-agent-framework: - name: .NET Agent Framework Agent - runs-on: windows-latest - if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'dotnet-agent-framework' }} - - env: - SAMPLE_PATH: dotnet/agent-framework/sample-agent - AGENT_PORT: 3978 - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup .NET - uses: actions/setup-dotnet@v4 - with: - dotnet-version: '8.0.x' - - - name: Restore dependencies - working-directory: ${{ env.SAMPLE_PATH }} - run: dotnet restore - - - name: Build - working-directory: ${{ env.SAMPLE_PATH }} - run: dotnet build --no-restore - - - name: Acquire Bearer Token (ROPC) - id: token - shell: pwsh - run: | - $token = & "${{ env.SCRIPTS_PATH }}/Acquire-BearerToken.ps1" ` - -ClientId "${{ secrets.MCP_CLIENT_ID }}" ` - -TenantId "${{ secrets.MCP_TENANT_ID }}" ` - -Username "${{ secrets.MCP_TEST_USERNAME }}" ` - -Password "${{ secrets.MCP_TEST_PASSWORD }}" - echo "BEARER_TOKEN=$token" >> $env:GITHUB_OUTPUT - echo "::add-mask::$token" - - - name: Copy ToolingManifest.json - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Copy-ToolingManifest.ps1" -TargetPath "${{ env.SAMPLE_PATH }}" - - - name: Generate appsettings.json - shell: pwsh - run: | - $configMappings = @{ - "AIServices:AzureOpenAI:Endpoint" = "${{ secrets.DOTNET_AF_AZURE_OPENAI_ENDPOINT }}" - "AIServices:AzureOpenAI:ApiKey" = "${{ secrets.DOTNET_AF_AZURE_OPENAI_API_KEY }}" - "AIServices:AzureOpenAI:DeploymentName" = "${{ secrets.DOTNET_AF_AZURE_OPENAI_DEPLOYMENT }}" - "TokenValidation:Enabled" = "false" - "Connections:ServiceConnection:Settings:AuthType" = "ClientSecret" - "Connections:ServiceConnection:Settings:ClientId" = "${{ secrets.DOTNET_AF_CLIENT_ID }}" - "Connections:ServiceConnection:Settings:ClientSecret" = "${{ secrets.DOTNET_AF_CLIENT_SECRET }}" - "Connections:ServiceConnection:Settings:TenantId" = "${{ secrets.TENANT_ID }}" - "Connections:ServiceConnection:Settings:AuthorityEndpoint" = "https://login.microsoftonline.com/${{ secrets.TENANT_ID }}" - "Connections:ServiceConnection:Settings:Scopes:0" = "5a807f24-c9de-44ee-a3a7-329e88a00ffc/.default" - } - & "${{ env.SCRIPTS_PATH }}/Generate-AppSettings.ps1" ` - -OutputPath "${{ env.SAMPLE_PATH }}/appsettings.json" ` - -ConfigMappings $configMappings - - - name: Start Agent - id: start-agent - shell: pwsh - run: | - $agentPid = & "${{ env.SCRIPTS_PATH }}/Start-Agent.ps1" ` - -AgentPath "${{ env.SAMPLE_PATH }}" ` - -StartCommand "dotnet run --no-build" ` - -Port ${{ env.AGENT_PORT }} ` - -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` - -Environment "Development" ` - -Runtime "dotnet" - echo "AGENT_PID=$agentPid" >> $env:GITHUB_OUTPUT - - - name: Verify Agent Running - shell: pwsh - run: | - $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" - if ($agentPid) { - try { - $proc = Get-Process -Id $agentPid -ErrorAction Stop - Write-Host "Agent process (PID: $agentPid) is running: $($proc.ProcessName)" -ForegroundColor Green - } catch { - Write-Host "ERROR: Agent process (PID: $agentPid) is NOT running!" -ForegroundColor Red - throw "Agent process has stopped" - } - } - $response = Invoke-WebRequest -Uri "http://localhost:${{ env.AGENT_PORT }}/api/health" -UseBasicParsing -ErrorAction SilentlyContinue - Write-Host "Health: $($response.StatusCode)" -ForegroundColor Green - - - name: Restore E2E Test Dependencies - run: dotnet restore "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" - - - name: Run .NET E2E Tests - shell: pwsh - run: | - dotnet test "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" ` - --no-restore ` - --filter "Category=HTTP" ` - --logger "trx;LogFileName=test-results.trx" ` - --results-directory "${{ runner.temp }}/TestResults" - env: - AGENT_PORT: ${{ env.AGENT_PORT }} - AGENT_URL: http://localhost:${{ env.AGENT_PORT }} - TEST_RESULTS_DIR: ${{ runner.temp }}/TestConversations - - - name: Upload Test Results - if: always() - uses: actions/upload-artifact@v4 - with: - name: test-results-dotnet-af - path: ${{ runner.temp }}/TestResults/ - retention-days: 30 - - - name: Capture Logs - if: always() - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Capture-AgentLogs.ps1" -AgentPath "${{ env.SAMPLE_PATH }}" - - - name: Cleanup - if: always() - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Stop-AgentProcess.ps1" ` - -AgentPID "${{ steps.start-agent.outputs.AGENT_PID }}" ` - -Port ${{ env.AGENT_PORT }} - - # =========================================================================== - # Summary - # =========================================================================== - summary: - name: Test Summary - runs-on: ubuntu-latest - needs: [python-openai, nodejs-openai, dotnet-semantic-kernel, dotnet-agent-framework] - if: always() - - steps: - - name: Generate Summary - run: | - echo "## E2E Test Results" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "| Sample | Status |" >> $GITHUB_STEP_SUMMARY - echo "|--------|--------|" >> $GITHUB_STEP_SUMMARY - echo "| Python OpenAI | ${{ needs.python-openai.result }} |" >> $GITHUB_STEP_SUMMARY - echo "| Node.js OpenAI | ${{ needs.nodejs-openai.result }} |" >> $GITHUB_STEP_SUMMARY - echo "| .NET Semantic Kernel | ${{ needs.dotnet-semantic-kernel.result }} |" >> $GITHUB_STEP_SUMMARY - echo "| .NET Agent Framework | ${{ needs.dotnet-agent-framework.result }} |" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/e2e-dotnet-agent-framework.yml b/.github/workflows/e2e-dotnet-agent-framework.yml new file mode 100644 index 00000000..8e5b9379 --- /dev/null +++ b/.github/workflows/e2e-dotnet-agent-framework.yml @@ -0,0 +1,153 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +name: E2E - .NET Agent Framework + +on: + workflow_call: + workflow_dispatch: + push: + branches: [main] + paths: + - 'dotnet/agent-framework/**' + - 'scripts/e2e/**' + - '.github/workflows/e2e-dotnet-agent-framework.yml' + pull_request: + branches: [main] + paths: + - 'dotnet/agent-framework/**' + - 'scripts/e2e/**' + +permissions: + contents: read + +env: + SAMPLE_PATH: dotnet/agent-framework/sample-agent + AGENT_PORT: 3978 + SCRIPTS_PATH: scripts/e2e + E2E_TESTS_PATH: tests/e2e + +jobs: + dotnet-agent-framework: + name: .NET Agent Framework Agent + runs-on: windows-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + + - name: Restore dependencies + working-directory: ${{ env.SAMPLE_PATH }} + run: dotnet restore + + - name: Build + working-directory: ${{ env.SAMPLE_PATH }} + run: dotnet build --no-restore + + - name: Acquire Bearer Token (ROPC) + id: token + shell: pwsh + run: | + $token = & "${{ env.SCRIPTS_PATH }}/Acquire-BearerToken.ps1" ` + -ClientId "${{ secrets.MCP_CLIENT_ID }}" ` + -TenantId "${{ secrets.MCP_TENANT_ID }}" ` + -Username "${{ secrets.MCP_TEST_USERNAME }}" ` + -Password "${{ secrets.MCP_TEST_PASSWORD }}" + echo "BEARER_TOKEN=$token" >> $env:GITHUB_OUTPUT + echo "::add-mask::$token" + + - name: Copy ToolingManifest.json + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Copy-ToolingManifest.ps1" -TargetPath "${{ env.SAMPLE_PATH }}" + + - name: Generate appsettings.json + shell: pwsh + run: | + $configMappings = @{ + "AIServices:AzureOpenAI:Endpoint" = "${{ secrets.DOTNET_AF_AZURE_OPENAI_ENDPOINT }}" + "AIServices:AzureOpenAI:ApiKey" = "${{ secrets.DOTNET_AF_AZURE_OPENAI_API_KEY }}" + "AIServices:AzureOpenAI:DeploymentName" = "${{ secrets.DOTNET_AF_AZURE_OPENAI_DEPLOYMENT }}" + "TokenValidation:Enabled" = "false" + "Connections:ServiceConnection:Settings:AuthType" = "ClientSecret" + "Connections:ServiceConnection:Settings:ClientId" = "${{ secrets.DOTNET_AF_CLIENT_ID }}" + "Connections:ServiceConnection:Settings:ClientSecret" = "${{ secrets.DOTNET_AF_CLIENT_SECRET }}" + "Connections:ServiceConnection:Settings:TenantId" = "${{ secrets.TENANT_ID }}" + "Connections:ServiceConnection:Settings:AuthorityEndpoint" = "https://login.microsoftonline.com/${{ secrets.TENANT_ID }}" + "Connections:ServiceConnection:Settings:Scopes:0" = "5a807f24-c9de-44ee-a3a7-329e88a00ffc/.default" + } + & "${{ env.SCRIPTS_PATH }}/Generate-AppSettings.ps1" ` + -OutputPath "${{ env.SAMPLE_PATH }}/appsettings.json" ` + -ConfigMappings $configMappings + + - name: Start Agent + id: start-agent + shell: pwsh + run: | + $agentPid = & "${{ env.SCRIPTS_PATH }}/Start-Agent.ps1" ` + -AgentPath "${{ env.SAMPLE_PATH }}" ` + -StartCommand "dotnet run --no-build" ` + -Port ${{ env.AGENT_PORT }} ` + -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` + -Environment "Development" ` + -Runtime "dotnet" + echo "AGENT_PID=$agentPid" >> $env:GITHUB_OUTPUT + + - name: Verify Agent Running + shell: pwsh + run: | + $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" + if ($agentPid) { + try { + $proc = Get-Process -Id $agentPid -ErrorAction Stop + Write-Host "Agent process (PID: $agentPid) is running: $($proc.ProcessName)" -ForegroundColor Green + } catch { + Write-Host "ERROR: Agent process (PID: $agentPid) is NOT running!" -ForegroundColor Red + throw "Agent process has stopped" + } + } + $response = Invoke-WebRequest -Uri "http://localhost:${{ env.AGENT_PORT }}/api/health" -UseBasicParsing -ErrorAction SilentlyContinue + Write-Host "Health: $($response.StatusCode)" -ForegroundColor Green + + - name: Restore E2E Test Dependencies + run: dotnet restore "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" + + - name: Run .NET E2E Tests + shell: pwsh + run: | + dotnet test "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" ` + --no-restore ` + --verbosity normal ` + --logger "console;verbosity=detailed" ` + --logger "trx;LogFileName=test-results-dotnet-af.trx" + + - name: Capture Agent Logs + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Capture-AgentLogs.ps1" ` + -AgentPath "${{ env.SAMPLE_PATH }}" + + - name: Stop Agent Process + if: always() + shell: pwsh + run: | + $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" + if ($agentPid) { + & "${{ env.SCRIPTS_PATH }}/Stop-AgentProcess.ps1" -AgentPID $agentPid + } + + - name: Upload Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-results-dotnet-af + path: | + ${{ env.E2E_TESTS_PATH }}/TestResults/**/*.trx + ${{ env.E2E_TESTS_PATH }}/TestResults/**/*-logs.txt + retention-days: 7 diff --git a/.github/workflows/e2e-dotnet-semantic-kernel.yml b/.github/workflows/e2e-dotnet-semantic-kernel.yml new file mode 100644 index 00000000..583ed034 --- /dev/null +++ b/.github/workflows/e2e-dotnet-semantic-kernel.yml @@ -0,0 +1,153 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +name: E2E - .NET Semantic Kernel + +on: + workflow_call: + workflow_dispatch: + push: + branches: [main] + paths: + - 'dotnet/semantic-kernel/**' + - 'scripts/e2e/**' + - '.github/workflows/e2e-dotnet-semantic-kernel.yml' + pull_request: + branches: [main] + paths: + - 'dotnet/semantic-kernel/**' + - 'scripts/e2e/**' + +permissions: + contents: read + +env: + SAMPLE_PATH: dotnet/semantic-kernel/sample-agent + AGENT_PORT: 3978 + SCRIPTS_PATH: scripts/e2e + E2E_TESTS_PATH: tests/e2e + +jobs: + dotnet-semantic-kernel: + name: .NET Semantic Kernel Agent + runs-on: windows-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + + - name: Restore dependencies + working-directory: ${{ env.SAMPLE_PATH }} + run: dotnet restore + + - name: Build + working-directory: ${{ env.SAMPLE_PATH }} + run: dotnet build --no-restore + + - name: Acquire Bearer Token (ROPC) + id: token + shell: pwsh + run: | + $token = & "${{ env.SCRIPTS_PATH }}/Acquire-BearerToken.ps1" ` + -ClientId "${{ secrets.MCP_CLIENT_ID }}" ` + -TenantId "${{ secrets.MCP_TENANT_ID }}" ` + -Username "${{ secrets.MCP_TEST_USERNAME }}" ` + -Password "${{ secrets.MCP_TEST_PASSWORD }}" + echo "BEARER_TOKEN=$token" >> $env:GITHUB_OUTPUT + echo "::add-mask::$token" + + - name: Copy ToolingManifest.json + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Copy-ToolingManifest.ps1" -TargetPath "${{ env.SAMPLE_PATH }}" + + - name: Generate appsettings.json + shell: pwsh + run: | + $configMappings = @{ + "AIServices:UseAzureOpenAI" = "true" + "AIServices:AzureOpenAI:Endpoint" = "${{ secrets.DOTNET_SK_AZURE_OPENAI_ENDPOINT }}" + "AIServices:AzureOpenAI:ApiKey" = "${{ secrets.DOTNET_SK_AZURE_OPENAI_API_KEY }}" + "AIServices:AzureOpenAI:DeploymentName" = "${{ secrets.DOTNET_SK_AZURE_OPENAI_DEPLOYMENT }}" + "TokenValidation:Enabled" = "false" + "TokenValidation:TenantId" = "${{ secrets.TENANT_ID }}" + "Connections:ServiceConnection:Settings:AuthType" = "ClientSecret" + "Connections:ServiceConnection:Settings:ClientId" = "${{ secrets.DOTNET_SK_CLIENT_ID }}" + "Connections:ServiceConnection:Settings:ClientSecret" = "${{ secrets.DOTNET_SK_CLIENT_SECRET }}" + "Connections:ServiceConnection:Settings:TenantId" = "${{ secrets.TENANT_ID }}" + } + & "${{ env.SCRIPTS_PATH }}/Generate-AppSettings.ps1" ` + -OutputPath "${{ env.SAMPLE_PATH }}/appsettings.json" ` + -ConfigMappings $configMappings + + - name: Start Agent + id: start-agent + shell: pwsh + run: | + $agentPid = & "${{ env.SCRIPTS_PATH }}/Start-Agent.ps1" ` + -AgentPath "${{ env.SAMPLE_PATH }}" ` + -StartCommand "dotnet run --no-build" ` + -Port ${{ env.AGENT_PORT }} ` + -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` + -Environment "Development" ` + -Runtime "dotnet" + echo "AGENT_PID=$agentPid" >> $env:GITHUB_OUTPUT + + - name: Verify Agent Running + shell: pwsh + run: | + $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" + if ($agentPid) { + try { + $proc = Get-Process -Id $agentPid -ErrorAction Stop + Write-Host "Agent process (PID: $agentPid) is running: $($proc.ProcessName)" -ForegroundColor Green + } catch { + Write-Host "ERROR: Agent process (PID: $agentPid) is NOT running!" -ForegroundColor Red + throw "Agent process has stopped" + } + } + $response = Invoke-WebRequest -Uri "http://localhost:${{ env.AGENT_PORT }}/api/health" -UseBasicParsing -ErrorAction SilentlyContinue + Write-Host "Health: $($response.StatusCode)" -ForegroundColor Green + + - name: Restore E2E Test Dependencies + run: dotnet restore "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" + + - name: Run .NET E2E Tests + shell: pwsh + run: | + dotnet test "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" ` + --no-restore ` + --verbosity normal ` + --logger "console;verbosity=detailed" ` + --logger "trx;LogFileName=test-results-dotnet-sk.trx" + + - name: Capture Agent Logs + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Capture-AgentLogs.ps1" ` + -AgentPath "${{ env.SAMPLE_PATH }}" + + - name: Stop Agent Process + if: always() + shell: pwsh + run: | + $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" + if ($agentPid) { + & "${{ env.SCRIPTS_PATH }}/Stop-AgentProcess.ps1" -AgentPID $agentPid + } + + - name: Upload Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-results-dotnet-sk + path: | + ${{ env.E2E_TESTS_PATH }}/TestResults/**/*.trx + ${{ env.E2E_TESTS_PATH }}/TestResults/**/*-logs.txt + retention-days: 7 diff --git a/.github/workflows/e2e-nodejs-openai.yml b/.github/workflows/e2e-nodejs-openai.yml new file mode 100644 index 00000000..8c207dd4 --- /dev/null +++ b/.github/workflows/e2e-nodejs-openai.yml @@ -0,0 +1,169 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +name: E2E - Node.js OpenAI + +on: + workflow_call: # Allow orchestrator to call this workflow + workflow_dispatch: # Allow manual triggering + push: + branches: [main] + paths: + - 'nodejs/openai/**' + - 'scripts/e2e/**' + - '.github/workflows/e2e-nodejs-openai.yml' + pull_request: + branches: [main] + paths: + - 'nodejs/openai/**' + - 'scripts/e2e/**' + +permissions: + contents: read + +env: + SAMPLE_PATH: nodejs/openai/sample-agent + AGENT_PORT: 3979 + SCRIPTS_PATH: scripts/e2e + E2E_TESTS_PATH: tests/e2e + +jobs: + nodejs-openai: + name: Node.js OpenAI Agent + runs-on: windows-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup .NET SDK + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Install dependencies + working-directory: ${{ env.SAMPLE_PATH }} + run: | + if (Test-Path "package-lock.json") { + npm ci + } else { + npm install + } + if (Test-Path "tsconfig.json") { + npm run build + } + shell: pwsh + + - name: Acquire Bearer Token (ROPC) + id: token + shell: pwsh + run: | + $token = & "${{ env.SCRIPTS_PATH }}/Acquire-BearerToken.ps1" ` + -ClientId "${{ secrets.MCP_CLIENT_ID }}" ` + -TenantId "${{ secrets.MCP_TENANT_ID }}" ` + -Username "${{ secrets.MCP_TEST_USERNAME }}" ` + -Password "${{ secrets.MCP_TEST_PASSWORD }}" + echo "BEARER_TOKEN=$token" >> $env:GITHUB_OUTPUT + echo "::add-mask::$token" + + - name: Copy ToolingManifest.json + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Copy-ToolingManifest.ps1" -TargetPath "${{ env.SAMPLE_PATH }}" + + - name: Generate .env configuration + shell: pwsh + run: | + $configMappings = @{ + "NODE_ENV" = "development" + "AZURE_OPENAI_API_KEY" = "${{ secrets.NODEJS_OPENAI_AZURE_OPENAI_API_KEY }}" + "AZURE_OPENAI_ENDPOINT" = "${{ secrets.NODEJS_OPENAI_AZURE_OPENAI_ENDPOINT }}" + "AZURE_OPENAI_DEPLOYMENT" = "${{ secrets.NODEJS_OPENAI_AZURE_OPENAI_DEPLOYMENT }}" + "AZURE_OPENAI_API_VERSION" = "2024-12-01-preview" + "connections__service_connection__settings__authType" = "ClientSecret" + "connections__service_connection__settings__clientId" = "${{ secrets.NODEJS_OPENAI_AGENT_ID }}" + "connections__service_connection__settings__clientSecret" = "${{ secrets.NODEJS_OPENAI_CLIENT_SECRET }}" + "connections__service_connection__settings__tenantId" = "${{ secrets.TENANT_ID }}" + "connectionsMap__0__serviceUrl" = "*" + "connectionsMap__0__connection" = "service_connection" + "agentic_scopes" = "ea9ffc3e-8a23-4a7d-836d-234d7c7565c1/.default" + } + & "${{ env.SCRIPTS_PATH }}/Generate-EnvConfig.ps1" ` + -OutputPath "${{ env.SAMPLE_PATH }}/.env" ` + -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` + -Port ${{ env.AGENT_PORT }} ` + -ConfigMappings $configMappings + + - name: Start Agent + id: start-agent + shell: pwsh + run: | + $agentPid = & "${{ env.SCRIPTS_PATH }}/Start-Agent.ps1" ` + -AgentPath "${{ env.SAMPLE_PATH }}" ` + -StartCommand "node dist/index.js" ` + -Port ${{ env.AGENT_PORT }} ` + -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` + -Environment "Development" ` + -Runtime "nodejs" + echo "AGENT_PID=$agentPid" >> $env:GITHUB_OUTPUT + + - name: Verify Agent Running + shell: pwsh + run: | + $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" + if ($agentPid) { + try { + $proc = Get-Process -Id $agentPid -ErrorAction Stop + Write-Host "Agent process (PID: $agentPid) is running: $($proc.ProcessName)" -ForegroundColor Green + } catch { + Write-Host "ERROR: Agent process (PID: $agentPid) is NOT running!" -ForegroundColor Red + throw "Agent process has stopped" + } + } + $agentUrl = "http://localhost:${{ env.AGENT_PORT }}" + $healthResponse = Invoke-WebRequest -Uri "$agentUrl/api/health" -Method GET -UseBasicParsing -ErrorAction SilentlyContinue + Write-Host "Health check: $($healthResponse.StatusCode)" -ForegroundColor Green + + - name: Restore E2E Test Dependencies + run: dotnet restore "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" + + - name: Run .NET E2E Tests + shell: pwsh + run: | + dotnet test "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" ` + --no-restore ` + --verbosity normal ` + --logger "console;verbosity=detailed" ` + --logger "trx;LogFileName=test-results-nodejs-openai.trx" ` + --filter "FullyQualifiedName~BasicConversation|FullyQualifiedName~Notification" + + - name: Capture Agent Logs + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Capture-AgentLogs.ps1" ` + -AgentPath "${{ env.SAMPLE_PATH }}" + + - name: Stop Agent Process + if: always() + shell: pwsh + run: | + $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" + if ($agentPid) { + & "${{ env.SCRIPTS_PATH }}/Stop-AgentProcess.ps1" -AgentPID $agentPid + } + + - name: Upload Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-results-nodejs-openai + path: | + ${{ env.E2E_TESTS_PATH }}/TestResults/**/*.trx + ${{ env.E2E_TESTS_PATH }}/TestResults/**/*-logs.txt + retention-days: 7 diff --git a/.github/workflows/e2e-orchestrator.yml b/.github/workflows/e2e-orchestrator.yml new file mode 100644 index 00000000..1cee986c --- /dev/null +++ b/.github/workflows/e2e-orchestrator.yml @@ -0,0 +1,67 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +name: E2E Test Orchestrator + +on: + schedule: + # Run every 6 hours + - cron: '0 */6 * * *' + push: + branches: [main] + paths: + - 'python/openai/**' + - 'nodejs/openai/**' + - 'dotnet/semantic-kernel/**' + - 'dotnet/agent-framework/**' + - '.github/workflows/e2e-*.yml' + - 'scripts/e2e/**' + pull_request: + branches: [main] + paths: + - 'python/openai/**' + - 'nodejs/openai/**' + - 'dotnet/semantic-kernel/**' + - 'dotnet/agent-framework/**' + - '.github/workflows/e2e-*.yml' + - 'scripts/e2e/**' + workflow_dispatch: + inputs: + sample: + description: 'Sample to test (leave empty for all)' + required: false + type: choice + options: + - '' + - 'python-openai' + - 'nodejs-openai' + - 'dotnet-sk' + - 'dotnet-af' + +permissions: + contents: read + +jobs: + python-openai: + name: Python OpenAI E2E + if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'python-openai' }} + uses: ./.github/workflows/e2e-python-openai.yml + secrets: inherit + + nodejs-openai: + name: Node.js OpenAI E2E + if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'nodejs-openai' }} + uses: ./.github/workflows/e2e-nodejs-openai.yml + secrets: inherit + + dotnet-sk: + name: .NET Semantic Kernel E2E + if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'dotnet-sk' }} + uses: ./.github/workflows/e2e-dotnet-semantic-kernel.yml + secrets: inherit + + dotnet-af: + name: .NET Agent Framework E2E + if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'dotnet-af' }} + uses: ./.github/workflows/e2e-dotnet-agent-framework.yml + secrets: inherit diff --git a/.github/workflows/e2e-python-openai.yml b/.github/workflows/e2e-python-openai.yml new file mode 100644 index 00000000..668f4e8d --- /dev/null +++ b/.github/workflows/e2e-python-openai.yml @@ -0,0 +1,187 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +name: E2E - Python OpenAI + +on: + workflow_call: # Allow orchestrator to call this workflow + workflow_dispatch: # Allow manual triggering + push: + branches: [main] + paths: + - 'python/openai/**' + - 'scripts/e2e/**' + - '.github/workflows/e2e-python-openai.yml' + pull_request: + branches: [main] + paths: + - 'python/openai/**' + - 'scripts/e2e/**' + +permissions: + contents: read + +env: + SAMPLE_PATH: python/openai/sample-agent + AGENT_PORT: 3979 + SCRIPTS_PATH: scripts/e2e + E2E_TESTS_PATH: tests/e2e + +jobs: + python-openai: + name: Python OpenAI Agent + runs-on: windows-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup .NET SDK + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install uv package manager + run: pip install uv + + - name: Install dependencies + working-directory: ${{ env.SAMPLE_PATH }} + run: uv sync + + - name: Acquire Bearer Token (ROPC) + id: token + shell: pwsh + run: | + $token = & "${{ env.SCRIPTS_PATH }}/Acquire-BearerToken.ps1" ` + -ClientId "${{ secrets.MCP_CLIENT_ID }}" ` + -TenantId "${{ secrets.MCP_TENANT_ID }}" ` + -Username "${{ secrets.MCP_TEST_USERNAME }}" ` + -Password "${{ secrets.MCP_TEST_PASSWORD }}" + echo "BEARER_TOKEN=$token" >> $env:GITHUB_OUTPUT + echo "::add-mask::$token" + + - name: Copy ToolingManifest.json + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Copy-ToolingManifest.ps1" -TargetPath "${{ env.SAMPLE_PATH }}" + + - name: Generate .env configuration + shell: pwsh + run: | + $configMappings = @{ + "AZURE_OPENAI_API_KEY" = "${{ secrets.PYTHON_OPENAI_AZURE_OPENAI_API_KEY }}" + "AZURE_OPENAI_ENDPOINT" = "${{ secrets.PYTHON_OPENAI_AZURE_OPENAI_ENDPOINT }}" + "AZURE_OPENAI_DEPLOYMENT" = "${{ secrets.PYTHON_OPENAI_AZURE_OPENAI_DEPLOYMENT }}" + "AZURE_OPENAI_API_VERSION" = "2024-12-01-preview" + "USE_AGENTIC_AUTH" = "false" + "MCP_PLATFORM_ENDPOINT" = "https://agent365.svc.cloud.microsoft" + "AGENT_ID" = "${{ secrets.PYTHON_OPENAI_AGENT_ID }}" + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__AUTHTYPE" = "ClientSecret" + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID" = "${{ secrets.PYTHON_OPENAI_AGENT_ID }}" + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET" = "${{ secrets.PYTHON_OPENAI_CLIENT_SECRET }}" + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID" = "${{ secrets.TENANT_ID }}" + "CONNECTIONSMAP__0__SERVICEURL" = "*" + "CONNECTIONSMAP__0__CONNECTION" = "SERVICE_CONNECTION" + "ENABLE_OBSERVABILITY" = "true" + } + & "${{ env.SCRIPTS_PATH }}/Generate-EnvConfig.ps1" ` + -OutputPath "${{ env.SAMPLE_PATH }}/.env" ` + -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` + -Port ${{ env.AGENT_PORT }} ` + -ConfigMappings $configMappings + + - name: Start Agent + id: start-agent + shell: pwsh + run: | + $agentPid = & "${{ env.SCRIPTS_PATH }}/Start-Agent.ps1" ` + -AgentPath "${{ env.SAMPLE_PATH }}" ` + -StartCommand "uv run python start_with_generic_host.py" ` + -Port ${{ env.AGENT_PORT }} ` + -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` + -Environment "Development" ` + -Runtime "python" + echo "AGENT_PID=$agentPid" >> $env:GITHUB_OUTPUT + + - name: Verify Agent Running + shell: pwsh + run: | + Write-Host "=== Verifying Agent ===" -ForegroundColor Cyan + + $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" + Write-Host "Checking agent process (PID: $agentPid)..." -ForegroundColor Gray + + if ($agentPid) { + try { + $proc = Get-Process -Id $agentPid -ErrorAction Stop + Write-Host "Agent process (PID: $agentPid) is running: $($proc.ProcessName)" -ForegroundColor Green + } catch { + Write-Host "ERROR: Agent process (PID: $agentPid) is NOT running!" -ForegroundColor Red + + $logFile = "${{ env.SAMPLE_PATH }}/agent.log" + if (Test-Path $logFile) { + Write-Host "Agent logs:" -ForegroundColor Yellow + Get-Content $logFile + } + throw "Agent process has stopped" + } + } + + $agentUrl = "http://localhost:${{ env.AGENT_PORT }}" + Write-Host "Verifying agent at $agentUrl..." -ForegroundColor Gray + $healthResponse = Invoke-WebRequest -Uri "$agentUrl/api/health" -Method GET -UseBasicParsing -ErrorAction SilentlyContinue + Write-Host "Health check: $($healthResponse.StatusCode)" -ForegroundColor Green + + - name: Restore E2E Test Dependencies + shell: pwsh + run: | + dotnet restore "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" + + - name: Run .NET E2E Tests + shell: pwsh + run: | + dotnet test "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" ` + --no-restore ` + --filter "Category=HTTP" ` + --logger "trx;LogFileName=test-results.trx" ` + --results-directory "${{ runner.temp }}/TestResults" + env: + AGENT_PORT: ${{ env.AGENT_PORT }} + AGENT_URL: http://localhost:${{ env.AGENT_PORT }} + TEST_RESULTS_DIR: ${{ runner.temp }}/TestConversations + + - name: Upload Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-results-python-openai + path: ${{ runner.temp }}/TestResults/ + retention-days: 30 + + - name: Upload Test Conversations + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-conversations-python-openai + path: ${{ runner.temp }}/TestConversations/ + retention-days: 30 + continue-on-error: true + + - name: Capture Agent Logs + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Capture-AgentLogs.ps1" -AgentPath "${{ env.SAMPLE_PATH }}" + + - name: Cleanup Agent Process + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Stop-AgentProcess.ps1" ` + -AgentPID "${{ steps.start-agent.outputs.AGENT_PID }}" ` + -Port ${{ env.AGENT_PORT }} diff --git a/.github/workflows/update-e2e-status.yml b/.github/workflows/update-e2e-status.yml deleted file mode 100644 index d20dac05..00000000 --- a/.github/workflows/update-e2e-status.yml +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -name: Update E2E Status - -on: - workflow_run: - workflows: ["E2E Agent Samples"] - types: - - completed - workflow_dispatch: - -permissions: - contents: write - -jobs: - update-status: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: main - - - name: Get job statuses and update README - uses: actions/github-script@v7 - with: - script: | - const fs = require('fs'); - - // Get the triggering workflow run or latest run - let runId; - if (context.payload.workflow_run) { - runId = context.payload.workflow_run.id; - } else { - // Manual trigger - get latest run - const runs = await github.rest.actions.listWorkflowRuns({ - owner: context.repo.owner, - repo: context.repo.repo, - workflow_id: 'e2e-agent-samples.yml', - branch: 'main', - per_page: 1 - }); - if (runs.data.workflow_runs.length === 0) { - console.log('No workflow runs found'); - return; - } - runId = runs.data.workflow_runs[0].id; - } - - console.log(`Processing workflow run: ${runId}`); - - // Get jobs for this run - const jobs = await github.rest.actions.listJobsForWorkflowRun({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: runId - }); - - // Map job names to their status badges - const jobStatusMap = { - 'Python OpenAI Agent': { key: 'python-openai', label: 'Python OpenAI' }, - 'Node.js OpenAI Agent': { key: 'nodejs-openai', label: 'Node.js OpenAI' }, - '.NET Semantic Kernel Agent': { key: 'dotnet-sk', label: '.NET Semantic Kernel' }, - '.NET Agent Framework Agent': { key: 'dotnet-af', label: '.NET Agent Framework' } - }; - - const statuses = {}; - for (const job of jobs.data.jobs) { - const mapping = jobStatusMap[job.name]; - if (mapping) { - const conclusion = job.conclusion || job.status; - let badge; - if (conclusion === 'success') { - badge = `![${mapping.label}](https://img.shields.io/badge/${encodeURIComponent(mapping.label)}-passing-brightgreen)`; - } else if (conclusion === 'failure') { - badge = `![${mapping.label}](https://img.shields.io/badge/${encodeURIComponent(mapping.label)}-failing-red)`; - } else if (conclusion === 'in_progress' || job.status === 'in_progress') { - badge = `![${mapping.label}](https://img.shields.io/badge/${encodeURIComponent(mapping.label)}-running-yellow)`; - } else { - badge = `![${mapping.label}](https://img.shields.io/badge/${encodeURIComponent(mapping.label)}-pending-lightgrey)`; - } - statuses[mapping.key] = badge; - console.log(`${job.name}: ${conclusion} -> ${badge}`); - } - } - - // Read current README - let readme = fs.readFileSync('README.md', 'utf8'); - - // Generate new status table - const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${runId}`; - const newTable = `## E2E Test Status - -| Sample | Status | -|--------|--------| -| Python OpenAI | ${statuses['python-openai'] || '![Python OpenAI](https://img.shields.io/badge/Python%20OpenAI-unknown-lightgrey)'} | -| Node.js OpenAI | ${statuses['nodejs-openai'] || '![Node.js OpenAI](https://img.shields.io/badge/Node.js%20OpenAI-unknown-lightgrey)'} | -| .NET Semantic Kernel | ${statuses['dotnet-sk'] || '![.NET SK](https://img.shields.io/badge/.NET%20SK-unknown-lightgrey)'} | -| .NET Agent Framework | ${statuses['dotnet-af'] || '![.NET AF](https://img.shields.io/badge/.NET%20AF-unknown-lightgrey)'} | - -*Last updated: ${new Date().toISOString().split('T')[0]} | [View Run](${runUrl})*`; - - // Replace the status section - const statusRegex = /## E2E Test Status[\s\S]*?\n(?=\n>|$)/; - if (statusRegex.test(readme)) { - readme = readme.replace(statusRegex, newTable + '\n'); - } - - fs.writeFileSync('README.md', readme); - console.log('README updated successfully'); - - - name: Commit and push changes - run: | - git config --local user.email "github-actions[bot]@users.noreply.github.com" - git config --local user.name "github-actions[bot]" - git add README.md - git diff --staged --quiet || (git commit -m "chore: Update E2E test status [skip ci]" && git push) diff --git a/README.md b/README.md index b6281d03..72eaca34 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Microsoft Agent 365 SDK Samples and Prompts -[![E2E Tests](https://img.shields.io/github/actions/workflow/status/microsoft/Agent365-Samples/e2e-agent-samples.yml?branch=main&label=E2E%20Agent%20Samples)](https://github.com/microsoft/Agent365-Samples/actions/workflows/e2e-agent-samples.yml) +[![E2E Tests](https://img.shields.io/github/actions/workflow/status/microsoft/Agent365-Samples/e2e-orchestrator.yml?branch=main&label=E2E%20All%20Samples)](https://github.com/microsoft/Agent365-Samples/actions/workflows/e2e-orchestrator.yml) This repository contains sample agents and prompts for building with the Microsoft Agent 365 SDK. The Microsoft Agent 365 SDK extends the Microsoft 365 Agents SDK with enterprise-grade capabilities for building sophisticated agents. It provides comprehensive tooling for observability, notifications, runtime utilities, and development tools that help developers create production-ready agents for platforms including M365, Teams, Copilot Studio, and Webchat. @@ -11,10 +11,10 @@ This repository contains sample agents and prompts for building with the Microso | Sample | Status | |--------|--------| -| Python OpenAI | [![Python OpenAI](https://img.shields.io/github/actions/workflow/status/microsoft/Agent365-Samples/e2e-agent-samples.yml?branch=main&label=E2E&job=python-openai)](https://github.com/microsoft/Agent365-Samples/actions/workflows/e2e-agent-samples.yml) | -| Node.js OpenAI | [![Node.js OpenAI](https://img.shields.io/github/actions/workflow/status/microsoft/Agent365-Samples/e2e-agent-samples.yml?branch=main&label=E2E&job=nodejs-openai)](https://github.com/microsoft/Agent365-Samples/actions/workflows/e2e-agent-samples.yml) | -| .NET Semantic Kernel | [![.NET SK](https://img.shields.io/github/actions/workflow/status/microsoft/Agent365-Samples/e2e-agent-samples.yml?branch=main&label=E2E&job=dotnet-semantic-kernel)](https://github.com/microsoft/Agent365-Samples/actions/workflows/e2e-agent-samples.yml) | -| .NET Agent Framework | [![.NET AF](https://img.shields.io/github/actions/workflow/status/microsoft/Agent365-Samples/e2e-agent-samples.yml?branch=main&label=E2E&job=dotnet-agent-framework)](https://github.com/microsoft/Agent365-Samples/actions/workflows/e2e-agent-samples.yml) | +| Python OpenAI | [![Python OpenAI](https://img.shields.io/github/actions/workflow/status/microsoft/Agent365-Samples/e2e-python-openai.yml?branch=main&label=E2E)](https://github.com/microsoft/Agent365-Samples/actions/workflows/e2e-python-openai.yml) | +| Node.js OpenAI | [![Node.js OpenAI](https://img.shields.io/github/actions/workflow/status/microsoft/Agent365-Samples/e2e-nodejs-openai.yml?branch=main&label=E2E)](https://github.com/microsoft/Agent365-Samples/actions/workflows/e2e-nodejs-openai.yml) | +| .NET Semantic Kernel | [![.NET SK](https://img.shields.io/github/actions/workflow/status/microsoft/Agent365-Samples/e2e-dotnet-semantic-kernel.yml?branch=main&label=E2E)](https://github.com/microsoft/Agent365-Samples/actions/workflows/e2e-dotnet-semantic-kernel.yml) | +| .NET Agent Framework | [![.NET AF](https://img.shields.io/github/actions/workflow/status/microsoft/Agent365-Samples/e2e-dotnet-agent-framework.yml?branch=main&label=E2E)](https://github.com/microsoft/Agent365-Samples/actions/workflows/e2e-dotnet-agent-framework.yml) | > #### Note: > Use the information in this README to contribute to this open-source project. To learn about using this SDK in your projects, refer to the [Microsoft Agent 365 Developer documentation](https://learn.microsoft.com/en-us/microsoft-agent-365/developer/).