diff --git a/.github/workflows/e2e-dotnet-agent-framework.yml b/.github/workflows/e2e-dotnet-agent-framework.yml index 8e5b9379..9a7c3349 100644 --- a/.github/workflows/e2e-dotnet-agent-framework.yml +++ b/.github/workflows/e2e-dotnet-agent-framework.yml @@ -49,6 +49,13 @@ jobs: working-directory: ${{ env.SAMPLE_PATH }} run: dotnet build --no-restore + - name: Log SDK Versions + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Get-SDKVersions.ps1" ` + -Runtime "dotnet" ` + -WorkingDirectory "${{ env.SAMPLE_PATH }}" + - name: Acquire Bearer Token (ROPC) id: token shell: pwsh @@ -125,6 +132,15 @@ jobs: --verbosity normal ` --logger "console;verbosity=detailed" ` --logger "trx;LogFileName=test-results-dotnet-af.trx" + env: + TEST_RESULTS_DIR: ${{ runner.temp }}/TestConversations + + - name: Emit Test Conversations + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Emit-TestConversations.ps1" ` + -TestResultsDir "${{ runner.temp }}/TestConversations" - name: Capture Agent Logs if: always() diff --git a/.github/workflows/e2e-dotnet-semantic-kernel.yml b/.github/workflows/e2e-dotnet-semantic-kernel.yml index 583ed034..1736e848 100644 --- a/.github/workflows/e2e-dotnet-semantic-kernel.yml +++ b/.github/workflows/e2e-dotnet-semantic-kernel.yml @@ -49,6 +49,13 @@ jobs: working-directory: ${{ env.SAMPLE_PATH }} run: dotnet build --no-restore + - name: Log SDK Versions + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Get-SDKVersions.ps1" ` + -Runtime "dotnet" ` + -WorkingDirectory "${{ env.SAMPLE_PATH }}" + - name: Acquire Bearer Token (ROPC) id: token shell: pwsh @@ -125,6 +132,15 @@ jobs: --verbosity normal ` --logger "console;verbosity=detailed" ` --logger "trx;LogFileName=test-results-dotnet-sk.trx" + env: + TEST_RESULTS_DIR: ${{ runner.temp }}/TestConversations + + - name: Emit Test Conversations + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Emit-TestConversations.ps1" ` + -TestResultsDir "${{ runner.temp }}/TestConversations" - name: Capture Agent Logs if: always() diff --git a/.github/workflows/e2e-nodejs-langchain.yml b/.github/workflows/e2e-nodejs-langchain.yml new file mode 100644 index 00000000..14a904d7 --- /dev/null +++ b/.github/workflows/e2e-nodejs-langchain.yml @@ -0,0 +1,185 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +name: E2E - Node.js LangChain + +on: + workflow_call: # Allow orchestrator to call this workflow + workflow_dispatch: # Allow manual triggering + push: + branches: [main] + paths: + - 'nodejs/langchain/**' + - 'scripts/e2e/**' + - '.github/workflows/e2e-nodejs-langchain.yml' + pull_request: + branches: [main] + paths: + - 'nodejs/langchain/**' + - 'scripts/e2e/**' + +permissions: + contents: read + +env: + SAMPLE_PATH: nodejs/langchain/sample-agent + AGENT_PORT: 3979 + SCRIPTS_PATH: scripts/e2e + E2E_TESTS_PATH: tests/e2e + +jobs: + nodejs-langchain: + name: Node.js LangChain Agent + runs-on: windows-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup .NET SDK + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Install dependencies + working-directory: ${{ env.SAMPLE_PATH }} + run: | + if (Test-Path "package-lock.json") { + npm ci + } else { + npm install + } + if (Test-Path "tsconfig.json") { + npm run build + } + shell: pwsh + + - name: Log SDK Versions + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Get-SDKVersions.ps1" ` + -Runtime "nodejs" ` + -WorkingDirectory "${{ env.SAMPLE_PATH }}" + + - name: Acquire Bearer Token (ROPC) + id: token + shell: pwsh + run: | + $token = & "${{ env.SCRIPTS_PATH }}/Acquire-BearerToken.ps1" ` + -ClientId "${{ secrets.MCP_CLIENT_ID }}" ` + -TenantId "${{ secrets.MCP_TENANT_ID }}" ` + -Username "${{ secrets.MCP_TEST_USERNAME }}" ` + -Password "${{ secrets.MCP_TEST_PASSWORD }}" + echo "BEARER_TOKEN=$token" >> $env:GITHUB_OUTPUT + echo "::add-mask::$token" + + - name: Copy ToolingManifest.json + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Copy-ToolingManifest.ps1" -TargetPath "${{ env.SAMPLE_PATH }}" + + - name: Generate .env configuration + shell: pwsh + run: | + $configMappings = @{ + "NODE_ENV" = "development" + "AZURE_OPENAI_API_KEY" = "${{ secrets.NODEJS_OPENAI_AZURE_OPENAI_API_KEY }}" + "AZURE_OPENAI_ENDPOINT" = "${{ secrets.NODEJS_OPENAI_AZURE_OPENAI_ENDPOINT }}" + "AZURE_OPENAI_DEPLOYMENT" = "${{ secrets.NODEJS_OPENAI_AZURE_OPENAI_DEPLOYMENT }}" + "AZURE_OPENAI_API_VERSION" = "2024-12-01-preview" + "connections__service_connection__settings__authType" = "ClientSecret" + "connections__service_connection__settings__clientId" = "${{ secrets.NODEJS_OPENAI_AGENT_ID }}" + "connections__service_connection__settings__clientSecret" = "${{ secrets.NODEJS_OPENAI_CLIENT_SECRET }}" + "connections__service_connection__settings__tenantId" = "${{ secrets.TENANT_ID }}" + "connectionsMap__0__serviceUrl" = "*" + "connectionsMap__0__connection" = "service_connection" + "agentic_scopes" = "ea9ffc3e-8a23-4a7d-836d-234d7c7565c1/.default" + } + & "${{ env.SCRIPTS_PATH }}/Generate-EnvConfig.ps1" ` + -OutputPath "${{ env.SAMPLE_PATH }}/.env" ` + -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` + -Port ${{ env.AGENT_PORT }} ` + -ConfigMappings $configMappings + + - name: Start Agent + id: start-agent + shell: pwsh + run: | + $agentPid = & "${{ env.SCRIPTS_PATH }}/Start-Agent.ps1" ` + -AgentPath "${{ env.SAMPLE_PATH }}" ` + -StartCommand "node dist/index.js" ` + -Port ${{ env.AGENT_PORT }} ` + -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` + -Environment "Development" ` + -Runtime "nodejs" + echo "AGENT_PID=$agentPid" >> $env:GITHUB_OUTPUT + + - name: Verify Agent Running + shell: pwsh + run: | + $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" + if ($agentPid) { + try { + $proc = Get-Process -Id $agentPid -ErrorAction Stop + Write-Host "Agent process (PID: $agentPid) is running: $($proc.ProcessName)" -ForegroundColor Green + } catch { + Write-Host "ERROR: Agent process (PID: $agentPid) is NOT running!" -ForegroundColor Red + throw "Agent process has stopped" + } + } + $agentUrl = "http://localhost:${{ env.AGENT_PORT }}" + $healthResponse = Invoke-WebRequest -Uri "$agentUrl/api/health" -Method GET -UseBasicParsing -ErrorAction SilentlyContinue + Write-Host "Health check: $($healthResponse.StatusCode)" -ForegroundColor Green + + - name: Restore E2E Test Dependencies + run: dotnet restore "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" + + - name: Run .NET E2E Tests + shell: pwsh + run: | + dotnet test "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" ` + --no-restore ` + --verbosity normal ` + --logger "console;verbosity=detailed" ` + --logger "trx;LogFileName=test-results-nodejs-langchain.trx" ` + --filter "FullyQualifiedName~BasicConversation|FullyQualifiedName~Notification" + env: + TEST_RESULTS_DIR: ${{ runner.temp }}/TestConversations + + - name: Emit Test Conversations + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Emit-TestConversations.ps1" ` + -TestResultsDir "${{ runner.temp }}/TestConversations" + + - name: Capture Agent Logs + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Capture-AgentLogs.ps1" ` + -AgentPath "${{ env.SAMPLE_PATH }}" + + - name: Stop Agent Process + if: always() + shell: pwsh + run: | + $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" + if ($agentPid) { + & "${{ env.SCRIPTS_PATH }}/Stop-AgentProcess.ps1" -AgentPID $agentPid + } + + - name: Upload Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-results-nodejs-langchain + path: | + ${{ env.E2E_TESTS_PATH }}/TestResults/**/*.trx + ${{ env.E2E_TESTS_PATH }}/TestResults/**/*-logs.txt + retention-days: 7 diff --git a/.github/workflows/e2e-nodejs-openai.yml b/.github/workflows/e2e-nodejs-openai.yml index 8c207dd4..2f7555cd 100644 --- a/.github/workflows/e2e-nodejs-openai.yml +++ b/.github/workflows/e2e-nodejs-openai.yml @@ -59,6 +59,13 @@ jobs: } shell: pwsh + - name: Log SDK Versions + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Get-SDKVersions.ps1" ` + -Runtime "nodejs" ` + -WorkingDirectory "${{ env.SAMPLE_PATH }}" + - name: Acquire Bearer Token (ROPC) id: token shell: pwsh @@ -141,6 +148,15 @@ jobs: --logger "console;verbosity=detailed" ` --logger "trx;LogFileName=test-results-nodejs-openai.trx" ` --filter "FullyQualifiedName~BasicConversation|FullyQualifiedName~Notification" + env: + TEST_RESULTS_DIR: ${{ runner.temp }}/TestConversations + + - name: Emit Test Conversations + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Emit-TestConversations.ps1" ` + -TestResultsDir "${{ runner.temp }}/TestConversations" - name: Capture Agent Logs if: always() diff --git a/.github/workflows/e2e-orchestrator.yml b/.github/workflows/e2e-orchestrator.yml index 1cee986c..e84be697 100644 --- a/.github/workflows/e2e-orchestrator.yml +++ b/.github/workflows/e2e-orchestrator.yml @@ -12,6 +12,7 @@ on: paths: - 'python/openai/**' - 'nodejs/openai/**' + - 'nodejs/langchain/**' - 'dotnet/semantic-kernel/**' - 'dotnet/agent-framework/**' - '.github/workflows/e2e-*.yml' @@ -21,6 +22,7 @@ on: paths: - 'python/openai/**' - 'nodejs/openai/**' + - 'nodejs/langchain/**' - 'dotnet/semantic-kernel/**' - 'dotnet/agent-framework/**' - '.github/workflows/e2e-*.yml' @@ -35,11 +37,13 @@ on: - '' - 'python-openai' - 'nodejs-openai' + - 'nodejs-langchain' - 'dotnet-sk' - 'dotnet-af' permissions: contents: read + pull-requests: write jobs: python-openai: @@ -53,6 +57,12 @@ jobs: if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'nodejs-openai' }} uses: ./.github/workflows/e2e-nodejs-openai.yml secrets: inherit + + nodejs-langchain: + name: Node.js LangChain E2E + if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'nodejs-langchain' }} + uses: ./.github/workflows/e2e-nodejs-langchain.yml + secrets: inherit dotnet-sk: name: .NET Semantic Kernel E2E @@ -65,3 +75,116 @@ jobs: if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'dotnet-af' }} uses: ./.github/workflows/e2e-dotnet-agent-framework.yml secrets: inherit + + # =========================================================================== + # Summary - Posts test results to PR + # =========================================================================== + summary: + name: Test Summary + runs-on: ubuntu-latest + needs: [python-openai, nodejs-openai, nodejs-langchain, dotnet-sk, dotnet-af] + if: always() + + steps: + - name: Generate Summary + id: summary + run: | + # Determine overall status + OVERALL_STATUS="✅ All Tests Passed" + if [[ "${{ needs.python-openai.result }}" != "success" ]] || \ + [[ "${{ needs.nodejs-openai.result }}" != "success" ]] || \ + [[ "${{ needs.nodejs-langchain.result }}" != "success" ]] || \ + [[ "${{ needs.dotnet-sk.result }}" != "success" ]] || \ + [[ "${{ needs.dotnet-af.result }}" != "success" ]]; then + OVERALL_STATUS="⚠️ Some Tests Failed" + fi + + # Generate status icons using if-else for reliability + if [[ "${{ needs.python-openai.result }}" == "success" ]]; then PYTHON_OPENAI_ICON="✅"; else PYTHON_OPENAI_ICON="❌"; fi + if [[ "${{ needs.nodejs-openai.result }}" == "success" ]]; then NODEJS_OPENAI_ICON="✅"; else NODEJS_OPENAI_ICON="❌"; fi + if [[ "${{ needs.nodejs-langchain.result }}" == "success" ]]; then NODEJS_LANGCHAIN_ICON="✅"; else NODEJS_LANGCHAIN_ICON="❌"; fi + if [[ "${{ needs.dotnet-sk.result }}" == "success" ]]; then DOTNET_SK_ICON="✅"; else DOTNET_SK_ICON="❌"; fi + if [[ "${{ needs.dotnet-af.result }}" == "success" ]]; then DOTNET_AF_ICON="✅"; else DOTNET_AF_ICON="❌"; fi + + echo "## E2E Test Results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Status:** $OVERALL_STATUS" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Sample | Status | Result |" >> $GITHUB_STEP_SUMMARY + echo "|--------|--------|--------|" >> $GITHUB_STEP_SUMMARY + echo "| Python OpenAI | $PYTHON_OPENAI_ICON | ${{ needs.python-openai.result }} |" >> $GITHUB_STEP_SUMMARY + echo "| Node.js OpenAI | $NODEJS_OPENAI_ICON | ${{ needs.nodejs-openai.result }} |" >> $GITHUB_STEP_SUMMARY + echo "| Node.js LangChain | $NODEJS_LANGCHAIN_ICON | ${{ needs.nodejs-langchain.result }} |" >> $GITHUB_STEP_SUMMARY + echo "| .NET Semantic Kernel | $DOTNET_SK_ICON | ${{ needs.dotnet-sk.result }} |" >> $GITHUB_STEP_SUMMARY + echo "| .NET Agent Framework | $DOTNET_AF_ICON | ${{ needs.dotnet-af.result }} |" >> $GITHUB_STEP_SUMMARY + + - name: Post PR Comment + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + script: | + const pythonOpenaiIcon = '${{ needs.python-openai.result }}' === 'success' ? '✅' : '❌'; + const nodejsOpenaiIcon = '${{ needs.nodejs-openai.result }}' === 'success' ? '✅' : '❌'; + const nodejsLangchainIcon = '${{ needs.nodejs-langchain.result }}' === 'success' ? '✅' : '❌'; + const dotnetSkIcon = '${{ needs.dotnet-sk.result }}' === 'success' ? '✅' : '❌'; + const dotnetAfIcon = '${{ needs.dotnet-af.result }}' === 'success' ? '✅' : '❌'; + + const allPassed = '${{ needs.python-openai.result }}' === 'success' && + '${{ needs.nodejs-openai.result }}' === 'success' && + '${{ needs.nodejs-langchain.result }}' === 'success' && + '${{ needs.dotnet-sk.result }}' === 'success' && + '${{ needs.dotnet-af.result }}' === 'success'; + + const overallStatus = allPassed ? '✅ All E2E Tests Passed' : '⚠️ Some E2E Tests Failed'; + + const body = [ + `## ${overallStatus}`, + '', + '| Sample | Status | Result |', + '|--------|--------|--------|', + `| Python OpenAI | ${pythonOpenaiIcon} | ${{ needs.python-openai.result }} |`, + `| Node.js OpenAI | ${nodejsOpenaiIcon} | ${{ needs.nodejs-openai.result }} |`, + `| Node.js LangChain | ${nodejsLangchainIcon} | ${{ needs.nodejs-langchain.result }} |`, + `| .NET Semantic Kernel | ${dotnetSkIcon} | ${{ needs.dotnet-sk.result }} |`, + `| .NET Agent Framework | ${dotnetAfIcon} | ${{ needs.dotnet-af.result }} |`, + '', + `[View full test details](${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID})` + ].join('\n'); + + // Find existing comment + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + + console.log(`Found ${comments.length} comments on PR #${context.issue.number}`); + + const botComment = comments.find(comment => + comment.user.login === 'github-actions[bot]' && + comment.body.includes('E2E') + ); + + console.log(`Bot comment found: ${!!botComment}, ID: ${botComment?.id}`); + + if (botComment) { + // Update existing comment + console.log(`Updating existing comment ${botComment.id}...`); + const result = await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + body: body + }); + console.log(`Update result: ${result.status}`); + } else { + // Create new comment + console.log('Creating new comment...'); + const result = await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: body + }); + console.log(`Create result: ${result.status}, comment ID: ${result.data.id}`); + } diff --git a/.github/workflows/e2e-python-openai.yml b/.github/workflows/e2e-python-openai.yml index 668f4e8d..e666053c 100644 --- a/.github/workflows/e2e-python-openai.yml +++ b/.github/workflows/e2e-python-openai.yml @@ -53,6 +53,13 @@ jobs: working-directory: ${{ env.SAMPLE_PATH }} run: uv sync + - name: Log SDK Versions + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Get-SDKVersions.ps1" ` + -Runtime "python" ` + -WorkingDirectory "${{ env.SAMPLE_PATH }}" + - name: Acquire Bearer Token (ROPC) id: token shell: pwsh @@ -163,14 +170,12 @@ jobs: path: ${{ runner.temp }}/TestResults/ retention-days: 30 - - name: Upload Test Conversations + - name: Emit Test Conversations if: always() - uses: actions/upload-artifact@v4 - with: - name: test-conversations-python-openai - path: ${{ runner.temp }}/TestConversations/ - retention-days: 30 - continue-on-error: true + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Emit-TestConversations.ps1" ` + -TestResultsDir "${{ runner.temp }}/TestConversations" - name: Capture Agent Logs if: always() diff --git a/nodejs/langchain/sample-agent/.env.example b/nodejs/langchain/sample-agent/.env.example index cbb85229..4345c0f2 100644 --- a/nodejs/langchain/sample-agent/.env.example +++ b/nodejs/langchain/sample-agent/.env.example @@ -1,5 +1,14 @@ -# OpenAI Configuration +# LLM Configuration (choose one option) + +# Option 1: Azure OpenAI (preferred for enterprise) +AZURE_OPENAI_API_KEY= +AZURE_OPENAI_ENDPOINT= +AZURE_OPENAI_DEPLOYMENT= +AZURE_OPENAI_API_VERSION=2024-12-01-preview + +# Option 2: OpenAI (if Azure OpenAI not configured) OPENAI_API_KEY= +OPENAI_MODEL=gpt-4o # MCP Tooling Configuration BEARER_TOKEN= diff --git a/nodejs/langchain/sample-agent/src/client.ts b/nodejs/langchain/sample-agent/src/client.ts index a6ef930f..3f0c6251 100644 --- a/nodejs/langchain/sample-agent/src/client.ts +++ b/nodejs/langchain/sample-agent/src/client.ts @@ -2,7 +2,8 @@ // Licensed under the MIT License. import { createAgent, ReactAgent } from "langchain"; -import { ChatOpenAI } from "@langchain/openai"; +import { AzureChatOpenAI, ChatOpenAI } from "@langchain/openai"; +import { BaseChatModel } from "@langchain/core/language_models/chat_models"; // Tooling Imports import { McpToolRegistrationService } from '@microsoft/agents-a365-tooling-extensions-langchain'; @@ -48,8 +49,41 @@ a365Observability.start(); const toolService = new McpToolRegistrationService(); const agentName = "LangChainA365Agent"; + +/** + * Creates the appropriate chat model based on available environment variables. + * Supports both Azure OpenAI and regular OpenAI. + */ +function createChatModel(): BaseChatModel { + // Check for Azure OpenAI configuration first + if (process.env.AZURE_OPENAI_API_KEY && process.env.AZURE_OPENAI_ENDPOINT && process.env.AZURE_OPENAI_DEPLOYMENT) { + console.log('Using Azure OpenAI'); + return new AzureChatOpenAI({ + azureOpenAIApiKey: process.env.AZURE_OPENAI_API_KEY, + azureOpenAIApiInstanceName: process.env.AZURE_OPENAI_ENDPOINT?.replace('https://', '').replace('.openai.azure.com/', '').replace('.openai.azure.com', ''), + azureOpenAIApiDeploymentName: process.env.AZURE_OPENAI_DEPLOYMENT, + azureOpenAIApiVersion: process.env.AZURE_OPENAI_API_VERSION || "2024-12-01-preview", + temperature: 0, + }); + } + + // Fall back to regular OpenAI + if (process.env.OPENAI_API_KEY) { + console.log('Using OpenAI'); + return new ChatOpenAI({ + openAIApiKey: process.env.OPENAI_API_KEY, + modelName: process.env.OPENAI_MODEL || "gpt-4o", + temperature: 0, + }); + } + + throw new Error('No OpenAI credentials found. Please set either AZURE_OPENAI_API_KEY + AZURE_OPENAI_ENDPOINT + AZURE_OPENAI_DEPLOYMENT, or OPENAI_API_KEY.'); +} + +const model = createChatModel(); + const agent = createAgent({ - model: new ChatOpenAI({ temperature: 0 }), + model, name: agentName, systemPrompt: `You are a helpful assistant with access to tools. diff --git a/nodejs/langchain/sample-agent/src/index.ts b/nodejs/langchain/sample-agent/src/index.ts index 584cb028..2e1930d7 100644 --- a/nodejs/langchain/sample-agent/src/index.ts +++ b/nodejs/langchain/sample-agent/src/index.ts @@ -13,6 +13,15 @@ const authConfig: AuthConfiguration = isProduction ? loadAuthConfigFromEnv() : { const server: Express = express() server.use(express.json()) + +// Health endpoint - placed BEFORE auth middleware so it doesn't require authentication +server.get('/api/health', (req, res: Response) => { + res.status(200).json({ + status: 'healthy', + timestamp: new Date().toISOString() + }); +}); + server.use(authorizeJWT(authConfig)) server.post('/api/messages', (req: Request, res: Response) => { @@ -22,7 +31,7 @@ server.post('/api/messages', (req: Request, res: Response) => { }) }) -const port = 3978 +const port = Number(process.env.PORT) || 3978 const host = isProduction ? '0.0.0.0' : '127.0.0.1'; server.listen(port, host, async () => { console.log(`\nServer listening on http://${host}:${port} for appId ${authConfig.clientId} debug ${process.env.DEBUG}`) diff --git a/scripts/e2e/Emit-TestConversations.ps1 b/scripts/e2e/Emit-TestConversations.ps1 new file mode 100644 index 00000000..4673d283 --- /dev/null +++ b/scripts/e2e/Emit-TestConversations.ps1 @@ -0,0 +1,215 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# +.SYNOPSIS + Emits test conversation results in a prettified format to GitHub Actions. + +.DESCRIPTION + This script reads the test-conversations.json file generated by the E2E tests + and outputs a formatted table showing prompts, responses, pass/fail status, + and any error messages. The output is written to both console and GitHub + Actions step summary. + +.PARAMETER TestResultsDir + The directory containing the test-conversations.json file + +.EXAMPLE + .\Emit-TestConversations.ps1 -TestResultsDir "$env:RUNNER_TEMP/TestConversations" +#> + +param( + [Parameter(Mandatory = $true)] + [string]$TestResultsDir +) + +$ErrorActionPreference = 'Continue' + +Write-Host "=== Test Conversations Report ===" -ForegroundColor Cyan +Write-Host "Results Directory: $TestResultsDir" -ForegroundColor Gray +Write-Host "" + +$jsonPath = Join-Path $TestResultsDir "test-conversations.json" + +if (-not (Test-Path $jsonPath)) { + Write-Host "No test-conversations.json found at $jsonPath" -ForegroundColor Yellow + Write-Host "This may indicate tests did not run or did not record conversations." -ForegroundColor Yellow + exit 0 +} + +try { + $results = Get-Content $jsonPath -Raw | ConvertFrom-Json +} +catch { + Write-Host "Failed to parse test-conversations.json: $_" -ForegroundColor Red + exit 1 +} + +# Console output +Write-Host "" +Write-Host "Summary:" -ForegroundColor Green +Write-Host " Total Tests: $($results.TotalTests)" +Write-Host " Passed: $($results.PassedTests)" -ForegroundColor Green +Write-Host " Failed: $($results.FailedTests)" -ForegroundColor $(if ($results.FailedTests -gt 0) { 'Red' } else { 'Green' }) +Write-Host " Generated At: $($results.GeneratedAt)" +Write-Host "" + +# Detailed console output +Write-Host "Test Details:" -ForegroundColor Yellow +Write-Host ("-" * 80) + +foreach ($conv in $results.Conversations) { + $statusIcon = if ($conv.Passed) { "✅" } else { "❌" } + $statusColor = if ($conv.Passed) { "Green" } else { "Red" } + + Write-Host "" + Write-Host "$statusIcon $($conv.TestName)" -ForegroundColor $statusColor + Write-Host " Duration: $($conv.Duration)" + + foreach ($turn in $conv.Turns) { + $promptPreview = if ($turn.UserMessage.Length -gt 60) { + $turn.UserMessage.Substring(0, 60) + "..." + } else { + $turn.UserMessage + } + + $responsePreview = if ($turn.AgentResponse) { + if ($turn.AgentResponse.Length -gt 80) { + $turn.AgentResponse.Substring(0, 80) + "..." + } else { + $turn.AgentResponse + } + } else { + "(no response)" + } + + Write-Host " Prompt: $promptPreview" -ForegroundColor Cyan + Write-Host " Response: $responsePreview" -ForegroundColor Gray + } + + if (-not $conv.Passed -and $conv.ErrorMessage) { + Write-Host " Error: $($conv.ErrorMessage)" -ForegroundColor Red + } +} + +Write-Host "" +Write-Host ("-" * 80) + +# GitHub Actions Step Summary +if ($env:GITHUB_STEP_SUMMARY) { + Write-Host "Writing to GitHub Step Summary..." -ForegroundColor Gray + + $passedIcon = if ($results.FailedTests -eq 0) { "✅" } else { "⚠️" } + + $summary = @" +## $passedIcon Test Conversations Report + +**Summary:** $($results.PassedTests)/$($results.TotalTests) tests passed + +| Test | Status | Prompt | Response | Duration | Error | +|------|--------|--------|----------|----------|-------| +"@ + + foreach ($conv in $results.Conversations) { + $statusIcon = if ($conv.Passed) { "✅ Pass" } else { "❌ Fail" } + + foreach ($turn in $conv.Turns) { + # Escape pipe characters and limit length for markdown table + $promptDisplay = $turn.UserMessage -replace '\|', '\\|' -replace '\n', ' ' + if ($promptDisplay.Length -gt 50) { + $promptDisplay = $promptDisplay.Substring(0, 47) + "..." + } + + $responseDisplay = if ($turn.AgentResponse) { + $resp = $turn.AgentResponse -replace '\|', '\\|' -replace '\n', ' ' + if ($resp.Length -gt 60) { + $resp.Substring(0, 57) + "..." + } else { + $resp + } + } else { + "_(no response)_" + } + + $durationDisplay = if ($conv.Duration) { + # Parse duration string and format + $conv.Duration -replace '00:00:', '' -replace '(\d+\.\d{2})\d+', '$1s' + } else { + "-" + } + + $errorDisplay = if ($conv.ErrorMessage) { + $err = $conv.ErrorMessage -replace '\|', '\\|' -replace '\n', ' ' + if ($err.Length -gt 40) { + $err.Substring(0, 37) + "..." + } else { + $err + } + } else { + "-" + } + + $summary += "`n| $($conv.TestName) | $statusIcon | $promptDisplay | $responseDisplay | $durationDisplay | $errorDisplay |" + } + } + + # Add expandable details section for full responses + $summary += @" + +
+📝 Full Test Details (click to expand) + +"@ + + foreach ($conv in $results.Conversations) { + $statusIcon = if ($conv.Passed) { "✅" } else { "❌" } + + $summary += @" + +### $statusIcon $($conv.TestName) + +- **Status:** $(if ($conv.Passed) { 'Passed' } else { 'Failed' }) +- **Duration:** $($conv.Duration) +- **Timestamp:** $($conv.Timestamp) + +"@ + + $turnNum = 1 + foreach ($turn in $conv.Turns) { + $summary += @" +**Turn ${turnNum}:** +- **Prompt:** ``$($turn.UserMessage)`` +- **Response:** +`````` +$($turn.AgentResponse ?? '(no response)') +`````` + +"@ + $turnNum++ + } + + if (-not $conv.Passed -and $conv.ErrorMessage) { + $summary += @" +> ⚠️ **Error:** $($conv.ErrorMessage) + +"@ + } + } + + $summary += @" +
+"@ + + $summary | Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Append -Encoding utf8 + Write-Host "Step summary written successfully" -ForegroundColor Green +} + +# Return exit code based on test results +if ($results.FailedTests -gt 0) { + Write-Host "" + Write-Host "⚠️ $($results.FailedTests) test(s) failed" -ForegroundColor Yellow +} +else { + Write-Host "" + Write-Host "✅ All tests passed!" -ForegroundColor Green +} diff --git a/scripts/e2e/Get-SDKVersions.ps1 b/scripts/e2e/Get-SDKVersions.ps1 new file mode 100644 index 00000000..a6017314 --- /dev/null +++ b/scripts/e2e/Get-SDKVersions.ps1 @@ -0,0 +1,242 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# +.SYNOPSIS + Gets the installed SDK versions for Agent 365 and Microsoft Agents packages. + +.DESCRIPTION + This script retrieves and logs the version numbers of installed SDK packages + for Python, Node.js, or .NET runtimes. It outputs the versions to the console + and to GitHub Actions step summary if running in CI. + +.PARAMETER Runtime + The runtime to check: 'python', 'nodejs', or 'dotnet' + +.PARAMETER WorkingDirectory + The working directory containing the project files + +.EXAMPLE + .\Get-SDKVersions.ps1 -Runtime python -WorkingDirectory "python/openai/sample-agent" +#> + +param( + [Parameter(Mandatory = $true)] + [ValidateSet('python', 'nodejs', 'dotnet')] + [string]$Runtime, + + [Parameter(Mandatory = $true)] + [string]$WorkingDirectory +) + +$ErrorActionPreference = 'Continue' + +Write-Host "=== SDK Version Information ===" -ForegroundColor Cyan +Write-Host "Runtime: $Runtime" -ForegroundColor Gray +Write-Host "Directory: $WorkingDirectory" -ForegroundColor Gray +Write-Host "" + +$versions = @() + +switch ($Runtime) { + 'python' { + Write-Host "Checking Python package versions..." -ForegroundColor Yellow + + Push-Location $WorkingDirectory + try { + # Get installed packages using uv + $pipList = uv pip list --format=json 2>$null | ConvertFrom-Json + + if ($pipList) { + # Agent 365 SDK packages (check first for exclusion from Agents SDK list) + $a365Packages = $pipList | Where-Object { + $_.name -like 'microsoft-agents-a365-*' -or + $_.name -like 'microsoft_agents_a365_*' + } + + # Microsoft Agents SDK packages (excluding A365 packages) + $agentsSdkPackages = $pipList | Where-Object { + $_.name -like 'microsoft-agents-*' -and + $_.name -notlike 'microsoft-agents-a365-*' -and + $_.name -notlike 'microsoft_agents_a365_*' + } + + Write-Host "" + Write-Host "Microsoft Agents SDK Packages:" -ForegroundColor Green + foreach ($pkg in $agentsSdkPackages) { + Write-Host " $($pkg.name): $($pkg.version)" + $versions += [PSCustomObject]@{ + Category = "Microsoft Agents SDK" + Package = $pkg.name + Version = $pkg.version + } + } + + Write-Host "" + Write-Host "Microsoft Agent 365 SDK Packages:" -ForegroundColor Green + foreach ($pkg in $a365Packages) { + Write-Host " $($pkg.name): $($pkg.version)" + $versions += [PSCustomObject]@{ + Category = "Microsoft Agent 365 SDK" + Package = $pkg.name + Version = $pkg.version + } + } + } + else { + Write-Host "Could not retrieve package list" -ForegroundColor Red + } + } + finally { + Pop-Location + } + } + + 'nodejs' { + Write-Host "Checking Node.js package versions..." -ForegroundColor Yellow + + Push-Location $WorkingDirectory + try { + # Read package.json + $packageJson = Get-Content "package.json" -Raw | ConvertFrom-Json + + # Get installed versions from node_modules + $nodeModulesPath = Join-Path $WorkingDirectory "node_modules" + + # Microsoft Agents SDK packages + $agentsPackages = @( + '@microsoft/agents-hosting', + '@microsoft/agents-activity' + ) + + # Agent 365 SDK packages + $a365Packages = @( + '@microsoft/agents-a365-notifications', + '@microsoft/agents-a365-observability', + '@microsoft/agents-a365-observability-hosting', + '@microsoft/agents-a365-runtime', + '@microsoft/agents-a365-tooling', + '@microsoft/agents-a365-tooling-extensions-claude', + '@microsoft/agents-a365-tooling-extensions-langchain', + '@microsoft/agents-a365-tooling-extensions-openai' + ) + + Write-Host "" + Write-Host "Microsoft Agents SDK Packages:" -ForegroundColor Green + foreach ($pkg in $agentsPackages) { + # Handle scoped packages (e.g., @microsoft/agents-sdk) + $pkgPathParts = $pkg -split '/' + $pkgPath = $nodeModulesPath + foreach ($part in $pkgPathParts) { + $pkgPath = Join-Path $pkgPath $part + } + $pkgPath = Join-Path $pkgPath "package.json" + if (Test-Path $pkgPath) { + $pkgJson = Get-Content $pkgPath -Raw | ConvertFrom-Json + Write-Host " $($pkg): $($pkgJson.version)" + $versions += [PSCustomObject]@{ + Category = "Microsoft Agents SDK" + Package = $pkg + Version = $pkgJson.version + } + } + } + + Write-Host "" + Write-Host "Microsoft Agent 365 SDK Packages:" -ForegroundColor Green + foreach ($pkg in $a365Packages) { + # Handle scoped packages (e.g., @microsoft/agents-a365-runtime) + $pkgPathParts = $pkg -split '/' + $pkgPath = $nodeModulesPath + foreach ($part in $pkgPathParts) { + $pkgPath = Join-Path $pkgPath $part + } + $pkgPath = Join-Path $pkgPath "package.json" + if (Test-Path $pkgPath) { + $pkgJson = Get-Content $pkgPath -Raw | ConvertFrom-Json + Write-Host " $($pkg): $($pkgJson.version)" + $versions += [PSCustomObject]@{ + Category = "Microsoft Agent 365 SDK" + Package = $pkg + Version = $pkgJson.version + } + } + } + } + finally { + Pop-Location + } + } + + 'dotnet' { + Write-Host "Checking .NET package versions..." -ForegroundColor Yellow + + Push-Location $WorkingDirectory + try { + # Find .csproj file + $csproj = Get-ChildItem -Filter "*.csproj" | Select-Object -First 1 + + if ($csproj) { + # Get package references from project + $packages = dotnet list package --format json 2>$null | ConvertFrom-Json + + if ($packages -and $packages.projects) { + foreach ($project in $packages.projects) { + foreach ($framework in $project.frameworks) { + foreach ($pkg in $framework.topLevelPackages) { + $isAgentsSdk = $pkg.id -like 'Microsoft.Agents.*' + $isA365Sdk = $pkg.id -like 'Microsoft.Agents.A365.*' + + if ($isAgentsSdk -and -not $isA365Sdk) { + Write-Host " $($pkg.id): $($pkg.resolvedVersion)" -ForegroundColor Gray + $versions += [PSCustomObject]@{ + Category = "Microsoft Agents SDK" + Package = $pkg.id + Version = $pkg.resolvedVersion + } + } + elseif ($isA365Sdk) { + Write-Host " $($pkg.id): $($pkg.resolvedVersion)" -ForegroundColor Gray + $versions += [PSCustomObject]@{ + Category = "Microsoft Agent 365 SDK" + Package = $pkg.id + Version = $pkg.resolvedVersion + } + } + } + } + } + } + } + } + finally { + Pop-Location + } + } +} + +# Output to GitHub Actions step summary if available +if ($env:GITHUB_STEP_SUMMARY) { + Write-Host "" + Write-Host "Writing to GitHub Step Summary..." -ForegroundColor Gray + + $summary = @" +## SDK Versions ($Runtime) + +| Category | Package | Version | +|----------|---------|---------| +"@ + + foreach ($v in $versions) { + $summary += "`n| $($v.Category) | ``$($v.Package)`` | ``$($v.Version)`` |" + } + + $summary | Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Append -Encoding utf8 +} + +# Output versions as JSON for potential downstream use +$versionsJson = $versions | ConvertTo-Json -Compress +Write-Host "" +Write-Host "SDK_VERSIONS_JSON=$versionsJson" -ForegroundColor Gray + +return $versions