Fix MCP gateway startup by extracting HTTP MCP server environment variables

pkg/workflow/http_mcp_env_vars_test.go

@@ -0,0 +1,93 @@
+package workflow
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// TestCollectMCPEnvironmentVariables_HTTPMCPWithSecrets tests that HTTP MCP servers
+// with secrets in headers get their environment variables added to the Start MCP gateway step
+func TestCollectMCPEnvironmentVariables_HTTPMCPWithSecrets(t *testing.T) {
+	tools := map[string]any{
+		"github": map[string]any{
+			"mode":     "remote",
+			"toolsets": []string{"repos", "issues"},
+		},
+		"tavily": map[string]any{
+			"type": "http",
+			"url":  "https://mcp.tavily.com/mcp/",
+			"headers": map[string]string{
+				"Authorization": "Bearer ${{ secrets.TAVILY_API_KEY }}",
+			},
+		},
+	}
+
+	mcpTools := []string{"github", "tavily"}
+	workflowData := &WorkflowData{
+		GitHubToken: "${{ secrets.GITHUB_TOKEN }}",
+	}
+	hasAgenticWorkflows := false
+
+	envVars := collectMCPEnvironmentVariables(tools, mcpTools, workflowData, hasAgenticWorkflows)
+
+	// Verify TAVILY_API_KEY is present
+	assert.Contains(t, envVars, "TAVILY_API_KEY", "TAVILY_API_KEY should be extracted from HTTP MCP headers")
+	assert.Equal(t, "${{ secrets.TAVILY_API_KEY }}", envVars["TAVILY_API_KEY"], "TAVILY_API_KEY should have the correct secret expression")
+}
+
+// TestCollectMCPEnvironmentVariables_MultipleHTTPMCPServers tests that multiple HTTP MCP servers
+// with different secrets all get their environment variables added
+func TestCollectMCPEnvironmentVariables_MultipleHTTPMCPServers(t *testing.T) {
+	tools := map[string]any{
+		"tavily": map[string]any{
+			"type": "http",
+			"url":  "https://mcp.tavily.com/mcp/",
+			"headers": map[string]string{
+				"Authorization": "Bearer ${{ secrets.TAVILY_API_KEY }}",
+			},
+		},
+		"datadog": map[string]any{
+			"type": "http",
+			"url":  "https://api.datadoghq.com/mcp",
+			"headers": map[string]string{
+				"DD-API-KEY":         "${{ secrets.DD_API_KEY }}",
+				"DD-APPLICATION-KEY": "${{ secrets.DD_APP_KEY }}",
+			},
+		},
+	}
+
+	mcpTools := []string{"tavily", "datadog"}
+	workflowData := &WorkflowData{}
+	hasAgenticWorkflows := false
+
+	envVars := collectMCPEnvironmentVariables(tools, mcpTools, workflowData, hasAgenticWorkflows)
+
+	// Verify all secrets are present
+	assert.Contains(t, envVars, "TAVILY_API_KEY", "TAVILY_API_KEY should be extracted")
+	assert.Contains(t, envVars, "DD_API_KEY", "DD_API_KEY should be extracted")
+	assert.Contains(t, envVars, "DD_APP_KEY", "DD_APP_KEY should be extracted")
+	assert.Len(t, envVars, 3, "Should have exactly 3 environment variables")
+}
+
+// TestCollectMCPEnvironmentVariables_HTTPMCPWithoutSecrets verifies that HTTP MCP servers without secrets don't add env vars
+func TestCollectMCPEnvironmentVariables_HTTPMCPWithoutSecrets(t *testing.T) {
+	tools := map[string]any{
+		"public_api": map[string]any{
+			"type": "http",
+			"url":  "https://api.example.com/mcp",
+			"headers": map[string]string{
+				"Content-Type": "application/json",
+			},
+		},
+	}
+
+	mcpTools := []string{"public_api"}
+	workflowData := &WorkflowData{}
+	hasAgenticWorkflows := false
+
+	envVars := collectMCPEnvironmentVariables(tools, mcpTools, workflowData, hasAgenticWorkflows)
+
+	// Should not add any env vars since there are no secrets
+	assert.Empty(t, envVars, "Should not add environment variables for HTTP MCP without secrets")
+}

pkg/workflow/mcp_servers.go

@@ -141,6 +141,49 @@ func collectMCPEnvironmentVariables(tools map[string]any, mcpTools []string, wor
 		}
 	}
 
+	// Check for HTTP MCP servers with secrets in headers (e.g., Tavily)
+	// These need to be available as environment variables when the MCP gateway starts
+	for toolName, toolValue := range tools {
+		// Skip standard tools that are handled above
+		if toolName == "github" || toolName == "playwright" || toolName == "serena" ||
+			toolName == "cache-memory" || toolName == "agentic-workflows" ||
+			toolName == "safe-outputs" || toolName == "safe-inputs" {
+			continue
+		}
+
+		// Check if this is an MCP tool
+		if toolConfig, ok := toolValue.(map[string]any); ok {
+			if hasMcp, _ := hasMCPConfig(toolConfig); !hasMcp {
+				continue
+			}
+
+			// Get MCP config and check if it's an HTTP type
+			mcpConfig, err := getMCPConfig(toolConfig, toolName)
+			if err != nil {
+				mcpServersLog.Printf("Failed to parse MCP config for tool %s: %v", toolName, err)
+				continue
+			}
+
+			// Extract secrets from headers for HTTP MCP servers
+			if mcpConfig.Type == "http" && len(mcpConfig.Headers) > 0 {
+				headerSecrets := ExtractSecretsFromMap(mcpConfig.Headers)
+				mcpServersLog.Printf("Extracted %d secrets from HTTP MCP server '%s'", len(headerSecrets), toolName)
+				for envVarName, secretExpr := range headerSecrets {
+					envVars[envVarName] = secretExpr
+				}
+			}
+
+			// Also extract secrets from env section if present
+			if len(mcpConfig.Env) > 0 {
+				envSecrets := ExtractSecretsFromMap(mcpConfig.Env)
+				mcpServersLog.Printf("Extracted %d secrets from env section of MCP server '%s'", len(envSecrets), toolName)
+				for envVarName, secretExpr := range envSecrets {
+					envVars[envVarName] = secretExpr
+				}
+			}
+		}
+	}
+
 	return envVars
 }