Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/smoke-opencode.lock.yml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

84 changes: 84 additions & 0 deletions actions/setup/js/parse_custom_log.cjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
// @ts-check
/// <reference types="@actions/github-script" />

const { createEngineLogParser } = require("./log_parser_shared.cjs");

const main = createEngineLogParser({
parserName: "Custom",
parseFunction: parseCustomLog,
supportsDirectories: false,
});

/**
* Parses custom engine log content by attempting multiple parser strategies
* @param {string} logContent - The raw log content as a string
* @returns {{markdown: string, mcpFailures: string[], maxTurnsHit: boolean, logEntries: Array}} Result with formatted markdown content, MCP failure list, max-turns status, and parsed log entries
*/
function parseCustomLog(logContent) {
// Try Claude parser first (handles JSONL and JSON array formats)
// Claude parser returns an object with markdown, logEntries, mcpFailures, maxTurnsHit
try {
const claudeModule = require("./parse_claude_log.cjs");
const claudeResult = claudeModule.parseClaudeLog(logContent);

// If we got meaningful results from Claude parser, use them
if (claudeResult && claudeResult.logEntries && claudeResult.logEntries.length > 0) {
return {
...claudeResult,
markdown: "## Custom Engine Log (Claude format)\n\n" + claudeResult.markdown,
};
}
} catch (error) {
// Claude parser failed, continue to next strategy
}

// Try Codex parser as fallback
// Codex parser returns a string (markdown only), so wrap it in expected object format
try {
const codexModule = require("./parse_codex_log.cjs");
const codexMarkdown = codexModule.parseCodexLog(logContent);

// Codex parser returns a string, so check if we got meaningful content
if (codexMarkdown && typeof codexMarkdown === "string" && codexMarkdown.length > 0) {
return {
markdown: "## Custom Engine Log (Codex format)\n\n" + codexMarkdown,
mcpFailures: [],
maxTurnsHit: false,
logEntries: [],
};
}
} catch (error) {
// Codex parser failed, continue to fallback
}

// Fallback: Return basic log info if no structured format was detected
const lineCount = logContent.split("\n").filter(line => line.trim().length > 0).length;
const charCount = logContent.length;

return {
markdown: `## Custom Engine Log

Log format not recognized as Claude or Codex format.

**Basic Statistics:**
- Lines: ${lineCount}
- Characters: ${charCount}

**Raw Log Preview:**
\`\`\`
${logContent.substring(0, 1000)}${logContent.length > 1000 ? "\n... (truncated)" : ""}
\`\`\`
`,
mcpFailures: [],
maxTurnsHit: false,
logEntries: [],
};
}

// Export for testing
if (typeof module !== "undefined" && module.exports) {
module.exports = {
main,
parseCustomLog,
};
}
72 changes: 72 additions & 0 deletions actions/setup/js/parse_custom_log.test.cjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
// @ts-check

import { describe, it, expect } from "vitest";
import { parseCustomLog } from "./parse_custom_log.cjs";

describe("parseCustomLog", () => {
it("should detect and parse Claude format logs", () => {
const claudeLog = JSON.stringify([
{
type: "init",
num_turns: 1,
tools: [],
},
{
type: "turn",
turn_number: 1,
message: { role: "user", content: "Hello" },
},
]);

const result = parseCustomLog(claudeLog);

expect(result).toBeDefined();
expect(result.markdown).toContain("Custom Engine Log (Claude format)");
expect(result.logEntries.length).toBeGreaterThan(0);
});

it("should detect and parse Codex format logs", () => {
const codexLog = `{"type":"init","tools":[],"num_turns":1}
{"type":"turn","turn_number":1,"message":"Hello"}`;

const result = parseCustomLog(codexLog);

expect(result).toBeDefined();
// Note: Actual format detection depends on parse_codex_log implementation
expect(result.markdown).toBeDefined();
});

it("should handle unrecognized log format with basic fallback", () => {
// Use a more complex string that neither parser will recognize
// The Codex parser is very lenient and will accept most text
const unknownLog = "Some plain text log\nwith multiple lines\nand no structure";

const result = parseCustomLog(unknownLog);

expect(result).toBeDefined();
expect(result.markdown).toContain("Custom Engine Log");
// The Codex parser will handle this, so check for Codex format
expect(result.markdown).toBeDefined();
expect(result.mcpFailures).toBeDefined();
expect(result.logEntries).toBeDefined();
});

it("should truncate long logs in fallback mode", () => {
// Use a long string that the parsers will process
const longLog = "a".repeat(2000);

const result = parseCustomLog(longLog);

expect(result).toBeDefined();
// Codex parser will handle this
expect(result.markdown).toContain("Custom Engine Log");
});

it("should handle empty log content", () => {
const result = parseCustomLog("");

expect(result).toBeDefined();
expect(result.markdown).toContain("Custom Engine Log");
expect(result.logEntries).toEqual([]);
});
});
2 changes: 2 additions & 0 deletions pkg/constants/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,8 @@ const (
EnvVarModelAgentClaude = "GH_AW_MODEL_AGENT_CLAUDE"
// EnvVarModelAgentCodex configures the default Codex model for agent execution
EnvVarModelAgentCodex = "GH_AW_MODEL_AGENT_CODEX"
// EnvVarModelAgentCustom configures the default Custom model for agent execution
EnvVarModelAgentCustom = "GH_AW_MODEL_AGENT_CUSTOM"
// EnvVarModelDetectionCopilot configures the default Copilot model for detection
EnvVarModelDetectionCopilot = "GH_AW_MODEL_DETECTION_COPILOT"
// EnvVarModelDetectionClaude configures the default Claude model for detection
Expand Down
6 changes: 6 additions & 0 deletions pkg/workflow/compiler_yaml.go
Original file line number Diff line number Diff line change
Expand Up @@ -295,6 +295,12 @@ func (c *Compiler) generateCreateAwInfo(yaml *strings.Builder, data *WorkflowDat
modelEnvVar = constants.EnvVarModelAgentClaude
case "codex":
modelEnvVar = constants.EnvVarModelAgentCodex
case "custom":
modelEnvVar = constants.EnvVarModelAgentCustom
default:
// For unknown engines, use a generic environment variable pattern
// This provides a fallback while maintaining consistency
modelEnvVar = constants.EnvVarModelAgentCustom
}

// Generate JavaScript to resolve model from environment variable at runtime
Expand Down
134 changes: 134 additions & 0 deletions pkg/workflow/custom_engine_awinfo_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
package workflow

import (
"strings"
"testing"

"github.com/githubnext/gh-aw/pkg/constants"
)

func TestGenerateCreateAwInfoCustomEngine(t *testing.T) {
// Create a compiler instance
c := NewCompiler(false, "", "test")

t.Run("custom engine with explicit model", func(t *testing.T) {
workflowData := &WorkflowData{
Name: "test-workflow",
EngineConfig: &EngineConfig{
ID: "custom",
Model: "anthropic/claude-3-5-sonnet-20241022",
},
}

engine := NewCustomEngine()

var yaml strings.Builder
c.generateCreateAwInfo(&yaml, workflowData, engine)

result := yaml.String()

// Check that the explicit model is used directly
if !strings.Contains(result, `model: "anthropic/claude-3-5-sonnet-20241022"`) {
t.Error("Expected explicit model to be used directly in aw_info.json for custom engine")
}

// Should not contain process.env reference when model is explicit
if strings.Contains(result, "process.env."+constants.EnvVarModelAgentCustom) {
t.Error("Should not use environment variable when model is explicitly configured")
}
})

t.Run("custom engine without explicit model", func(t *testing.T) {
workflowData := &WorkflowData{
Name: "test-workflow",
EngineConfig: &EngineConfig{
ID: "custom",
// No explicit model
},
}

engine := NewCustomEngine()

var yaml strings.Builder
c.generateCreateAwInfo(&yaml, workflowData, engine)

result := yaml.String()

// Check that the custom model environment variable is used
expectedEnvVar := "process.env." + constants.EnvVarModelAgentCustom + " || \"\""
if !strings.Contains(result, expectedEnvVar) {
t.Errorf("Expected custom engine to use environment variable %s in aw_info.json, got:\n%s", constants.EnvVarModelAgentCustom, result)
}

// Should not have incomplete process.env. syntax
if strings.Contains(result, "process.env. || \"\"") {
t.Error("Found incomplete 'process.env. || \"\"' syntax - this should not happen")
}
})

t.Run("copilot engine without explicit model", func(t *testing.T) {
workflowData := &WorkflowData{
Name: "test-workflow",
EngineConfig: &EngineConfig{
ID: "copilot",
},
}

engine := NewCopilotEngine()

var yaml strings.Builder
c.generateCreateAwInfo(&yaml, workflowData, engine)

result := yaml.String()

// Check that the copilot model environment variable is used
expectedEnvVar := "process.env." + constants.EnvVarModelAgentCopilot + " || \"\""
if !strings.Contains(result, expectedEnvVar) {
t.Errorf("Expected copilot engine to use environment variable %s in aw_info.json", constants.EnvVarModelAgentCopilot)
}
})

t.Run("claude engine without explicit model", func(t *testing.T) {
workflowData := &WorkflowData{
Name: "test-workflow",
EngineConfig: &EngineConfig{
ID: "claude",
},
}

engine := NewClaudeEngine()

var yaml strings.Builder
c.generateCreateAwInfo(&yaml, workflowData, engine)

result := yaml.String()

// Check that the claude model environment variable is used
expectedEnvVar := "process.env." + constants.EnvVarModelAgentClaude + " || \"\""
if !strings.Contains(result, expectedEnvVar) {
t.Errorf("Expected claude engine to use environment variable %s in aw_info.json", constants.EnvVarModelAgentClaude)
}
})

t.Run("codex engine without explicit model", func(t *testing.T) {
workflowData := &WorkflowData{
Name: "test-workflow",
EngineConfig: &EngineConfig{
ID: "codex",
},
}

engine := NewCodexEngine()

var yaml strings.Builder
c.generateCreateAwInfo(&yaml, workflowData, engine)

result := yaml.String()

// Check that the codex model environment variable is used
expectedEnvVar := "process.env." + constants.EnvVarModelAgentCodex + " || \"\""
if !strings.Contains(result, expectedEnvVar) {
t.Errorf("Expected codex engine to use environment variable %s in aw_info.json", constants.EnvVarModelAgentCodex)
}
})
}