Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/patch-add-parse-option-to-audit.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

33 changes: 21 additions & 12 deletions pkg/cli/audit.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,8 @@ Examples:
` + constants.CLIExtensionPrefix + ` audit https://github.com/owner/repo/actions/runs/1234567890 # Audit from run URL
` + constants.CLIExtensionPrefix + ` audit https://github.com/owner/repo/actions/runs/1234567890/job/9876543210 # Audit from job URL
` + constants.CLIExtensionPrefix + ` audit 1234567890 -o ./audit-reports # Custom output directory
` + constants.CLIExtensionPrefix + ` audit 1234567890 -v # Verbose output`,
` + constants.CLIExtensionPrefix + ` audit 1234567890 -v # Verbose output
` + constants.CLIExtensionPrefix + ` audit 1234567890 --parse # Parse agent logs and generate log.md`,
Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {
runIDOrURL := args[0]
Expand All @@ -58,8 +59,9 @@ Examples:
outputDir, _ := cmd.Flags().GetString("output")
verbose, _ := cmd.Flags().GetBool("verbose")
jsonOutput, _ := cmd.Flags().GetBool("json")
parse, _ := cmd.Flags().GetBool("parse")

if err := AuditWorkflowRun(runID, outputDir, verbose, jsonOutput); err != nil {
if err := AuditWorkflowRun(runID, outputDir, verbose, parse, jsonOutput); err != nil {
fmt.Fprintln(os.Stderr, console.FormatErrorMessage(err.Error()))
os.Exit(1)
}
Expand All @@ -69,6 +71,7 @@ Examples:
// Add flags to audit command
auditCmd.Flags().StringP("output", "o", "./logs", "Output directory for downloaded logs and artifacts")
auditCmd.Flags().Bool("json", false, "Output audit report as JSON instead of formatted console tables")
auditCmd.Flags().Bool("parse", false, "Run JavaScript parser on agent logs and write markdown to log.md")

return auditCmd
}
Expand Down Expand Up @@ -113,7 +116,7 @@ func isPermissionError(err error) bool {
}

// AuditWorkflowRun audits a single workflow run and generates a report
func AuditWorkflowRun(runID int64, outputDir string, verbose bool, jsonOutput bool) error {
func AuditWorkflowRun(runID int64, outputDir string, verbose bool, parse bool, jsonOutput bool) error {
if verbose {
fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Auditing workflow run %d...", runID)))
}
Expand Down Expand Up @@ -259,20 +262,26 @@ func AuditWorkflowRun(runID int64, outputDir string, verbose bool, jsonOutput bo
renderConsole(auditData, runOutputDir)
}

// Always attempt to render agentic log (similar to `logs --parse`) if engine & logs are available
// Conditionally attempt to render agentic log (similar to `logs --parse`) if --parse flag is set
// This creates a log.md file in the run directory for a rich, human-readable agent session summary.
// We intentionally do not fail the audit on parse errors; they are reported as warnings.
awInfoPath := filepath.Join(runOutputDir, "aw_info.json")
if engine := extractEngineFromAwInfo(awInfoPath, verbose); engine != nil { // reuse existing helper in same package
if err := parseAgentLog(runOutputDir, engine, verbose); err != nil {
if verbose {
fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to parse agent log for run %d: %v", runID, err)))
if parse {
awInfoPath := filepath.Join(runOutputDir, "aw_info.json")
if engine := extractEngineFromAwInfo(awInfoPath, verbose); engine != nil { // reuse existing helper in same package
if err := parseAgentLog(runOutputDir, engine, verbose); err != nil {
if verbose {
fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to parse agent log for run %d: %v", runID, err)))
}
} else {
// Always show success message for parsing, not just in verbose mode
logMdPath := filepath.Join(runOutputDir, "log.md")
if _, err := os.Stat(logMdPath); err == nil {
fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("✓ Parsed log for run %d → %s", runID, logMdPath)))
}
}
} else if verbose {
fmt.Fprintln(os.Stderr, console.FormatInfoMessage("No agent logs found to parse or no parser available"))
fmt.Fprintln(os.Stderr, console.FormatInfoMessage("No engine detected (aw_info.json missing or invalid); skipping agent log rendering"))
}
} else if verbose {
fmt.Fprintln(os.Stderr, console.FormatInfoMessage("No engine detected (aw_info.json missing or invalid); skipping agent log rendering"))
}

// Display logs location (only for console output)
Expand Down
80 changes: 80 additions & 0 deletions pkg/cli/audit_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -720,3 +720,83 @@ func TestRenderJSON(t *testing.T) {
t.Errorf("Expected 1 warning, got %d", len(parsed.Warnings))
}
}

func TestAuditParseFlagBehavior(t *testing.T) {
// Create a temporary directory for test artifacts
tempDir := t.TempDir()
runDir := filepath.Join(tempDir, "run-12345")
if err := os.MkdirAll(runDir, 0755); err != nil {
t.Fatalf("Failed to create run directory: %v", err)
}

// Create a mock agent-stdio.log file with Claude log format
agentStdioPath := filepath.Join(runDir, "agent-stdio.log")
mockLogContent := `[
{"type": "text", "text": "Starting task"},
{"type": "tool_use", "id": "1", "name": "bash", "input": {"command": "echo hello"}},
{"type": "tool_result", "tool_use_id": "1", "content": "hello"}
]`
if err := os.WriteFile(agentStdioPath, []byte(mockLogContent), 0644); err != nil {
t.Fatalf("Failed to create mock agent-stdio.log: %v", err)
}

// Create a mock aw_info.json with Claude engine
awInfoPath := filepath.Join(runDir, "aw_info.json")
awInfoContent := `{"engine_id": "claude", "workflow_name": "test-workflow"}`
if err := os.WriteFile(awInfoPath, []byte(awInfoContent), 0644); err != nil {
t.Fatalf("Failed to create mock aw_info.json: %v", err)
}

logMdPath := filepath.Join(runDir, "log.md")

// Test with parse=false - log.md should NOT be created
t.Run("parse=false does not create log.md", func(t *testing.T) {
// Clean up any existing log.md
os.Remove(logMdPath)

// Simulate the audit logic with parse=false
parse := false
if parse {
engine := extractEngineFromAwInfo(awInfoPath, false)
if engine != nil {
parseAgentLog(runDir, engine, false)
}
}

// Verify log.md was NOT created
if _, err := os.Stat(logMdPath); !os.IsNotExist(err) {
t.Errorf("log.md should not be created when parse=false")
}
})

// Test with parse=true - log.md SHOULD be created
t.Run("parse=true creates log.md", func(t *testing.T) {
// Clean up any existing log.md
os.Remove(logMdPath)

// Simulate the audit logic with parse=true
parse := true
if parse {
engine := extractEngineFromAwInfo(awInfoPath, false)
if engine != nil {
if err := parseAgentLog(runDir, engine, false); err != nil {
t.Fatalf("parseAgentLog failed: %v", err)
}
}
}

// Verify log.md was created
if _, err := os.Stat(logMdPath); os.IsNotExist(err) {
t.Errorf("log.md should be created when parse=true")
}

// Verify log.md has content
content, err := os.ReadFile(logMdPath)
if err != nil {
t.Fatalf("Failed to read log.md: %v", err)
}
if len(content) == 0 {
t.Errorf("log.md should not be empty")
}
})
}
Loading