diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 798396de14..faab1b4d12 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2142,6 +2142,57 @@ jobs: echo "$WORKFLOWS" > /tmp/workflow-list.txt echo "workflow_count=$WORKFLOW_COUNT" >> $GITHUB_OUTPUT + - name: Compare gh aw list with git clone + run: | + set -e + echo "## Comparing 'gh aw list' output with git clone results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + # List workflows using gh aw list command with custom path + echo "Running: ./gh-aw list --repo githubnext/agentics --path workflows --json" + ./gh-aw list --repo githubnext/agentics --path workflows --json > /tmp/gh-aw-list.json + + # Extract workflow names from JSON output + echo "Extracting workflow names from gh aw list output..." + jq -r '.[].workflow' /tmp/gh-aw-list.json | sort > /tmp/gh-aw-workflows.txt + + # Get workflow names from git clone (already in /tmp/workflow-list.txt) + echo "Sorting git clone workflow list..." + sort /tmp/workflow-list.txt > /tmp/git-workflows-sorted.txt + + # Display both lists + echo "### Workflows from 'gh aw list --repo githubnext/agentics --path workflows'" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + cat /tmp/gh-aw-workflows.txt >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + echo "### Workflows from git clone" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + cat /tmp/git-workflows-sorted.txt >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + # Compare the two lists + if diff -u /tmp/git-workflows-sorted.txt /tmp/gh-aw-workflows.txt > /tmp/diff-output.txt; then + echo "✅ **SUCCESS**: Workflow lists match!" >> $GITHUB_STEP_SUMMARY + echo "The 'gh aw list' command returned the same workflows as the git clone." >> $GITHUB_STEP_SUMMARY + echo "" + echo "✅ Workflow lists match!" + else + echo "❌ **FAILURE**: Workflow lists do not match!" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Differences" >> $GITHUB_STEP_SUMMARY + echo '```diff' >> $GITHUB_STEP_SUMMARY + cat /tmp/diff-output.txt >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" + echo "❌ Workflow lists do not match!" + echo "Differences:" + cat /tmp/diff-output.txt + exit 1 + fi + - name: Add workflows one by one id: add-workflows env: diff --git a/pkg/cli/list_workflows_command.go b/pkg/cli/list_workflows_command.go index ed816d8275..428c68cb52 100644 --- a/pkg/cli/list_workflows_command.go +++ b/pkg/cli/list_workflows_command.go @@ -30,7 +30,7 @@ func NewListCommand() *cobra.Command { cmd := &cobra.Command{ Use: "list [pattern]", Short: "List agentic workflows in the repository", - Long: `List all agentic workflows in the repository without checking their status. + Long: `List all agentic workflows in a repository without checking their status. Displays a simplified table with workflow name, AI engine, and compilation status. Unlike 'status', this command does not check GitHub workflow state or time remaining. @@ -38,24 +38,32 @@ Unlike 'status', this command does not check GitHub workflow state or time remai The optional pattern argument filters workflows by name (case-insensitive substring match). Examples: - ` + string(constants.CLIExtensionPrefix) + ` list # List all workflows - ` + string(constants.CLIExtensionPrefix) + ` list ci- # List workflows with 'ci-' in name - ` + string(constants.CLIExtensionPrefix) + ` list --json # Output in JSON format - ` + string(constants.CLIExtensionPrefix) + ` list --label automation # List workflows with 'automation' label`, + ` + string(constants.CLIExtensionPrefix) + ` list # List all workflows in current repo + ` + string(constants.CLIExtensionPrefix) + ` list --repo github/gh-aw # List workflows from github/gh-aw repo + ` + string(constants.CLIExtensionPrefix) + ` list --repo org/repo --path workflows # List from custom path + ` + string(constants.CLIExtensionPrefix) + ` list ci- # List workflows with 'ci-' in name + ` + string(constants.CLIExtensionPrefix) + ` list --repo github/gh-aw ci- # List workflows from github/gh-aw with 'ci-' in name + ` + string(constants.CLIExtensionPrefix) + ` list --json # Output in JSON format + ` + string(constants.CLIExtensionPrefix) + ` list --label automation # List workflows with 'automation' label`, RunE: func(cmd *cobra.Command, args []string) error { var pattern string if len(args) > 0 { pattern = args[0] } + + repo, _ := cmd.Flags().GetString("repo") + path, _ := cmd.Flags().GetString("path") verbose, _ := cmd.Flags().GetBool("verbose") jsonFlag, _ := cmd.Flags().GetBool("json") labelFilter, _ := cmd.Flags().GetString("label") - return RunListWorkflows(pattern, verbose, jsonFlag, labelFilter) + return RunListWorkflows(repo, path, pattern, verbose, jsonFlag, labelFilter) }, } + addRepoFlag(cmd) addJSONFlag(cmd) cmd.Flags().String("label", "", "Filter workflows by label") + cmd.Flags().String("path", ".github/workflows", "Path to workflows directory in the repository") // Register completions for list command cmd.ValidArgsFunction = CompleteWorkflowNames @@ -64,16 +72,31 @@ Examples: } // RunListWorkflows lists workflows without checking GitHub status -func RunListWorkflows(pattern string, verbose bool, jsonOutput bool, labelFilter string) error { - listWorkflowsLog.Printf("Listing workflows: pattern=%s, jsonOutput=%v, labelFilter=%s", pattern, jsonOutput, labelFilter) - if verbose && !jsonOutput { - fmt.Fprintf(os.Stderr, "Listing workflow files\n") - if pattern != "" { - fmt.Fprintf(os.Stderr, "Filtering by pattern: %s\n", pattern) +func RunListWorkflows(repo, path, pattern string, verbose bool, jsonOutput bool, labelFilter string) error { + listWorkflowsLog.Printf("Listing workflows: repo=%s, path=%s, pattern=%s, jsonOutput=%v, labelFilter=%s", repo, path, pattern, jsonOutput, labelFilter) + + var mdFiles []string + var err error + var isRemote bool + + if repo != "" { + // List workflows from remote repository + isRemote = true + if verbose && !jsonOutput { + fmt.Fprintf(os.Stderr, "Listing workflow files from %s\n", repo) } + mdFiles, err = getRemoteWorkflowFiles(repo, path, verbose, jsonOutput) + } else { + // List workflows from local repository + if verbose && !jsonOutput { + fmt.Fprintf(os.Stderr, "Listing workflow files\n") + if pattern != "" { + fmt.Fprintf(os.Stderr, "Filtering by pattern: %s\n", pattern) + } + } + mdFiles, err = getMarkdownWorkflowFiles("") } - mdFiles, err := getMarkdownWorkflowFiles("") if err != nil { listWorkflowsLog.Printf("Failed to get markdown workflow files: %v", err) fmt.Fprintln(os.Stderr, console.FormatErrorMessage(err.Error())) @@ -108,67 +131,80 @@ func RunListWorkflows(pattern string, verbose bool, jsonOutput bool, labelFilter continue } - // Extract engine ID from workflow file - agent := extractEngineIDFromFile(file) - - // Check if compiled (.lock.yml file is in .github/workflows) - lockFile := stringutil.MarkdownToLockFile(file) - compiled := "N/A" - - if _, err := os.Stat(lockFile); err == nil { - // Check if up to date - mdStat, _ := os.Stat(file) - lockStat, _ := os.Stat(lockFile) - if mdStat.ModTime().After(lockStat.ModTime()) { - compiled = "No" - } else { - compiled = "Yes" + // For remote repos, we can't check compilation status or read local files + if isRemote { + // For remote repos, skip fetching individual file metadata to avoid slowness + // Just show file name with minimal info + workflows = append(workflows, WorkflowListItem{ + Workflow: name, + EngineID: "N/A", // Skip fetching to avoid slow API/git calls + Compiled: "N/A", // Cannot determine for remote repos + Labels: nil, + On: nil, + }) + } else { + // Extract engine ID from workflow file + agent := extractEngineIDFromFile(file) + + // Check if compiled (.lock.yml file is in .github/workflows) + lockFile := stringutil.MarkdownToLockFile(file) + compiled := "N/A" + + if _, err := os.Stat(lockFile); err == nil { + // Check if up to date + mdStat, _ := os.Stat(file) + lockStat, _ := os.Stat(lockFile) + if mdStat.ModTime().After(lockStat.ModTime()) { + compiled = "No" + } else { + compiled = "Yes" + } } - } - // Extract "on" field and labels from frontmatter - var onField any - var labels []string - if content, err := os.ReadFile(file); err == nil { - if result, err := parser.ExtractFrontmatterFromContent(string(content)); err == nil { - if result.Frontmatter != nil { - onField = result.Frontmatter["on"] - // Extract labels field if present - if labelsField, ok := result.Frontmatter["labels"]; ok { - if labelsArray, ok := labelsField.([]any); ok { - for _, label := range labelsArray { - if labelStr, ok := label.(string); ok { - labels = append(labels, labelStr) + // Extract "on" field and labels from frontmatter + var onField any + var labels []string + if content, err := os.ReadFile(file); err == nil { + if result, err := parser.ExtractFrontmatterFromContent(string(content)); err == nil { + if result.Frontmatter != nil { + onField = result.Frontmatter["on"] + // Extract labels field if present + if labelsField, ok := result.Frontmatter["labels"]; ok { + if labelsArray, ok := labelsField.([]any); ok { + for _, label := range labelsArray { + if labelStr, ok := label.(string); ok { + labels = append(labels, labelStr) + } } } } } } } - } - // Skip if label filter specified and workflow doesn't have the label - if labelFilter != "" { - hasLabel := false - for _, label := range labels { - if strings.EqualFold(label, labelFilter) { - hasLabel = true - break + // Skip if label filter specified and workflow doesn't have the label + if labelFilter != "" { + hasLabel := false + for _, label := range labels { + if strings.EqualFold(label, labelFilter) { + hasLabel = true + break + } + } + if !hasLabel { + continue } } - if !hasLabel { - continue - } - } - // Build workflow list item - workflows = append(workflows, WorkflowListItem{ - Workflow: name, - EngineID: agent, - Compiled: compiled, - Labels: labels, - On: onField, - }) + // Build workflow list item + workflows = append(workflows, WorkflowListItem{ + Workflow: name, + EngineID: agent, + Compiled: compiled, + Labels: labels, + On: onField, + }) + } } // Output results @@ -194,3 +230,36 @@ func RunListWorkflows(pattern string, verbose bool, jsonOutput bool, labelFilter return nil } + +// getRemoteWorkflowFiles fetches the list of workflow files from a remote repository +func getRemoteWorkflowFiles(repoSpec, workflowPath string, verbose bool, jsonOutput bool) ([]string, error) { + // Parse repo spec: owner/repo[@ref] + var owner, repo, ref string + parts := strings.SplitN(repoSpec, "@", 2) + repoPart := parts[0] + if len(parts) == 2 { + ref = parts[1] + } else { + ref = "main" // default to main branch + } + + // Parse owner/repo + repoParts := strings.Split(repoPart, "/") + if len(repoParts) != 2 { + return nil, fmt.Errorf("invalid repository format: %s (expected owner/repo or owner/repo@ref)", repoSpec) + } + owner = repoParts[0] + repo = repoParts[1] + + if verbose && !jsonOutput { + fmt.Fprintf(os.Stderr, "Fetching workflow files from %s/%s@%s (path: %s)\n", owner, repo, ref, workflowPath) + } + + // Use the parser package to list workflow files + files, err := parser.ListWorkflowFiles(owner, repo, ref, workflowPath) + if err != nil { + return nil, fmt.Errorf("failed to list workflow files from %s/%s: %w", owner, repo, err) + } + + return files, nil +} diff --git a/pkg/cli/list_workflows_command_test.go b/pkg/cli/list_workflows_command_test.go index 8b01c93ffb..4ae06bbc53 100644 --- a/pkg/cli/list_workflows_command_test.go +++ b/pkg/cli/list_workflows_command_test.go @@ -25,19 +25,19 @@ func TestRunListWorkflows_JSONOutput(t *testing.T) { // Test JSON output without pattern t.Run("JSON output without pattern", func(t *testing.T) { - err := RunListWorkflows("", false, true, "") + err := RunListWorkflows("", ".github/workflows", "", false, true, "") assert.NoError(t, err, "RunListWorkflows with JSON flag should not error") }) // Test JSON output with pattern t.Run("JSON output with pattern", func(t *testing.T) { - err := RunListWorkflows("smoke", false, true, "") + err := RunListWorkflows("", ".github/workflows", "smoke", false, true, "") assert.NoError(t, err, "RunListWorkflows with JSON flag and pattern should not error") }) // Test JSON output with label filter t.Run("JSON output with label filter", func(t *testing.T) { - err := RunListWorkflows("", false, true, "test") + err := RunListWorkflows("", ".github/workflows", "", false, true, "test") assert.NoError(t, err, "RunListWorkflows with JSON flag and label filter should not error") }) } @@ -93,13 +93,13 @@ func TestRunListWorkflows_TextOutput(t *testing.T) { // Test text output t.Run("Text output without pattern", func(t *testing.T) { - err := RunListWorkflows("", false, false, "") + err := RunListWorkflows("", ".github/workflows", "", false, false, "") assert.NoError(t, err, "RunListWorkflows without JSON flag should not error") }) // Test text output with pattern t.Run("Text output with pattern", func(t *testing.T) { - err := RunListWorkflows("ci-", false, false, "") + err := RunListWorkflows("", ".github/workflows", "ci-", false, false, "") assert.NoError(t, err, "RunListWorkflows with pattern should not error") }) } diff --git a/pkg/parser/remote_fetch.go b/pkg/parser/remote_fetch.go index 7e0f89104f..e3960cbc1a 100644 --- a/pkg/parser/remote_fetch.go +++ b/pkg/parser/remote_fetch.go @@ -679,3 +679,108 @@ func downloadFileFromGitHubWithDepth(owner, repo, path, ref string, symlinkDepth return content, nil } + +// ListWorkflowFiles lists workflow files from a remote GitHub repository +// Returns a list of .md files in the specified directory (excluding subdirectories) +func ListWorkflowFiles(owner, repo, ref, workflowPath string) ([]string, error) { + remoteLog.Printf("Listing workflow files for %s/%s@%s (path: %s)", owner, repo, ref, workflowPath) + + // Create REST client + client, err := api.DefaultRESTClient() + if err != nil { + remoteLog.Printf("Failed to create REST client, attempting git fallback: %v", err) + return listWorkflowFilesViaGit(owner, repo, ref, workflowPath) + } + + // Define response struct for GitHub contents API (array of file objects) + var contents []struct { + Name string `json:"name"` + Path string `json:"path"` + Type string `json:"type"` + } + + // Fetch directory contents from GitHub API + endpoint := fmt.Sprintf("repos/%s/%s/contents/%s?ref=%s", owner, repo, workflowPath, ref) + err = client.Get(endpoint, &contents) + if err != nil { + errStr := err.Error() + + // Check if this is an authentication error + if gitutil.IsAuthError(errStr) { + remoteLog.Printf("GitHub API authentication failed, attempting git fallback for %s/%s@%s", owner, repo, ref) + // Try fallback using git commands for public repositories + files, gitErr := listWorkflowFilesViaGit(owner, repo, ref, workflowPath) + if gitErr != nil { + // If git fallback also fails, return both errors + return nil, fmt.Errorf("failed to list workflow files via GitHub API (auth error) and git fallback: API error: %w, Git error: %v", err, gitErr) + } + return files, nil + } + + return nil, fmt.Errorf("failed to list workflow files from %s/%s@%s (path: %s): %w", owner, repo, ref, workflowPath, err) + } + + // Filter to only .md files (not in subdirectories) + var workflowFiles []string + for _, item := range contents { + if item.Type == "file" && strings.HasSuffix(strings.ToLower(item.Name), ".md") { + workflowFiles = append(workflowFiles, item.Path) + } + } + + remoteLog.Printf("Found %d workflow files in %s/%s@%s (path: %s)", len(workflowFiles), owner, repo, ref, workflowPath) + return workflowFiles, nil +} + +// listWorkflowFilesViaGit lists workflow files using git commands (fallback for auth errors) +func listWorkflowFilesViaGit(owner, repo, ref, workflowPath string) ([]string, error) { + remoteLog.Printf("Attempting git fallback for listing workflow files: %s/%s@%s (path: %s)", owner, repo, ref, workflowPath) + + githubHost := GetGitHubHostForRepo(owner, repo) + repoURL := fmt.Sprintf("%s/%s/%s.git", githubHost, owner, repo) + + // Create a temporary directory for minimal clone + tmpDir, err := os.MkdirTemp("", "gh-aw-list-*") + if err != nil { + return nil, fmt.Errorf("failed to create temp directory: %w", err) + } + defer os.RemoveAll(tmpDir) + + // Do a minimal clone using filter=blob:none for faster cloning (metadata only, no blobs) + // Use --depth=1 for shallow clone and --no-checkout to skip checkout initially + cloneCmd := exec.Command("git", "clone", "--depth", "1", "--branch", ref, "--single-branch", "--filter=blob:none", "--no-checkout", repoURL, tmpDir) + cloneOutput, err := cloneCmd.CombinedOutput() + if err != nil { + remoteLog.Printf("Failed to clone repository: %s", string(cloneOutput)) + return nil, fmt.Errorf("failed to clone repository for %s/%s@%s: %w", owner, repo, ref, err) + } + + // Use git ls-tree to list files in the specified workflows directory + lsTreeCmd := exec.Command("git", "-C", tmpDir, "ls-tree", "-r", "--name-only", "HEAD", workflowPath+"/") + lsTreeOutput, err := lsTreeCmd.CombinedOutput() + if err != nil { + remoteLog.Printf("Failed to list files: %s", string(lsTreeOutput)) + return nil, fmt.Errorf("failed to list workflow files: %w", err) + } + + // Parse output and filter for .md files (not in subdirectories) + lines := strings.Split(strings.TrimSpace(string(lsTreeOutput)), "\n") + var workflowFiles []string + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" { + continue + } + // Only include .md files directly in the workflow path (not in subdirectories) + if strings.HasSuffix(strings.ToLower(line), ".md") { + // Check if it's a top-level file (no additional slashes after workflowPath/) + afterWorkflowPath := strings.TrimPrefix(line, workflowPath+"/") + if !strings.Contains(afterWorkflowPath, "/") { + workflowFiles = append(workflowFiles, line) + } + } + } + + remoteLog.Printf("Found %d workflow files via git for %s/%s@%s (path: %s)", len(workflowFiles), owner, repo, ref, workflowPath) + return workflowFiles, nil +}