diff --git a/actions/setup/js/handle_agent_failure.cjs b/actions/setup/js/handle_agent_failure.cjs index 61f9c000a9..935c3187e8 100644 --- a/actions/setup/js/handle_agent_failure.cjs +++ b/actions/setup/js/handle_agent_failure.cjs @@ -239,6 +239,7 @@ async function linkSubIssue(parentNodeId, subIssueNodeId, parentNumber, subIssue /** * Handle agent job failure by creating or updating a failure tracking issue * This script is called from the conclusion job when the agent job has failed + * or when the agent succeeded but produced no safe outputs */ async function main() { try { @@ -260,9 +261,21 @@ async function main() { // Check if there are assignment errors (regardless of agent job status) const hasAssignmentErrors = parseInt(assignmentErrorCount, 10) > 0; - // Only proceed if the agent job actually failed OR there are assignment errors - if (agentConclusion !== "failure" && !hasAssignmentErrors) { - core.info(`Agent job did not fail and no assignment errors (conclusion: ${agentConclusion}), skipping failure handling`); + // Check if agent succeeded but produced no safe outputs + let hasMissingSafeOutputs = false; + if (agentConclusion === "success") { + const { loadAgentOutput } = require("./load_agent_output.cjs"); + const agentOutputResult = loadAgentOutput(); + + if (!agentOutputResult.success || !agentOutputResult.items || agentOutputResult.items.length === 0) { + hasMissingSafeOutputs = true; + core.info("Agent succeeded but produced no safe outputs"); + } + } + + // Only proceed if the agent job actually failed OR there are assignment errors OR missing safe outputs + if (agentConclusion !== "failure" && !hasAssignmentErrors && !hasMissingSafeOutputs) { + core.info(`Agent job did not fail and no assignment errors and has safe outputs (conclusion: ${agentConclusion}), skipping failure handling`); return; } @@ -329,6 +342,15 @@ async function main() { assignmentErrorsContext += "\n"; } + // Build missing safe outputs context + let missingSafeOutputsContext = ""; + if (hasMissingSafeOutputs) { + missingSafeOutputsContext = "\n**⚠️ No Safe Outputs Generated**: The agent job succeeded but did not produce any safe outputs. This typically indicates:\n"; + missingSafeOutputsContext += "- The safe output server failed to run\n"; + missingSafeOutputsContext += "- The prompt failed to generate any meaningful result\n"; + missingSafeOutputsContext += "- The agent should have called `noop` to explicitly indicate no action was taken\n\n"; + } + // Create template context const templateContext = { run_url: runUrl, @@ -340,6 +362,7 @@ async function main() { secret_verification_context: secretVerificationResult === "failed" ? "\n**⚠️ Secret Verification Failed**: The workflow's secret validation step failed. Please check that the required secrets are configured in your repository settings.\n" : "", assignment_errors_context: assignmentErrorsContext, + missing_safe_outputs_context: missingSafeOutputsContext, }; // Render the comment template @@ -394,6 +417,15 @@ async function main() { assignmentErrorsContext += "\n"; } + // Build missing safe outputs context + let missingSafeOutputsContext = ""; + if (hasMissingSafeOutputs) { + missingSafeOutputsContext = "\n**⚠️ No Safe Outputs Generated**: The agent job succeeded but did not produce any safe outputs. This typically indicates:\n"; + missingSafeOutputsContext += "- The safe output server failed to run\n"; + missingSafeOutputsContext += "- The prompt failed to generate any meaningful result\n"; + missingSafeOutputsContext += "- The agent should have called `noop` to explicitly indicate no action was taken\n\n"; + } + // Create template context with sanitized workflow name const templateContext = { workflow_name: sanitizedWorkflowName, @@ -405,6 +437,7 @@ async function main() { secret_verification_context: secretVerificationResult === "failed" ? "\n**⚠️ Secret Verification Failed**: The workflow's secret validation step failed. Please check that the required secrets are configured in your repository settings.\n" : "", assignment_errors_context: assignmentErrorsContext, + missing_safe_outputs_context: missingSafeOutputsContext, }; // Render the issue template diff --git a/actions/setup/js/handle_agent_failure.test.cjs b/actions/setup/js/handle_agent_failure.test.cjs index ae7a75d8a6..6fbd08aefe 100644 --- a/actions/setup/js/handle_agent_failure.test.cjs +++ b/actions/setup/js/handle_agent_failure.test.cjs @@ -26,7 +26,7 @@ describe("handle_agent_failure.cjs", () => { **Branch:** {branch} **Run URL:** {run_url}{pull_request_info} -{secret_verification_context} +{secret_verification_context}{assignment_errors_context}{missing_safe_outputs_context} ### Action Required @@ -40,7 +40,7 @@ When prompted, instruct the agent to debug this workflow failure.`; } else if (filePath.includes("agent_failure_comment.md")) { return `Agent job [{run_id}]({run_url}) failed. -{secret_verification_context}`; +{secret_verification_context}{assignment_errors_context}{missing_safe_outputs_context}`; } return originalReadFileSync.call(fs, filePath, encoding); }); @@ -550,14 +550,55 @@ When prompted, instruct the agent to debug this workflow failure.`; }); describe("when agent job did not fail", () => { - it("should skip processing when agent conclusion is success", async () => { + it("should skip processing when agent conclusion is success with safe outputs", async () => { process.env.GH_AW_AGENT_CONCLUSION = "success"; + // Set up agent output file with safe outputs + const tempFilePath = "/tmp/test_agent_output_success.json"; + fs.writeFileSync( + tempFilePath, + JSON.stringify({ + items: [{ type: "noop", message: "No action taken" }], + }) + ); + process.env.GH_AW_AGENT_OUTPUT = tempFilePath; + + try { + await main(); + + expect(mockCore.info).toHaveBeenCalledWith(expect.stringContaining("Agent job did not fail")); + expect(mockGithub.rest.search.issuesAndPullRequests).not.toHaveBeenCalled(); + expect(mockGithub.rest.issues.create).not.toHaveBeenCalled(); + } finally { + // Clean up + if (fs.existsSync(tempFilePath)) { + fs.unlinkSync(tempFilePath); + } + } + }); + + it("should process when agent conclusion is success but no safe outputs", async () => { + process.env.GH_AW_AGENT_CONCLUSION = "success"; + // Don't set up GH_AW_AGENT_OUTPUT to simulate missing safe outputs + + // Mock API responses + mockGithub.rest.search.issuesAndPullRequests.mockResolvedValue({ + data: { total_count: 0, items: [] }, + }); + + mockGithub.rest.issues.create.mockResolvedValue({ + data: { + number: 1, + html_url: "https://github.com/test-owner/test-repo/issues/1", + node_id: "test-node-id", + }, + }); + await main(); - expect(mockCore.info).toHaveBeenCalledWith(expect.stringContaining("Agent job did not fail")); - expect(mockGithub.rest.search.issuesAndPullRequests).not.toHaveBeenCalled(); - expect(mockGithub.rest.issues.create).not.toHaveBeenCalled(); + expect(mockCore.info).toHaveBeenCalledWith(expect.stringContaining("Agent succeeded but produced no safe outputs")); + expect(mockGithub.rest.search.issuesAndPullRequests).toHaveBeenCalled(); + expect(mockGithub.rest.issues.create).toHaveBeenCalled(); }); it("should skip processing when agent conclusion is cancelled", async () => { diff --git a/actions/setup/md/agent_failure_comment.md b/actions/setup/md/agent_failure_comment.md index baeebdcc4a..572d0d7053 100644 --- a/actions/setup/md/agent_failure_comment.md +++ b/actions/setup/md/agent_failure_comment.md @@ -1,3 +1,3 @@ Agent job [{run_id}]({run_url}) failed. -{secret_verification_context}{assignment_errors_context} +{secret_verification_context}{assignment_errors_context}{missing_safe_outputs_context} diff --git a/actions/setup/md/agent_failure_issue.md b/actions/setup/md/agent_failure_issue.md index 935e2cd2c5..22e2d840d9 100644 --- a/actions/setup/md/agent_failure_issue.md +++ b/actions/setup/md/agent_failure_issue.md @@ -4,7 +4,7 @@ **Branch:** {branch} **Run URL:** {run_url}{pull_request_info} -{secret_verification_context}{assignment_errors_context} +{secret_verification_context}{assignment_errors_context}{missing_safe_outputs_context} ### Action Required