From a7b3216613e37d8efc0e699932f72ec4a8234de1 Mon Sep 17 00:00:00 2001 From: Kathan Sanghavi <52504721+KathanS@users.noreply.github.com> Date: Fri, 6 Dec 2024 12:48:05 +0530 Subject: [PATCH] Handle failTaskOnFailedTests option - Azure Test Plan (#20712) * Gradle - do not fail here if test fails, we fail from publish test results * Maven - do not fail due to test fail here, we do that from publish test results * Test task should fail on test failures by default. * Gen * default value to be true. * added comment * documentation link * send back error through call back function. --- .../AzureTestPlanV0/Invokers/maveninvoker.ts | 5 ++- Tasks/AzureTestPlanV0/task.json | 4 +-- Tasks/AzureTestPlanV0/task.loc.json | 4 +-- Tasks/AzureTestPlanV0/testLibExecutor.ts | 34 ++++++++++++++++--- _generated/AzureTestPlanV0.versionmap.txt | 4 +-- .../AzureTestPlanV0/Invokers/maveninvoker.ts | 5 ++- _generated/AzureTestPlanV0/task.json | 9 ++--- _generated/AzureTestPlanV0/task.loc.json | 9 ++--- _generated/AzureTestPlanV0/testLibExecutor.ts | 34 ++++++++++++++++--- .../Invokers/maveninvoker.ts | 5 ++- _generated/AzureTestPlanV0_Node20/task.json | 9 ++--- .../AzureTestPlanV0_Node20/task.loc.json | 9 ++--- .../AzureTestPlanV0_Node20/testLibExecutor.ts | 34 ++++++++++++++++--- 13 files changed, 125 insertions(+), 40 deletions(-) diff --git a/Tasks/AzureTestPlanV0/Invokers/maveninvoker.ts b/Tasks/AzureTestPlanV0/Invokers/maveninvoker.ts index 4f56bdc3c286..34baa61174a3 100644 --- a/Tasks/AzureTestPlanV0/Invokers/maveninvoker.ts +++ b/Tasks/AzureTestPlanV0/Invokers/maveninvoker.ts @@ -33,10 +33,13 @@ export async function executeMavenTests(testsToBeExecuted: string[], pomFilePath args.push(pomFilePath); } + //for returning success exit code incase of test failure and later we detect test failure from PTR command, documentation: https://maven.apache.org/surefire/maven-failsafe-plugin/verify-mojo.html, https://maven.apache.org/archives/maven-1.x/plugins/test/announcements/announcement-1.8.txt + args.push('-Dmaven.test.failure.ignore=true'); + tl.debug("Executing java maven tests with executable : " + executable); tl.debug("Executing java maven tests with args :" + args); let status = await execMavenBuild(args); return status ?? 1; -} \ No newline at end of file +} diff --git a/Tasks/AzureTestPlanV0/task.json b/Tasks/AzureTestPlanV0/task.json index 6b9ea848c7ac..d16efa1d196c 100644 --- a/Tasks/AzureTestPlanV0/task.json +++ b/Tasks/AzureTestPlanV0/task.json @@ -13,7 +13,7 @@ "author": "Microsoft Corporation", "version": { "Major": 0, - "Minor": 248, + "Minor": 250, "Patch": 0 }, "preview": true, @@ -123,7 +123,7 @@ "name": "failTaskOnFailedTests", "type": "boolean", "label": "Fail if there are test failures", - "defaultValue": "false", + "defaultValue": "true", "required": false, "helpMarkDown": "Fail the task if there are any test failures. Check this option to fail the task if test failures are detected in the result files." }, diff --git a/Tasks/AzureTestPlanV0/task.loc.json b/Tasks/AzureTestPlanV0/task.loc.json index bc55ba96f681..15da12187537 100644 --- a/Tasks/AzureTestPlanV0/task.loc.json +++ b/Tasks/AzureTestPlanV0/task.loc.json @@ -13,7 +13,7 @@ "author": "Microsoft Corporation", "version": { "Major": 0, - "Minor": 248, + "Minor": 250, "Patch": 0 }, "preview": true, @@ -123,7 +123,7 @@ "name": "failTaskOnFailedTests", "type": "boolean", "label": "ms-resource:loc.input.label.failTaskOnFailedTests", - "defaultValue": "false", + "defaultValue": "true", "required": false, "helpMarkDown": "ms-resource:loc.input.help.failTaskOnFailedTests" }, diff --git a/Tasks/AzureTestPlanV0/testLibExecutor.ts b/Tasks/AzureTestPlanV0/testLibExecutor.ts index dc148fe4ce06..69a8219d2331 100644 --- a/Tasks/AzureTestPlanV0/testLibExecutor.ts +++ b/Tasks/AzureTestPlanV0/testLibExecutor.ts @@ -4,6 +4,7 @@ import fs = require('fs'); import * as path from 'path'; import * as tl from 'azure-pipelines-task-lib/task'; import * as tr from 'azure-pipelines-task-lib/toolrunner'; +import { Writable } from 'stream'; var isWindows = os.type().match(/^Win/); @@ -40,11 +41,29 @@ function getMavenExec() { return mvnExec; } -function getExecOptions(): tr.IExecOptions { +function getExecOptions(output?: { stdout: string}): tr.IExecOptions { var env = process.env; - return { + + var execOptions: tr.IExecOptions = output + ? { + env: env, + outStream: new Writable({ + write(chunk, encoding, callback) { + try { + output.stdout += chunk.toString(); + process.stdout.write(chunk); + callback(); + } catch (error) { + callback(error); + } + }, + }), + } + : { env: env, }; + + return execOptions; } /**Maven orchestration occurs as follows: @@ -149,12 +168,17 @@ export async function execGradleBuild(args: string[]): Promise { gradleRunner.arg('clean'); gradleRunner.arg(args); + let runnerOutput = { stdout: ''}; + try { - await gradleRunner.exec(getExecOptions()); + await gradleRunner.exec(getExecOptions(runnerOutput)); // Gradle build succeeded return 0; // Return 0 indicating success } catch (err) { - console.error(err.message); + // we read stdout and return success incase of error due to test failure as later we detect test failure from PTR command + if (runnerOutput.stdout.includes('There were failing tests')) { + return 0; + } tl.setResult(tl.TaskResult.Failed, "Build failed."); // Set the step result to Failed return 1; // Return 1 indicating failure } @@ -170,4 +194,4 @@ export async function executeJestCommand(jestPath: string, argument: string): Pr let jest: tr.ToolRunner = tl.tool(jestPath); jest.line(argument); return await jest.exec({ cwd: "" }); -} \ No newline at end of file +} diff --git a/_generated/AzureTestPlanV0.versionmap.txt b/_generated/AzureTestPlanV0.versionmap.txt index c22c7fbf7195..86308b7e2f89 100644 --- a/_generated/AzureTestPlanV0.versionmap.txt +++ b/_generated/AzureTestPlanV0.versionmap.txt @@ -1,2 +1,2 @@ -Default|0.248.0 -Node20-225|0.248.1 +Default|0.250.0 +Node20-225|0.250.1 diff --git a/_generated/AzureTestPlanV0/Invokers/maveninvoker.ts b/_generated/AzureTestPlanV0/Invokers/maveninvoker.ts index 4f56bdc3c286..34baa61174a3 100644 --- a/_generated/AzureTestPlanV0/Invokers/maveninvoker.ts +++ b/_generated/AzureTestPlanV0/Invokers/maveninvoker.ts @@ -33,10 +33,13 @@ export async function executeMavenTests(testsToBeExecuted: string[], pomFilePath args.push(pomFilePath); } + //for returning success exit code incase of test failure and later we detect test failure from PTR command, documentation: https://maven.apache.org/surefire/maven-failsafe-plugin/verify-mojo.html, https://maven.apache.org/archives/maven-1.x/plugins/test/announcements/announcement-1.8.txt + args.push('-Dmaven.test.failure.ignore=true'); + tl.debug("Executing java maven tests with executable : " + executable); tl.debug("Executing java maven tests with args :" + args); let status = await execMavenBuild(args); return status ?? 1; -} \ No newline at end of file +} diff --git a/_generated/AzureTestPlanV0/task.json b/_generated/AzureTestPlanV0/task.json index c837b6b88fb5..175a9f1bd567 100644 --- a/_generated/AzureTestPlanV0/task.json +++ b/_generated/AzureTestPlanV0/task.json @@ -13,7 +13,7 @@ "author": "Microsoft Corporation", "version": { "Major": 0, - "Minor": 248, + "Minor": 250, "Patch": 0 }, "preview": true, @@ -123,7 +123,7 @@ "name": "failTaskOnFailedTests", "type": "boolean", "label": "Fail if there are test failures", - "defaultValue": "false", + "defaultValue": "true", "required": false, "helpMarkDown": "Fail the task if there are any test failures. Check this option to fail the task if test failures are detected in the result files." }, @@ -196,7 +196,8 @@ "MultipleMatchingGradlewFound": "Multiple gradlew files found. Selecting the first matched instance" }, "_buildConfigMapping": { - "Default": "0.248.0", - "Node20-225": "0.248.1" + "Default": "0.250.0", + "LocalPackages": "0.249.4", + "Node20-225": "0.250.1" } } \ No newline at end of file diff --git a/_generated/AzureTestPlanV0/task.loc.json b/_generated/AzureTestPlanV0/task.loc.json index 27c78ad0af6a..b8f82d3f7660 100644 --- a/_generated/AzureTestPlanV0/task.loc.json +++ b/_generated/AzureTestPlanV0/task.loc.json @@ -13,7 +13,7 @@ "author": "Microsoft Corporation", "version": { "Major": 0, - "Minor": 248, + "Minor": 250, "Patch": 0 }, "preview": true, @@ -123,7 +123,7 @@ "name": "failTaskOnFailedTests", "type": "boolean", "label": "ms-resource:loc.input.label.failTaskOnFailedTests", - "defaultValue": "false", + "defaultValue": "true", "required": false, "helpMarkDown": "ms-resource:loc.input.help.failTaskOnFailedTests" }, @@ -196,7 +196,8 @@ "MultipleMatchingGradlewFound": "ms-resource:loc.messages.MultipleMatchingGradlewFound" }, "_buildConfigMapping": { - "Default": "0.248.0", - "Node20-225": "0.248.1" + "Default": "0.250.0", + "LocalPackages": "0.249.4", + "Node20-225": "0.250.1" } } \ No newline at end of file diff --git a/_generated/AzureTestPlanV0/testLibExecutor.ts b/_generated/AzureTestPlanV0/testLibExecutor.ts index dc148fe4ce06..69a8219d2331 100644 --- a/_generated/AzureTestPlanV0/testLibExecutor.ts +++ b/_generated/AzureTestPlanV0/testLibExecutor.ts @@ -4,6 +4,7 @@ import fs = require('fs'); import * as path from 'path'; import * as tl from 'azure-pipelines-task-lib/task'; import * as tr from 'azure-pipelines-task-lib/toolrunner'; +import { Writable } from 'stream'; var isWindows = os.type().match(/^Win/); @@ -40,11 +41,29 @@ function getMavenExec() { return mvnExec; } -function getExecOptions(): tr.IExecOptions { +function getExecOptions(output?: { stdout: string}): tr.IExecOptions { var env = process.env; - return { + + var execOptions: tr.IExecOptions = output + ? { + env: env, + outStream: new Writable({ + write(chunk, encoding, callback) { + try { + output.stdout += chunk.toString(); + process.stdout.write(chunk); + callback(); + } catch (error) { + callback(error); + } + }, + }), + } + : { env: env, }; + + return execOptions; } /**Maven orchestration occurs as follows: @@ -149,12 +168,17 @@ export async function execGradleBuild(args: string[]): Promise { gradleRunner.arg('clean'); gradleRunner.arg(args); + let runnerOutput = { stdout: ''}; + try { - await gradleRunner.exec(getExecOptions()); + await gradleRunner.exec(getExecOptions(runnerOutput)); // Gradle build succeeded return 0; // Return 0 indicating success } catch (err) { - console.error(err.message); + // we read stdout and return success incase of error due to test failure as later we detect test failure from PTR command + if (runnerOutput.stdout.includes('There were failing tests')) { + return 0; + } tl.setResult(tl.TaskResult.Failed, "Build failed."); // Set the step result to Failed return 1; // Return 1 indicating failure } @@ -170,4 +194,4 @@ export async function executeJestCommand(jestPath: string, argument: string): Pr let jest: tr.ToolRunner = tl.tool(jestPath); jest.line(argument); return await jest.exec({ cwd: "" }); -} \ No newline at end of file +} diff --git a/_generated/AzureTestPlanV0_Node20/Invokers/maveninvoker.ts b/_generated/AzureTestPlanV0_Node20/Invokers/maveninvoker.ts index 4f56bdc3c286..34baa61174a3 100644 --- a/_generated/AzureTestPlanV0_Node20/Invokers/maveninvoker.ts +++ b/_generated/AzureTestPlanV0_Node20/Invokers/maveninvoker.ts @@ -33,10 +33,13 @@ export async function executeMavenTests(testsToBeExecuted: string[], pomFilePath args.push(pomFilePath); } + //for returning success exit code incase of test failure and later we detect test failure from PTR command, documentation: https://maven.apache.org/surefire/maven-failsafe-plugin/verify-mojo.html, https://maven.apache.org/archives/maven-1.x/plugins/test/announcements/announcement-1.8.txt + args.push('-Dmaven.test.failure.ignore=true'); + tl.debug("Executing java maven tests with executable : " + executable); tl.debug("Executing java maven tests with args :" + args); let status = await execMavenBuild(args); return status ?? 1; -} \ No newline at end of file +} diff --git a/_generated/AzureTestPlanV0_Node20/task.json b/_generated/AzureTestPlanV0_Node20/task.json index 36dfa1cf9943..7aa8812d946b 100644 --- a/_generated/AzureTestPlanV0_Node20/task.json +++ b/_generated/AzureTestPlanV0_Node20/task.json @@ -13,7 +13,7 @@ "author": "Microsoft Corporation", "version": { "Major": 0, - "Minor": 248, + "Minor": 250, "Patch": 1 }, "preview": true, @@ -123,7 +123,7 @@ "name": "failTaskOnFailedTests", "type": "boolean", "label": "Fail if there are test failures", - "defaultValue": "false", + "defaultValue": "true", "required": false, "helpMarkDown": "Fail the task if there are any test failures. Check this option to fail the task if test failures are detected in the result files." }, @@ -200,7 +200,8 @@ "MultipleMatchingGradlewFound": "Multiple gradlew files found. Selecting the first matched instance" }, "_buildConfigMapping": { - "Default": "0.248.0", - "Node20-225": "0.248.1" + "Default": "0.250.0", + "LocalPackages": "0.249.4", + "Node20-225": "0.250.1" } } \ No newline at end of file diff --git a/_generated/AzureTestPlanV0_Node20/task.loc.json b/_generated/AzureTestPlanV0_Node20/task.loc.json index 4870f2435c3a..7f1b820f1aae 100644 --- a/_generated/AzureTestPlanV0_Node20/task.loc.json +++ b/_generated/AzureTestPlanV0_Node20/task.loc.json @@ -13,7 +13,7 @@ "author": "Microsoft Corporation", "version": { "Major": 0, - "Minor": 248, + "Minor": 250, "Patch": 1 }, "preview": true, @@ -123,7 +123,7 @@ "name": "failTaskOnFailedTests", "type": "boolean", "label": "ms-resource:loc.input.label.failTaskOnFailedTests", - "defaultValue": "false", + "defaultValue": "true", "required": false, "helpMarkDown": "ms-resource:loc.input.help.failTaskOnFailedTests" }, @@ -200,7 +200,8 @@ "MultipleMatchingGradlewFound": "ms-resource:loc.messages.MultipleMatchingGradlewFound" }, "_buildConfigMapping": { - "Default": "0.248.0", - "Node20-225": "0.248.1" + "Default": "0.250.0", + "LocalPackages": "0.249.4", + "Node20-225": "0.250.1" } } \ No newline at end of file diff --git a/_generated/AzureTestPlanV0_Node20/testLibExecutor.ts b/_generated/AzureTestPlanV0_Node20/testLibExecutor.ts index dc148fe4ce06..69a8219d2331 100644 --- a/_generated/AzureTestPlanV0_Node20/testLibExecutor.ts +++ b/_generated/AzureTestPlanV0_Node20/testLibExecutor.ts @@ -4,6 +4,7 @@ import fs = require('fs'); import * as path from 'path'; import * as tl from 'azure-pipelines-task-lib/task'; import * as tr from 'azure-pipelines-task-lib/toolrunner'; +import { Writable } from 'stream'; var isWindows = os.type().match(/^Win/); @@ -40,11 +41,29 @@ function getMavenExec() { return mvnExec; } -function getExecOptions(): tr.IExecOptions { +function getExecOptions(output?: { stdout: string}): tr.IExecOptions { var env = process.env; - return { + + var execOptions: tr.IExecOptions = output + ? { + env: env, + outStream: new Writable({ + write(chunk, encoding, callback) { + try { + output.stdout += chunk.toString(); + process.stdout.write(chunk); + callback(); + } catch (error) { + callback(error); + } + }, + }), + } + : { env: env, }; + + return execOptions; } /**Maven orchestration occurs as follows: @@ -149,12 +168,17 @@ export async function execGradleBuild(args: string[]): Promise { gradleRunner.arg('clean'); gradleRunner.arg(args); + let runnerOutput = { stdout: ''}; + try { - await gradleRunner.exec(getExecOptions()); + await gradleRunner.exec(getExecOptions(runnerOutput)); // Gradle build succeeded return 0; // Return 0 indicating success } catch (err) { - console.error(err.message); + // we read stdout and return success incase of error due to test failure as later we detect test failure from PTR command + if (runnerOutput.stdout.includes('There were failing tests')) { + return 0; + } tl.setResult(tl.TaskResult.Failed, "Build failed."); // Set the step result to Failed return 1; // Return 1 indicating failure } @@ -170,4 +194,4 @@ export async function executeJestCommand(jestPath: string, argument: string): Pr let jest: tr.ToolRunner = tl.tool(jestPath); jest.line(argument); return await jest.exec({ cwd: "" }); -} \ No newline at end of file +}