Handle failTaskOnFailedTests option - Azure Test Plan (#20712)

* Gradle - do not fail here if test fails, we fail from publish test results * Maven - do not fail due to test fail here, we do that from publish test results * Test task should fail on test failures by default. * Gen * default value to be true. * added comment * documentation link * send back error through call back function.
microsoft · Dec 6, 2024 · a7b3216 · a7b3216
1 parent 1444497
commit a7b3216
Show file tree

Hide file tree

Showing 13 changed files with 125 additions and 40 deletions.
diff --git a/Tasks/AzureTestPlanV0/Invokers/maveninvoker.ts b/Tasks/AzureTestPlanV0/Invokers/maveninvoker.ts
@@ -33,10 +33,13 @@ export async function executeMavenTests(testsToBeExecuted: string[], pomFilePath
         args.push(pomFilePath);
     }
 
+    //for returning success exit code incase of test failure and later we detect test failure from PTR command, documentation: https://maven.apache.org/surefire/maven-failsafe-plugin/verify-mojo.html, https://maven.apache.org/archives/maven-1.x/plugins/test/announcements/announcement-1.8.txt
+    args.push('-Dmaven.test.failure.ignore=true');
+
     tl.debug("Executing java maven tests with executable : " + executable);
     tl.debug("Executing java maven tests with args :" + args);
 
    let status = await execMavenBuild(args);
 
    return status ?? 1;
-}
+}
diff --git a/Tasks/AzureTestPlanV0/task.json b/Tasks/AzureTestPlanV0/task.json
@@ -13,7 +13,7 @@
   "author": "Microsoft Corporation",
   "version": {
     "Major": 0,
-    "Minor": 248,
+    "Minor": 250,
     "Patch": 0
   },
   "preview": true,
@@ -123,7 +123,7 @@
       "name": "failTaskOnFailedTests",
       "type": "boolean",
       "label": "Fail if there are test failures",
-      "defaultValue": "false",
+      "defaultValue": "true",
       "required": false,
       "helpMarkDown": "Fail the task if there are any test failures. Check this option to fail the task if test failures are detected in the result files."
     },

diff --git a/Tasks/AzureTestPlanV0/task.loc.json b/Tasks/AzureTestPlanV0/task.loc.json
@@ -13,7 +13,7 @@
   "author": "Microsoft Corporation",
   "version": {
     "Major": 0,
-    "Minor": 248,
+    "Minor": 250,
     "Patch": 0
   },
   "preview": true,
@@ -123,7 +123,7 @@
       "name": "failTaskOnFailedTests",
       "type": "boolean",
       "label": "ms-resource:loc.input.label.failTaskOnFailedTests",
-      "defaultValue": "false",
+      "defaultValue": "true",
       "required": false,
       "helpMarkDown": "ms-resource:loc.input.help.failTaskOnFailedTests"
     },

diff --git a/Tasks/AzureTestPlanV0/testLibExecutor.ts b/Tasks/AzureTestPlanV0/testLibExecutor.ts
@@ -4,6 +4,7 @@ import fs = require('fs');
 import * as path from 'path';
 import * as tl from 'azure-pipelines-task-lib/task';
 import * as tr from 'azure-pipelines-task-lib/toolrunner';
+import { Writable } from 'stream';
 
 var isWindows = os.type().match(/^Win/);
 
@@ -40,11 +41,29 @@ function getMavenExec() {
     return mvnExec;
 }
 
-function getExecOptions(): tr.IExecOptions {
+function getExecOptions(output?: { stdout: string}): tr.IExecOptions {
     var env = process.env;
-    return <tr.IExecOptions>{
+
+    var execOptions: tr.IExecOptions = output
+    ? {
+        env: env,
+        outStream: new Writable({
+            write(chunk, encoding, callback) {
+                try {
+                    output.stdout += chunk.toString();
+                    process.stdout.write(chunk);
+                    callback();
+                } catch (error) {
+                    callback(error);
+                }
+            },
+        }),
+    }
+    : {
         env: env,
     };
+
+    return execOptions;
 }
 
 /**Maven orchestration occurs as follows:
@@ -149,12 +168,17 @@ export async function execGradleBuild(args: string[]): Promise<number> {
     gradleRunner.arg('clean');
     gradleRunner.arg(args);
 
+    let runnerOutput = { stdout: ''};
+
     try {
-        await gradleRunner.exec(getExecOptions());
+        await gradleRunner.exec(getExecOptions(runnerOutput));
         // Gradle build succeeded
         return 0; // Return 0 indicating success
     } catch (err) {
-        console.error(err.message);
+        // we read stdout and return success incase of error due to test failure as later we detect test failure from PTR command
+        if (runnerOutput.stdout.includes('There were failing tests')) {
+            return 0;
+        }
         tl.setResult(tl.TaskResult.Failed, "Build failed."); // Set the step result to Failed
         return 1; // Return 1 indicating failure
     }
@@ -170,4 +194,4 @@ export async function executeJestCommand(jestPath: string, argument: string): Pr
     let jest: tr.ToolRunner = tl.tool(jestPath);
     jest.line(argument);
     return await jest.exec(<tr.IExecOptions>{ cwd: "" });
-}
+}
diff --git a/_generated/AzureTestPlanV0.versionmap.txt b/_generated/AzureTestPlanV0.versionmap.txt
@@ -1,2 +1,2 @@
-Default|0.248.0
-Node20-225|0.248.1
+Default|0.250.0
+Node20-225|0.250.1
diff --git a/_generated/AzureTestPlanV0/Invokers/maveninvoker.ts b/_generated/AzureTestPlanV0/Invokers/maveninvoker.ts
@@ -33,10 +33,13 @@ export async function executeMavenTests(testsToBeExecuted: string[], pomFilePath
         args.push(pomFilePath);
     }
 
+    //for returning success exit code incase of test failure and later we detect test failure from PTR command, documentation: https://maven.apache.org/surefire/maven-failsafe-plugin/verify-mojo.html, https://maven.apache.org/archives/maven-1.x/plugins/test/announcements/announcement-1.8.txt
+    args.push('-Dmaven.test.failure.ignore=true');
+
     tl.debug("Executing java maven tests with executable : " + executable);
     tl.debug("Executing java maven tests with args :" + args);
 
    let status = await execMavenBuild(args);
 
    return status ?? 1;
-}
+}
diff --git a/_generated/AzureTestPlanV0/task.json b/_generated/AzureTestPlanV0/task.json
@@ -13,7 +13,7 @@
   "author": "Microsoft Corporation",
   "version": {
     "Major": 0,
-    "Minor": 248,
+    "Minor": 250,
     "Patch": 0
   },
   "preview": true,
@@ -123,7 +123,7 @@
       "name": "failTaskOnFailedTests",
       "type": "boolean",
       "label": "Fail if there are test failures",
-      "defaultValue": "false",
+      "defaultValue": "true",
       "required": false,
       "helpMarkDown": "Fail the task if there are any test failures. Check this option to fail the task if test failures are detected in the result files."
     },
@@ -196,7 +196,8 @@
     "MultipleMatchingGradlewFound": "Multiple gradlew files found. Selecting the first matched instance"
   },
   "_buildConfigMapping": {
-    "Default": "0.248.0",
-    "Node20-225": "0.248.1"
+    "Default": "0.250.0",
+    "LocalPackages": "0.249.4",
+    "Node20-225": "0.250.1"
   }
 }
diff --git a/_generated/AzureTestPlanV0/task.loc.json b/_generated/AzureTestPlanV0/task.loc.json
@@ -13,7 +13,7 @@
   "author": "Microsoft Corporation",
   "version": {
     "Major": 0,
-    "Minor": 248,
+    "Minor": 250,
     "Patch": 0
   },
   "preview": true,
@@ -123,7 +123,7 @@
       "name": "failTaskOnFailedTests",
       "type": "boolean",
       "label": "ms-resource:loc.input.label.failTaskOnFailedTests",
-      "defaultValue": "false",
+      "defaultValue": "true",
       "required": false,
       "helpMarkDown": "ms-resource:loc.input.help.failTaskOnFailedTests"
     },
@@ -196,7 +196,8 @@
     "MultipleMatchingGradlewFound": "ms-resource:loc.messages.MultipleMatchingGradlewFound"
   },
   "_buildConfigMapping": {
-    "Default": "0.248.0",
-    "Node20-225": "0.248.1"
+    "Default": "0.250.0",
+    "LocalPackages": "0.249.4",
+    "Node20-225": "0.250.1"
   }
 }
diff --git a/_generated/AzureTestPlanV0/testLibExecutor.ts b/_generated/AzureTestPlanV0/testLibExecutor.ts
@@ -4,6 +4,7 @@ import fs = require('fs');
 import * as path from 'path';
 import * as tl from 'azure-pipelines-task-lib/task';
 import * as tr from 'azure-pipelines-task-lib/toolrunner';
+import { Writable } from 'stream';
 
 var isWindows = os.type().match(/^Win/);
 
@@ -40,11 +41,29 @@ function getMavenExec() {
     return mvnExec;
 }
 
-function getExecOptions(): tr.IExecOptions {
+function getExecOptions(output?: { stdout: string}): tr.IExecOptions {
     var env = process.env;
-    return <tr.IExecOptions>{
+
+    var execOptions: tr.IExecOptions = output
+    ? {
+        env: env,
+        outStream: new Writable({
+            write(chunk, encoding, callback) {
+                try {
+                    output.stdout += chunk.toString();
+                    process.stdout.write(chunk);
+                    callback();
+                } catch (error) {
+                    callback(error);
+                }
+            },
+        }),
+    }
+    : {
         env: env,
     };
+
+    return execOptions;
 }
 
 /**Maven orchestration occurs as follows:
@@ -149,12 +168,17 @@ export async function execGradleBuild(args: string[]): Promise<number> {
     gradleRunner.arg('clean');
     gradleRunner.arg(args);
 
+    let runnerOutput = { stdout: ''};
+
     try {
-        await gradleRunner.exec(getExecOptions());
+        await gradleRunner.exec(getExecOptions(runnerOutput));
         // Gradle build succeeded
         return 0; // Return 0 indicating success
     } catch (err) {
-        console.error(err.message);
+        // we read stdout and return success incase of error due to test failure as later we detect test failure from PTR command
+        if (runnerOutput.stdout.includes('There were failing tests')) {
+            return 0;
+        }
         tl.setResult(tl.TaskResult.Failed, "Build failed."); // Set the step result to Failed
         return 1; // Return 1 indicating failure
     }
@@ -170,4 +194,4 @@ export async function executeJestCommand(jestPath: string, argument: string): Pr
     let jest: tr.ToolRunner = tl.tool(jestPath);
     jest.line(argument);
     return await jest.exec(<tr.IExecOptions>{ cwd: "" });
-}
+}
diff --git a/_generated/AzureTestPlanV0_Node20/Invokers/maveninvoker.ts b/_generated/AzureTestPlanV0_Node20/Invokers/maveninvoker.ts
@@ -33,10 +33,13 @@ export async function executeMavenTests(testsToBeExecuted: string[], pomFilePath
         args.push(pomFilePath);
     }
 
+    //for returning success exit code incase of test failure and later we detect test failure from PTR command, documentation: https://maven.apache.org/surefire/maven-failsafe-plugin/verify-mojo.html, https://maven.apache.org/archives/maven-1.x/plugins/test/announcements/announcement-1.8.txt
+    args.push('-Dmaven.test.failure.ignore=true');
+
     tl.debug("Executing java maven tests with executable : " + executable);
     tl.debug("Executing java maven tests with args :" + args);
 
    let status = await execMavenBuild(args);
 
    return status ?? 1;
-}
+}
diff --git a/_generated/AzureTestPlanV0_Node20/task.json b/_generated/AzureTestPlanV0_Node20/task.json
@@ -13,7 +13,7 @@
   "author": "Microsoft Corporation",
   "version": {
     "Major": 0,
-    "Minor": 248,
+    "Minor": 250,
     "Patch": 1
   },
   "preview": true,
@@ -123,7 +123,7 @@
       "name": "failTaskOnFailedTests",
       "type": "boolean",
       "label": "Fail if there are test failures",
-      "defaultValue": "false",
+      "defaultValue": "true",
       "required": false,
       "helpMarkDown": "Fail the task if there are any test failures. Check this option to fail the task if test failures are detected in the result files."
     },
@@ -200,7 +200,8 @@
     "MultipleMatchingGradlewFound": "Multiple gradlew files found. Selecting the first matched instance"
   },
   "_buildConfigMapping": {
-    "Default": "0.248.0",
-    "Node20-225": "0.248.1"
+    "Default": "0.250.0",
+    "LocalPackages": "0.249.4",
+    "Node20-225": "0.250.1"
   }
 }
diff --git a/_generated/AzureTestPlanV0_Node20/task.loc.json b/_generated/AzureTestPlanV0_Node20/task.loc.json
@@ -13,7 +13,7 @@
   "author": "Microsoft Corporation",
   "version": {
     "Major": 0,
-    "Minor": 248,
+    "Minor": 250,
     "Patch": 1
   },
   "preview": true,
@@ -123,7 +123,7 @@
       "name": "failTaskOnFailedTests",
       "type": "boolean",
       "label": "ms-resource:loc.input.label.failTaskOnFailedTests",
-      "defaultValue": "false",
+      "defaultValue": "true",
       "required": false,
       "helpMarkDown": "ms-resource:loc.input.help.failTaskOnFailedTests"
     },
@@ -200,7 +200,8 @@
     "MultipleMatchingGradlewFound": "ms-resource:loc.messages.MultipleMatchingGradlewFound"
   },
   "_buildConfigMapping": {
-    "Default": "0.248.0",
-    "Node20-225": "0.248.1"
+    "Default": "0.250.0",
+    "LocalPackages": "0.249.4",
+    "Node20-225": "0.250.1"
   }
 }