Add reportFailedTest (#1067)

I realized that we don't have reportFailedTest which is very convenient for tracking especially flaky tests. This PR copies all the work in line/armeria#4292

.github/workflows/actions_build.yml

@@ -75,11 +75,34 @@ jobs:
         -Porg.gradle.java.installations.paths=${{ steps.setup-build-jdk.outputs.path }},${{ steps.setup-test-jdk.outputs.path }}
       shell: bash
 
+    - name: Summarize the failed tests
+      if: failure()
+      run: |
+        ./gradlew --no-daemon --stacktrace --max-workers=1 reportFailedTests \
+        -PnoLint \
+        -PflakyTests=false \
+        -PbuildJdkVersion=${{ env.BUILD_JDK_VERSION }} \
+        -PtestJavaVersion=${{ matrix.java }} \
+        ${{ matrix.min-java && format('-PminimumJavaVersion={0}', matrix.min-java) || '' }} \
+        -Porg.gradle.java.installations.paths=${{ steps.setup-build-jdk.outputs.path }},${{ steps.setup-jdk.outputs.path }}
+
+        SUMMARY_FILE="build/failed-tests-result.txt"
+        if test -f "$SUMMARY_FILE"; then
+          echo '### 🔴 Failed tests' >> $GITHUB_STEP_SUMMARY
+          cat $SUMMARY_FILE >> $GITHUB_STEP_SUMMARY
+        fi
+      shell: bash
+
+    - name: Dump stuck threads
+      if: always()
+      run: jps | grep -vi "jps" | awk '{ print $1 }' | xargs -I'{}' jstack -l {} || true
+      shell: bash
+
     - name: Upload coverage to Codecov
       if: ${{ matrix.coverage }}
       uses: codecov/codecov-action@v3
 
-    - name: Collecting the test reports ..
+    - name: Collect the test reports ..
       if: failure()
       run: |
         find . '(' \
@@ -96,11 +119,6 @@ jobs:
         path: reports-JVM-${{ matrix.java }}.tar
         retention-days: 3
 
-    - name: Dump stuck threads
-      if: always()
-      run: jps | grep -vi "jps" | awk '{ print $1 }' | xargs -I'{}' jstack -l {} || true
-      shell: bash
-
   lint:
     if: github.repository == 'line/centraldogma'
     runs-on: ubuntu-latest

build.gradle

@@ -1,3 +1,15 @@
+import org.jsoup.Jsoup
+import org.jsoup.nodes.Element
+import org.jsoup.select.Elements
+
+import static java.lang.Math.min
+
+buildscript {
+    dependencies {
+        classpath libs.jsoup
+    }
+}
+
 plugins {
     alias libs.plugins.nexus.publish
     alias libs.plugins.osdetector apply false
@@ -127,6 +139,85 @@ configure(projectsWithFlags('java')) {
     tasks.processResources.duplicatesStrategy = DuplicatesStrategy.INCLUDE
 }
 
+tasks.register("reportFailedTests", TestsReportTask)
+
+/**
+ * Summarizes the failed tests and reports as a file with the Markdown syntax.
+ */
+class TestsReportTask extends DefaultTask {
+    @OutputFile
+    final def reportFile = project.file("${project.buildDir}/failed-tests-result.txt")
+
+    @TaskAction
+    def run() {
+        // Collect up to 20 error results
+        int maxErrorSize = 20
+        List<Map> failedTests = []
+        Set<String> handledFiles = []
+
+        project.allprojects {
+            tasks.withType(Test) { testTask ->
+
+                def xmlFiles = testTask.reports.junitXml.outputLocation.asFileTree.files
+                if (xmlFiles.isEmpty()) {
+                    return
+                }
+                xmlFiles.each { file ->
+                    if (!handledFiles.add(file.name)) {
+                        return
+                    }
+
+                    Elements failures = Jsoup.parse(file, 'UTF-8').select("testsuite failure")
+                    if (failures.isEmpty() || failedTests.size() > maxErrorSize) {
+                        return
+                    }
+                    failures.each { failure ->
+                        Element parent = failure.parent()
+                        String fullMethodName = "${parent.attr("classname")}.${parent.attr("name")}"
+                        String detail = failure.wholeText()
+                        failedTests += [method: fullMethodName, detail: detail]
+                    }
+                }
+            }
+        }
+
+        if (failedTests.isEmpty()) {
+            return
+        }
+
+        reportFile.withPrintWriter('UTF-8') { writer ->
+            failedTests.each { it ->
+                String method = it.method
+                String detail = it.detail
+
+                // Create an link to directly create an issue from the error message
+                String ghIssueTitle = URLEncoder.encode("Test failure: `$method`", "UTF-8")
+                // 8k is the maximum allowed URL length for GitHub
+                String ghIssueBody = URLEncoder.encode(
+                        "```\n${detail.substring(0, min(6000, detail.length()))}\n```\n", "UTF-8")
+                String ghIssueLink =
+                        "https://github.com/line/centraldogma/issues/new?title=$ghIssueTitle&body=$ghIssueBody"
+                String ghSearchQuery = URLEncoder.encode("is:issue $method", "UTF-8")
+                String ghSearchLink = "https://github.com/line/centraldogma/issues?q=$ghSearchQuery"
+                writer.print("- $it.method - [Search similar issues]($ghSearchLink) | ")
+                writer.println("[Create an issue?]($ghIssueLink)")
+
+                writer.println("  ```")
+                List<String> lines = detail.split("\n") as List
+                def summary = lines.take(8)
+                summary.each { line -> writer.println("  $line") }
+                writer.println("  ```")
+                if (lines.size() > 8) {
+                    writer.println("  <details><summary>Full error messages</summary>")
+                    writer.println("  <pre>")
+                    lines.each { line -> writer.println("  $line") }
+                    writer.println("  </pre></details>\n")
+                }
+            }
+        }
+    }
+}
+
 // Configure the Javadoc tasks of all projects.
 allprojects {
     tasks.withType(Javadoc) {

dependencies.toml

@@ -49,6 +49,7 @@ jsch = "0.1.55"
 json-path = "2.2.0"
 # 3.0.0 requires java 17
 json-unit = "2.38.0"
+jsoup = "1.18.1" # JSoup is only used for Gradle script.
 jmh-core = "1.37"
 jmh-gradle-plugin = "0.7.2"
 jxr = "0.2.1"
@@ -281,6 +282,10 @@ version.ref = "json-unit"
 module = "net.javacrumbs.json-unit:json-unit-fluent"
 version.ref = "json-unit"
 
+[libraries.jsoup]
+module = "org.jsoup:jsoup"
+version.ref = "jsoup"
+
 [libraries.junit4]
 module = "junit:junit"
 version.ref = "junit4"