[NSE-274] Comment to trigger tpc-h RAM test

.github/workflows/tpch.yml

@@ -18,14 +18,23 @@
 name: Native SQL Engine TPC-H Suite
 
 on:
-  pull_request
+  issue_comment:
+    types: [created, edited]
 
 jobs:
   ram-usage-test:
-    if: ${{ contains(github.event.pull_request.labels.*.name, 'RAM Report') }}
+    if: ${{ github.event.issue.pull_request && startsWith(github.event.comment.body, '@github-actions ram-usage-test') }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+      - name: Checkout Pull Request
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          PR_URL="${{ github.event.issue.pull_request.url }}"
+          PR_NUM=${PR_URL##*/}
+          echo "Checking out from PR #$PR_NUM based on URL: $PR_URL"
+          hub pr checkout $PR_NUM
       - name: Set up JDK 1.8
         uses: actions/setup-java@v1
         with:
@@ -50,7 +59,7 @@ jobs:
           mvn clean install -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -P arrow-jni -am -Darrow.cpp.build.dir=/tmp/arrow/cpp/build/release/ -DskipTests -Dcheckstyle.skip
       - name: Run Maven tests - BHJ
         run: |
-          mvn test -B -pl native-sql-engine/core/ -am -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DmembersOnlySuites=com.intel.oap.tpc.h -DtagsToInclude=com.intel.oap.tags.BroadcastHashJoinMode -DargLine="-Xmx1G -XX:MaxDirectMemorySize=500M -Dio.netty.allocator.numDirectArena=1"
+          mvn test -B -Dbuild_arrow=OFF -pl native-sql-engine/core/ -am -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DmembersOnlySuites=com.intel.oap.tpc.h -DtagsToInclude=com.intel.oap.tags.BroadcastHashJoinMode -DargLine="-Xmx1G -XX:MaxDirectMemorySize=500M -Dio.netty.allocator.numDirectArena=1"
         env:
           MALLOC_ARENA_MAX: "4"
           MAVEN_OPTS: "-Xmx1G"
@@ -59,7 +68,7 @@ jobs:
           ENABLE_TPCH_TESTS: "true"
       - name: Run Maven tests - SMJ
         run: |
-          mvn test -B -pl native-sql-engine/core/ -am -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DmembersOnlySuites=com.intel.oap.tpc.h -DtagsToInclude=com.intel.oap.tags.SortMergeJoinMode -DargLine="-Xmx1G -XX:MaxDirectMemorySize=500M -Dio.netty.allocator.numDirectArena=1"
+          mvn test -B -Dbuild_arrow=OFF -pl native-sql-engine/core/ -am -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DmembersOnlySuites=com.intel.oap.tpc.h -DtagsToInclude=com.intel.oap.tags.SortMergeJoinMode -DargLine="-Xmx1G -XX:MaxDirectMemorySize=500M -Dio.netty.allocator.numDirectArena=1"
         env:
           MALLOC_ARENA_MAX: "4"
           MAVEN_OPTS: "-Xmx1G"
@@ -69,14 +78,12 @@ jobs:
       - run: |
           cml-publish /tmp/comment_image_1.png --md > /tmp/comment.md
           cml-publish /tmp/comment_image_2.png --md >> /tmp/comment.md
-      - run: echo "::set-output name=event_path::${GITHUB_EVENT_PATH}"
-        id: output-envs
-      - uses: actions/upload-artifact@v2
-        with:
-          name: comment_content
-          path: /tmp/comment.md
-      - uses: actions/upload-artifact@v2
-        with:
-          name: pr_event
-          path: ${{steps.output-envs.outputs.event_path}}
-
+      - name: Run Maven tests - Report
+        run: |
+          mvn test -B -Dbuild_arrow=OFF -Dbuild_protobuf=OFF -pl native-sql-engine/core/ -am -DmembersOnlySuites=com.intel.oap.tpc.h -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DtagsToInclude=com.intel.oap.tags.CommentOnContextPR -Dexec.skip=true
+        env:
+          PR_URL: ${{ github.event.issue.pull_request.url }}
+          MAVEN_OPTS: "-Xmx1G"
+          COMMENT_CONTENT_PATH: "/tmp/comment.md"
+          GITHUB_TOKEN: ${{ github.token }}
+          ENABLE_TPCH_TESTS: "true"

native-sql-engine/core/src/test/scala/com/intel/oap/tpc/ds/TPCDSSuite.scala

@@ -34,7 +34,6 @@ class TPCDSSuite extends QueryTest with SharedSparkSession {
   override protected def sparkConf: SparkConf = {
     val conf = super.sparkConf
     conf.set("spark.memory.offHeap.size", String.valueOf(MAX_DIRECT_MEMORY))
-        .set("spark.driver.bindAddress", "127.0.0.1")
         .set("spark.sql.extensions", "com.intel.oap.ColumnarPlugin")
         .set("spark.sql.codegen.wholeStage", "true")
         .set("spark.sql.sources.useV1SourceList", "")

native-sql-engine/core/src/test/scala/com/intel/oap/tpc/h/GitHubActionEntries.scala

@@ -17,12 +17,13 @@
 package com.intel.oap.tpc.h
 
 import java.io.File
+import java.util.regex.Pattern
 
 import com.intel.oap.tags.CommentOnContextPR
 import org.apache.commons.io.FileUtils
 import org.apache.commons.lang.StringUtils
-import org.codehaus.jackson.map.ObjectMapper
-import org.kohsuke.github.{GHIssueComment, GitHubBuilder}
+import org.kohsuke.github.GHIssueComment
+import org.kohsuke.github.GitHubBuilder
 import org.scalatest.FunSuite
 
 class GitHubActionEntries extends FunSuite {
@@ -48,12 +49,12 @@ class GitHubActionEntries extends FunSuite {
         throw new IllegalArgumentException("No GITHUB_REPOSITORY set")
       }
 
-      val eventPath = System.getenv("PREVIOUS_EVENT_PATH")
+      val eventPath = System.getenv("GITHUB_EVENT_PATH")
       println("Reading essential env variables... " +
-          "Envs: PREVIOUS_EVENT_PATH: %s" .format(eventPath))
+          "Envs: GITHUB_EVENT_PATH: %s" .format(eventPath))
 
       if (StringUtils.isEmpty(eventPath)) {
-        throw new IllegalArgumentException("No PREVIOUS_EVENT_PATH set")
+        throw new IllegalArgumentException("No GITHUB_EVENT_PATH set")
       }
 
       val token = System.getenv("GITHUB_TOKEN")
@@ -62,8 +63,13 @@ class GitHubActionEntries extends FunSuite {
         throw new IllegalArgumentException("No GITHUB_TOKEN set")
       }
 
-      val ghEventPayloadJson = new ObjectMapper().readTree(FileUtils.readFileToString(new File(eventPath)))
-      val prId = ghEventPayloadJson.get("number").asInt()
+      val prUrl = System.getenv("PR_URL")
+      val pattern = Pattern.compile("^.*/(\\d+)$")
+      val matcher = pattern.matcher(prUrl)
+      if (!matcher.matches()) {
+        throw new IllegalArgumentException("Unable to find pull request number in URL: " + prUrl)
+      }
+      val prId = matcher.group(1).toInt
 
       GitHubActionEntries.commentOnContextPR(repoSlug, prId, token,
         FileUtils.readFileToString(new File(commentContentPath)))
@@ -73,10 +79,11 @@ class GitHubActionEntries extends FunSuite {
 }
 
 object GitHubActionEntries {
-  def commentOnContextPR(repoSlug: String, prId: Int, token: String, comment: String): Option[GHIssueComment] = {
+  def commentOnContextPR(repoSlug: String, prId: Int, token: String,
+                         comment: String): Option[GHIssueComment] = {
     val inst = new GitHubBuilder()
-        .withAppInstallationToken(token)
-        .build()
+      .withAppInstallationToken(token)
+      .build()
 
     val repository = inst.getRepository(repoSlug)
     val pr = repository.getPullRequest(prId)

native-sql-engine/core/src/test/scala/com/intel/oap/tpc/h/TPCHSuite.scala

@@ -49,7 +49,6 @@ class TPCHSuite extends QueryTest with SharedSparkSession {
   override protected def sparkConf: SparkConf = {
     val conf = super.sparkConf
     conf.set("spark.memory.offHeap.size", String.valueOf(MAX_DIRECT_MEMORY))
-        .set("spark.driver.bindAddress", "127.0.0.1")
         .set("spark.sql.extensions", "com.intel.oap.ColumnarPlugin")
         .set("spark.sql.codegen.wholeStage", "false")
         .set("spark.sql.sources.useV1SourceList", "")

scalastyle-config.xml

@@ -135,12 +135,6 @@ This file is divided into 3 sections:
   <!-- ??? usually shouldn't be checked into the code base. -->
   <check level="error" class="org.scalastyle.scalariform.NotImplementedErrorUsage" enabled="true"></check>
 
-  <!-- As of SPARK-7558, all tests in Spark should extend o.a.s.SparkFunSuite instead of FunSuite directly -->
-  <check customId="funsuite" level="error" class="org.scalastyle.scalariform.TokenChecker" enabled="true">
-    <parameters><parameter name="regex">^FunSuite[A-Za-z]*$</parameter></parameters>
-    <customMessage>Tests must extend org.apache.spark.SparkFunSuite instead.</customMessage>
-  </check>
-
   <!-- As of SPARK-7977 all printlns need to be wrapped in '// scalastyle:off/on println' -->
   <check customId="println" level="error" class="org.scalastyle.scalariform.TokenChecker" enabled="true">
     <parameters><parameter name="regex">^println$</parameter></parameters>