From 08bb11c5b516b39345cef49c69e61b094ef199b3 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Thu, 22 Apr 2021 19:20:47 +0800 Subject: [PATCH 01/14] [NSE-274] Comment to trigger tpc-h RAM test --- .github/workflows/tpch.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tpch.yml b/.github/workflows/tpch.yml index d0f655dbb..3bdc1e306 100644 --- a/.github/workflows/tpch.yml +++ b/.github/workflows/tpch.yml @@ -18,11 +18,12 @@ name: Native SQL Engine TPC-H Suite on: - pull_request + issue_comment: + types: [created, edited] jobs: ram-usage-test: - if: ${{ contains(github.event.pull_request.labels.*.name, 'RAM Report') }} + if: startsWith(github.event.comment.body, '@github-actions ram-usage-test') runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 From 0687120bed3031c9ba584a2eeb55e5f01b36e5cc Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Thu, 22 Apr 2021 19:53:25 +0800 Subject: [PATCH 02/14] empty From b0524bafb1e96716bc74b5a3709efd077cb0c377 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Thu, 22 Apr 2021 20:40:05 +0800 Subject: [PATCH 03/14] fix --- .github/workflows/report_ram_log.yml | 65 ------------------- .github/workflows/tpch.yml | 31 +++++---- .../intel/oap/tpc/h/GitHubActionEntries.scala | 9 ++- 3 files changed, 23 insertions(+), 82 deletions(-) delete mode 100644 .github/workflows/report_ram_log.yml diff --git a/.github/workflows/report_ram_log.yml b/.github/workflows/report_ram_log.yml deleted file mode 100644 index 3067e2e73..000000000 --- a/.github/workflows/report_ram_log.yml +++ /dev/null @@ -1,65 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Report RAM Log - -on: - workflow_run: - workflows: ["Native SQL Engine TPC-H Suite"] - types: - - completed - -jobs: - comment-on-pr: - if: ${{ github.event.workflow_run.conclusion == 'success' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - - name: Download log - uses: dawidd6/action-download-artifact@v2 - with: - workflow: tpch.yml - run_id: ${{ github.event.workflow_run.id }} - name: comment_content - path: /tmp/ - - name: Download previous event payload - uses: dawidd6/action-download-artifact@v2 - with: - workflow: tpch.yml - run_id: ${{ github.event.workflow_run.id }} - name: pr_event - path: /tmp/ - - name: Install OAP optimized Arrow - run: | - cd /tmp - git clone -b arrow-3.0.0-oap https://github.com/oap-project/arrow.git - cd arrow/java - mvn clean install -B -P arrow-jni -am -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -Darrow.cpp.build.dir=/tmp/arrow/cpp/build/release/ -DskipTests -Dcheckstyle.skip - - name: Run Maven tests - run: | - mvn test -B -pl native-sql-engine/core/ -am -DmembersOnlySuites=com.intel.oap.tpc.h -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DtagsToInclude=com.intel.oap.tags.CommentOnContextPR -Dexec.skip=true - env: - MAVEN_OPTS: "-Xmx2048m" - COMMENT_CONTENT_PATH: "/tmp/comment.md" - PREVIOUS_EVENT_PATH: "/tmp/event.json" - GITHUB_TOKEN: ${{ github.token }} - ENABLE_TPCH_TESTS: "true" - diff --git a/.github/workflows/tpch.yml b/.github/workflows/tpch.yml index 3bdc1e306..8c88bb1aa 100644 --- a/.github/workflows/tpch.yml +++ b/.github/workflows/tpch.yml @@ -23,10 +23,18 @@ on: jobs: ram-usage-test: - if: startsWith(github.event.comment.body, '@github-actions ram-usage-test') + if: ${{ github.event.issue.pull_request && startsWith(github.event.comment.body, '@github-actions ram-usage-test') }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - name: Checkout Pull Request + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_URL: "${{ github.event.issue.pull_request.url }}" + PR_NUM: ${PR_URL##*/} + run: | + echo "Checking out from PR #$PR_NUM based on URL: $PR_URL" + hub pr checkout $PR_NUM - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: @@ -70,14 +78,13 @@ jobs: - run: | cml-publish /tmp/comment_image_1.png --md > /tmp/comment.md cml-publish /tmp/comment_image_2.png --md >> /tmp/comment.md - - run: echo "::set-output name=event_path::${GITHUB_EVENT_PATH}" - id: output-envs - - uses: actions/upload-artifact@v2 - with: - name: comment_content - path: /tmp/comment.md - - uses: actions/upload-artifact@v2 - with: - name: pr_event - path: ${{steps.output-envs.outputs.event_path}} - + - name: Run Maven tests - Report + run: | + mvn test -B -pl native-sql-engine/core/ -am -DmembersOnlySuites=com.intel.oap.tpc.h -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DtagsToInclude=com.intel.oap.tags.CommentOnContextPR -Dexec.skip=true + env: + PR_URL: "${{ github.event.issue.pull_request.url }}" + PR_NUM: ${PR_URL##*/} + MAVEN_OPTS: "-Xmx1G" + COMMENT_CONTENT_PATH: "/tmp/comment.md" + GITHUB_TOKEN: ${{ github.token }} + ENABLE_TPCH_TESTS: "true" diff --git a/native-sql-engine/core/src/test/scala/com/intel/oap/tpc/h/GitHubActionEntries.scala b/native-sql-engine/core/src/test/scala/com/intel/oap/tpc/h/GitHubActionEntries.scala index a89931393..1982116d8 100644 --- a/native-sql-engine/core/src/test/scala/com/intel/oap/tpc/h/GitHubActionEntries.scala +++ b/native-sql-engine/core/src/test/scala/com/intel/oap/tpc/h/GitHubActionEntries.scala @@ -48,12 +48,12 @@ class GitHubActionEntries extends FunSuite { throw new IllegalArgumentException("No GITHUB_REPOSITORY set") } - val eventPath = System.getenv("PREVIOUS_EVENT_PATH") + val eventPath = System.getenv("GITHUB_EVENT_PATH") println("Reading essential env variables... " + - "Envs: PREVIOUS_EVENT_PATH: %s" .format(eventPath)) + "Envs: GITHUB_EVENT_PATH: %s" .format(eventPath)) if (StringUtils.isEmpty(eventPath)) { - throw new IllegalArgumentException("No PREVIOUS_EVENT_PATH set") + throw new IllegalArgumentException("No GITHUB_EVENT_PATH set") } val token = System.getenv("GITHUB_TOKEN") @@ -62,8 +62,7 @@ class GitHubActionEntries extends FunSuite { throw new IllegalArgumentException("No GITHUB_TOKEN set") } - val ghEventPayloadJson = new ObjectMapper().readTree(FileUtils.readFileToString(new File(eventPath))) - val prId = ghEventPayloadJson.get("number").asInt() + val prId = System.getenv("PR_NUM").toInt GitHubActionEntries.commentOnContextPR(repoSlug, prId, token, FileUtils.readFileToString(new File(commentContentPath))) From d67a2a036d20768a9d917f21b37b434fef333006 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Thu, 22 Apr 2021 21:00:43 +0800 Subject: [PATCH 04/14] empty From fc554365847fb1d9fbea7c3cf6b5106c8e0102e1 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Thu, 22 Apr 2021 21:16:43 +0800 Subject: [PATCH 05/14] fix --- .github/workflows/tpch.yml | 7 +++---- .../scala/com/intel/oap/tpc/h/GitHubActionEntries.scala | 9 ++++++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tpch.yml b/.github/workflows/tpch.yml index 8c88bb1aa..2a7409dc1 100644 --- a/.github/workflows/tpch.yml +++ b/.github/workflows/tpch.yml @@ -30,9 +30,9 @@ jobs: - name: Checkout Pull Request env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PR_URL: "${{ github.event.issue.pull_request.url }}" - PR_NUM: ${PR_URL##*/} run: | + PR_URL="${{ github.event.issue.pull_request.url }}" + PR_NUM=${PR_URL##*/} echo "Checking out from PR #$PR_NUM based on URL: $PR_URL" hub pr checkout $PR_NUM - name: Set up JDK 1.8 @@ -82,8 +82,7 @@ jobs: run: | mvn test -B -pl native-sql-engine/core/ -am -DmembersOnlySuites=com.intel.oap.tpc.h -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DtagsToInclude=com.intel.oap.tags.CommentOnContextPR -Dexec.skip=true env: - PR_URL: "${{ github.event.issue.pull_request.url }}" - PR_NUM: ${PR_URL##*/} + PR_URL: ${{ github.event.issue.pull_request.url }} MAVEN_OPTS: "-Xmx1G" COMMENT_CONTENT_PATH: "/tmp/comment.md" GITHUB_TOKEN: ${{ github.token }} diff --git a/native-sql-engine/core/src/test/scala/com/intel/oap/tpc/h/GitHubActionEntries.scala b/native-sql-engine/core/src/test/scala/com/intel/oap/tpc/h/GitHubActionEntries.scala index 1982116d8..2028dc915 100644 --- a/native-sql-engine/core/src/test/scala/com/intel/oap/tpc/h/GitHubActionEntries.scala +++ b/native-sql-engine/core/src/test/scala/com/intel/oap/tpc/h/GitHubActionEntries.scala @@ -17,6 +17,7 @@ package com.intel.oap.tpc.h import java.io.File +import java.util.regex.Pattern import com.intel.oap.tags.CommentOnContextPR import org.apache.commons.io.FileUtils @@ -62,7 +63,13 @@ class GitHubActionEntries extends FunSuite { throw new IllegalArgumentException("No GITHUB_TOKEN set") } - val prId = System.getenv("PR_NUM").toInt + val prUrl = System.getenv("PR_URL") + val pattern = new Pattern("^.*/(\d+)$") + val matcher = pattern.matcher(prUrl) + if (!matcher.matches()) { + throw new IllegalArgumentException("Unable to find pull request number in URL: " + prUrl) + } + val prId = matcher.group(1).toInt GitHubActionEntries.commentOnContextPR(repoSlug, prId, token, FileUtils.readFileToString(new File(commentContentPath))) From 01e007806de98c4e6af574121cc6ffcfe9dc4d31 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Thu, 22 Apr 2021 21:16:55 +0800 Subject: [PATCH 06/14] empty From e8f89545ced15f96afa03d1841dbb0b4caa7bbcd Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Thu, 22 Apr 2021 22:52:31 +0800 Subject: [PATCH 07/14] fix --- .github/workflows/tpch.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tpch.yml b/.github/workflows/tpch.yml index 2a7409dc1..ae159eb8b 100644 --- a/.github/workflows/tpch.yml +++ b/.github/workflows/tpch.yml @@ -59,7 +59,7 @@ jobs: mvn clean install -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -P arrow-jni -am -Darrow.cpp.build.dir=/tmp/arrow/cpp/build/release/ -DskipTests -Dcheckstyle.skip - name: Run Maven tests - BHJ run: | - mvn test -B -pl native-sql-engine/core/ -am -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DmembersOnlySuites=com.intel.oap.tpc.h -DtagsToInclude=com.intel.oap.tags.BroadcastHashJoinMode -DargLine="-Xmx1G -XX:MaxDirectMemorySize=500M -Dio.netty.allocator.numDirectArena=1" + mvn test -B -Dbuild_arrow=OFF -pl native-sql-engine/core/ -am -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DmembersOnlySuites=com.intel.oap.tpc.h -DtagsToInclude=com.intel.oap.tags.BroadcastHashJoinMode -DargLine="-Xmx1G -XX:MaxDirectMemorySize=500M -Dio.netty.allocator.numDirectArena=1" env: MALLOC_ARENA_MAX: "4" MAVEN_OPTS: "-Xmx1G" @@ -68,7 +68,7 @@ jobs: ENABLE_TPCH_TESTS: "true" - name: Run Maven tests - SMJ run: | - mvn test -B -pl native-sql-engine/core/ -am -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DmembersOnlySuites=com.intel.oap.tpc.h -DtagsToInclude=com.intel.oap.tags.SortMergeJoinMode -DargLine="-Xmx1G -XX:MaxDirectMemorySize=500M -Dio.netty.allocator.numDirectArena=1" + mvn test -B -Dbuild_arrow=OFF -pl native-sql-engine/core/ -am -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DmembersOnlySuites=com.intel.oap.tpc.h -DtagsToInclude=com.intel.oap.tags.SortMergeJoinMode -DargLine="-Xmx1G -XX:MaxDirectMemorySize=500M -Dio.netty.allocator.numDirectArena=1" env: MALLOC_ARENA_MAX: "4" MAVEN_OPTS: "-Xmx1G" @@ -80,7 +80,7 @@ jobs: cml-publish /tmp/comment_image_2.png --md >> /tmp/comment.md - name: Run Maven tests - Report run: | - mvn test -B -pl native-sql-engine/core/ -am -DmembersOnlySuites=com.intel.oap.tpc.h -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DtagsToInclude=com.intel.oap.tags.CommentOnContextPR -Dexec.skip=true + mvn test -B -Dbuild_arrow=OFF -Dbuild_protobuf=OFF -pl native-sql-engine/core/ -am -DmembersOnlySuites=com.intel.oap.tpc.h -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DtagsToInclude=com.intel.oap.tags.CommentOnContextPR -Dexec.skip=true env: PR_URL: ${{ github.event.issue.pull_request.url }} MAVEN_OPTS: "-Xmx1G" From ca43f3724598f65904bff822a96e42103f7f518a Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Thu, 22 Apr 2021 22:52:44 +0800 Subject: [PATCH 08/14] empty From b869f153cb81a8ebbd298022fca8376c5aee2dfe Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Fri, 23 Apr 2021 09:11:10 +0800 Subject: [PATCH 09/14] fx --- .../scala/com/intel/oap/tpc/h/GitHubActionEntries.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/native-sql-engine/core/src/test/scala/com/intel/oap/tpc/h/GitHubActionEntries.scala b/native-sql-engine/core/src/test/scala/com/intel/oap/tpc/h/GitHubActionEntries.scala index 2028dc915..26b31d2ee 100644 --- a/native-sql-engine/core/src/test/scala/com/intel/oap/tpc/h/GitHubActionEntries.scala +++ b/native-sql-engine/core/src/test/scala/com/intel/oap/tpc/h/GitHubActionEntries.scala @@ -22,8 +22,8 @@ import java.util.regex.Pattern import com.intel.oap.tags.CommentOnContextPR import org.apache.commons.io.FileUtils import org.apache.commons.lang.StringUtils -import org.codehaus.jackson.map.ObjectMapper -import org.kohsuke.github.{GHIssueComment, GitHubBuilder} +import org.kohsuke.github.GHIssueComment +import org.kohsuke.github.GitHubBuilder import org.scalatest.FunSuite class GitHubActionEntries extends FunSuite { @@ -64,7 +64,7 @@ class GitHubActionEntries extends FunSuite { } val prUrl = System.getenv("PR_URL") - val pattern = new Pattern("^.*/(\d+)$") + val pattern = new Pattern("^.*/(\\d+)$") val matcher = pattern.matcher(prUrl) if (!matcher.matches()) { throw new IllegalArgumentException("Unable to find pull request number in URL: " + prUrl) From d35d9fdcab62841831e24f8d1dae4c32daf5402f Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Fri, 23 Apr 2021 09:11:22 +0800 Subject: [PATCH 10/14] empty From f47a0c698f326711c1f6f80f26816fdf1999e4cb Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Fri, 23 Apr 2021 10:27:54 +0800 Subject: [PATCH 11/14] fix --- .../scala/com/intel/oap/tpc/h/GitHubActionEntries.scala | 9 +++++---- scalastyle-config.xml | 6 ------ 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/native-sql-engine/core/src/test/scala/com/intel/oap/tpc/h/GitHubActionEntries.scala b/native-sql-engine/core/src/test/scala/com/intel/oap/tpc/h/GitHubActionEntries.scala index 26b31d2ee..5dad63cdf 100644 --- a/native-sql-engine/core/src/test/scala/com/intel/oap/tpc/h/GitHubActionEntries.scala +++ b/native-sql-engine/core/src/test/scala/com/intel/oap/tpc/h/GitHubActionEntries.scala @@ -64,7 +64,7 @@ class GitHubActionEntries extends FunSuite { } val prUrl = System.getenv("PR_URL") - val pattern = new Pattern("^.*/(\\d+)$") + val pattern = Pattern.compile("^.*/(\\d+)$") val matcher = pattern.matcher(prUrl) if (!matcher.matches()) { throw new IllegalArgumentException("Unable to find pull request number in URL: " + prUrl) @@ -79,10 +79,11 @@ class GitHubActionEntries extends FunSuite { } object GitHubActionEntries { - def commentOnContextPR(repoSlug: String, prId: Int, token: String, comment: String): Option[GHIssueComment] = { + def commentOnContextPR(repoSlug: String, prId: Int, token: String, + comment: String): Option[GHIssueComment] = { val inst = new GitHubBuilder() - .withAppInstallationToken(token) - .build() + .withAppInstallationToken(token) + .build() val repository = inst.getRepository(repoSlug) val pr = repository.getPullRequest(prId) diff --git a/scalastyle-config.xml b/scalastyle-config.xml index 4892819ae..1f2280dcf 100644 --- a/scalastyle-config.xml +++ b/scalastyle-config.xml @@ -135,12 +135,6 @@ This file is divided into 3 sections: - - - ^FunSuite[A-Za-z]*$ - Tests must extend org.apache.spark.SparkFunSuite instead. - - ^println$ From 2d7c8903b8310688678a2a155d54a1a152201779 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Fri, 23 Apr 2021 10:27:56 +0800 Subject: [PATCH 12/14] empty From 92c3bbb7846a204acc7ba8754812f3c091227313 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Fri, 23 Apr 2021 11:35:33 +0800 Subject: [PATCH 13/14] fix --- .../core/src/test/scala/com/intel/oap/tpc/ds/TPCDSSuite.scala | 1 - .../core/src/test/scala/com/intel/oap/tpc/h/TPCHSuite.scala | 1 - 2 files changed, 2 deletions(-) diff --git a/native-sql-engine/core/src/test/scala/com/intel/oap/tpc/ds/TPCDSSuite.scala b/native-sql-engine/core/src/test/scala/com/intel/oap/tpc/ds/TPCDSSuite.scala index 88c25a16f..e4ef15cfd 100644 --- a/native-sql-engine/core/src/test/scala/com/intel/oap/tpc/ds/TPCDSSuite.scala +++ b/native-sql-engine/core/src/test/scala/com/intel/oap/tpc/ds/TPCDSSuite.scala @@ -34,7 +34,6 @@ class TPCDSSuite extends QueryTest with SharedSparkSession { override protected def sparkConf: SparkConf = { val conf = super.sparkConf conf.set("spark.memory.offHeap.size", String.valueOf(MAX_DIRECT_MEMORY)) - .set("spark.driver.bindAddress", "127.0.0.1") .set("spark.sql.extensions", "com.intel.oap.ColumnarPlugin") .set("spark.sql.codegen.wholeStage", "true") .set("spark.sql.sources.useV1SourceList", "") diff --git a/native-sql-engine/core/src/test/scala/com/intel/oap/tpc/h/TPCHSuite.scala b/native-sql-engine/core/src/test/scala/com/intel/oap/tpc/h/TPCHSuite.scala index bc634ed57..a8fc06e86 100644 --- a/native-sql-engine/core/src/test/scala/com/intel/oap/tpc/h/TPCHSuite.scala +++ b/native-sql-engine/core/src/test/scala/com/intel/oap/tpc/h/TPCHSuite.scala @@ -49,7 +49,6 @@ class TPCHSuite extends QueryTest with SharedSparkSession { override protected def sparkConf: SparkConf = { val conf = super.sparkConf conf.set("spark.memory.offHeap.size", String.valueOf(MAX_DIRECT_MEMORY)) - .set("spark.driver.bindAddress", "127.0.0.1") .set("spark.sql.extensions", "com.intel.oap.ColumnarPlugin") .set("spark.sql.codegen.wholeStage", "false") .set("spark.sql.sources.useV1SourceList", "") From 395da8c6bc9c6bc8269099c650c9ece3eeed2a16 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Fri, 23 Apr 2021 11:35:37 +0800 Subject: [PATCH 14/14] empty