From 71a2245c188b2ba0c36c73fb5a8bda8e6b39662f Mon Sep 17 00:00:00 2001 From: Yuan Date: Tue, 26 Aug 2025 09:11:49 +0100 Subject: [PATCH] fix: Enable CI on IBM repo Signed-off-by: Yuan --- .github/workflows/gluten.yml | 212 +++++++++++++++++++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 .github/workflows/gluten.yml diff --git a/.github/workflows/gluten.yml b/.github/workflows/gluten.yml new file mode 100644 index 000000000000..136e21a61c4d --- /dev/null +++ b/.github/workflows/gluten.yml @@ -0,0 +1,212 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Gluten Build + +on: + pull_request: + paths: + - .github/workflows/gluten.yml +env: + MVN_CMD: mvn -ntp + +jobs: + + gluten-cpp-build: + name: gluten cpp build + # prevent errors when forks ff their main branch + if: ${{ github.repository == 'IBM/velox' }} + runs-on: ubuntu-22.04 + env: + CCACHE_DIR: "${{ github.workspace }}/.ccache" + steps: + - uses: actions/checkout@v4 + - name: Get Ccache + uses: actions/cache/restore@v4 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-centos7-release-default-${{github.sha}} + restore-keys: | + ccache-centos7-release-default + - name: Setup Gluten + run: | + git clone --depth 1 https://github.com/apache/incubator-gluten gluten && cd gluten + BRANCH=$(echo ${GITHUB_REF#refs/heads/}) + sed -i 's/oap-project/IBM/g' ep/build-velox/src/get_velox.sh + # sed -i 's/VELOX_BRANCH=2025.*/VELOX_BRANCH=${BRANCH}/g' ep/build-velox/src/get_velox.sh + - name: Build Gluten native libraries + run: | + docker pull apache/gluten:vcpkg-centos-7 + docker run -v $GITHUB_WORKSPACE:/work -w /work apache/gluten:vcpkg-centos-7 bash -c " + git config --global --add safe.directory /work + set -e + df -a + cd /work/gluten + export CCACHE_DIR=/work/.ccache + export CCACHE_SLOPPINESS=file_macro,locale,time_macros + mkdir -p /work/.ccache + ccache -sz + source /opt/rh/devtoolset-11/enable + source /opt/rh/rh-git227/enable + export NUM_THREADS=4 + ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF --build_tests=OFF --build_benchmarks=OFF \ + --build_examples=OFF --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON --velox_home=/work + ccache -s + mkdir -p /work/.m2/repository/org/apache/arrow/ + cp -r /root/.m2/repository/org/apache/arrow/* /work/.m2/repository/org/apache/arrow/ + " + - uses: actions/upload-artifact@v4 + with: + name: velox-native-lib-centos-7-${{github.sha}} + path: ./gluten/cpp/build/releases/ + if-no-files-found: error + - uses: actions/upload-artifact@v4 + with: + name: arrow-jars-centos-7-${{github.sha}} + path: .m2/repository/org/apache/arrow/ + if-no-files-found: error + + spark-test-spark32: + needs: gluten-cpp-build + runs-on: ubuntu-22.04 + container: apache/gluten:centos-8-jdk8 + steps: + - name: Setup Gluten + run: | + git clone --depth 1 https://github.com/apache/incubator-gluten gluten && cd gluten + - name: Download All Artifacts + uses: actions/download-artifact@v4 + with: + name: velox-native-lib-centos-7-${{github.sha}} + path: ./gluten/cpp/build/releases + - name: Download Arrow Jars + uses: actions/download-artifact@v4 + with: + name: arrow-jars-centos-7-${{github.sha}} + path: /root/.m2/repository/org/apache/arrow/ + - name: Build package for Spark 3.2 + run: | + cd $GITHUB_WORKSPACE/gluten + export SPARK_SCALA_VERSION=2.12 + yum install -y java-17-openjdk-devel + export JAVA_HOME=/usr/lib/jvm/java-17-openjdk + export PATH=$JAVA_HOME/bin:$PATH + java -version + $MVN_CMD clean package -Pspark-3.2 -Pbackends-velox -Piceberg -Pdelta -Phudi -DskipTests + + spark-test-spark34: + needs: gluten-cpp-build + runs-on: ubuntu-22.04 + container: apache/gluten:centos-8-jdk8 + steps: + - name: Setup Gluten + run: | + git clone --depth 1 https://github.com/apache/incubator-gluten gluten && cd gluten + - name: Download All Artifacts + uses: actions/download-artifact@v4 + with: + name: velox-native-lib-centos-7-${{github.sha}} + path: ./gluten/cpp/build/releases + - name: Download Arrow Jars + uses: actions/download-artifact@v4 + with: + name: arrow-jars-centos-7-${{github.sha}} + path: /root/.m2/repository/org/apache/arrow/ + - name: Prepare spark.test.home for Spark 3.4.4 (other tests) + run: | + dnf module -y install python39 && \ + alternatives --set python3 /usr/bin/python3.9 && \ + pip3 install setuptools==77.0.3 && \ + pip3 install pyspark==3.4.4 cython && \ + pip3 install pandas==2.2.3 pyarrow==20.0.0 + - name: Build and Run unit test for Spark 3.4.4 (other tests) + run: | + cd $GITHUB_WORKSPACE/gluten + export SPARK_SCALA_VERSION=2.12 + yum install -y java-17-openjdk-devel + export JAVA_HOME=/usr/lib/jvm/java-17-openjdk + export PATH=$JAVA_HOME/bin:$PATH + java -version + export SPARK_HOME=/opt/shims/spark34/spark_home/ + ls -l $SPARK_HOME + $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Pspark-ut \ + -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest \ + -DargLine="-Dspark.test.home=$SPARK_HOME ${EXTRA_FLAGS}" + - name: Upload test report + if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ github.job }}-report + path: '**/surefire-reports/TEST-*.xml' + - name: Upload unit tests log files + if: ${{ !success() }} + uses: actions/upload-artifact@v4 + with: + name: ${{ github.job }}-test-log + path: | + **/target/*.log + **/gluten-ut/**/hs_err_*.log + **/gluten-ut/**/core.* + - name: Upload golden files + if: failure() + uses: actions/upload-artifact@v4 + with: + name: ${{ github.job }}-golden-files + path: /tmp/tpch-approved-plan/** + + spark-test-spark34-slow: + needs: gluten-cpp-build + runs-on: ubuntu-22.04 + container: apache/gluten:centos-8-jdk8 + steps: + - name: Setup Gluten + run: | + git clone --depth 1 https://github.com/apache/incubator-gluten gluten && cd gluten + - name: Download All Artifacts + uses: actions/download-artifact@v4 + with: + name: velox-native-lib-centos-7-${{github.sha}} + path: ./gluten/cpp/build/releases + - name: Download Arrow Jars + uses: actions/download-artifact@v4 + with: + name: arrow-jars-centos-7-${{github.sha}} + path: /root/.m2/repository/org/apache/arrow/ + - name: Build and Run unit test for Spark 3.4.4 (slow tests) + run: | + cd $GITHUB_WORKSPACE/gluten + yum install -y java-17-openjdk-devel + export JAVA_HOME=/usr/lib/jvm/java-17-openjdk + export PATH=$JAVA_HOME/bin:$PATH + java -version + export SPARK_HOME=/opt/shims/spark34/spark_home/ + ls -l $SPARK_HOME + $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Pspark-ut -Phudi \ + -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest \ + -DargLine="-Dspark.test.home=$SPARK_HOME ${EXTRA_FLAGS}" + - name: Upload test report + if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ github.job }}-report + path: '**/surefire-reports/TEST-*.xml' + - name: Upload unit tests log files + if: ${{ !success() }} + uses: actions/upload-artifact@v4 + with: + name: ${{ github.job }}-test-log + path: | + **/target/*.log + **/gluten-ut/**/hs_err_*.log + **/gluten-ut/**/core.*