[GLUTEN-8709][VL] Support build on openEuler 24.03 LTS with Velox backend

dev/build_helper_functions.sh

@@ -173,6 +173,17 @@ function setup_linux {
       exit 1
       ;;
     esac
+  elif [[ "$LINUX_DISTRIBUTION" == "openEuler" ]]; then
+    case "$LINUX_VERSION_ID" in
+      24.03)
+        $GLUTEN_VELOX_SCRIPT_HOME/setup-openeuler24.sh
+        export re2_SOURCE=BUNDLED
+        ;;
+      *)
+        echo "Unsupported openEuler version: $LINUX_VERSION_ID"
+        exit 1
+        ;;
+    esac
   elif [[ "$LINUX_DISTRIBUTION" == "alinux" ]]; then
     case "${LINUX_VERSION_ID:0:1}" in
     2)

ep/build-velox/src/build_velox.sh

@@ -98,7 +98,7 @@ function compile {
   CXX_FLAGS='-Wno-missing-field-initializers'
   COMPILE_OPTION="-DCMAKE_CXX_FLAGS=\"$CXX_FLAGS\" -DVELOX_ENABLE_PARQUET=ON -DVELOX_BUILD_TESTING=OFF -DVELOX_MONO_LIBRARY=ON -DVELOX_BUILD_RUNNER=OFF -DVELOX_SIMDJSON_SKIPUTF8VALIDATION=ON"
   if [ $BUILD_TEST_UTILS == "ON" ]; then
-      COMPILE_OPTION="$COMPILE_OPTION -DVELOX_BUILD_TEST_UTILS=ON"
+    COMPILE_OPTION="$COMPILE_OPTION -DVELOX_BUILD_TEST_UTILS=ON"
   fi
   if [ $ENABLE_HDFS == "ON" ]; then
     COMPILE_OPTION="$COMPILE_OPTION -DVELOX_ENABLE_HDFS=ON"
@@ -109,7 +109,7 @@ function compile {
   # If ENABLE_BENCHMARK == ON, Velox disables tests and connectors
   if [ $ENABLE_BENCHMARK == "OFF" ]; then
     if [ $ENABLE_TESTS == "ON" ]; then
-        COMPILE_OPTION="$COMPILE_OPTION -DVELOX_BUILD_TESTING=ON "
+      COMPILE_OPTION="$COMPILE_OPTION -DVELOX_BUILD_TESTING=ON "
     fi
     if [ $ENABLE_ABFS == "ON" ]; then
       COMPILE_OPTION="$COMPILE_OPTION -DVELOX_ENABLE_ABFS=ON"

ep/build-velox/src/get_velox.sh

@@ -195,6 +195,14 @@ function setup_linux {
         exit 1
       ;;
     esac
+  elif [[ "$LINUX_DISTRIBUTION" == "openEuler" ]]; then
+    case "$LINUX_VERSION_ID" in
+      24.03) ;;
+      *)
+        echo "Unsupported openEuler version: $LINUX_VERSION_ID"
+        exit 1
+      ;;
+    esac
   elif [[ "$LINUX_DISTRIBUTION" == "alinux" ]]; then
     case "${LINUX_VERSION_ID:0:1}" in
       2) ;;

ep/build-velox/src/setup-openeuler24.sh

@@ -0,0 +1,199 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script documents setting up a openEuler24 host for Velox
+# development.  Running it should make you ready to compile.
+#
+# Environment variables:
+# * INSTALL_PREREQUISITES="N": Skip installation of packages for build.
+# * PROMPT_ALWAYS_RESPOND="n": Automatically respond to interactive prompts.
+#     Use "n" to never wipe directories.
+#
+# You can also run individual functions below by specifying them as arguments:
+# $ ./setup-openeuler24.sh install_googletest install_fmt
+#
+
+set -efx -o pipefail
+# Some of the packages must be build with the same compiler flags
+# so that some low level types are the same size. Also, disable warnings.
+SCRIPTDIR=./scripts
+source $SCRIPTDIR/setup-helper-functions.sh
+CPU_TARGET="${CPU_TARGET:-avx}"
+NPROC=$(getconf _NPROCESSORS_ONLN)
+export CFLAGS=$(get_cxx_flags $CPU_TARGET)  # Used by LZO.
+export CXXFLAGS=$CFLAGS  # Used by boost.
+export CPPFLAGS=$CFLAGS  # Used by LZO.
+CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}"
+VELOX_BUILD_SHARED=${VELOX_BUILD_SHARED:-"OFF"} #Build folly and gflags shared for use in libvelox.so.
+BUILD_DUCKDB="${BUILD_DUCKDB:-true}"
+VERSION=$(cat /etc/os-release | grep VERSION_ID)
+export INSTALL_PREFIX=${INSTALL_PREFIX:-"/usr/local"}
+DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)/deps-download}
+
+FB_OS_VERSION="v2024.07.01.00"
+FMT_VERSION="10.1.1"
+BOOST_VERSION="boost-1.84.0"
+DUCKDB_VERSION="v0.8.1"
+
+function dnf_install {
+  dnf install -y -q --setopt=install_weak_deps=False "$@"
+}
+
+# Install packages required for build.
+function install_build_prerequisites {
+  dnf update -y
+  dnf_install dnf-plugins-core # For ccache, ninja
+  dnf update -y
+  dnf_install ninja-build cmake ccache gcc g++ git wget which patch
+  dnf_install autoconf automake python3-devel python3-pip libtool
+
+  pip install cmake==3.28.3
+}
+
+# Install dependencies from the package managers.
+function install_velox_deps_from_dnf {
+  dnf_install libevent-devel \
+    openssl-devel re2-devel libzstd-devel lz4-devel double-conversion-devel \
+    libdwarf-devel elfutils-libelf-devel curl-devel libicu-devel bison flex \
+    libsodium-devel zlib-devel
+
+  # install sphinx for doc gen
+  pip install sphinx sphinx-tabs breathe sphinx_rtd_theme
+}
+
+function install_gflags {
+  # Remove an older version if present.
+  dnf remove -y gflags
+  wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags
+  cmake_install_dir gflags -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64
+}
+
+function install_glog {
+  wget_and_untar https://github.com/google/glog/archive/v0.6.0.tar.gz glog
+  cmake_install_dir glog -DBUILD_SHARED_LIBS=ON
+}
+
+function install_lzo {
+  wget_and_untar http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz lzo
+  (
+    cd ${DEPENDENCY_DIR}/lzo
+    ./configure --prefix=${INSTALL_PREFIX} --enable-shared --disable-static --docdir=/usr/share/doc/lzo-2.10
+    make "-j${NPROC}"
+    make install
+  )
+}
+
+function install_boost {
+  wget_and_untar https://github.com/boostorg/boost/releases/download/${BOOST_VERSION}/${BOOST_VERSION}.tar.gz boost
+  (
+    cd ${DEPENDENCY_DIR}/boost
+    ./bootstrap.sh --prefix=${INSTALL_PREFIX}
+    ${SUDO} ./b2 "-j${NPROC}" -d0 install threading=multi --without-python
+  )
+}
+
+function install_snappy {
+  wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy
+  cmake_install_dir snappy -DSNAPPY_BUILD_TESTS=OFF
+}
+
+function install_fmt {
+  wget_and_untar https://github.com/fmtlib/fmt/archive/${FMT_VERSION}.tar.gz fmt
+  cmake_install_dir fmt -DFMT_TEST=OFF
+}
+
+function install_protobuf {
+  wget_and_untar https://github.com/protocolbuffers/protobuf/releases/download/v21.8/protobuf-all-21.8.tar.gz protobuf
+  (
+    cd ${DEPENDENCY_DIR}/protobuf
+    ./configure CXXFLAGS="-fPIC" --prefix=${INSTALL_PREFIX}
+    make "-j${NPROC}"
+    make install
+    ldconfig
+  )
+}
+
+function install_fizz {
+  wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz
+  cmake_install_dir fizz/fizz -DBUILD_TESTS=OFF
+}
+
+function install_folly {
+  wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly
+  cmake_install_dir folly -DBUILD_SHARED_LIBS="$VELOX_BUILD_SHARED" -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON
+}
+
+function install_wangle {
+  wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle
+  cmake_install_dir wangle/wangle -DBUILD_TESTS=OFF
+}
+
+function install_fbthrift {
+  wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift
+  cmake_install_dir fbthrift -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF
+}
+
+function install_mvfst {
+  wget_and_untar https://github.com/facebook/mvfst/archive/refs/tags/${FB_OS_VERSION}.tar.gz mvfst
+  cmake_install_dir mvfst -DBUILD_TESTS=OFF
+}
+
+function install_duckdb {
+  if $BUILD_DUCKDB ; then
+    echo 'Building DuckDB'
+    wget_and_untar https://github.com/duckdb/duckdb/archive/refs/tags/${DUCKDB_VERSION}.tar.gz duckdb
+    cmake_install_dir duckdb -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=Release
+  fi
+}
+
+function install_velox_deps {
+  run_and_time install_velox_deps_from_dnf
+  run_and_time install_gflags
+  run_and_time install_glog
+  run_and_time install_lzo
+  run_and_time install_snappy
+  run_and_time install_boost
+  run_and_time install_protobuf
+  run_and_time install_fmt
+  run_and_time install_folly
+  run_and_time install_fizz
+  run_and_time install_wangle
+  run_and_time install_mvfst
+  run_and_time install_fbthrift
+  run_and_time install_duckdb
+}
+
+(return 2> /dev/null) && return # If script was sourced, don't run commands.
+
+(
+  if [[ $# -ne 0 ]]; then
+    for cmd in "$@"; do
+      run_and_time "${cmd}"
+    done
+    echo "All specified dependencies installed!"
+  else
+    if [ "${INSTALL_PREREQUISITES:-Y}" == "Y" ]; then
+      echo "Installing build dependencies"
+      run_and_time install_build_prerequisites
+    else
+      echo "Skipping installation of build dependencies since INSTALL_PREREQUISITES is not set"
+    fi
+
+    install_velox_deps
+    echo "All dependencies for Velox installed!"
+    dnf clean all
+  fi
+)