From 4eeb03b783f413e4964e7f4f32498ee5dc1ada7a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Jul 2025 08:42:31 +0000 Subject: [PATCH 1/7] MINOR: Bump org.apache:apache from 33 to 35 Bumps [org.apache:apache](https://github.com/apache/maven-apache-parent) from 33 to 35. - [Release notes](https://github.com/apache/maven-apache-parent/releases) - [Commits](https://github.com/apache/maven-apache-parent/commits) --- updated-dependencies: - dependency-name: org.apache:apache dependency-version: '35' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- bom/pom.xml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bom/pom.xml b/bom/pom.xml index 61b452b9c..93ce38f65 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache apache - 33 + 35 diff --git a/pom.xml b/pom.xml index 49e0c47c6..1d05c2a99 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache apache - 34 + 35 org.apache.arrow From 061442fc780c04638ade80df6fd939724af4b75b Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 29 Jul 2025 15:03:37 +0900 Subject: [PATCH 2/7] Use bundled xsimd Because vcpkg's xsimd is old. --- ci/scripts/jni_manylinux_build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/scripts/jni_manylinux_build.sh b/ci/scripts/jni_manylinux_build.sh index a34ec0f42..2332d7177 100755 --- a/ci/scripts/jni_manylinux_build.sh +++ b/ci/scripts/jni_manylinux_build.sh @@ -117,6 +117,7 @@ cmake \ -DPARQUET_REQUIRE_ENCRYPTION=OFF \ -DVCPKG_MANIFEST_MODE=OFF \ -DVCPKG_TARGET_TRIPLET="${VCPKG_TARGET_TRIPLET}" \ + -Dxsimd_SOURCE=BUNDLED \ -GNinja cmake --build "${build_dir}/cpp" cmake --install "${build_dir}/cpp" From d52e397c0ea834581e7709c356fbf0c7f1a85197 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 29 Jul 2025 15:22:26 +0900 Subject: [PATCH 3/7] Use manylinux-2-28 --- compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose.yaml b/compose.yaml index b125c3c98..7b94b119b 100644 --- a/compose.yaml +++ b/compose.yaml @@ -99,7 +99,7 @@ services: cache_from: - ${REPO}:${ARCH}-vcpkg-jni-${VCPKG} args: - base: ${ARROW_REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2014-vcpkg-${VCPKG} + base: ${ARROW_REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2-28-vcpkg-${VCPKG} volumes: - .:/arrow-java:delegated - ${ARROW_REPO_ROOT}:/arrow:delegated From 0c12487e42a11c53ec2759317ad337f220d46b02 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 29 Jul 2025 16:15:42 +0900 Subject: [PATCH 4/7] Ensure using the latest image --- .github/workflows/rc.yml | 5 +++++ ci/docker/vcpkg-jni.dockerfile | 21 +++++++++++++++++---- compose.yaml | 12 ++++++++++++ 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 72456fa55..0cb887efc 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -138,6 +138,11 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Build Docker image + env: + VCPKG_BINARY_SOURCES: "clear;nuget,GitHub,readwrite" + run: | + docker compose build --pull vcpkg-jni - name: Cache uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: diff --git a/ci/docker/vcpkg-jni.dockerfile b/ci/docker/vcpkg-jni.dockerfile index 55fa35e0d..895bf92c1 100644 --- a/ci/docker/vcpkg-jni.dockerfile +++ b/ci/docker/vcpkg-jni.dockerfile @@ -18,9 +18,21 @@ ARG base FROM ${base} -# Install the libraries required by Gandiva to run -# Use enable llvm[enable-rtti] in the vcpkg.json to avoid link problems in Gandiva -RUN vcpkg install \ +# For --mount=type=secret: The GITHUB_TOKEN is the only real secret +# but we use --mount=type=secret for GITHUB_REPOSITORY_OWNER and +# VCPKG_BINARY_SOURCES too because we don't want to store them into +# the built image in order to easily reuse the built image cache. +# +# Install the libraries required by Gandiva to run. Use enable +# llvm[enable-rtti] in the vcpkg.json to avoid link problems in +# Gandiva. +RUN --mount=type=secret,id=github_repository_owner \ + --mount=type=secret,id=github_token \ + --mount=type=secret,id=vcpkg_binary_sources \ + export GITHUB_REPOSITORY_OWNER=$(cat /run/secrets/github_repository_owner); \ + export GITHUB_TOKEN=$(cat /run/secrets/github_token); \ + export VCPKG_BINARY_SOURCES=$(cat /run/secrets/vcpkg_binary_sources); \ + vcpkg install \ --clean-after-build \ --x-install-root=${VCPKG_ROOT}/installed \ --x-manifest-root=/arrow/ci/vcpkg \ @@ -30,7 +42,8 @@ RUN vcpkg install \ --x-feature=json \ --x-feature=parquet \ --x-feature=gandiva \ - --x-feature=s3 + --x-feature=s3 && \ + rm -rf ~/.config/NuGet/ # Install Java # We need Java for JNI headers, but we don't invoke Maven in this build. diff --git a/compose.yaml b/compose.yaml index 7b94b119b..424177878 100644 --- a/compose.yaml +++ b/compose.yaml @@ -100,6 +100,10 @@ services: - ${REPO}:${ARCH}-vcpkg-jni-${VCPKG} args: base: ${ARROW_REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2-28-vcpkg-${VCPKG} + secrets: + - github_repository_owner + - github_token + - vcpkg_binary_sources volumes: - .:/arrow-java:delegated - ${ARROW_REPO_ROOT}:/arrow:delegated @@ -111,3 +115,11 @@ services: command: ["git config --global --add safe.directory /arrow-java && \ /arrow-java/ci/scripts/jni_manylinux_build.sh /arrow-java /arrow /build/java /arrow-java/jni"] + +secrets: + github_repository_owner: + environment: GITHUB_REPOSITORY_OWNER + github_token: + environment: GITHUB_TOKEN + vcpkg_binary_sources: + environment: VCPKG_BINARY_SOURCES From 6543b1631e5273eaac71c366fe3f3de5c5664dff Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 12 Aug 2025 06:12:08 +0900 Subject: [PATCH 5/7] Update for new manylinux image --- ci/scripts/jni_manylinux_build.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/ci/scripts/jni_manylinux_build.sh b/ci/scripts/jni_manylinux_build.sh index 2332d7177..b8ab4ccd2 100755 --- a/ci/scripts/jni_manylinux_build.sh +++ b/ci/scripts/jni_manylinux_build.sh @@ -53,8 +53,16 @@ if [ "${ARROW_USE_CCACHE}" == "ON" ]; then fi github_actions_group_begin "Building Arrow C++ libraries" -devtoolset_version="$(rpm -qa "devtoolset-*-gcc" --queryformat '%{VERSION}' | grep -o "^[0-9]*")" -devtoolset_include_cpp="/opt/rh/devtoolset-${devtoolset_version}/root/usr/include/c++/${devtoolset_version}" +case "$(arch)" in + x86_64) + vcpkg_arch=x64 + ;; + aarch64) + vcpkg_arch=arm64 + ;; +esac +gcc_toolset_version="$(rpm -qa "gcc-toolset-*-runtime" --queryformat '%{VERSION}' | grep -o "^[0-9]*")" +gcc_toolset_include_cpp="/opt/rh/gcc-toolset-${gcc_toolset_version}/root/usr/include/c++/${gcc_toolset_version}" : "${ARROW_ACERO:=ON}" export ARROW_ACERO : "${ARROW_BUILD_TESTS:=OFF}" @@ -75,8 +83,8 @@ export ARROW_ORC : "${CMAKE_UNITY_BUILD:=ON}" : "${VCPKG_ROOT:=/opt/vcpkg}" : "${VCPKG_FEATURE_FLAGS:=-manifests}" -: "${VCPKG_TARGET_TRIPLET:=${VCPKG_DEFAULT_TRIPLET:-x64-linux-static-${CMAKE_BUILD_TYPE}}}" -: "${GANDIVA_CXX_FLAGS:=-isystem;${devtoolset_include_cpp};-isystem;${devtoolset_include_cpp}/x86_64-redhat-linux;-lpthread}" +: "${VCPKG_TARGET_TRIPLET:=${VCPKG_DEFAULT_TRIPLET:-${vcpkg_arch}-linux-static-${CMAKE_BUILD_TYPE}}}" +: "${GANDIVA_CXX_FLAGS:=-isystem;${gcc_toolset_include_cpp};-isystem;${gcc_toolset_include_cpp}/$(arch)-redhat-linux;-lpthread}" export ARROW_TEST_DATA="${arrow_dir}/testing/data" export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data" From e9b245d359c2eb917680b1cfb481eae2da9955a2 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 12 Aug 2025 06:56:59 +0900 Subject: [PATCH 6/7] Use amd64 for vcpkg --- ci/scripts/jni_manylinux_build.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/scripts/jni_manylinux_build.sh b/ci/scripts/jni_manylinux_build.sh index b8ab4ccd2..2c902380c 100755 --- a/ci/scripts/jni_manylinux_build.sh +++ b/ci/scripts/jni_manylinux_build.sh @@ -54,12 +54,12 @@ fi github_actions_group_begin "Building Arrow C++ libraries" case "$(arch)" in - x86_64) - vcpkg_arch=x64 - ;; - aarch64) - vcpkg_arch=arm64 - ;; +x86_64) + vcpkg_arch=amd64 + ;; +aarch64) + vcpkg_arch=arm64 + ;; esac gcc_toolset_version="$(rpm -qa "gcc-toolset-*-runtime" --queryformat '%{VERSION}' | grep -o "^[0-9]*")" gcc_toolset_include_cpp="/opt/rh/gcc-toolset-${gcc_toolset_version}/root/usr/include/c++/${gcc_toolset_version}" From d99ac0a8fc3df7649e8948de3b63ceced1e35180 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 12 Aug 2025 09:33:45 +0900 Subject: [PATCH 7/7] Remove unused parameters --- ci/scripts/jni_manylinux_build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/scripts/jni_manylinux_build.sh b/ci/scripts/jni_manylinux_build.sh index 2c902380c..082a40656 100755 --- a/ci/scripts/jni_manylinux_build.sh +++ b/ci/scripts/jni_manylinux_build.sh @@ -119,7 +119,6 @@ cmake \ -DCMAKE_UNITY_BUILD="${CMAKE_UNITY_BUILD}" \ -DGTest_SOURCE=BUNDLED \ -DORC_SOURCE=BUNDLED \ - -DORC_PROTOBUF_EXECUTABLE="${VCPKG_ROOT}/installed/${VCPKG_TARGET_TRIPLET}/tools/protobuf/protoc" \ -DPARQUET_BUILD_EXAMPLES=OFF \ -DPARQUET_BUILD_EXECUTABLES=OFF \ -DPARQUET_REQUIRE_ENCRYPTION=OFF \