ARROW-11340: [C++] Add vcpkg.json manifest to cpp project root

@kszucs could you review this please? My main purpose in adding this is to improve the experience for Arrow C++ devs using Windows, but I noticed it also relates to your [TODO in apache#9096](https://github.com/apache/arrow/pull/9096/files#diff-990134cce6657dbbcf95457cf1a56810a7efa1f6cd58ecc27557c7d6ff45b533R67-R68). vcpkg does not have any `requirements.txt`-style package enumeration mechanism, but it supports this JSON manifest as a mechanism of defining dependencies. In the `vcpkg install` command, you can specify the path to the directory containing this manifest file with `--x-manifest-root` which later will change to `--manifest-root`. See details at https://vcpkg.readthedocs.io/en/stable/specifications/manifests/. There are some differences between the packages listed in this manifest versus the packages you listed in the `vcpkg install` commands in apache#9096 - This installs `gtest` and `benchmark` - This installs `boost` instead of separate `boost-filesystem`, `boost-regex`, etc. - This does not explicitly include the `core` feature of `aws-sdk-cpp` because explicitly including it causes an error, and it gets installed anyway Closes apache#9287 from ianmcook/ARROW-11340 Lead-authored-by: Ian Cook <ianmcook@gmail.com> Co-authored-by: Sutou Kouhei <kou@clear-code.com> Signed-off-by: Krisztián Szűcs <szucs.krisztian@gmail.com>
JerAguilon · Feb 17, 2021 · ad4504e · ad4504e
1 parent bca7d2f
commit ad4504e
Show file tree

Hide file tree

Showing 11 changed files with 261 additions and 2 deletions.
diff --git a/ci/docker/python-wheel-manylinux-201x.dockerfile b/ci/docker/python-wheel-manylinux-201x.dockerfile
@@ -62,7 +62,8 @@ ARG build_type=release
 ENV CMAKE_BUILD_TYPE=${build_type} \
     VCPKG_FORCE_SYSTEM_BINARIES=1 \
     VCPKG_OVERLAY_TRIPLETS=/arrow/ci/vcpkg \
-    VCPKG_DEFAULT_TRIPLET=x64-linux-static-${build_type}
+    VCPKG_DEFAULT_TRIPLET=x64-linux-static-${build_type} \
+    VCPKG_FEATURE_FLAGS=-manifests
 
 # TODO(kszucs): factor out the package enumeration to a text file and reuse it
 # from the windows image and potentially in a future macos wheel build

diff --git a/ci/docker/python-wheel-windows-vs2017.dockerfile b/ci/docker/python-wheel-windows-vs2017.dockerfile
@@ -46,7 +46,8 @@ COPY ci/vcpkg arrow/ci/vcpkg
 ARG build_type=release
 ENV CMAKE_BUILD_TYPE=${build_type} \
     VCPKG_OVERLAY_TRIPLETS=C:\\arrow\\ci\\vcpkg \
-    VCPKG_DEFAULT_TRIPLET=x64-windows-static-md-${build_type}
+    VCPKG_DEFAULT_TRIPLET=x64-windows-static-md-${build_type} \
+    VCPKG_FEATURE_FLAGS=-manifests
 RUN vcpkg install --clean-after-build \
         abseil \
         aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \

diff --git a/ci/scripts/python_wheel_manylinux_build.sh b/ci/scripts/python_wheel_manylinux_build.sh
@@ -67,6 +67,7 @@ echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
 : ${ARROW_WITH_ZSTD:=ON}
 : ${CMAKE_BUILD_TYPE:=release}
 : ${CMAKE_GENERATOR:=Ninja}
+: ${VCPKG_FEATURE_FLAGS:=-manifests}
 
 mkdir /tmp/arrow-build
 pushd /tmp/arrow-build
@@ -106,6 +107,8 @@ cmake \
     -DCMAKE_UNITY_BUILD=ON \
     -DOPENSSL_USE_STATIC_LIBS=ON \
     -DThrift_ROOT=/opt/vcpkg/installed/x64-linux/lib \
+    -D_VCPKG_INSTALLED_DIR=/opt/vcpkg/installed \
+    -DVCPKG_MANIFEST_MODE=OFF \
     -DVCPKG_TARGET_TRIPLET=x64-linux-static-${CMAKE_BUILD_TYPE} \
     -G ${CMAKE_GENERATOR} \
     /arrow/cpp

diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat
@@ -47,6 +47,7 @@ set ARROW_WITH_ZLIB=ON
 set ARROW_WITH_ZSTD=ON
 set CMAKE_UNITY_BUILD=ON
 set CMAKE_GENERATOR=Visual Studio 15 2017 Win64
+set VCPKG_FEATURE_FLAGS=-manifests
 
 mkdir C:\arrow-build
 pushd C:\arrow-build
@@ -83,6 +84,8 @@ cmake ^
     -DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake ^
     -DCMAKE_UNITY_BUILD=%CMAKE_UNITY_BUILD% ^
     -DMSVC_LINK_VERBOSE=ON ^
+    -D_VCPKG_INSTALLED_DIR=C:\vcpkg\installed ^
+    -DVCPKG_MANIFEST_MODE=OFF ^
     -DVCPKG_TARGET_TRIPLET=x64-windows-static-md-%CMAKE_BUILD_TYPE% ^
     -G "%CMAKE_GENERATOR%" ^
     C:\arrow\cpp || exit /B

diff --git a/cpp/CMakeSettings.json b/cpp/CMakeSettings.json
@@ -0,0 +1,21 @@
+{
+  "configurations": [
+  {
+    "name": "x64-Debug (default)",
+    "generator": "Ninja",
+    "configurationType": "Debug",
+    "inheritEnvironments": [ "msvc_x64_x64" ],
+    "buildRoot": "${projectDir}\\out\\build\\${name}",
+    "installRoot": "${projectDir}\\out\\install\\${name}",
+    "cmakeCommandArgs": "",
+    "buildCommandArgs": "",
+    "ctestCommandArgs": "",
+    "variables": [
+        {
+          "name":"VCPKG_MANIFEST_MODE",
+          "value":"OFF"
+        }
+      ]
+    }
+  ]
+}
diff --git a/cpp/vcpkg.json b/cpp/vcpkg.json
@@ -0,0 +1,40 @@
+{
+  "name": "arrow",
+  "version-string": "4.0.0-SNAPSHOT",
+  "dependencies": [
+    "abseil",
+    {
+      "name": "aws-sdk-cpp",
+      "features": [
+        "config",
+        "cognito-identity",
+        "identity-management",
+        "s3",
+        "sts",
+        "transfer"
+      ]
+    },
+    "benchmark",
+    "boost",
+    "brotli",
+    "bzip2",
+    "c-ares",
+    "curl",
+    "flatbuffers",
+    "gflags",
+    "glog",
+    "grpc",
+    "gtest",
+    "lz4",
+    "openssl",
+    "orc",
+    "protobuf",
+    "rapidjson",
+    "re2",
+    "snappy",
+    "thrift",
+    "utf8proc",
+    "zlib",
+    "zstd"
+  ]
+}
diff --git a/dev/release/00-prepare-test.rb b/dev/release/00-prepare-test.rb
@@ -164,6 +164,13 @@ def test_version_pre_tag
                         "+set(ARROW_VERSION \"#{@release_version}\")"],
                      ],
                    },
+                   {
+                     path: "cpp/vcpkg.json",
+                     hunks: [
+                       ["-  \"version-string\": \"#{@snapshot_version}\",",
+                        "+  \"version-string\": \"#{@release_version}\","],
+                     ],
+                   },
                    {
                      path: "csharp/Directory.Build.props",
                      hunks: [
@@ -410,6 +417,13 @@ def test_version_post_tag
                         "+set(ARROW_VERSION \"#{@next_snapshot_version}\")"],
                      ],
                    },
+                   {
+                     path: "cpp/vcpkg.json",
+                     hunks: [
+                       ["-  \"version-string\": \"#{@release_version}\",",
+                        "+  \"version-string\": \"#{@next_snapshot_version}\","],
+                     ],
+                   },
                    {
                      path: "csharp/Directory.Build.props",
                      hunks: [

diff --git a/dev/release/00-prepare.sh b/dev/release/00-prepare.sh
@@ -67,6 +67,12 @@ update_versions() {
     CMakeLists.txt
   rm -f CMakeLists.txt.bak
   git add CMakeLists.txt
+
+  sed -i.bak -E -e \
+    "s/\"version-string\": \".+\"/\"version-string\": \"${version}\"/" \
+    vcpkg.json
+  rm -f vcpkg.json.bak
+  git add vcpkg.json
   cd -
 
   cd "${SOURCE_DIR}/../../csharp"

diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
@@ -82,6 +82,9 @@ groups:
   ruby:
     - test-*ruby*
 
+  vcpkg:
+    - test-*vcpkg*
+
   integration:
     - test-*dask*
     - test-*hdfs*
@@ -1809,6 +1812,12 @@ tasks:
         UBUNTU: 18.04
       run: ubuntu-docs
 
+  ############################## vcpkg tests ##################################
+
+  test-build-vcpkg-win:
+    ci: github
+    template: vcpkg-tests/github.windows.yml
+
   ############################## Integration tests ############################
 
   test-conda-python-3.7-pandas-latest:

diff --git a/dev/tasks/vcpkg-tests/cpp-build-vcpkg.bat b/dev/tasks/vcpkg-tests/cpp-build-vcpkg.bat
@@ -0,0 +1,98 @@
+@rem Licensed to the Apache Software Foundation (ASF) under one
+@rem or more contributor license agreements.  See the NOTICE file
+@rem distributed with this work for additional information
+@rem regarding copyright ownership.  The ASF licenses this file
+@rem to you under the Apache License, Version 2.0 (the
+@rem "License"); you may not use this file except in compliance
+@rem with the License.  You may obtain a copy of the License at
+@rem
+@rem   http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing,
+@rem software distributed under the License is distributed on an
+@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@rem KIND, either express or implied.  See the License for the
+@rem specific language governing permissions and limitations
+@rem under the License.
+
+@rem Run VsDevCmd.bat to set Visual Studio environment variables for building
+@rem on the command line. This is the path for Visual Studio Enterprise 2019
+
+call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=amd64
+
+
+@rem Install build dependencies with vcpkg
+
+@rem TODO(ianmcook): change --x-manifest-root to --manifest-root after it
+@rem changes in vcpkg
+
+vcpkg install ^
+    --triplet x64-windows ^
+    --x-manifest-root cpp  ^
+    --clean-after-build ^
+    || exit /B 1
+
+
+@rem Set environment variables
+
+set ARROW_TEST_DATA=%cd%\testing\data
+set PARQUET_TEST_DATA=%cd%\cpp\submodules\parquet-testing\data
+set VCPKG_INSTALLED=%cd%\cpp\vcpkg_installed
+
+
+@rem Build Arrow C++ library
+
+mkdir cpp\build
+pushd cpp\build
+
+@rem TODO(ianmcook): test using --parallel %NUMBER_OF_PROCESSORS% with
+@rem cmake --build instead of specifying -DARROW_CXXFLAGS="/MP" here
+@rem (see https://gitlab.kitware.com/cmake/cmake/-/issues/20564)
+
+@rem TODO(ianmcook): Add -DARROW_BUILD_BENCHMARKS=ON after the issue described
+@rem at https://github.com/google/benchmark/issues/1046 is resolved
+
+cmake -G "Visual Studio 16 2019" -A x64 ^
+      -DARROW_BOOST_USE_SHARED=ON ^
+      -DARROW_BUILD_SHARED=ON ^
+      -DARROW_BUILD_STATIC=OFF ^
+      -DARROW_BUILD_TESTS=ON ^
+      -DARROW_CXXFLAGS="/MP" ^
+      -DARROW_DATASET=ON ^
+      -DARROW_DEPENDENCY_SOURCE=SYSTEM ^
+      -DARROW_FLIGHT=ON ^
+      -DARROW_MIMALLOC=ON ^
+      -DARROW_PACKAGE_PREFIX="%VCPKG_INSTALLED%\x64-windows" ^
+      -DARROW_PARQUET=ON ^
+      -DARROW_PYTHON=OFF ^
+      -DARROW_WITH_BROTLI=ON ^
+      -DARROW_WITH_BZ2=ON ^
+      -DARROW_WITH_LZ4=ON ^
+      -DARROW_WITH_SNAPPY=ON ^
+      -DARROW_WITH_ZLIB=ON ^
+      -DARROW_WITH_ZSTD=ON ^
+      -DCMAKE_BUILD_TYPE=release ^
+      -DCMAKE_TOOLCHAIN_FILE="C:\vcpkg\scripts\buildsystems\vcpkg.cmake" ^
+      -DCMAKE_UNITY_BUILD=ON ^
+      -DLZ4_MSVC_LIB_PREFIX="" ^
+      -DLZ4_MSVC_STATIC_LIB_SUFFIX="" ^
+      -D_VCPKG_INSTALLED_DIR="%VCPKG_INSTALLED%" ^
+      -DVCPKG_MANIFEST_MODE=ON ^
+      -DVCPKG_TARGET_TRIPLET="x64-windows" ^
+      -DZSTD_MSVC_LIB_PREFIX="" ^
+      .. || exit /B 1
+
+cmake --build . --target INSTALL --config Release || exit /B 1
+
+
+@rem Test Arrow C++ library
+
+@rem TODO(ianmcook): Troubleshoot two test failures:
+@rem  - TestStatisticsSortOrder/0.MinMax
+@rem  - TestStatistic.Int32Extremums
+
+ctest --output-on-failure ^
+      --parallel %NUMBER_OF_PROCESSORS% ^
+      --timeout 300 || exit /B 1
+
+popd
diff --git a/dev/tasks/vcpkg-tests/github.windows.yml b/dev/tasks/vcpkg-tests/github.windows.yml
@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# NOTE: must set "Crossbow" as name to have the badge links working in the
+# github comment reports!
+name: Crossbow
+
+on:
+  push:
+    branches:
+      - "*-github-*"
+
+jobs:
+  test-vcpkg-win:
+    name: Install build deps with vcpkg and build Arrow C++
+    runs-on: windows-2019
+    steps:
+      - name: Checkout Arrow
+        run: |
+          git clone --no-checkout {{ arrow.remote }} arrow
+          git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
+          git -C arrow checkout FETCH_HEAD
+          git -C arrow submodule update --init --recursive
+      - name: Remove and Reinstall vcpkg
+        # As of January 2021, the version of vcpkg that is preinstalled on the
+        # Github Actions windows-2019 image is 2020.11.12, as noted at
+        # https://github.com/actions/virtual-environments/blob/main/images/win/Windows2019-Readme.md
+        # This version of vcpkg has a bug that causes the installation of
+        # aws-cpp-sdk to fail. See details at
+        # https://github.com/awslabs/aws-c-common/issues/734
+        # and https://github.com/microsoft/vcpkg/pull/14716.
+        # When running vcpkg in Github Actions on Windows, remove the
+        # preinstalled vcpkg and install the newest version from source.
+        shell: cmd
+        run: |
+          CALL vcpkg integrate remove 2>NUL
+          CALL C:
+          CALL cd \
+          CALL rmdir /s /q vcpkg 2>NUL
+          CALL git clone https://github.com/microsoft/vcpkg.git vcpkg
+          CALL cd vcpkg
+          CALL bootstrap-vcpkg.bat -win64 -disableMetrics
+          CALL vcpkg integrate install
+          CALL setx PATH "%PATH%;C:\vcpkg"
+      - name: Install Dependencies with vcpkg and Build Arrow C++
+        shell: cmd
+        run: |
+          CALL cd arrow
+          CALL dev\tasks\vcpkg-tests\cpp-build-vcpkg.bat