From 8d73fcce5221c6174c185194c1c44cf4a1b4f29a Mon Sep 17 00:00:00 2001 From: Jin Shang Date: Fri, 20 Jan 2023 15:56:03 +0000 Subject: [PATCH 01/16] GH-20272: [C++] Bump version of bundled AWS SDK --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 6 ++++-- cpp/thirdparty/versions.txt | 8 ++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index dd1cf41346e9f..b643e49a993b6 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -4630,6 +4630,7 @@ endif() macro(build_awssdk) message(STATUS "Building AWS C++ SDK from source") set(AWSSDK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/awssdk_ep-install") + set(AWSSDK_SOURCE "${CMAKE_CURRENT_BINARY_DIR}/awssdk_ep-prefix/src/awssdk_ep") set(AWSSDK_INCLUDE_DIR "${AWSSDK_PREFIX}/include") if(WIN32) @@ -4661,7 +4662,7 @@ macro(build_awssdk) set(AWSSDK_CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} -DOPENSSL_ROOT_DIR=${OPENSSL_ROOT_HINT} - -DBUILD_DEPS=OFF + -DBUILD_DEPS=ON # We need to build aws-crt-cpp from source -DBUILD_ONLY=config\\$s3\\$transfer\\$identity-management\\$sts -DMINIMIZE_SIZE=ON) @@ -4746,8 +4747,9 @@ macro(build_awssdk) "10") # Workaround for https://github.com/aws/aws-sdk-cpp/issues/1750 set(AWSSDK_PATCH_COMMAND "sed" "-i.bak" "-e" "s/\"-Werror\"//g" - "/cmake/compiler_settings.cmake") + "/cmake/compiler_settings.cmake" "&&") endif() + list(APPEND AWSSDK_PATCH_COMMAND "${AWSSDK_SOURCE}/prefetch_crt_dependency.sh") externalproject_add(awssdk_ep ${EP_COMMON_OPTIONS} diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 69073f3a47581..82c2fc63851ff 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -25,12 +25,12 @@ ARROW_ABSL_BUILD_VERSION=20211102.0 ARROW_ABSL_BUILD_SHA256_CHECKSUM=dcf71b9cba8dc0ca9940c4b316a0c796be8fab42b070bb6b7cab62b48f0e66c4 -ARROW_AWSSDK_BUILD_VERSION=1.8.133 -ARROW_AWSSDK_BUILD_SHA256_CHECKSUM=d6c495bc06be5e21dac716571305d77437e7cfd62a2226b8fe48d9ab5785a8d6 +ARROW_AWSSDK_BUILD_VERSION=1.10.55 +ARROW_AWSSDK_BUILD_SHA256_CHECKSUM=2d552fb1a84bef4a9b65e34aa7031851ed2aef5319e02cc6e4cb735c48aa30de ARROW_AWS_CHECKSUMS_BUILD_VERSION=v0.1.12 ARROW_AWS_CHECKSUMS_BUILD_SHA256_CHECKSUM=394723034b81cc7cd528401775bc7aca2b12c7471c92350c80a0e2fb9d2909fe -ARROW_AWS_C_COMMON_BUILD_VERSION=v0.6.9 -ARROW_AWS_C_COMMON_BUILD_SHA256_CHECKSUM=928a3e36f24d1ee46f9eec360ec5cebfe8b9b8994fe39d4fa74ff51aebb12717 +ARROW_AWS_C_COMMON_BUILD_VERSION=v0.8.9 +ARROW_AWS_C_COMMON_BUILD_SHA256_CHECKSUM=2f3fbaf7c38eae5a00e2a816d09b81177f93529ae8ba1b82dc8f31407565327a ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION=v0.1.5 ARROW_AWS_C_EVENT_STREAM_BUILD_SHA256_CHECKSUM=f1b423a487b5d6dca118bfc0d0c6cc596dc476b282258a3228e73a8f730422d4 ARROW_BOOST_BUILD_VERSION=1.81.0 From a1f0cf46fc2c20b0feb6859624ac92222074e476 Mon Sep 17 00:00:00 2001 From: Jin Shang Date: Sun, 22 Jan 2023 13:53:23 +0000 Subject: [PATCH 02/16] compile each dependency manually --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 234 ++++++++++++++++++-- cpp/thirdparty/versions.txt | 41 +++- 2 files changed, 258 insertions(+), 17 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index b643e49a993b6..f2ebaf11b5728 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -457,6 +457,93 @@ else() ) endif() +if(DEFINED ENV{ARROW_AWS_C_AUTH_URL}) + set(AWS_C_AUTH_SOURCE_URL "$ENV{ARROW_AWS_C_AUTH_URL}") +else() + set_urls(AWS_C_AUTH_SOURCE_URL + "https://github.com/awslabs/aws-c-auth/archive/${ARROW_AWS_C_AUTH_BUILD_VERSION}.tar.gz" + ) +endif() + +if(DEFINED ENV{ARROW_AWS_C_CAL_URL}) + set(AWS_C_CAL_SOURCE_URL "$ENV{ARROW_AWS_C_CAL_URL}") +else() + set_urls(AWS_C_CAL_SOURCE_URL + "https://github.com/awslabs/aws-c-cal/archive/${ARROW_AWS_C_CAL_BUILD_VERSION}.tar.gz" + ) +endif() + +if(DEFINED ENV{ARROW_AWS_C_COMPRESSION_URL}) + set(AWS_C_COMPRESSION_SOURCE_URL "$ENV{ARROW_AWS_C_COMPRESSION_URL}") +else() + set_urls(AWS_C_COMPRESSION_SOURCE_URL + "https://github.com/awslabs/aws-c-compression/archive/${ARROW_AWS_C_COMPRESSION_BUILD_VERSION}.tar.gz" + ) +endif() + +if(DEFINED ENV{ARROW_AWS_C_HTTP_URL}) + set(AWS_C_HTTP_SOURCE_URL "$ENV{ARROW_AWS_C_HTTP_URL}") +else() + set_urls(AWS_C_HTTP_SOURCE_URL + "https://github.com/awslabs/aws-c-http/archive/${ARROW_AWS_C_HTTP_BUILD_VERSION}.tar.gz" + ) +endif() + +if(DEFINED ENV{ARROW_AWS_C_IO_URL}) + set(AWS_C_IO_SOURCE_URL "$ENV{ARROW_AWS_C_IO_URL}") +else() + set_urls(AWS_C_IO_SOURCE_URL + "https://github.com/awslabs/aws-c-io/archive/${ARROW_AWS_C_IO_BUILD_VERSION}.tar.gz" + ) +endif() + +if(DEFINED ENV{ARROW_AWS_C_MQTT_URL}) + set(AWS_C_MQTT_SOURCE_URL "$ENV{ARROW_AWS_C_MQTT_URL}") +else() + set_urls(AWS_C_MQTT_SOURCE_URL + "https://github.com/awslabs/aws-c-mqtt/archive/${ARROW_AWS_C_MQTT_BUILD_VERSION}.tar.gz" + ) +endif() + +if(DEFINED ENV{ARROW_AWS_C_S3_URL}) + set(AWS_C_S3_SOURCE_URL "$ENV{ARROW_AWS_C_S3_URL}") +else() + set_urls(AWS_C_S3_SOURCE_URL + "https://github.com/awslabs/aws-c-s3/archive/${ARROW_AWS_C_S3_BUILD_VERSION}.tar.gz" + ) +endif() + +if(DEFINED ENV{ARROW_AWS_C_SDKUTILS_URL}) + set(AWS_C_SDKUTILS_SOURCE_URL "$ENV{ARROW_AWS_C_SDKUTILS_URL}") +else() + set_urls(AWS_C_SDKUTILS_SOURCE_URL + "https://github.com/awslabs/aws-c-sdkutils/archive/${ARROW_AWS_C_SDKUTILS_BUILD_VERSION}.tar.gz" + ) +endif() + +if(DEFINED ENV{ARROW_AWS_LC_URL}) + set(AWS_LC_SOURCE_URL "$ENV{ARROW_AWS_LC_URL}") +else() + set_urls(AWS_LC_SOURCE_URL + "https://github.com/awslabs/aws-lc/archive/${ARROW_AWS_LC_BUILD_VERSION}.tar.gz" + ) +endif() + +if(DEFINED ENV{ARROW_S2N_URL}) + set(S2N_SOURCE_URL "$ENV{ARROW_S2N_URL}") +else() + set_urls(S2N_SOURCE_URL + "https://github.com/awslabs/s2n/archive/${ARROW_S2N_BUILD_VERSION}.tar.gz") +endif() + +if(DEFINED ENV{ARROW_AWS_CRT_CPP_URL}) + set(AWS_CRT_CPP_SOURCE_URL "$ENV{ARROW_AWS_CRT_CPP_URL}") +else() + set_urls(AWS_CRT_CPP_SOURCE_URL + "https://github.com/awslabs/aws-crt-cpp/archive/${ARROW_AWS_CRT_CPP_BUILD_VERSION}.tar.gz" + ) +endif() + if(DEFINED ENV{ARROW_AWSSDK_URL}) set(AWSSDK_SOURCE_URL "$ENV{ARROW_AWSSDK_URL}") else() @@ -4662,7 +4749,7 @@ macro(build_awssdk) set(AWSSDK_CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} -DOPENSSL_ROOT_DIR=${OPENSSL_ROOT_HINT} - -DBUILD_DEPS=ON # We need to build aws-crt-cpp from source + -DBUILD_DEPS=OFF -DBUILD_ONLY=config\\$s3\\$transfer\\$identity-management\\$sts -DMINIMIZE_SIZE=ON) @@ -4692,7 +4779,19 @@ macro(build_awssdk) aws-cpp-sdk-core aws-c-event-stream aws-checksums - aws-c-common) + aws-c-common + aws-c-auth + aws-c-cal + aws-c-compression + aws-c-http + aws-c-io + aws-c-mqtt + aws-c-s3 + aws-c-sdkutils + s2n + crypto + ssl + aws-crt-cpp) set(AWSSDK_LIBRARIES) foreach(_AWSSDK_LIB ${_AWSSDK_LIBS}) # aws-c-common -> AWS-C-COMMON @@ -4733,36 +4832,145 @@ macro(build_awssdk) DEPENDS aws_c_common_ep) add_dependencies(AWS::aws-checksums aws_checksums_ep) + externalproject_add(s2n_ep + ${EP_COMMON_OPTIONS} + URL ${S2N_SOURCE_URL} + URL_HASH "SHA256=${ARROW_S2N_BUILD_SHA256_CHECKSUM}" + CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} + BUILD_BYPRODUCTS ${S2N_STATIC_LIBRARY}) + add_dependencies(AWS::s2n s2n_ep) + + externalproject_add(aws_c_io_ep + ${EP_COMMON_OPTIONS} + URL ${AWS_C_IO_SOURCE_URL} + URL_HASH "SHA256=${ARROW_AWS_C_IO_BUILD_SHA256_CHECKSUM}" + CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} + BUILD_BYPRODUCTS ${AWS_C_IO_STATIC_LIBRARY} + DEPENDS aws_c_common_ep s2n_ep) + add_dependencies(AWS::aws-c-io aws_c_io_ep) + externalproject_add(aws_c_event_stream_ep ${EP_COMMON_OPTIONS} URL ${AWS_C_EVENT_STREAM_SOURCE_URL} URL_HASH "SHA256=${ARROW_AWS_C_EVENT_STREAM_BUILD_SHA256_CHECKSUM}" CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} BUILD_BYPRODUCTS ${AWS_C_EVENT_STREAM_STATIC_LIBRARY} - DEPENDS aws_checksums_ep) + DEPENDS aws_checksums_ep s2n_ep aws_c_io_ep) add_dependencies(AWS::aws-c-event-stream aws_c_event_stream_ep) - set(AWSSDK_PATCH_COMMAND) - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER - "10") - # Workaround for https://github.com/aws/aws-sdk-cpp/issues/1750 - set(AWSSDK_PATCH_COMMAND "sed" "-i.bak" "-e" "s/\"-Werror\"//g" - "/cmake/compiler_settings.cmake" "&&") - endif() - list(APPEND AWSSDK_PATCH_COMMAND "${AWSSDK_SOURCE}/prefetch_crt_dependency.sh") + externalproject_add(aws_c_sdkutils_ep + ${EP_COMMON_OPTIONS} + URL ${AWS_C_SDKUTILS_SOURCE_URL} + URL_HASH "SHA256=${ARROW_AWS_C_SDKUTILS_BUILD_SHA256_CHECKSUM}" + CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} + BUILD_BYPRODUCTS ${AWS_C_SDKUTILS_STATIC_LIBRARY} + DEPENDS aws_c_common_ep) + add_dependencies(AWS::aws-c-sdkutils aws_c_sdkutils_ep) + + externalproject_add(aws_c_cal_ep + ${EP_COMMON_OPTIONS} + URL ${AWS_C_CAL_SOURCE_URL} + URL_HASH "SHA256=${ARROW_AWS_C_CAL_BUILD_SHA256_CHECKSUM}" + CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} + BUILD_BYPRODUCTS ${AWS_C_CAL_STATIC_LIBRARY} + DEPENDS aws_c_common_ep) + add_dependencies(AWS::aws-c-cal aws_c_cal_ep) + + externalproject_add(aws_c_compression_ep + ${EP_COMMON_OPTIONS} + URL ${AWS_C_COMPRESSION_SOURCE_URL} + URL_HASH "SHA256=${ARROW_AWS_C_COMPRESSION_BUILD_SHA256_CHECKSUM}" + CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} + BUILD_BYPRODUCTS ${AWS_C_COMPRESSION_STATIC_LIBRARY} + DEPENDS aws_c_common_ep) + add_dependencies(AWS::aws-c-compression aws_c_compression_ep) + + externalproject_add(aws_c_http_ep + ${EP_COMMON_OPTIONS} + URL ${AWS_C_HTTP_SOURCE_URL} + URL_HASH "SHA256=${ARROW_AWS_C_HTTP_BUILD_SHA256_CHECKSUM}" + CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} + BUILD_BYPRODUCTS ${AWS_C_HTTP_STATIC_LIBRARY} + DEPENDS aws_c_common_ep s2n_ep aws_c_io_ep) + add_dependencies(AWS::aws-c-http aws_c_http_ep) + + externalproject_add(aws_c_mqtt_ep + ${EP_COMMON_OPTIONS} + URL ${AWS_C_MQTT_SOURCE_URL} + URL_HASH "SHA256=${ARROW_AWS_C_MQTT_BUILD_SHA256_CHECKSUM}" + CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} + BUILD_BYPRODUCTS ${AWS_C_MQTT_STATIC_LIBRARY} + DEPENDS aws_c_common_ep s2n_ep aws_c_io_ep aws_c_http_ep) + add_dependencies(AWS::aws-c-mqtt aws_c_mqtt_ep) + + externalproject_add(aws_c_auth_ep + ${EP_COMMON_OPTIONS} + URL ${AWS_C_AUTH_SOURCE_URL} + URL_HASH "SHA256=${ARROW_AWS_C_AUTH_BUILD_SHA256_CHECKSUM}" + CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} + BUILD_BYPRODUCTS ${AWS_C_AUTH_STATIC_LIBRARY} + DEPENDS aws_c_common_ep + aws_c_sdkutils_ep + s2n_ep + aws_c_io_ep + aws_c_http_ep) + add_dependencies(AWS::aws-c-auth aws_c_auth_ep) + + externalproject_add(aws_c_s3_ep + ${EP_COMMON_OPTIONS} + URL ${AWS_C_S3_SOURCE_URL} + URL_HASH "SHA256=${ARROW_AWS_C_S3_BUILD_SHA256_CHECKSUM}" + CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} + BUILD_BYPRODUCTS ${AWS_C_S3_STATIC_LIBRARY} + DEPENDS aws_c_common_ep + aws_c_cal_ep + s2n_ep + aws_c_io_ep + aws_c_http_ep + aws_c_auth_ep) + add_dependencies(AWS::aws-c-s3 aws_c_s3_ep) + + externalproject_add(aws_lc_ep + ${EP_COMMON_OPTIONS} + URL ${AWS_LC_SOURCE_URL} + URL_HASH "SHA256=${ARROW_AWS_LC_BUILD_SHA256_CHECKSUM}" + CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} + BUILD_BYPRODUCTS ${AWS_LC_STATIC_LIBRARY}) + add_dependencies(AWS::crypto aws_lc_ep) + add_dependencies(AWS::ssl aws_lc_ep) + + externalproject_add(aws_crt_cpp_ep + ${EP_COMMON_OPTIONS} + URL ${AWS_CRT_CPP_SOURCE_URL} + URL_HASH "SHA256=${ARROW_AWS_CRT_CPP_BUILD_SHA256_CHECKSUM}" + CMAKE_ARGS ${AWSSDK_CMAKE_ARGS} + BUILD_BYPRODUCTS ${AWS_CRT_CPP_STATIC_LIBRARY} + DEPENDS aws_c_auth_ep + aws_c_cal_ep + aws_c_common_ep + aws_c_compression_ep + aws_c_event_stream_ep + aws_c_http_ep + aws_c_io_ep + aws_c_mqtt_ep + aws_c_s3_ep + aws_c_sdkutils_ep + aws_checksums_ep + aws_lc_ep + s2n_ep) + add_dependencies(AWS::aws-crt-cpp aws_crt_cpp_ep) externalproject_add(awssdk_ep ${EP_COMMON_OPTIONS} URL ${AWSSDK_SOURCE_URL} URL_HASH "SHA256=${ARROW_AWSSDK_BUILD_SHA256_CHECKSUM}" CMAKE_ARGS ${AWSSDK_CMAKE_ARGS} - PATCH_COMMAND ${AWSSDK_PATCH_COMMAND} BUILD_BYPRODUCTS ${AWS_CPP_SDK_COGNITO_IDENTITY_STATIC_LIBRARY} ${AWS_CPP_SDK_CORE_STATIC_LIBRARY} ${AWS_CPP_SDK_IDENTITY_MANAGEMENT_STATIC_LIBRARY} ${AWS_CPP_SDK_S3_STATIC_LIBRARY} ${AWS_CPP_SDK_STS_STATIC_LIBRARY} - DEPENDS aws_c_event_stream_ep) + DEPENDS aws_c_event_stream_ep aws_crt_cpp_ep) add_dependencies(toolchain awssdk_ep) foreach(_AWSSDK_LIB ${_AWSSDK_LIBS}) if(${_AWSSDK_LIB} MATCHES "^aws-cpp-sdk-") diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 82c2fc63851ff..8710f0763701a 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -27,12 +27,34 @@ ARROW_ABSL_BUILD_VERSION=20211102.0 ARROW_ABSL_BUILD_SHA256_CHECKSUM=dcf71b9cba8dc0ca9940c4b316a0c796be8fab42b070bb6b7cab62b48f0e66c4 ARROW_AWSSDK_BUILD_VERSION=1.10.55 ARROW_AWSSDK_BUILD_SHA256_CHECKSUM=2d552fb1a84bef4a9b65e34aa7031851ed2aef5319e02cc6e4cb735c48aa30de -ARROW_AWS_CHECKSUMS_BUILD_VERSION=v0.1.12 -ARROW_AWS_CHECKSUMS_BUILD_SHA256_CHECKSUM=394723034b81cc7cd528401775bc7aca2b12c7471c92350c80a0e2fb9d2909fe +ARROW_AWS_CHECKSUMS_BUILD_VERSION=v0.1.13 +ARROW_AWS_CHECKSUMS_BUILD_SHA256_CHECKSUM=0f897686f1963253c5069a0e495b85c31635ba146cd3ac38cc2ea31eaf54694d ARROW_AWS_C_COMMON_BUILD_VERSION=v0.8.9 ARROW_AWS_C_COMMON_BUILD_SHA256_CHECKSUM=2f3fbaf7c38eae5a00e2a816d09b81177f93529ae8ba1b82dc8f31407565327a -ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION=v0.1.5 -ARROW_AWS_C_EVENT_STREAM_BUILD_SHA256_CHECKSUM=f1b423a487b5d6dca118bfc0d0c6cc596dc476b282258a3228e73a8f730422d4 +ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION=v0.2.15 +ARROW_AWS_C_EVENT_STREAM_BUILD_SHA256_CHECKSUM=4ff2ada07ede3c6afa4b8e6e20de541e717038307f29b38c27efa7c4d875ee26 +ARROW_AWS_C_AUTH_BUILD_VERSION=v0.6.19 +ARROW_AWS_C_AUTH_BUILD_SHA256_CHECKSUM=2b6b05e046227f87c39d0dd821759273c80e864689dfb36bbc64bed86dc8336f +ARROW_AWS_C_CAL_BUILD_VERSION=v0.5.20 +ARROW_AWS_C_CAL_BUILD_SHA256_CHECKSUM=acc352359bd06f8597415c366cf4ec4f00d0b0da92d637039a73323dd55b6cd0 +ARROW_AWS_C_COMPRESSION_BUILD_VERSION=v0.2.16 +ARROW_AWS_C_COMPRESSION_BUILD_SHA256_CHECKSUM=044b1dbbca431a07bde8255ef9ec443c300fc60d4c9408d4b862f65e496687f4 +ARROW_AWS_C_HTTP_BUILD_VERSION=v0.6.25 +ARROW_AWS_C_HTTP_BUILD_SHA256_CHECKSUM=8c4b4dcf49ef1abec48cfa6748983e352b490d557fa06e0565107c826d589fb0 +ARROW_AWS_C_IO_BUILD_VERSION=v0.13.10 +ARROW_AWS_C_IO_BUILD_SHA256_CHECKSUM=4e7caa335ece916c00f4c9896e7413bb6ed52abc77e2c3dda3ffe5bcf3fc8655 +ARROW_AWS_C_MQTT_BUILD_VERSION=v0.7.13 +ARROW_AWS_C_MQTT_BUILD_SHA256_CHECKSUM=04503c704f6d4fba5b0a470a01a96fa2ed702bfa897b0d38071477b0b70ccf82 +ARROW_AWS_C_S3_BUILD_VERSION=v0.1.51 +ARROW_AWS_C_S3_BUILD_SHA256_CHECKSUM=46b8ef3d42a1973ffcd17ea25540baecc4b4e35216a276c7f93bc8e8e9dc3d9e +ARROW_AWS_C_SDKUTILS_BUILD_VERSION=v0.1.6 +ARROW_AWS_C_SDKUTILS_BUILD_SHA256_CHECKSUM=8a2951344b2fb541eab1e9ca17c18a7fcbfd2aaff4cdd31d362d1fad96111b91 +ARROW_AWS_LC_BUILD_VERSION=v1.3.0 +ARROW_AWS_LC_BUILD_SHA256_CHECKSUM=ae96a3567161552744fc0cae8b4d68ed88b1ec0f3d3c98700070115356da5a37 +ARROW_S2N_BUILD_VERSION=v1.3.27 +ARROW_S2N_BUILD_SHA256_CHECKSUM=58dde9979867a3209a19098ffcd748a144b2462ec81edca5baf864313ff4f589 +ARROW_AWS_CRT_CPP_BUILD_VERSION=v0.18.16 +ARROW_AWS_CRT_CPP_BUILD_SHA256_CHECKSUM=9e69bc1dc4b50871d1038aa9ff6ddeb4c9b28f7d6b5e5b1b69041ccf50a13483 ARROW_BOOST_BUILD_VERSION=1.81.0 ARROW_BOOST_BUILD_SHA256_CHECKSUM=9e0ffae35528c35f90468997bc8d99500bf179cbae355415a89a600c38e13574 ARROW_BROTLI_BUILD_VERSION=v1.0.9 @@ -107,6 +129,17 @@ DEPENDENCIES=( "ARROW_AWS_CHECKSUMS_URL aws-checksums-${ARROW_AWS_CHECKSUMS_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-checksums/archive/${ARROW_AWS_CHECKSUMS_BUILD_VERSION}.tar.gz" "ARROW_AWS_C_COMMON_URL aws-c-common-${ARROW_AWS_C_COMMON_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-c-common/archive/${ARROW_AWS_C_COMMON_BUILD_VERSION}.tar.gz" "ARROW_AWS_C_EVENT_STREAM_URL aws-c-event-stream-${ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-c-event-stream/archive/${ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION}.tar.gz" + "ARROW_AWS_C_AUTH_URL aws-c-auth-${ARROW_AWS_C_AUTH_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-c-auth/archive/${ARROW_AWS_C_AUTH_BUILD_VERSION}.tar.gz" + "ARROW_AWS_C_CAL_URL aws-c-cal-${ARROW_AWS_C_CAL_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-c-cal/archive/${ARROW_AWS_C_CAL_BUILD_VERSION}.tar.gz" + "ARROW_AWS_C_COMPRESSION_URL aws-c-compression-${ARROW_AWS_C_COMPRESSION_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-c-compression/archive/${ARROW_AWS_C_COMPRESSION_BUILD_VERSION}.tar.gz" + "ARROW_AWS_C_HTTP_URL aws-c-http-${ARROW_AWS_C_HTTP_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-c-http/archive/${ARROW_AWS_C_HTTP_BUILD_VERSION}.tar.gz" + "ARROW_AWS_C_IO_URL aws-c-io-${ARROW_AWS_C_IO_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-c-io/archive/${ARROW_AWS_C_IO_BUILD_VERSION}.tar.gz" + "ARROW_AWS_C_MQTT_URL aws-c-mqtt-${ARROW_AWS_C_MQTT_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-c-mqtt/archive/${ARROW_AWS_C_MQTT_BUILD_VERSION}.tar.gz" + "ARROW_AWS_C_S3_URL aws-c-s3-${ARROW_AWS_C_S3_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-c-s3/archive/${ARROW_AWS_C_S3_BUILD_VERSION}.tar.gz" + "ARROW_AWS_C_SDKUTILS_URL aws-c-sdkutils-${ARROW_AWS_C_SDKUTILS_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-c-sdkutils/archive/${ARROW_AWS_C_SDKUTILS_BUILD_VERSION}.tar.gz" + "ARROW_AWS_LC_URL aws-lc-${ARROW_AWS_LC_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-lc/archive/${ARROW_AWS_LC_BUILD_VERSION}.tar.gz" + "ARROW_S2N_URL s2n-${ARROW_S2N_BUILD_VERSION}.tar.gz https://github.com/awslabs/s2n/archive/${ARROW_S2N_BUILD_VERSION}.tar.gz" + "ARROW_AWS_CRT_CPP_URL aws-crt-cpp-${ARROW_AWS_CRT_CPP_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-crt-cpp/archive/${ARROW_AWS_CRT_CPP_BUILD_VERSION}.tar.gz" "ARROW_BOOST_URL boost-${ARROW_BOOST_BUILD_VERSION}.tar.gz https://apache.jfrog.io/artifactory/arrow/thirdparty/7.0.0/boost_${ARROW_BOOST_BUILD_VERSION//./_}.tar.gz" "ARROW_BROTLI_URL brotli-${ARROW_BROTLI_BUILD_VERSION}.tar.gz https://github.com/google/brotli/archive/${ARROW_BROTLI_BUILD_VERSION}.tar.gz" "ARROW_BZIP2_URL bzip2-${ARROW_BZIP2_BUILD_VERSION}.tar.gz https://sourceware.org/pub/bzip2/bzip2-${ARROW_BZIP2_BUILD_VERSION}.tar.gz" From dacce2390f9638ad02e2d514fbda440bad25d158 Mon Sep 17 00:00:00 2001 From: Jin Shang Date: Mon, 23 Jan 2023 17:13:09 +0000 Subject: [PATCH 03/16] fix aws crypto & ssl by product --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 22 ++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index f2ebaf11b5728..ef2f0272305b1 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -4840,13 +4840,22 @@ macro(build_awssdk) BUILD_BYPRODUCTS ${S2N_STATIC_LIBRARY}) add_dependencies(AWS::s2n s2n_ep) + externalproject_add(aws_c_cal_ep + ${EP_COMMON_OPTIONS} + URL ${AWS_C_CAL_SOURCE_URL} + URL_HASH "SHA256=${ARROW_AWS_C_CAL_BUILD_SHA256_CHECKSUM}" + CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} + BUILD_BYPRODUCTS ${AWS_C_CAL_STATIC_LIBRARY} + DEPENDS aws_c_common_ep) + add_dependencies(AWS::aws-c-cal aws_c_cal_ep) + externalproject_add(aws_c_io_ep ${EP_COMMON_OPTIONS} URL ${AWS_C_IO_SOURCE_URL} URL_HASH "SHA256=${ARROW_AWS_C_IO_BUILD_SHA256_CHECKSUM}" CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} BUILD_BYPRODUCTS ${AWS_C_IO_STATIC_LIBRARY} - DEPENDS aws_c_common_ep s2n_ep) + DEPENDS aws_c_common_ep s2n_ep aws_c_cal_ep) add_dependencies(AWS::aws-c-io aws_c_io_ep) externalproject_add(aws_c_event_stream_ep @@ -4867,15 +4876,6 @@ macro(build_awssdk) DEPENDS aws_c_common_ep) add_dependencies(AWS::aws-c-sdkutils aws_c_sdkutils_ep) - externalproject_add(aws_c_cal_ep - ${EP_COMMON_OPTIONS} - URL ${AWS_C_CAL_SOURCE_URL} - URL_HASH "SHA256=${ARROW_AWS_C_CAL_BUILD_SHA256_CHECKSUM}" - CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWS_C_CAL_STATIC_LIBRARY} - DEPENDS aws_c_common_ep) - add_dependencies(AWS::aws-c-cal aws_c_cal_ep) - externalproject_add(aws_c_compression_ep ${EP_COMMON_OPTIONS} URL ${AWS_C_COMPRESSION_SOURCE_URL} @@ -4935,7 +4935,7 @@ macro(build_awssdk) URL ${AWS_LC_SOURCE_URL} URL_HASH "SHA256=${ARROW_AWS_LC_BUILD_SHA256_CHECKSUM}" CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWS_LC_STATIC_LIBRARY}) + BUILD_BYPRODUCTS ${CRYPTO_STATIC_LIBRARY} ${SSL_STATIC_LIBRARY}) add_dependencies(AWS::crypto aws_lc_ep) add_dependencies(AWS::ssl aws_lc_ep) From 2cb9e36f2d6472ee3844ce587556cc11fab4ac37 Mon Sep 17 00:00:00 2001 From: Jin Shang Date: Mon, 23 Jan 2023 17:25:53 +0000 Subject: [PATCH 04/16] update versions --- cpp/thirdparty/versions.txt | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 8710f0763701a..b846aaecce619 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -31,22 +31,22 @@ ARROW_AWS_CHECKSUMS_BUILD_VERSION=v0.1.13 ARROW_AWS_CHECKSUMS_BUILD_SHA256_CHECKSUM=0f897686f1963253c5069a0e495b85c31635ba146cd3ac38cc2ea31eaf54694d ARROW_AWS_C_COMMON_BUILD_VERSION=v0.8.9 ARROW_AWS_C_COMMON_BUILD_SHA256_CHECKSUM=2f3fbaf7c38eae5a00e2a816d09b81177f93529ae8ba1b82dc8f31407565327a -ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION=v0.2.15 -ARROW_AWS_C_EVENT_STREAM_BUILD_SHA256_CHECKSUM=4ff2ada07ede3c6afa4b8e6e20de541e717038307f29b38c27efa7c4d875ee26 -ARROW_AWS_C_AUTH_BUILD_VERSION=v0.6.19 -ARROW_AWS_C_AUTH_BUILD_SHA256_CHECKSUM=2b6b05e046227f87c39d0dd821759273c80e864689dfb36bbc64bed86dc8336f +ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION=v0.2.18 +ARROW_AWS_C_EVENT_STREAM_BUILD_SHA256_CHECKSUM=310ca617f713bf664e4c7485a3d42c1fb57813abd0107e49790d107def7cde4f +ARROW_AWS_C_AUTH_BUILD_VERSION=v0.6.22 +ARROW_AWS_C_AUTH_BUILD_SHA256_CHECKSUM=691a6b4418afcd3dc141351b6ad33fccd8e3ff84df0e9e045b42295d284ee14c ARROW_AWS_C_CAL_BUILD_VERSION=v0.5.20 ARROW_AWS_C_CAL_BUILD_SHA256_CHECKSUM=acc352359bd06f8597415c366cf4ec4f00d0b0da92d637039a73323dd55b6cd0 ARROW_AWS_C_COMPRESSION_BUILD_VERSION=v0.2.16 ARROW_AWS_C_COMPRESSION_BUILD_SHA256_CHECKSUM=044b1dbbca431a07bde8255ef9ec443c300fc60d4c9408d4b862f65e496687f4 -ARROW_AWS_C_HTTP_BUILD_VERSION=v0.6.25 -ARROW_AWS_C_HTTP_BUILD_SHA256_CHECKSUM=8c4b4dcf49ef1abec48cfa6748983e352b490d557fa06e0565107c826d589fb0 -ARROW_AWS_C_IO_BUILD_VERSION=v0.13.10 -ARROW_AWS_C_IO_BUILD_SHA256_CHECKSUM=4e7caa335ece916c00f4c9896e7413bb6ed52abc77e2c3dda3ffe5bcf3fc8655 -ARROW_AWS_C_MQTT_BUILD_VERSION=v0.7.13 -ARROW_AWS_C_MQTT_BUILD_SHA256_CHECKSUM=04503c704f6d4fba5b0a470a01a96fa2ed702bfa897b0d38071477b0b70ccf82 -ARROW_AWS_C_S3_BUILD_VERSION=v0.1.51 -ARROW_AWS_C_S3_BUILD_SHA256_CHECKSUM=46b8ef3d42a1973ffcd17ea25540baecc4b4e35216a276c7f93bc8e8e9dc3d9e +ARROW_AWS_C_HTTP_BUILD_VERSION=v0.7.3 +ARROW_AWS_C_HTTP_BUILD_SHA256_CHECKSUM=07e16c6bf5eba6f0dea96b6f55eae312a7c95b736f4d2e4a210000f45d8265ae +ARROW_AWS_C_IO_BUILD_VERSION=v0.13.14 +ARROW_AWS_C_IO_BUILD_SHA256_CHECKSUM=12b66510c3d9a4f7e9b714e9cfab2a5bf835f8b9ce2f909d20ae2a2128608c71 +ARROW_AWS_C_MQTT_BUILD_VERSION=v0.8.4 +ARROW_AWS_C_MQTT_BUILD_SHA256_CHECKSUM=232eeac63e72883d460c686a09b98cdd811d24579affac47c5c3f696f956773f +ARROW_AWS_C_S3_BUILD_VERSION=v0.2.3 +ARROW_AWS_C_S3_BUILD_SHA256_CHECKSUM=a00b3c9f319cd1c9aa2c3fa15098864df94b066dcba0deaccbb3caa952d902fe ARROW_AWS_C_SDKUTILS_BUILD_VERSION=v0.1.6 ARROW_AWS_C_SDKUTILS_BUILD_SHA256_CHECKSUM=8a2951344b2fb541eab1e9ca17c18a7fcbfd2aaff4cdd31d362d1fad96111b91 ARROW_AWS_LC_BUILD_VERSION=v1.3.0 From 1145c4cdd0990f7bddb9429339022725e9a80049 Mon Sep 17 00:00:00 2001 From: Jin Shang Date: Thu, 26 Jan 2023 02:35:52 +0000 Subject: [PATCH 05/16] review feedback --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 20 +------------------- cpp/thirdparty/versions.txt | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 29 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index ef2f0272305b1..dfcce2a8202bf 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -4717,7 +4717,6 @@ endif() macro(build_awssdk) message(STATUS "Building AWS C++ SDK from source") set(AWSSDK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/awssdk_ep-install") - set(AWSSDK_SOURCE "${CMAKE_CURRENT_BINARY_DIR}/awssdk_ep-prefix/src/awssdk_ep") set(AWSSDK_INCLUDE_DIR "${AWSSDK_PREFIX}/include") if(WIN32) @@ -4737,11 +4736,6 @@ macro(build_awssdk) -DENABLE_UNITY_BUILD=ON "-DCMAKE_INSTALL_PREFIX=${AWSSDK_PREFIX}" "-DCMAKE_PREFIX_PATH=${AWSSDK_PREFIX}") - if(NOT MSVC) - list(APPEND AWSSDK_COMMON_CMAKE_ARGS - # Workaround for https://github.com/aws/aws-sdk-cpp/issues/1582 - "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS} -Wno-error=deprecated-declarations") - endif() # provide hint for AWS SDK to link with the already located openssl get_filename_component(OPENSSL_ROOT_HINT "${OPENSSL_INCLUDE_DIR}" DIRECTORY) @@ -4789,8 +4783,6 @@ macro(build_awssdk) aws-c-s3 aws-c-sdkutils s2n - crypto - ssl aws-crt-cpp) set(AWSSDK_LIBRARIES) foreach(_AWSSDK_LIB ${_AWSSDK_LIBS}) @@ -4930,15 +4922,6 @@ macro(build_awssdk) aws_c_auth_ep) add_dependencies(AWS::aws-c-s3 aws_c_s3_ep) - externalproject_add(aws_lc_ep - ${EP_COMMON_OPTIONS} - URL ${AWS_LC_SOURCE_URL} - URL_HASH "SHA256=${ARROW_AWS_LC_BUILD_SHA256_CHECKSUM}" - CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} - BUILD_BYPRODUCTS ${CRYPTO_STATIC_LIBRARY} ${SSL_STATIC_LIBRARY}) - add_dependencies(AWS::crypto aws_lc_ep) - add_dependencies(AWS::ssl aws_lc_ep) - externalproject_add(aws_crt_cpp_ep ${EP_COMMON_OPTIONS} URL ${AWS_CRT_CPP_SOURCE_URL} @@ -4956,7 +4939,6 @@ macro(build_awssdk) aws_c_s3_ep aws_c_sdkutils_ep aws_checksums_ep - aws_lc_ep s2n_ep) add_dependencies(AWS::aws-crt-cpp aws_crt_cpp_ep) @@ -4970,7 +4952,7 @@ macro(build_awssdk) ${AWS_CPP_SDK_IDENTITY_MANAGEMENT_STATIC_LIBRARY} ${AWS_CPP_SDK_S3_STATIC_LIBRARY} ${AWS_CPP_SDK_STS_STATIC_LIBRARY} - DEPENDS aws_c_event_stream_ep aws_crt_cpp_ep) + DEPENDS aws_crt_cpp_ep) add_dependencies(toolchain awssdk_ep) foreach(_AWSSDK_LIB ${_AWSSDK_LIBS}) if(${_AWSSDK_LIB} MATCHES "^aws-cpp-sdk-") diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index b846aaecce619..b5b755ce845ba 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -29,32 +29,30 @@ ARROW_AWSSDK_BUILD_VERSION=1.10.55 ARROW_AWSSDK_BUILD_SHA256_CHECKSUM=2d552fb1a84bef4a9b65e34aa7031851ed2aef5319e02cc6e4cb735c48aa30de ARROW_AWS_CHECKSUMS_BUILD_VERSION=v0.1.13 ARROW_AWS_CHECKSUMS_BUILD_SHA256_CHECKSUM=0f897686f1963253c5069a0e495b85c31635ba146cd3ac38cc2ea31eaf54694d -ARROW_AWS_C_COMMON_BUILD_VERSION=v0.8.9 -ARROW_AWS_C_COMMON_BUILD_SHA256_CHECKSUM=2f3fbaf7c38eae5a00e2a816d09b81177f93529ae8ba1b82dc8f31407565327a -ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION=v0.2.18 -ARROW_AWS_C_EVENT_STREAM_BUILD_SHA256_CHECKSUM=310ca617f713bf664e4c7485a3d42c1fb57813abd0107e49790d107def7cde4f +ARROW_AWS_CRT_CPP_BUILD_VERSION=v0.18.16 +ARROW_AWS_CRT_CPP_BUILD_SHA256_CHECKSUM=9e69bc1dc4b50871d1038aa9ff6ddeb4c9b28f7d6b5e5b1b69041ccf50a13483 ARROW_AWS_C_AUTH_BUILD_VERSION=v0.6.22 ARROW_AWS_C_AUTH_BUILD_SHA256_CHECKSUM=691a6b4418afcd3dc141351b6ad33fccd8e3ff84df0e9e045b42295d284ee14c ARROW_AWS_C_CAL_BUILD_VERSION=v0.5.20 ARROW_AWS_C_CAL_BUILD_SHA256_CHECKSUM=acc352359bd06f8597415c366cf4ec4f00d0b0da92d637039a73323dd55b6cd0 +ARROW_AWS_C_COMMON_BUILD_VERSION=v0.8.9 +ARROW_AWS_C_COMMON_BUILD_SHA256_CHECKSUM=2f3fbaf7c38eae5a00e2a816d09b81177f93529ae8ba1b82dc8f31407565327a ARROW_AWS_C_COMPRESSION_BUILD_VERSION=v0.2.16 ARROW_AWS_C_COMPRESSION_BUILD_SHA256_CHECKSUM=044b1dbbca431a07bde8255ef9ec443c300fc60d4c9408d4b862f65e496687f4 +ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION=v0.2.18 +ARROW_AWS_C_EVENT_STREAM_BUILD_SHA256_CHECKSUM=310ca617f713bf664e4c7485a3d42c1fb57813abd0107e49790d107def7cde4f ARROW_AWS_C_HTTP_BUILD_VERSION=v0.7.3 ARROW_AWS_C_HTTP_BUILD_SHA256_CHECKSUM=07e16c6bf5eba6f0dea96b6f55eae312a7c95b736f4d2e4a210000f45d8265ae ARROW_AWS_C_IO_BUILD_VERSION=v0.13.14 ARROW_AWS_C_IO_BUILD_SHA256_CHECKSUM=12b66510c3d9a4f7e9b714e9cfab2a5bf835f8b9ce2f909d20ae2a2128608c71 ARROW_AWS_C_MQTT_BUILD_VERSION=v0.8.4 ARROW_AWS_C_MQTT_BUILD_SHA256_CHECKSUM=232eeac63e72883d460c686a09b98cdd811d24579affac47c5c3f696f956773f -ARROW_AWS_C_S3_BUILD_VERSION=v0.2.3 -ARROW_AWS_C_S3_BUILD_SHA256_CHECKSUM=a00b3c9f319cd1c9aa2c3fa15098864df94b066dcba0deaccbb3caa952d902fe ARROW_AWS_C_SDKUTILS_BUILD_VERSION=v0.1.6 ARROW_AWS_C_SDKUTILS_BUILD_SHA256_CHECKSUM=8a2951344b2fb541eab1e9ca17c18a7fcbfd2aaff4cdd31d362d1fad96111b91 +ARROW_AWS_C_S3_BUILD_VERSION=v0.2.3 +ARROW_AWS_C_S3_BUILD_SHA256_CHECKSUM=a00b3c9f319cd1c9aa2c3fa15098864df94b066dcba0deaccbb3caa952d902fe ARROW_AWS_LC_BUILD_VERSION=v1.3.0 ARROW_AWS_LC_BUILD_SHA256_CHECKSUM=ae96a3567161552744fc0cae8b4d68ed88b1ec0f3d3c98700070115356da5a37 -ARROW_S2N_BUILD_VERSION=v1.3.27 -ARROW_S2N_BUILD_SHA256_CHECKSUM=58dde9979867a3209a19098ffcd748a144b2462ec81edca5baf864313ff4f589 -ARROW_AWS_CRT_CPP_BUILD_VERSION=v0.18.16 -ARROW_AWS_CRT_CPP_BUILD_SHA256_CHECKSUM=9e69bc1dc4b50871d1038aa9ff6ddeb4c9b28f7d6b5e5b1b69041ccf50a13483 ARROW_BOOST_BUILD_VERSION=1.81.0 ARROW_BOOST_BUILD_SHA256_CHECKSUM=9e0ffae35528c35f90468997bc8d99500bf179cbae355415a89a600c38e13574 ARROW_BROTLI_BUILD_VERSION=v1.0.9 @@ -105,6 +103,8 @@ ARROW_SNAPPY_BUILD_VERSION=1.1.9 ARROW_SNAPPY_BUILD_SHA256_CHECKSUM=75c1fbb3d618dd3a0483bff0e26d0a92b495bbe5059c8b4f1c962b478b6e06e7 ARROW_SUBSTRAIT_BUILD_VERSION=v0.20.0 ARROW_SUBSTRAIT_BUILD_SHA256_CHECKSUM=5ceaa559ccef29a7825b5e5d4b5e7eed384830294f08bec913feecdd903a94cf +ARROW_S2N_BUILD_VERSION=v1.3.27 +ARROW_S2N_BUILD_SHA256_CHECKSUM=58dde9979867a3209a19098ffcd748a144b2462ec81edca5baf864313ff4f589 ARROW_THRIFT_BUILD_VERSION=0.16.0 ARROW_THRIFT_BUILD_SHA256_CHECKSUM=f460b5c1ca30d8918ff95ea3eb6291b3951cf518553566088f3f2be8981f6209 ARROW_UCX_BUILD_VERSION=1.12.1 From fe07499f9900367035d1e2d704628ae331b53089 Mon Sep 17 00:00:00 2001 From: Jin Shang Date: Thu, 26 Jan 2023 03:01:44 +0000 Subject: [PATCH 06/16] fix link order --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index dfcce2a8202bf..6d6a03b5cbec1 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -4771,19 +4771,19 @@ macro(build_awssdk) aws-cpp-sdk-cognito-identity aws-cpp-sdk-s3 aws-cpp-sdk-core - aws-c-event-stream - aws-checksums - aws-c-common + aws-crt-cpp + aws-c-s3 aws-c-auth - aws-c-cal - aws-c-compression - aws-c-http - aws-c-io aws-c-mqtt - aws-c-s3 + aws-c-http + aws-c-compression aws-c-sdkutils + aws-c-event-stream + aws-c-io + aws-c-cal s2n - aws-crt-cpp) + aws-checksums + aws-c-common) set(AWSSDK_LIBRARIES) foreach(_AWSSDK_LIB ${_AWSSDK_LIBS}) # aws-c-common -> AWS-C-COMMON @@ -4883,7 +4883,7 @@ macro(build_awssdk) URL_HASH "SHA256=${ARROW_AWS_C_HTTP_BUILD_SHA256_CHECKSUM}" CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} BUILD_BYPRODUCTS ${AWS_C_HTTP_STATIC_LIBRARY} - DEPENDS aws_c_common_ep s2n_ep aws_c_io_ep) + DEPENDS aws_c_common_ep s2n_ep aws_c_io_ep aws_c_compression_ep) add_dependencies(AWS::aws-c-http aws_c_http_ep) externalproject_add(aws_c_mqtt_ep From 66d7111b47db31c765bffaa4fbd50ed68849550e Mon Sep 17 00:00:00 2001 From: Jin Shang Date: Thu, 26 Jan 2023 03:23:03 +0000 Subject: [PATCH 07/16] clean dependency; lex order --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 87 ++++++++------------- cpp/thirdparty/versions.txt | 29 +++---- 2 files changed, 47 insertions(+), 69 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 6d6a03b5cbec1..a01f3ecc6dfd1 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -433,30 +433,6 @@ else() ) endif() -if(DEFINED ENV{ARROW_AWS_C_COMMON_URL}) - set(AWS_C_COMMON_SOURCE_URL "$ENV{ARROW_AWS_C_COMMON_URL}") -else() - set_urls(AWS_C_COMMON_SOURCE_URL - "https://github.com/awslabs/aws-c-common/archive/${ARROW_AWS_C_COMMON_BUILD_VERSION}.tar.gz" - ) -endif() - -if(DEFINED ENV{ARROW_AWS_CHECKSUMS_URL}) - set(AWS_CHECKSUMS_SOURCE_URL "$ENV{ARROW_AWS_CHECKSUMS_URL}") -else() - set_urls(AWS_CHECKSUMS_SOURCE_URL - "https://github.com/awslabs/aws-checksums/archive/${ARROW_AWS_CHECKSUMS_BUILD_VERSION}.tar.gz" - ) -endif() - -if(DEFINED ENV{ARROW_AWS_C_EVENT_STREAM_URL}) - set(AWS_C_EVENT_STREAM_SOURCE_URL "$ENV{ARROW_AWS_C_EVENT_STREAM_URL}") -else() - set_urls(AWS_C_EVENT_STREAM_SOURCE_URL - "https://github.com/awslabs/aws-c-event-stream/archive/${ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION}.tar.gz" - ) -endif() - if(DEFINED ENV{ARROW_AWS_C_AUTH_URL}) set(AWS_C_AUTH_SOURCE_URL "$ENV{ARROW_AWS_C_AUTH_URL}") else() @@ -473,12 +449,29 @@ else() ) endif() +if(DEFINED ENV{ARROW_AWS_C_COMMON_URL}) + set(AWS_C_COMMON_SOURCE_URL "$ENV{ARROW_AWS_C_COMMON_URL}") +else() + set_urls(AWS_C_COMMON_SOURCE_URL + "https://github.com/awslabs/aws-c-common/archive/${ARROW_AWS_C_COMMON_BUILD_VERSION}.tar.gz" + ) +endif() + if(DEFINED ENV{ARROW_AWS_C_COMPRESSION_URL}) set(AWS_C_COMPRESSION_SOURCE_URL "$ENV{ARROW_AWS_C_COMPRESSION_URL}") else() set_urls(AWS_C_COMPRESSION_SOURCE_URL "https://github.com/awslabs/aws-c-compression/archive/${ARROW_AWS_C_COMPRESSION_BUILD_VERSION}.tar.gz" ) + + if(DEFINED ENV{ARROW_AWS_C_EVENT_STREAM_URL}) + set(AWS_C_EVENT_STREAM_SOURCE_URL "$ENV{ARROW_AWS_C_EVENT_STREAM_URL}") + else() + set_urls(AWS_C_EVENT_STREAM_SOURCE_URL + "https://github.com/awslabs/aws-c-event-stream/archive/${ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION}.tar.gz" + ) + endif() + endif() if(DEFINED ENV{ARROW_AWS_C_HTTP_URL}) @@ -521,21 +514,14 @@ else() ) endif() -if(DEFINED ENV{ARROW_AWS_LC_URL}) - set(AWS_LC_SOURCE_URL "$ENV{ARROW_AWS_LC_URL}") +if(DEFINED ENV{ARROW_AWS_CHECKSUMS_URL}) + set(AWS_CHECKSUMS_SOURCE_URL "$ENV{ARROW_AWS_CHECKSUMS_URL}") else() - set_urls(AWS_LC_SOURCE_URL - "https://github.com/awslabs/aws-lc/archive/${ARROW_AWS_LC_BUILD_VERSION}.tar.gz" + set_urls(AWS_CHECKSUMS_SOURCE_URL + "https://github.com/awslabs/aws-checksums/archive/${ARROW_AWS_CHECKSUMS_BUILD_VERSION}.tar.gz" ) endif() -if(DEFINED ENV{ARROW_S2N_URL}) - set(S2N_SOURCE_URL "$ENV{ARROW_S2N_URL}") -else() - set_urls(S2N_SOURCE_URL - "https://github.com/awslabs/s2n/archive/${ARROW_S2N_BUILD_VERSION}.tar.gz") -endif() - if(DEFINED ENV{ARROW_AWS_CRT_CPP_URL}) set(AWS_CRT_CPP_SOURCE_URL "$ENV{ARROW_AWS_CRT_CPP_URL}") else() @@ -738,6 +724,13 @@ else() "${THIRDPARTY_MIRROR_URL}/rapidjson-${ARROW_RAPIDJSON_BUILD_VERSION}.tar.gz") endif() +if(DEFINED ENV{ARROW_S2N_URL}) + set(S2N_SOURCE_URL "$ENV{ARROW_S2N_URL}") +else() + set_urls(S2N_SOURCE_URL + "https://github.com/awslabs/s2n/archive/${ARROW_S2N_BUILD_VERSION}.tar.gz") +endif() + if(DEFINED ENV{ARROW_SNAPPY_URL}) set(SNAPPY_SOURCE_URL "$ENV{ARROW_SNAPPY_URL}") else() @@ -4856,7 +4849,7 @@ macro(build_awssdk) URL_HASH "SHA256=${ARROW_AWS_C_EVENT_STREAM_BUILD_SHA256_CHECKSUM}" CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} BUILD_BYPRODUCTS ${AWS_C_EVENT_STREAM_STATIC_LIBRARY} - DEPENDS aws_checksums_ep s2n_ep aws_c_io_ep) + DEPENDS aws_checksums_ep aws_c_common_ep aws_c_io_ep) add_dependencies(AWS::aws-c-event-stream aws_c_event_stream_ep) externalproject_add(aws_c_sdkutils_ep @@ -4883,7 +4876,7 @@ macro(build_awssdk) URL_HASH "SHA256=${ARROW_AWS_C_HTTP_BUILD_SHA256_CHECKSUM}" CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} BUILD_BYPRODUCTS ${AWS_C_HTTP_STATIC_LIBRARY} - DEPENDS aws_c_common_ep s2n_ep aws_c_io_ep aws_c_compression_ep) + DEPENDS aws_c_io_ep aws_c_compression_ep) add_dependencies(AWS::aws-c-http aws_c_http_ep) externalproject_add(aws_c_mqtt_ep @@ -4892,7 +4885,7 @@ macro(build_awssdk) URL_HASH "SHA256=${ARROW_AWS_C_MQTT_BUILD_SHA256_CHECKSUM}" CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} BUILD_BYPRODUCTS ${AWS_C_MQTT_STATIC_LIBRARY} - DEPENDS aws_c_common_ep s2n_ep aws_c_io_ep aws_c_http_ep) + DEPENDS aws_c_io_ep aws_c_http_ep) add_dependencies(AWS::aws-c-mqtt aws_c_mqtt_ep) externalproject_add(aws_c_auth_ep @@ -4901,11 +4894,7 @@ macro(build_awssdk) URL_HASH "SHA256=${ARROW_AWS_C_AUTH_BUILD_SHA256_CHECKSUM}" CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} BUILD_BYPRODUCTS ${AWS_C_AUTH_STATIC_LIBRARY} - DEPENDS aws_c_common_ep - aws_c_sdkutils_ep - s2n_ep - aws_c_io_ep - aws_c_http_ep) + DEPENDS aws_c_sdkutils_ep aws_c_cal_ep aws_c_http_ep) add_dependencies(AWS::aws-c-auth aws_c_auth_ep) externalproject_add(aws_c_s3_ep @@ -4914,12 +4903,7 @@ macro(build_awssdk) URL_HASH "SHA256=${ARROW_AWS_C_S3_BUILD_SHA256_CHECKSUM}" CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} BUILD_BYPRODUCTS ${AWS_C_S3_STATIC_LIBRARY} - DEPENDS aws_c_common_ep - aws_c_cal_ep - s2n_ep - aws_c_io_ep - aws_c_http_ep - aws_c_auth_ep) + DEPENDS aws_checksums_ep aws_c_auth_ep) add_dependencies(AWS::aws-c-s3 aws_c_s3_ep) externalproject_add(aws_crt_cpp_ep @@ -4931,15 +4915,12 @@ macro(build_awssdk) DEPENDS aws_c_auth_ep aws_c_cal_ep aws_c_common_ep - aws_c_compression_ep aws_c_event_stream_ep aws_c_http_ep aws_c_io_ep aws_c_mqtt_ep aws_c_s3_ep - aws_c_sdkutils_ep - aws_checksums_ep - s2n_ep) + aws_checksums_ep) add_dependencies(AWS::aws-crt-cpp aws_crt_cpp_ep) externalproject_add(awssdk_ep diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index b5b755ce845ba..d55222186ffba 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -25,12 +25,6 @@ ARROW_ABSL_BUILD_VERSION=20211102.0 ARROW_ABSL_BUILD_SHA256_CHECKSUM=dcf71b9cba8dc0ca9940c4b316a0c796be8fab42b070bb6b7cab62b48f0e66c4 -ARROW_AWSSDK_BUILD_VERSION=1.10.55 -ARROW_AWSSDK_BUILD_SHA256_CHECKSUM=2d552fb1a84bef4a9b65e34aa7031851ed2aef5319e02cc6e4cb735c48aa30de -ARROW_AWS_CHECKSUMS_BUILD_VERSION=v0.1.13 -ARROW_AWS_CHECKSUMS_BUILD_SHA256_CHECKSUM=0f897686f1963253c5069a0e495b85c31635ba146cd3ac38cc2ea31eaf54694d -ARROW_AWS_CRT_CPP_BUILD_VERSION=v0.18.16 -ARROW_AWS_CRT_CPP_BUILD_SHA256_CHECKSUM=9e69bc1dc4b50871d1038aa9ff6ddeb4c9b28f7d6b5e5b1b69041ccf50a13483 ARROW_AWS_C_AUTH_BUILD_VERSION=v0.6.22 ARROW_AWS_C_AUTH_BUILD_SHA256_CHECKSUM=691a6b4418afcd3dc141351b6ad33fccd8e3ff84df0e9e045b42295d284ee14c ARROW_AWS_C_CAL_BUILD_VERSION=v0.5.20 @@ -47,12 +41,16 @@ ARROW_AWS_C_IO_BUILD_VERSION=v0.13.14 ARROW_AWS_C_IO_BUILD_SHA256_CHECKSUM=12b66510c3d9a4f7e9b714e9cfab2a5bf835f8b9ce2f909d20ae2a2128608c71 ARROW_AWS_C_MQTT_BUILD_VERSION=v0.8.4 ARROW_AWS_C_MQTT_BUILD_SHA256_CHECKSUM=232eeac63e72883d460c686a09b98cdd811d24579affac47c5c3f696f956773f -ARROW_AWS_C_SDKUTILS_BUILD_VERSION=v0.1.6 -ARROW_AWS_C_SDKUTILS_BUILD_SHA256_CHECKSUM=8a2951344b2fb541eab1e9ca17c18a7fcbfd2aaff4cdd31d362d1fad96111b91 ARROW_AWS_C_S3_BUILD_VERSION=v0.2.3 ARROW_AWS_C_S3_BUILD_SHA256_CHECKSUM=a00b3c9f319cd1c9aa2c3fa15098864df94b066dcba0deaccbb3caa952d902fe -ARROW_AWS_LC_BUILD_VERSION=v1.3.0 -ARROW_AWS_LC_BUILD_SHA256_CHECKSUM=ae96a3567161552744fc0cae8b4d68ed88b1ec0f3d3c98700070115356da5a37 +ARROW_AWS_C_SDKUTILS_BUILD_VERSION=v0.1.6 +ARROW_AWS_C_SDKUTILS_BUILD_SHA256_CHECKSUM=8a2951344b2fb541eab1e9ca17c18a7fcbfd2aaff4cdd31d362d1fad96111b91 +ARROW_AWS_CHECKSUMS_BUILD_VERSION=v0.1.13 +ARROW_AWS_CHECKSUMS_BUILD_SHA256_CHECKSUM=0f897686f1963253c5069a0e495b85c31635ba146cd3ac38cc2ea31eaf54694d +ARROW_AWS_CRT_CPP_BUILD_VERSION=v0.18.16 +ARROW_AWS_CRT_CPP_BUILD_SHA256_CHECKSUM=9e69bc1dc4b50871d1038aa9ff6ddeb4c9b28f7d6b5e5b1b69041ccf50a13483 +ARROW_AWSSDK_BUILD_VERSION=1.10.55 +ARROW_AWSSDK_BUILD_SHA256_CHECKSUM=2d552fb1a84bef4a9b65e34aa7031851ed2aef5319e02cc6e4cb735c48aa30de ARROW_BOOST_BUILD_VERSION=1.81.0 ARROW_BOOST_BUILD_SHA256_CHECKSUM=9e0ffae35528c35f90468997bc8d99500bf179cbae355415a89a600c38e13574 ARROW_BROTLI_BUILD_VERSION=v1.0.9 @@ -125,21 +123,19 @@ ARROW_ZSTD_BUILD_SHA256_CHECKSUM=f7de13462f7a82c29ab865820149e778cbfe01087b3a55b # given version. DEPENDENCIES=( "ARROW_ABSL_URL absl-${ARROW_ABSL_BUILD_VERSION}.tar.gz https://github.com/abseil/abseil-cpp/archive/${ARROW_ABSL_BUILD_VERSION}.tar.gz" - "ARROW_AWSSDK_URL aws-sdk-cpp-${ARROW_AWSSDK_BUILD_VERSION}.tar.gz https://github.com/aws/aws-sdk-cpp/archive/${ARROW_AWSSDK_BUILD_VERSION}.tar.gz" - "ARROW_AWS_CHECKSUMS_URL aws-checksums-${ARROW_AWS_CHECKSUMS_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-checksums/archive/${ARROW_AWS_CHECKSUMS_BUILD_VERSION}.tar.gz" - "ARROW_AWS_C_COMMON_URL aws-c-common-${ARROW_AWS_C_COMMON_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-c-common/archive/${ARROW_AWS_C_COMMON_BUILD_VERSION}.tar.gz" - "ARROW_AWS_C_EVENT_STREAM_URL aws-c-event-stream-${ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-c-event-stream/archive/${ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION}.tar.gz" "ARROW_AWS_C_AUTH_URL aws-c-auth-${ARROW_AWS_C_AUTH_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-c-auth/archive/${ARROW_AWS_C_AUTH_BUILD_VERSION}.tar.gz" "ARROW_AWS_C_CAL_URL aws-c-cal-${ARROW_AWS_C_CAL_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-c-cal/archive/${ARROW_AWS_C_CAL_BUILD_VERSION}.tar.gz" + "ARROW_AWS_C_COMMON_URL aws-c-common-${ARROW_AWS_C_COMMON_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-c-common/archive/${ARROW_AWS_C_COMMON_BUILD_VERSION}.tar.gz" "ARROW_AWS_C_COMPRESSION_URL aws-c-compression-${ARROW_AWS_C_COMPRESSION_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-c-compression/archive/${ARROW_AWS_C_COMPRESSION_BUILD_VERSION}.tar.gz" + "ARROW_AWS_C_EVENT_STREAM_URL aws-c-event-stream-${ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-c-event-stream/archive/${ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION}.tar.gz" "ARROW_AWS_C_HTTP_URL aws-c-http-${ARROW_AWS_C_HTTP_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-c-http/archive/${ARROW_AWS_C_HTTP_BUILD_VERSION}.tar.gz" "ARROW_AWS_C_IO_URL aws-c-io-${ARROW_AWS_C_IO_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-c-io/archive/${ARROW_AWS_C_IO_BUILD_VERSION}.tar.gz" "ARROW_AWS_C_MQTT_URL aws-c-mqtt-${ARROW_AWS_C_MQTT_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-c-mqtt/archive/${ARROW_AWS_C_MQTT_BUILD_VERSION}.tar.gz" "ARROW_AWS_C_S3_URL aws-c-s3-${ARROW_AWS_C_S3_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-c-s3/archive/${ARROW_AWS_C_S3_BUILD_VERSION}.tar.gz" "ARROW_AWS_C_SDKUTILS_URL aws-c-sdkutils-${ARROW_AWS_C_SDKUTILS_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-c-sdkutils/archive/${ARROW_AWS_C_SDKUTILS_BUILD_VERSION}.tar.gz" - "ARROW_AWS_LC_URL aws-lc-${ARROW_AWS_LC_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-lc/archive/${ARROW_AWS_LC_BUILD_VERSION}.tar.gz" - "ARROW_S2N_URL s2n-${ARROW_S2N_BUILD_VERSION}.tar.gz https://github.com/awslabs/s2n/archive/${ARROW_S2N_BUILD_VERSION}.tar.gz" + "ARROW_AWS_CHECKSUMS_URL aws-checksums-${ARROW_AWS_CHECKSUMS_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-checksums/archive/${ARROW_AWS_CHECKSUMS_BUILD_VERSION}.tar.gz" "ARROW_AWS_CRT_CPP_URL aws-crt-cpp-${ARROW_AWS_CRT_CPP_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-crt-cpp/archive/${ARROW_AWS_CRT_CPP_BUILD_VERSION}.tar.gz" + "ARROW_AWSSDK_URL aws-sdk-cpp-${ARROW_AWSSDK_BUILD_VERSION}.tar.gz https://github.com/aws/aws-sdk-cpp/archive/${ARROW_AWSSDK_BUILD_VERSION}.tar.gz" "ARROW_BOOST_URL boost-${ARROW_BOOST_BUILD_VERSION}.tar.gz https://apache.jfrog.io/artifactory/arrow/thirdparty/7.0.0/boost_${ARROW_BOOST_BUILD_VERSION//./_}.tar.gz" "ARROW_BROTLI_URL brotli-${ARROW_BROTLI_BUILD_VERSION}.tar.gz https://github.com/google/brotli/archive/${ARROW_BROTLI_BUILD_VERSION}.tar.gz" "ARROW_BZIP2_URL bzip2-${ARROW_BZIP2_BUILD_VERSION}.tar.gz https://sourceware.org/pub/bzip2/bzip2-${ARROW_BZIP2_BUILD_VERSION}.tar.gz" @@ -161,6 +157,7 @@ DEPENDENCIES=( "ARROW_PROTOBUF_URL protobuf-${ARROW_PROTOBUF_BUILD_VERSION}.tar.gz https://github.com/google/protobuf/releases/download/${ARROW_PROTOBUF_BUILD_VERSION}/protobuf-all-${ARROW_PROTOBUF_BUILD_VERSION:1}.tar.gz" "ARROW_RAPIDJSON_URL rapidjson-${ARROW_RAPIDJSON_BUILD_VERSION}.tar.gz https://github.com/miloyip/rapidjson/archive/${ARROW_RAPIDJSON_BUILD_VERSION}.tar.gz" "ARROW_RE2_URL re2-${ARROW_RE2_BUILD_VERSION}.tar.gz https://github.com/google/re2/archive/${ARROW_RE2_BUILD_VERSION}.tar.gz" + "ARROW_S2N_URL s2n-${ARROW_S2N_BUILD_VERSION}.tar.gz https://github.com/awslabs/s2n/archive/${ARROW_S2N_BUILD_VERSION}.tar.gz" "ARROW_SNAPPY_URL snappy-${ARROW_SNAPPY_BUILD_VERSION}.tar.gz https://github.com/google/snappy/archive/${ARROW_SNAPPY_BUILD_VERSION}.tar.gz" "ARROW_THRIFT_URL thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz https://archive.apache.org/dist/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz" "ARROW_UCX_URL ucx-${ARROW_UCX_BUILD_VERSION}.tar.gz https://github.com/openucx/ucx/archive/v${ARROW_UCX_BUILD_VERSION}.tar.gz" From e1f32b03690cb79f39e0adb4614b15a09da3b81e Mon Sep 17 00:00:00 2001 From: Jin Shang Date: Thu, 26 Jan 2023 19:19:25 +0000 Subject: [PATCH 08/16] s2n internalize libcrypto --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 9 ++++++++- cpp/src/arrow/filesystem/s3fs_test.cc | 6 +++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index a01f3ecc6dfd1..2bd1c840a2042 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -4817,11 +4817,18 @@ macro(build_awssdk) DEPENDS aws_c_common_ep) add_dependencies(AWS::aws-checksums aws_checksums_ep) + set(S2N_CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}) + # When arrow is dynamically linked, arrow-s3fs-test and libarrow.so + # each has its own copy of AWS SDK. + # Need S2N to statically link OpenSSL::Crypto and internalize it to avoid conflicts. + if(ARROW_BUILD_TESTS AND ARROW_TEST_LINKAGE STREQUAL "shared") + list(APPEND S2N_CMAKE_ARGS -DS2N_INTERN_LIBCRYPTO=ON) + endif() externalproject_add(s2n_ep ${EP_COMMON_OPTIONS} URL ${S2N_SOURCE_URL} URL_HASH "SHA256=${ARROW_S2N_BUILD_SHA256_CHECKSUM}" - CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} + CMAKE_ARGS ${S2N_CMAKE_ARGS} BUILD_BYPRODUCTS ${S2N_STATIC_LIBRARY}) add_dependencies(AWS::s2n s2n_ep) diff --git a/cpp/src/arrow/filesystem/s3fs_test.cc b/cpp/src/arrow/filesystem/s3fs_test.cc index c6957fa933ec8..6295705785580 100644 --- a/cpp/src/arrow/filesystem/s3fs_test.cc +++ b/cpp/src/arrow/filesystem/s3fs_test.cc @@ -191,7 +191,11 @@ class S3TestMixin : public AwsTestMixin { ASSERT_OK(connect_status); } - void TearDown() override { AwsTestMixin::TearDown(); } + void TearDown() override { + client_.reset(); // Aws::S3::S3Client destruction relies on AWS SDK, so it must be + // reset before Aws::ShutdownAPI + AwsTestMixin::TearDown(); + } protected: void InitServerAndClient() { From 4904021c9a2646292001b9675bdcf0d370b702f8 Mon Sep 17 00:00:00 2001 From: Jin Shang Date: Thu, 26 Jan 2023 19:34:56 +0000 Subject: [PATCH 09/16] lint cmake --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 2bd1c840a2042..af767acf99dfe 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -4818,8 +4818,7 @@ macro(build_awssdk) add_dependencies(AWS::aws-checksums aws_checksums_ep) set(S2N_CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}) - # When arrow is dynamically linked, arrow-s3fs-test and libarrow.so - # each has its own copy of AWS SDK. + # When arrow is dynamically linked, arrow-s3fs-test and libarrow.so each has its own copy of AWS SDK. # Need S2N to statically link OpenSSL::Crypto and internalize it to avoid conflicts. if(ARROW_BUILD_TESTS AND ARROW_TEST_LINKAGE STREQUAL "shared") list(APPEND S2N_CMAKE_ARGS -DS2N_INTERN_LIBCRYPTO=ON) From 9869c7a8337836ab858830ae786aea9d63265f77 Mon Sep 17 00:00:00 2001 From: Jin Shang Date: Tue, 31 Jan 2023 14:45:18 +0000 Subject: [PATCH 10/16] fix indent --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index af767acf99dfe..23d131ace4bff 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -463,15 +463,14 @@ else() set_urls(AWS_C_COMPRESSION_SOURCE_URL "https://github.com/awslabs/aws-c-compression/archive/${ARROW_AWS_C_COMPRESSION_BUILD_VERSION}.tar.gz" ) +endif() - if(DEFINED ENV{ARROW_AWS_C_EVENT_STREAM_URL}) - set(AWS_C_EVENT_STREAM_SOURCE_URL "$ENV{ARROW_AWS_C_EVENT_STREAM_URL}") - else() - set_urls(AWS_C_EVENT_STREAM_SOURCE_URL - "https://github.com/awslabs/aws-c-event-stream/archive/${ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION}.tar.gz" - ) - endif() - +if(DEFINED ENV{ARROW_AWS_C_EVENT_STREAM_URL}) + set(AWS_C_EVENT_STREAM_SOURCE_URL "$ENV{ARROW_AWS_C_EVENT_STREAM_URL}") +else() + set_urls(AWS_C_EVENT_STREAM_SOURCE_URL + "https://github.com/awslabs/aws-c-event-stream/archive/${ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION}.tar.gz" + ) endif() if(DEFINED ENV{ARROW_AWS_C_HTTP_URL}) @@ -4817,9 +4816,9 @@ macro(build_awssdk) DEPENDS aws_c_common_ep) add_dependencies(AWS::aws-checksums aws_checksums_ep) - set(S2N_CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}) # When arrow is dynamically linked, arrow-s3fs-test and libarrow.so each has its own copy of AWS SDK. # Need S2N to statically link OpenSSL::Crypto and internalize it to avoid conflicts. + set(S2N_CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}) if(ARROW_BUILD_TESTS AND ARROW_TEST_LINKAGE STREQUAL "shared") list(APPEND S2N_CMAKE_ARGS -DS2N_INTERN_LIBCRYPTO=ON) endif() From 5340bda2fcad1299697bf2e7c57388a033c73953 Mon Sep 17 00:00:00 2001 From: Jin Shang Date: Wed, 1 Feb 2023 12:26:38 +0800 Subject: [PATCH 11/16] Update cpp/cmake_modules/ThirdpartyToolchain.cmake Co-authored-by: Sutou Kouhei --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 23d131ace4bff..10c91a4ccf954 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -4890,7 +4890,7 @@ macro(build_awssdk) URL_HASH "SHA256=${ARROW_AWS_C_MQTT_BUILD_SHA256_CHECKSUM}" CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} BUILD_BYPRODUCTS ${AWS_C_MQTT_STATIC_LIBRARY} - DEPENDS aws_c_io_ep aws_c_http_ep) + DEPENDS aws_c_http_ep) add_dependencies(AWS::aws-c-mqtt aws_c_mqtt_ep) externalproject_add(aws_c_auth_ep From f3c930d971e1796f42e98d971f53bede5874de82 Mon Sep 17 00:00:00 2001 From: Jin Shang Date: Wed, 1 Feb 2023 12:26:52 +0800 Subject: [PATCH 12/16] Update cpp/cmake_modules/ThirdpartyToolchain.cmake Co-authored-by: Sutou Kouhei --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 10c91a4ccf954..8502eb6c22383 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -4854,7 +4854,7 @@ macro(build_awssdk) URL_HASH "SHA256=${ARROW_AWS_C_EVENT_STREAM_BUILD_SHA256_CHECKSUM}" CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} BUILD_BYPRODUCTS ${AWS_C_EVENT_STREAM_STATIC_LIBRARY} - DEPENDS aws_checksums_ep aws_c_common_ep aws_c_io_ep) + DEPENDS aws_checksums_ep aws_c_io_ep) add_dependencies(AWS::aws-c-event-stream aws_c_event_stream_ep) externalproject_add(aws_c_sdkutils_ep From 0f80ec8eb70bc53cc6c0a1a4b3ae63ac9b2339ef Mon Sep 17 00:00:00 2001 From: Jin Shang Date: Thu, 2 Feb 2023 20:26:31 +0000 Subject: [PATCH 13/16] change s2n to s2n-tls --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 35 ++++++++++++--------- cpp/thirdparty/versions.txt | 6 ++-- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 8502eb6c22383..f9e121ad21ce8 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -723,11 +723,11 @@ else() "${THIRDPARTY_MIRROR_URL}/rapidjson-${ARROW_RAPIDJSON_BUILD_VERSION}.tar.gz") endif() -if(DEFINED ENV{ARROW_S2N_URL}) - set(S2N_SOURCE_URL "$ENV{ARROW_S2N_URL}") +if(DEFINED ENV{ARROW_S2N_TLS_URL}) + set(S2N_TLS_SOURCE_URL "$ENV{ARROW_S2N_TLS_URL}") else() - set_urls(S2N_SOURCE_URL - "https://github.com/awslabs/s2n/archive/${ARROW_S2N_BUILD_VERSION}.tar.gz") + set_urls(S2N_TLS_SOURCE_URL + "https://github.com/aws/s2n-tls/archive/${ARROW_S2N_TLS_BUILD_VERSION}.tar.gz") endif() if(DEFINED ENV{ARROW_SNAPPY_URL}) @@ -4773,7 +4773,7 @@ macro(build_awssdk) aws-c-event-stream aws-c-io aws-c-cal - s2n + s2n-tls aws-checksums aws-c-common) set(AWSSDK_LIBRARIES) @@ -4785,6 +4785,11 @@ macro(build_awssdk) set(_AWSSDK_STATIC_LIBRARY "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${_AWSSDK_LIB}${CMAKE_STATIC_LIBRARY_SUFFIX}" ) + if(${_AWSSDK_LIB} STREQUAL "s2n-tls") # Build output of s2n-tls is libs2n.a + set(_AWSSDK_STATIC_LIBRARY + "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}s2n${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + endif() if(${_AWSSDK_LIB} MATCHES "^aws-cpp-sdk-") set(_AWSSDK_TARGET_NAME ${_AWSSDK_LIB}) else() @@ -4817,18 +4822,18 @@ macro(build_awssdk) add_dependencies(AWS::aws-checksums aws_checksums_ep) # When arrow is dynamically linked, arrow-s3fs-test and libarrow.so each has its own copy of AWS SDK. - # Need S2N to statically link OpenSSL::Crypto and internalize it to avoid conflicts. - set(S2N_CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}) + # Need S2N_TLS to statically link OpenSSL::Crypto and internalize it to avoid conflicts. + set(S2N_TLS_CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}) if(ARROW_BUILD_TESTS AND ARROW_TEST_LINKAGE STREQUAL "shared") - list(APPEND S2N_CMAKE_ARGS -DS2N_INTERN_LIBCRYPTO=ON) + list(APPEND S2N_TLS_CMAKE_ARGS -DS2N_INTERN_LIBCRYPTO=ON) endif() - externalproject_add(s2n_ep + externalproject_add(s2n_tls_ep ${EP_COMMON_OPTIONS} - URL ${S2N_SOURCE_URL} - URL_HASH "SHA256=${ARROW_S2N_BUILD_SHA256_CHECKSUM}" - CMAKE_ARGS ${S2N_CMAKE_ARGS} - BUILD_BYPRODUCTS ${S2N_STATIC_LIBRARY}) - add_dependencies(AWS::s2n s2n_ep) + URL ${S2N_TLS_SOURCE_URL} + URL_HASH "SHA256=${ARROW_S2N_TLS_BUILD_SHA256_CHECKSUM}" + CMAKE_ARGS ${S2N_TLS_CMAKE_ARGS} + BUILD_BYPRODUCTS ${S2N_TLS_STATIC_LIBRARY}) + add_dependencies(AWS::s2n-tls s2n_tls_ep) externalproject_add(aws_c_cal_ep ${EP_COMMON_OPTIONS} @@ -4845,7 +4850,7 @@ macro(build_awssdk) URL_HASH "SHA256=${ARROW_AWS_C_IO_BUILD_SHA256_CHECKSUM}" CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} BUILD_BYPRODUCTS ${AWS_C_IO_STATIC_LIBRARY} - DEPENDS aws_c_common_ep s2n_ep aws_c_cal_ep) + DEPENDS aws_c_common_ep s2n_tls_ep aws_c_cal_ep) add_dependencies(AWS::aws-c-io aws_c_io_ep) externalproject_add(aws_c_event_stream_ep diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index d55222186ffba..92509a3dd3fee 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -101,8 +101,8 @@ ARROW_SNAPPY_BUILD_VERSION=1.1.9 ARROW_SNAPPY_BUILD_SHA256_CHECKSUM=75c1fbb3d618dd3a0483bff0e26d0a92b495bbe5059c8b4f1c962b478b6e06e7 ARROW_SUBSTRAIT_BUILD_VERSION=v0.20.0 ARROW_SUBSTRAIT_BUILD_SHA256_CHECKSUM=5ceaa559ccef29a7825b5e5d4b5e7eed384830294f08bec913feecdd903a94cf -ARROW_S2N_BUILD_VERSION=v1.3.27 -ARROW_S2N_BUILD_SHA256_CHECKSUM=58dde9979867a3209a19098ffcd748a144b2462ec81edca5baf864313ff4f589 +ARROW_S2N_TLS_BUILD_VERSION=v1.3.35 +ARROW_S2N_TLS_BUILD_SHA256_CHECKSUM=9d32b26e6bfcc058d98248bf8fc231537e347395dd89cf62bb432b55c5da990d ARROW_THRIFT_BUILD_VERSION=0.16.0 ARROW_THRIFT_BUILD_SHA256_CHECKSUM=f460b5c1ca30d8918ff95ea3eb6291b3951cf518553566088f3f2be8981f6209 ARROW_UCX_BUILD_VERSION=1.12.1 @@ -157,7 +157,7 @@ DEPENDENCIES=( "ARROW_PROTOBUF_URL protobuf-${ARROW_PROTOBUF_BUILD_VERSION}.tar.gz https://github.com/google/protobuf/releases/download/${ARROW_PROTOBUF_BUILD_VERSION}/protobuf-all-${ARROW_PROTOBUF_BUILD_VERSION:1}.tar.gz" "ARROW_RAPIDJSON_URL rapidjson-${ARROW_RAPIDJSON_BUILD_VERSION}.tar.gz https://github.com/miloyip/rapidjson/archive/${ARROW_RAPIDJSON_BUILD_VERSION}.tar.gz" "ARROW_RE2_URL re2-${ARROW_RE2_BUILD_VERSION}.tar.gz https://github.com/google/re2/archive/${ARROW_RE2_BUILD_VERSION}.tar.gz" - "ARROW_S2N_URL s2n-${ARROW_S2N_BUILD_VERSION}.tar.gz https://github.com/awslabs/s2n/archive/${ARROW_S2N_BUILD_VERSION}.tar.gz" + "ARROW_S2N_TLS_URL s2n-${ARROW_S2N_TLS_BUILD_VERSION}.tar.gz https://github.com/aws/s2n-tls/archive/${ARROW_S2N_TLS_BUILD_VERSION}.tar.gz" "ARROW_SNAPPY_URL snappy-${ARROW_SNAPPY_BUILD_VERSION}.tar.gz https://github.com/google/snappy/archive/${ARROW_SNAPPY_BUILD_VERSION}.tar.gz" "ARROW_THRIFT_URL thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz https://archive.apache.org/dist/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz" "ARROW_UCX_URL ucx-${ARROW_UCX_BUILD_VERSION}.tar.gz https://github.com/openucx/ucx/archive/v${ARROW_UCX_BUILD_VERSION}.tar.gz" From 8d2eebe74d25a1a7bcd066bf9507bc0664e8163a Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 8 Feb 2023 12:25:43 +0900 Subject: [PATCH 14/16] Disable s3fs-test with bundled aws-sdk-cpp --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 8 +--- cpp/src/arrow/filesystem/CMakeLists.txt | 50 ++++++++++----------- cpp/src/arrow/filesystem/s3fs_test.cc | 14 ------ 3 files changed, 24 insertions(+), 48 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index f9e121ad21ce8..5205aaa0b3fa0 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -4821,17 +4821,11 @@ macro(build_awssdk) DEPENDS aws_c_common_ep) add_dependencies(AWS::aws-checksums aws_checksums_ep) - # When arrow is dynamically linked, arrow-s3fs-test and libarrow.so each has its own copy of AWS SDK. - # Need S2N_TLS to statically link OpenSSL::Crypto and internalize it to avoid conflicts. - set(S2N_TLS_CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}) - if(ARROW_BUILD_TESTS AND ARROW_TEST_LINKAGE STREQUAL "shared") - list(APPEND S2N_TLS_CMAKE_ARGS -DS2N_INTERN_LIBCRYPTO=ON) - endif() externalproject_add(s2n_tls_ep ${EP_COMMON_OPTIONS} URL ${S2N_TLS_SOURCE_URL} URL_HASH "SHA256=${ARROW_S2N_TLS_BUILD_SHA256_CHECKSUM}" - CMAKE_ARGS ${S2N_TLS_CMAKE_ARGS} + CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} BUILD_BYPRODUCTS ${S2N_TLS_STATIC_LIBRARY}) add_dependencies(AWS::s2n-tls s2n_tls_ep) diff --git a/cpp/src/arrow/filesystem/CMakeLists.txt b/cpp/src/arrow/filesystem/CMakeLists.txt index 6888231a35a6e..8e2960eec2097 100644 --- a/cpp/src/arrow/filesystem/CMakeLists.txt +++ b/cpp/src/arrow/filesystem/CMakeLists.txt @@ -47,7 +47,26 @@ if(ARROW_GCS) Boost::system) endif() +set(ARROW_S3_BUILD_TESTS ${ARROW_BUILD_TESTS}) if(ARROW_S3) + get_target_property(AWS_CPP_SDK_S3_TYPE aws-cpp-sdk-s3 TYPE) + # We need to initialize AWS C++ SDK for direct use (not via + # arrow::fs::S3FileSystem) in arrow-s3fs-test if we use static AWS + # C++ SDK. Because AWS C++ SDK has internal static variables that + # aren't shared in libarrow and arrow-s3fs-test. It means that + # arrow::fs::InitializeS3() doesn't initialize AWS C++ SDK that is + # directly used in arrow-s3fs-test. + # + # But it seems that internal static variables in AWS C++ SDK are + # shared on macOS even if we link static AWS C++ SDK to both + # libarrow and arrow-s3fs-test. So we don't need to initialize AWS + # C++ SDK in arrow-s3fs-test on macOS. + if(AWS_CPP_SDK_S3_TYPE STREQUAL "STATIC_LIBRARY" AND NOT APPLE) + set(ARROW_S3_BUILD_TESTS OFF) + endif() +endif() + +if(ARROW_S3_BUILD_TESTS) add_arrow_test(s3fs_test SOURCES s3fs_test.cc @@ -57,33 +76,10 @@ if(ARROW_S3) EXTRA_LINK_LIBS Boost::filesystem Boost::system) - if(TARGET arrow-s3fs-test) - set(ARROW_S3FS_TEST_COMPILE_DEFINITIONS) - get_target_property(AWS_CPP_SDK_S3_TYPE aws-cpp-sdk-s3 TYPE) - # We need to initialize AWS C++ SDK for direct use (not via - # arrow::fs::S3FileSystem) in arrow-s3fs-test if we use static AWS - # C++ SDK. Because AWS C++ SDK has internal static variables that - # aren't shared in libarrow and arrow-s3fs-test. It means that - # arrow::fs::InitializeS3() doesn't initialize AWS C++ SDK that is - # directly used in arrow-s3fs-test. - # - # But it seems that internal static variables in AWS C++ SDK are - # shared on macOS even if we link static AWS C++ SDK to both - # libarrow and arrow-s3fs-test. So we don't need to initialize AWS - # C++ SDK in arrow-s3fs-test on macOS. - if(AWS_CPP_SDK_S3_TYPE STREQUAL "STATIC_LIBRARY" AND NOT APPLE) - list(APPEND ARROW_S3FS_TEST_COMPILE_DEFINITIONS "AWS_CPP_SDK_S3_NOT_SHARED") - endif() - target_compile_definitions(arrow-s3fs-test - PRIVATE ${ARROW_S3FS_TEST_COMPILE_DEFINITIONS}) - endif() - - if(ARROW_BUILD_TESTS) - add_executable(arrow-s3fs-narrative-test s3fs_narrative_test.cc) - target_link_libraries(arrow-s3fs-narrative-test ${ARROW_TEST_LINK_LIBS} - ${GFLAGS_LIBRARIES} GTest::gtest) - add_dependencies(arrow-tests arrow-s3fs-narrative-test) - endif() + add_executable(arrow-s3fs-narrative-test s3fs_narrative_test.cc) + target_link_libraries(arrow-s3fs-narrative-test ${ARROW_TEST_LINK_LIBS} + ${GFLAGS_LIBRARIES} GTest::gtest) + add_dependencies(arrow-tests arrow-s3fs-narrative-test) if(ARROW_BUILD_BENCHMARKS AND ARROW_PARQUET) add_arrow_benchmark(s3fs_benchmark diff --git a/cpp/src/arrow/filesystem/s3fs_test.cc b/cpp/src/arrow/filesystem/s3fs_test.cc index 6295705785580..97c7f70f32ba3 100644 --- a/cpp/src/arrow/filesystem/s3fs_test.cc +++ b/cpp/src/arrow/filesystem/s3fs_test.cc @@ -151,27 +151,13 @@ class AwsTestMixin : public ::testing::Test { AwsTestMixin() : ec2_metadata_disabled_guard_("AWS_EC2_METADATA_DISABLED", "true") {} void SetUp() override { -#ifdef AWS_CPP_SDK_S3_NOT_SHARED - auto aws_log_level = Aws::Utils::Logging::LogLevel::Fatal; - aws_options_.loggingOptions.logLevel = aws_log_level; - aws_options_.loggingOptions.logger_create_fn = [&aws_log_level] { - return std::make_shared(aws_log_level); - }; - Aws::InitAPI(aws_options_); -#endif } void TearDown() override { -#ifdef AWS_CPP_SDK_S3_NOT_SHARED - Aws::ShutdownAPI(aws_options_); -#endif } private: EnvVarGuard ec2_metadata_disabled_guard_; -#ifdef AWS_CPP_SDK_S3_NOT_SHARED - Aws::SDKOptions aws_options_; -#endif }; class S3TestMixin : public AwsTestMixin { From 376264baa6feaf76fde806dfaad0fd0a92f3b36c Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 8 Feb 2023 13:51:15 +0900 Subject: [PATCH 15/16] Fix condition --- cpp/src/arrow/filesystem/CMakeLists.txt | 20 +++++++------------- cpp/src/arrow/filesystem/s3fs_test.cc | 6 ------ 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/cpp/src/arrow/filesystem/CMakeLists.txt b/cpp/src/arrow/filesystem/CMakeLists.txt index 8e2960eec2097..04b8b1e355749 100644 --- a/cpp/src/arrow/filesystem/CMakeLists.txt +++ b/cpp/src/arrow/filesystem/CMakeLists.txt @@ -47,23 +47,17 @@ if(ARROW_GCS) Boost::system) endif() -set(ARROW_S3_BUILD_TESTS ${ARROW_BUILD_TESTS}) if(ARROW_S3) + set(ARROW_S3_BUILD_TESTS ${ARROW_BUILD_TESTS}) get_target_property(AWS_CPP_SDK_S3_TYPE aws-cpp-sdk-s3 TYPE) - # We need to initialize AWS C++ SDK for direct use (not via - # arrow::fs::S3FileSystem) in arrow-s3fs-test if we use static AWS - # C++ SDK. Because AWS C++ SDK has internal static variables that - # aren't shared in libarrow and arrow-s3fs-test. It means that - # arrow::fs::InitializeS3() doesn't initialize AWS C++ SDK that is - # directly used in arrow-s3fs-test. - # - # But it seems that internal static variables in AWS C++ SDK are - # shared on macOS even if we link static AWS C++ SDK to both - # libarrow and arrow-s3fs-test. So we don't need to initialize AWS - # C++ SDK in arrow-s3fs-test on macOS. - if(AWS_CPP_SDK_S3_TYPE STREQUAL "STATIC_LIBRARY" AND NOT APPLE) + # We disable S3 filesystem tests with bundled (static linked) AWS + # C++ SDK because it's difficult to initialize underlying S3 library + # properly. + if(AWS_CPP_SDK_S3_TYPE STREQUAL "STATIC_LIBRARY") set(ARROW_S3_BUILD_TESTS OFF) endif() +else() + set(ARROW_S3_BUILD_TESTS OFF) endif() if(ARROW_S3_BUILD_TESTS) diff --git a/cpp/src/arrow/filesystem/s3fs_test.cc b/cpp/src/arrow/filesystem/s3fs_test.cc index 97c7f70f32ba3..e7de2ddd30e11 100644 --- a/cpp/src/arrow/filesystem/s3fs_test.cc +++ b/cpp/src/arrow/filesystem/s3fs_test.cc @@ -150,12 +150,6 @@ class AwsTestMixin : public ::testing::Test { // EC2 metadata endpoint AwsTestMixin() : ec2_metadata_disabled_guard_("AWS_EC2_METADATA_DISABLED", "true") {} - void SetUp() override { - } - - void TearDown() override { - } - private: EnvVarGuard ec2_metadata_disabled_guard_; }; From a9b2c3d3de20351f480245cabf72f3b31cfadd08 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 8 Feb 2023 14:14:53 +0900 Subject: [PATCH 16/16] Detect Homebrew's OpenSSL automatically --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 25 ++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 5205aaa0b3fa0..58fd1de3ed677 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -4821,11 +4821,34 @@ macro(build_awssdk) DEPENDS aws_c_common_ep) add_dependencies(AWS::aws-checksums aws_checksums_ep) + set(S2N_TLS_CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}) + if(APPLE AND NOT OPENSSL_ROOT_DIR) + find_program(BREW brew) + if(BREW) + execute_process(COMMAND ${BREW} --prefix "openssl@1.1" + OUTPUT_VARIABLE OPENSSL11_BREW_PREFIX + OUTPUT_STRIP_TRAILING_WHITESPACE) + if(OPENSSL11_BREW_PREFIX) + set(OPENSSL_ROOT_DIR ${OPENSSL11_BREW_PREFIX}) + else() + execute_process(COMMAND ${BREW} --prefix "openssl" + OUTPUT_VARIABLE OPENSSL_BREW_PREFIX + OUTPUT_STRIP_TRAILING_WHITESPACE) + if(OPENSSL_BREW_PREFIX) + set(OPENSSL_ROOT_DIR ${OPENSSL_BREW_PREFIX}) + endif() + endif() + endif() + endif() + if(OPENSSL_ROOT_DIR) + # For Findcrypto.cmake in s2n-tls. + list(APPEND S2N_TLS_CMAKE_ARGS -DCMAKE_PREFIX_PATH=${OPENSSL_ROOT_DIR}) + endif() externalproject_add(s2n_tls_ep ${EP_COMMON_OPTIONS} URL ${S2N_TLS_SOURCE_URL} URL_HASH "SHA256=${ARROW_S2N_TLS_BUILD_SHA256_CHECKSUM}" - CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} + CMAKE_ARGS ${S2N_TLS_CMAKE_ARGS} BUILD_BYPRODUCTS ${S2N_TLS_STATIC_LIBRARY}) add_dependencies(AWS::s2n-tls s2n_tls_ep)