diff --git a/cmake/Modules/FindAWSSDK_EP.cmake b/cmake/Modules/FindAWSSDK_EP.cmake index c276b17185c..1769e26fd3a 100644 --- a/cmake/Modules/FindAWSSDK_EP.cmake +++ b/cmake/Modules/FindAWSSDK_EP.cmake @@ -58,7 +58,6 @@ endif() if (NOT AWSSDK_FOUND) if (TILEDB_SUPERBUILD) - message(STATUS "Could NOT find AWSSDK") message(STATUS "Adding AWSSDK as an external project") set(DEPENDS) @@ -96,19 +95,19 @@ if (NOT AWSSDK_FOUND) if (WIN32) find_package(Git REQUIRED) - set(CONDITIONAL_PATCH cd ${CMAKE_SOURCE_DIR} && ${GIT_EXECUTABLE} apply --ignore-whitespace -p1 --unsafe-paths --verbose --directory=${TILEDB_EP_SOURCE_DIR}/ep_awssdk < ${TILEDB_CMAKE_INPUTS_DIR}/patches/ep_awssdk/awsccommon.patch && - ${GIT_EXECUTABLE} apply --ignore-whitespace -p1 --unsafe-paths --verbose --directory=${TILEDB_EP_SOURCE_DIR}/ep_awssdk < ${TILEDB_CMAKE_INPUTS_DIR}/patches/ep_awssdk/awsconfig_cmake_3.22.patch) + set(CONDITIONAL_PATCH cd ${CMAKE_SOURCE_DIR} && ${GIT_EXECUTABLE} apply --ignore-whitespace -p1 --unsafe-paths --verbose --directory=${TILEDB_EP_SOURCE_DIR}/ep_awssdk < ${TILEDB_CMAKE_INPUTS_DIR}/patches/ep_awssdk/aws_event_loop_windows.v1.9.291.patch) else() - set(CONDITIONAL_PATCH patch -N -p1 < ${TILEDB_CMAKE_INPUTS_DIR}/patches/ep_awssdk/awsccommon.patch && - patch -N -p1 < ${TILEDB_CMAKE_INPUTS_DIR}/patches/ep_awssdk/awsconfig_cmake_3.22.patch) + set(CONDITIONAL_PATCH "") endif() ExternalProject_Add(ep_awssdk PREFIX "externals" # Set download name to avoid collisions with only the version number in the filename DOWNLOAD_NAME ep_awssdk.zip - URL "https://github.com/aws/aws-sdk-cpp/archive/1.8.84.zip" - URL_HASH SHA1=e32a53a01c75ca7fdfe9feed9c5bbcedd98708e3 + GIT_REPOSITORY + "https://github.com/aws/aws-sdk-cpp" + GIT_TAG 1.9.291 + GIT_SUBMODULES_RECURSE ON #may be default, specifying for certainty. PATCH_COMMAND ${CONDITIONAL_PATCH} CMAKE_ARGS @@ -151,15 +150,37 @@ endif () if (AWSSDK_FOUND) set(AWS_SERVICES s3) - AWSSDK_DETERMINE_LIBS_TO_LINK(AWS_SERVICES AWS_LINKED_LIBS) - list(APPEND AWS_LINKED_LIBS aws-c-common - aws-c-event-stream - aws-checksums - aws-cpp-sdk-sts - aws-cpp-sdk-identity-management) - + # append these libs in an order manually, by trial and error, determined to support successful + # linking of dependencies on *nix. (windows doesn't seem to care or 'just lucked out'.) + list(APPEND AWS_LINKED_LIBS + aws-cpp-sdk-s3 + aws-cpp-sdk-core + aws-crt-cpp + aws-c-event-stream + aws-c-mqtt + aws-c-s3 + aws-c-auth + aws-c-http + aws-c-compression + aws-checksums + aws-c-sdkutils + aws-c-io + aws-c-cal + aws-cpp-sdk-identity-management + aws-cpp-sdk-sts + aws-c-common + ) + if (NOT WIN32) + list(APPEND AWS_LINKED_LIBS + s2n # There is a libs2n.a generated and in install/lib path... but not an INTERFACE? hmm... + ) + endif() + set(TILEDB_AWS_LINK_TARGETS "") + message(STATUS "findaws, processing AWS_LINKED_LIBS (${AWS_LINKED_LIBS})") foreach (LIB ${AWS_LINKED_LIBS}) - if (NOT ${LIB} MATCHES "aws-*") + if ((NOT ${LIB} MATCHES "aws-*" ) + AND (NOT LIB STREQUAL "s2n") ) + message(STATUS "AWS_LINKED_LIBS, skipping ${LIB}") continue() endif() @@ -168,7 +189,12 @@ if (AWSSDK_FOUND) PATHS ${AWSSDK_LIB_DIR} ${TILEDB_DEPS_NO_DEFAULT_PATH} ) - message(STATUS "Found AWS lib: ${LIB} (${AWS_FOUND_${LIB}})") + if ("${AWS_FOUND_${LIB}}" MATCHES ".*-NOTFOUND") + message(STATUS "not found, ${LIB}") + continue() + else() + message(STATUS "for ${LIB} adding link to ${AWS_FOUND_${LIB}}") + endif() if (NOT TARGET AWSSDK::${LIB}) add_library(AWSSDK::${LIB} UNKNOWN IMPORTED) set_target_properties(AWSSDK::${LIB} PROPERTIES @@ -177,6 +203,8 @@ if (AWSSDK_FOUND) ) endif() + list(APPEND TILEDB_AWS_LINK_TARGETS "AWSSDK::${LIB}") + # If we built a static EP, install it if required. if (TILEDB_AWSSDK_EP_BUILT AND TILEDB_INSTALL_STATIC_DEPS) install_target_libs(AWSSDK::${LIB}) @@ -184,9 +212,11 @@ if (AWSSDK_FOUND) endforeach () + message(STATUS "findaws, TILEDB_AWS_LINK_TARGETS is ${TILEDB_AWS_LINK_TARGETS}") + # the AWSSDK does not include links to some transitive dependencies # ref: githubcom/awsaws-sdk-cpp/issues/1074#issuecomment-466252911 if (WIN32) - list(APPEND AWS_EXTRA_LIBS userenv ws2_32 wininet winhttp bcrypt version) + list(APPEND AWS_EXTRA_LIBS userenv ws2_32 wininet winhttp bcrypt version crypt32 secur32 Ncrypt) endif() endif () diff --git a/cmake/inputs/patches/ep_awssdk/aws_event_loop_windows.v1.9.291.patch b/cmake/inputs/patches/ep_awssdk/aws_event_loop_windows.v1.9.291.patch new file mode 100644 index 00000000000..d61665c6c35 --- /dev/null +++ b/cmake/inputs/patches/ep_awssdk/aws_event_loop_windows.v1.9.291.patch @@ -0,0 +1,21 @@ +diff --git "a/crt/aws-crt-cpp/crt/aws-c-io/source/event_loop.c" "b/crt/aws-crt-cpp/crt/aws-c-io/source/event_loop.c" +index 5eb8a084ac..e46cec1dfd 100644 +--- "a/crt/aws-crt-cpp/crt/aws-c-io/source/event_loop.c" ++++ "b/crt/aws-crt-cpp/crt/aws-c-io/source/event_loop.c" +@@ -70,9 +70,16 @@ static void s_aws_event_loop_group_shutdown_async(struct aws_event_loop_group *e + thread_options.cpu_id = -1; + thread_options.join_strategy = AWS_TJS_MANAGED; + ++ /* ++ * tiledb encountered issue similar to https://github.com/huggingface/datasets/issues/3310#issuecomment-1155285967. ++ */ ++#if 1 ++ aws_thread_launch(&cleanup_thread, s_event_loop_destroy_async_thread_fn, el_group, &thread_options); ++#else + AWS_FATAL_ASSERT( + aws_thread_launch(&cleanup_thread, s_event_loop_destroy_async_thread_fn, el_group, &thread_options) == + AWS_OP_SUCCESS); ++#endif + } + + static struct aws_event_loop_group *s_event_loop_group_new( diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index 1c41021d581..f7e52a0da69 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -466,14 +466,8 @@ if (TILEDB_S3) message(STATUS "The TileDB library is compiled with S3 support.") find_package(AWSSDK_EP REQUIRED COMPONENTS s3) target_link_libraries(TILEDB_CORE_OBJECTS_ILIB - INTERFACE - AWSSDK::aws-cpp-sdk-s3 - AWSSDK::aws-cpp-sdk-core - AWSSDK::aws-c-event-stream - AWSSDK::aws-checksums - AWSSDK::aws-c-common - AWSSDK::aws-cpp-sdk-identity-management - AWSSDK::aws-cpp-sdk-sts) + INTERFACE ${TILEDB_AWS_LINK_TARGETS} + ) add_definitions(-DHAVE_S3) endif() @@ -625,6 +619,7 @@ if(APPLE) if(TILEDB_S3) # this is a transitive dependency for statically linking S3 SDK target_link_libraries(TILEDB_CORE_OBJECTS_ILIB INTERFACE "-framework CoreFoundation") + target_link_libraries(TILEDB_CORE_OBJECTS_ILIB INTERFACE "-framework Security") endif() endif() @@ -846,13 +841,6 @@ if (TILEDB_STATIC) append_dep_lib(OpenSSL::SSL) append_dep_lib(OpenSSL::Crypto) append_dep_lib(CURL::libcurl) - append_dep_lib(AWSSDK::aws-cpp-sdk-s3) - append_dep_lib(AWSSDK::aws-cpp-sdk-core) - append_dep_lib(AWSSDK::aws-c-event-stream) - append_dep_lib(AWSSDK::aws-checksums) - append_dep_lib(AWSSDK::aws-c-common) - append_dep_lib(AWSSDK::aws-cpp-sdk-sts) - append_dep_lib(AWSSDK::aws-cpp-sdk-identity-management) append_dep_lib(AzureSDK::AzureSDK) append_dep_lib(GCSSDK::storage_client) append_dep_lib(GCSSDK::google_cloud_cpp_common) @@ -860,6 +848,9 @@ if (TILEDB_STATIC) append_dep_lib(spdlog::spdlog) append_dep_lib(libmagic) append_dep_lib(WebP::webp) + foreach(AWSTARG ${TILEDB_AWS_LINK_TARGETS}) + append_dep_lib(${AWSTARG}) + endforeach() endif() ############################################################