From 9d107e9c7ea8c4406a53bc38e7249e05b7ad5c53 Mon Sep 17 00:00:00 2001 From: Peter Dillinger Date: Thu, 12 Mar 2020 13:55:11 -0700 Subject: [PATCH 1/2] Use an Amazon S3 bucket for downloading deps Summary: After we had a lot of failures with maven.org downloads, we wanted an alternative location for downloading binary dependencies. Hosting them through github would have been good in terms of organizational and network dependencies, but that approach seems to be awkward (fake releases, so would need a 'rocksdb-deps' repo) and strangely complicated for Facebook policy on open source repositories. This commit moves the downloads (that are not officially hosted by others on github) from my personal rocksdb fork to an S3 bucket owned by the Facebook RocksDB AWS account. Facebook employees can access this through an internal tool, and we should be able to grant permission to outside collaborators. Assuming this works out, I will back-port to older branches to stabilize their CI testing as well. Test Plan: CI --- .travis.yml | 2 +- java/CMakeLists.txt | 8 +++----- java/Makefile | 8 +++----- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7b3c7c13897..278e2c7c6ce 100644 --- a/.travis.yml +++ b/.travis.yml @@ -87,7 +87,7 @@ install: sudo apt-get install -y mingw-w64 ; fi - if [[ "${JOB_NAME}" == cmake* ]] && [ "${TRAVIS_OS_NAME}" == linux ]; then - CMAKE_DIST_URL="https://github.com/pdillinger/rocksdb/releases/download/cmake-3.14.5/cmake-3.14.5-Linux-$(uname -m).tar.bz2"; + CMAKE_DIST_URL="https://rocksdb-deps.s3-us-west-2.amazonaws.com/cmake/cmake-1.14.5-Linux-$(uname -m).tar.bz2"; TAR_OPT="--strip-components=1 -xj"; mkdir cmake-dist && curl --silent --fail --show-error --location "${CMAKE_DIST_URL}" | tar -C cmake-dist ${TAR_OPT} && export PATH=$PWD/cmake-dist/bin:$PATH; fi diff --git a/java/CMakeLists.txt b/java/CMakeLists.txt index dc419117172..8fc7bea9b51 100644 --- a/java/CMakeLists.txt +++ b/java/CMakeLists.txt @@ -318,11 +318,9 @@ endif() if (DEFINED CUSTOM_DEPS_URL) set(DEPS_URL ${CUSTOM_DEPS_URL}/) else () - # This is a URL for artifacts from a "fake" release on pdillinger's fork, - # so as not to put binaries in git (ew). We should move to hosting these - # under the facebook account on github, or something else more reliable - # than maven.org, which has been failing frequently from Travis. - set(DEPS_URL "https://github.com/pdillinger/rocksdb/releases/download/v6.6.x-java-deps") + # Using a Facebook AWS account for S3 storage. (maven.org has a history + # of failing in Travis builds.) + set(DEPS_URL "https://rocksdb-deps.s3-us-west-2.amazonaws.com/jars") endif() if(NOT EXISTS ${JAVA_JUNIT_JAR}) diff --git a/java/Makefile b/java/Makefile index 89e96c458fc..c2600cf98fc 100644 --- a/java/Makefile +++ b/java/Makefile @@ -217,11 +217,9 @@ ifneq ($(DEBUG_LEVEL),0) JAVAC_ARGS += -Xlint:deprecation -Xlint:unchecked endif -# This is a URL for artifacts from a "fake" release on pdillinger's fork, -# so as not to put binaries in git (ew). We should move to hosting these -# under the facebook account on github, or something else more reliable -# than maven.org, which has been failing frequently from Travis. -DEPS_URL?=https://github.com/pdillinger/rocksdb/releases/download/v6.6.x-java-deps +# Using a Facebook AWS account for S3 storage. (maven.org has a history +# of failing in Travis builds.) +DEPS_URL?=https://rocksdb-deps.s3-us-west-2.amazonaws.com/jars clean: clean-not-downloaded clean-downloaded From 4186b621aba2e446d8e04e061507cdd145050119 Mon Sep 17 00:00:00 2001 From: Peter Dillinger Date: Thu, 12 Mar 2020 15:11:21 -0700 Subject: [PATCH 2/2] Fix typo (I blame VIM - I didn't type it) --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 278e2c7c6ce..e8d0bf9724c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -87,7 +87,7 @@ install: sudo apt-get install -y mingw-w64 ; fi - if [[ "${JOB_NAME}" == cmake* ]] && [ "${TRAVIS_OS_NAME}" == linux ]; then - CMAKE_DIST_URL="https://rocksdb-deps.s3-us-west-2.amazonaws.com/cmake/cmake-1.14.5-Linux-$(uname -m).tar.bz2"; + CMAKE_DIST_URL="https://rocksdb-deps.s3-us-west-2.amazonaws.com/cmake/cmake-3.14.5-Linux-$(uname -m).tar.bz2"; TAR_OPT="--strip-components=1 -xj"; mkdir cmake-dist && curl --silent --fail --show-error --location "${CMAKE_DIST_URL}" | tar -C cmake-dist ${TAR_OPT} && export PATH=$PWD/cmake-dist/bin:$PATH; fi