From eb832b2a3244166c81d8e00d94805525f3bd7526 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 12 Aug 2019 09:41:06 +0200 Subject: [PATCH] ci: move mirrors to their standalone bucket Currently mirrors are stored in the rust-lang-ci2 S3 bucket along with CI toolchains. This is problematic for multiple reasons: - CI IAM credentials are allowed to both edit and delete those files. A malicious user gaining access to those credentials would be able to change our mirrored dependencies, possibly backdooring the compiler. - Contents of the rust-lang-ci2 bucket are disposable except for the mirrors' content. When we implement backups for S3 buckets we'd have to replicate just that part of the bucket, complicating the backup logic and increasing the chance of mistakes. A standalone bucket will be way easier to backup. This commit switches our CI to use the new rust-lang-ci-mirrors bucket. --- src/ci/azure-pipelines/auto.yml | 12 ++++++------ src/ci/azure-pipelines/steps/install-clang.yml | 2 +- src/ci/azure-pipelines/steps/install-sccache.yml | 4 ++-- .../steps/install-windows-build-deps.yml | 6 +++--- src/ci/docker/armhf-gnu/Dockerfile | 2 +- src/ci/docker/dist-various-1/install-mips-musl.sh | 2 +- src/ci/docker/dist-various-2/build-wasi-toolchain.sh | 2 +- src/ci/docker/dist-x86_64-linux/build-openssl.sh | 2 +- .../dist-x86_64-netbsd/build-netbsd-toolchain.sh | 2 +- src/ci/docker/scripts/android-sdk-manager.py | 8 +++++--- src/ci/docker/scripts/freebsd-toolchain.sh | 2 +- src/ci/docker/scripts/sccache.sh | 2 +- src/ci/install-awscli.sh | 2 +- 13 files changed, 25 insertions(+), 23 deletions(-) diff --git a/src/ci/azure-pipelines/auto.yml b/src/ci/azure-pipelines/auto.yml index 687856cca6b62..06fa3bd9f4347 100644 --- a/src/ci/azure-pipelines/auto.yml +++ b/src/ci/azure-pipelines/auto.yml @@ -273,7 +273,7 @@ jobs: MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu SCRIPT: make ci-subset-1 - MINGW_URL: https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror + MINGW_URL: https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z MINGW_DIR: mingw32 # FIXME(#59637) @@ -283,14 +283,14 @@ jobs: MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu SCRIPT: make ci-subset-2 - MINGW_URL: https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror + MINGW_URL: https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z MINGW_DIR: mingw32 x86_64-mingw-1: MSYS_BITS: 64 SCRIPT: make ci-subset-1 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu - MINGW_URL: https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror + MINGW_URL: https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z MINGW_DIR: mingw64 # FIXME(#59637) @@ -300,7 +300,7 @@ jobs: MSYS_BITS: 64 SCRIPT: make ci-subset-2 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu - MINGW_URL: https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror + MINGW_URL: https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z MINGW_DIR: mingw64 @@ -327,7 +327,7 @@ jobs: MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools --enable-profiler SCRIPT: python x.py dist - MINGW_URL: https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror + MINGW_URL: https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z MINGW_DIR: mingw32 DIST_REQUIRE_ALL_TOOLS: 1 @@ -336,7 +336,7 @@ jobs: MSYS_BITS: 64 SCRIPT: python x.py dist RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler - MINGW_URL: https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror + MINGW_URL: https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z MINGW_DIR: mingw64 DIST_REQUIRE_ALL_TOOLS: 1 diff --git a/src/ci/azure-pipelines/steps/install-clang.yml b/src/ci/azure-pipelines/steps/install-clang.yml index 45ec767e0b875..14daf81b43075 100644 --- a/src/ci/azure-pipelines/steps/install-clang.yml +++ b/src/ci/azure-pipelines/steps/install-clang.yml @@ -36,7 +36,7 @@ steps: set -e mkdir -p citools cd citools - curl -f https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/LLVM-7.0.0-win64.tar.gz | tar xzf - + curl -f https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/LLVM-7.0.0-win64.tar.gz | tar xzf - echo "##vso[task.setvariable variable=RUST_CONFIGURE_ARGS]$RUST_CONFIGURE_ARGS --set llvm.clang-cl=`pwd`/clang-rust/bin/clang-cl.exe" condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['MINGW_URL'],'')) displayName: Install clang (Windows) diff --git a/src/ci/azure-pipelines/steps/install-sccache.yml b/src/ci/azure-pipelines/steps/install-sccache.yml index 427e50f571f76..d4679c1c6733e 100644 --- a/src/ci/azure-pipelines/steps/install-sccache.yml +++ b/src/ci/azure-pipelines/steps/install-sccache.yml @@ -2,14 +2,14 @@ steps: - bash: | set -e - curl -fo /usr/local/bin/sccache https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/2018-04-02-sccache-x86_64-apple-darwin + curl -fo /usr/local/bin/sccache https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2018-04-02-sccache-x86_64-apple-darwin chmod +x /usr/local/bin/sccache displayName: Install sccache (OSX) condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) - script: | md sccache - powershell -Command "$ProgressPreference = 'SilentlyContinue'; iwr -outf sccache\sccache.exe https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/2018-04-26-sccache-x86_64-pc-windows-msvc" + powershell -Command "$ProgressPreference = 'SilentlyContinue'; iwr -outf sccache\sccache.exe https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2018-04-26-sccache-x86_64-pc-windows-msvc" echo ##vso[task.prependpath]%CD%\sccache displayName: Install sccache (Windows) condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) diff --git a/src/ci/azure-pipelines/steps/install-windows-build-deps.yml b/src/ci/azure-pipelines/steps/install-windows-build-deps.yml index c42c2311b493f..9aaeb4b79d634 100644 --- a/src/ci/azure-pipelines/steps/install-windows-build-deps.yml +++ b/src/ci/azure-pipelines/steps/install-windows-build-deps.yml @@ -4,7 +4,7 @@ steps: # https://github.com/wixtoolset/wix3 originally - bash: | set -e - curl -O https://rust-lang-ci2.s3-us-west-1.amazonaws.com/rust-ci-mirror/wix311-binaries.zip + curl -O https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/wix311-binaries.zip echo "##vso[task.setvariable variable=WIX]`pwd`/wix" mkdir -p wix/bin cd wix/bin @@ -18,7 +18,7 @@ steps: # one is MSI installers and one is EXE, but they're not used so frequently at # this point anyway so perhaps it's a wash! - script: | - powershell -Command "$ProgressPreference = 'SilentlyContinue'; iwr -outf is-install.exe https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/2017-08-22-is.exe" + powershell -Command "$ProgressPreference = 'SilentlyContinue'; iwr -outf is-install.exe https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2017-08-22-is.exe" is-install.exe /VERYSILENT /SUPPRESSMSGBOXES /NORESTART /SP- echo ##vso[task.prependpath]C:\Program Files (x86)\Inno Setup 5 displayName: Install InnoSetup @@ -109,7 +109,7 @@ steps: # Note that this is originally from the github releases patch of Ninja - script: | md ninja - powershell -Command "$ProgressPreference = 'SilentlyContinue'; iwr -outf 2017-03-15-ninja-win.zip https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/2017-03-15-ninja-win.zip" + powershell -Command "$ProgressPreference = 'SilentlyContinue'; iwr -outf 2017-03-15-ninja-win.zip https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2017-03-15-ninja-win.zip" 7z x -oninja 2017-03-15-ninja-win.zip del 2017-03-15-ninja-win.zip set RUST_CONFIGURE_ARGS=%RUST_CONFIGURE_ARGS% --enable-ninja diff --git a/src/ci/docker/armhf-gnu/Dockerfile b/src/ci/docker/armhf-gnu/Dockerfile index 235920833f839..9493b33698708 100644 --- a/src/ci/docker/armhf-gnu/Dockerfile +++ b/src/ci/docker/armhf-gnu/Dockerfile @@ -72,7 +72,7 @@ RUN arm-linux-gnueabihf-gcc addentropy.c -o rootfs/addentropy -static # TODO: What is this?! # Source of the file: https://github.com/vfdev-5/qemu-rpi2-vexpress/raw/master/vexpress-v2p-ca15-tc1.dtb -RUN curl -O https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/vexpress-v2p-ca15-tc1.dtb +RUN curl -O https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/vexpress-v2p-ca15-tc1.dtb COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh diff --git a/src/ci/docker/dist-various-1/install-mips-musl.sh b/src/ci/docker/dist-various-1/install-mips-musl.sh index 60a96e3b8e952..29cfb5d96083e 100755 --- a/src/ci/docker/dist-various-1/install-mips-musl.sh +++ b/src/ci/docker/dist-various-1/install-mips-musl.sh @@ -5,7 +5,7 @@ mkdir /usr/local/mips-linux-musl # originally from # https://downloads.openwrt.org/snapshots/trunk/ar71xx/generic/ # OpenWrt-Toolchain-ar71xx-generic_gcc-5.3.0_musl-1.1.16.Linux-x86_64.tar.bz2 -URL="https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror" +URL="https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc" FILE="OpenWrt-Toolchain-ar71xx-generic_gcc-5.3.0_musl-1.1.16.Linux-x86_64.tar.bz2" curl -L "$URL/$FILE" | tar xjf - -C /usr/local/mips-linux-musl --strip-components=2 diff --git a/src/ci/docker/dist-various-2/build-wasi-toolchain.sh b/src/ci/docker/dist-various-2/build-wasi-toolchain.sh index 7bf8946c4f136..f04ee78157167 100755 --- a/src/ci/docker/dist-various-2/build-wasi-toolchain.sh +++ b/src/ci/docker/dist-various-2/build-wasi-toolchain.sh @@ -5,7 +5,7 @@ set -ex # Originally from https://releases.llvm.org/8.0.0/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz -curl https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/clang%2Bllvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz | \ +curl https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/clang%2Bllvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz | \ tar xJf - export PATH=`pwd`/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/bin:$PATH diff --git a/src/ci/docker/dist-x86_64-linux/build-openssl.sh b/src/ci/docker/dist-x86_64-linux/build-openssl.sh index 13dae6169053a..be8a6c93945e9 100755 --- a/src/ci/docker/dist-x86_64-linux/build-openssl.sh +++ b/src/ci/docker/dist-x86_64-linux/build-openssl.sh @@ -4,7 +4,7 @@ set -ex source shared.sh VERSION=1.0.2k -URL=https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/openssl-$VERSION.tar.gz +URL=https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/openssl-$VERSION.tar.gz curl $URL | tar xzf - diff --git a/src/ci/docker/dist-x86_64-netbsd/build-netbsd-toolchain.sh b/src/ci/docker/dist-x86_64-netbsd/build-netbsd-toolchain.sh index 2e9b9dcc2344e..797f674b954f2 100755 --- a/src/ci/docker/dist-x86_64-netbsd/build-netbsd-toolchain.sh +++ b/src/ci/docker/dist-x86_64-netbsd/build-netbsd-toolchain.sh @@ -25,7 +25,7 @@ cd netbsd mkdir -p /x-tools/x86_64-unknown-netbsd/sysroot -URL=https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror +URL=https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc # Originally from ftp://ftp.netbsd.org/pub/NetBSD/NetBSD-$BSD/source/sets/*.tgz curl $URL/2018-03-01-netbsd-src.tgz | tar xzf - diff --git a/src/ci/docker/scripts/android-sdk-manager.py b/src/ci/docker/scripts/android-sdk-manager.py index 7c9a8b82e9282..c9e2961f6eb15 100755 --- a/src/ci/docker/scripts/android-sdk-manager.py +++ b/src/ci/docker/scripts/android-sdk-manager.py @@ -23,8 +23,9 @@ HOST_OS = "linux" # Mirroring options -MIRROR_BUCKET = "rust-lang-ci2" -MIRROR_BASE_DIR = "rust-ci-mirror/android/" +MIRROR_BUCKET = "rust-lang-ci-mirrors" +MIRROR_BUCKET_REGION = "us-west-1" +MIRROR_BASE_DIR = "rustc/android/" import argparse import hashlib @@ -144,7 +145,8 @@ def cli_install(args): lockfile = Lockfile(args.lockfile) for package in lockfile.packages.values(): # Download the file from the mirror into a temp file - url = "https://" + MIRROR_BUCKET + ".s3.amazonaws.com/" + MIRROR_BASE_DIR + url = "https://" + MIRROR_BUCKET + ".s3-" + MIRROR_BUCKET_REGION + \ + ".amazonaws.com/" + MIRROR_BASE_DIR downloaded = package.download(url) # Extract the file in a temporary directory extract_dir = tempfile.mkdtemp() diff --git a/src/ci/docker/scripts/freebsd-toolchain.sh b/src/ci/docker/scripts/freebsd-toolchain.sh index 8cef69d9c26bb..70155e770a960 100755 --- a/src/ci/docker/scripts/freebsd-toolchain.sh +++ b/src/ci/docker/scripts/freebsd-toolchain.sh @@ -59,7 +59,7 @@ done # Originally downloaded from: # https://download.freebsd.org/ftp/releases/${freebsd_arch}/${freebsd_version}-RELEASE/base.txz -URL=https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/2019-04-04-freebsd-${freebsd_arch}-${freebsd_version}-RELEASE-base.txz +URL=https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2019-04-04-freebsd-${freebsd_arch}-${freebsd_version}-RELEASE-base.txz curl "$URL" | tar xJf - -C "$sysroot" --wildcards "${files_to_extract[@]}" # Fix up absolute symlinks from the system image. This can be removed diff --git a/src/ci/docker/scripts/sccache.sh b/src/ci/docker/scripts/sccache.sh index 194de3c339f8c..efeb0ed0d72d0 100644 --- a/src/ci/docker/scripts/sccache.sh +++ b/src/ci/docker/scripts/sccache.sh @@ -1,6 +1,6 @@ set -ex curl -fo /usr/local/bin/sccache \ - https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/2018-04-02-sccache-x86_64-unknown-linux-musl + https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2018-04-02-sccache-x86_64-unknown-linux-musl chmod +x /usr/local/bin/sccache diff --git a/src/ci/install-awscli.sh b/src/ci/install-awscli.sh index d491b9fbcdcf8..69c8d2e3099ab 100755 --- a/src/ci/install-awscli.sh +++ b/src/ci/install-awscli.sh @@ -16,7 +16,7 @@ set -euo pipefail IFS=$'\n\t' -MIRROR="https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/2019-07-27-awscli.tar" +MIRROR="https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2019-07-27-awscli.tar" DEPS_DIR="/tmp/awscli-deps" pip="pip"