Skip to content

Commit 6225262

Browse files
borsbors
committed
Auto merge of #90457 - rusticstuff:use-curl-ca-bundle, r=Mark-Simulacrum
CI: Use ubuntu image to download openssl, curl sources, cacert.pem for x86 dist builds The dist-x86_64 and dist-i686 docker builds are failing again (see [try build](https://github.com/rust-lang-ci/rust/runs/4060836540?check_suite_focus=true)) because python.org renewed its certificate with a CA cert that is too new for debian:6. In order to solve this once and for all this PR moves the curl and openssl downloads to a new ubuntu:20.04 "stage-0" docker build and copies the downloaded tarballs over to build them in the "stage-1" debian:6 context. It also downloads the cacert.pem file from the curl website and uses it by pointing the `CURL_CA_BUNDLE` environment variable to it.
2 parents db062de + aef51a0 commit 6225262

File tree

7 files changed

+56
-68
lines changed

7 files changed

+56
-68
lines changed

‎src/ci/docker/host-x86_64/dist-i686-linux/Dockerfile

+19-8
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,15 @@
1+
# We need recent curl, OpenSSL and CA certificates, so we can download further
2+
# dependencies in the debian:6 image. We use an ubuntu 20.04 image download
3+
# those.
4+
FROM ubuntu:20.04
5+
RUN apt-get update && \
6+
apt-get install -y --no-install-recommends \
7+
curl \
8+
ca-certificates
9+
WORKDIR /tmp
10+
COPY host-x86_64/dist-x86_64-linux/download-openssl-curl.sh /tmp/
11+
RUN ./download-openssl-curl.sh
12+
113
# We use Debian 6 (glibc 2.11, kernel 2.6.32) as a common base for other
214
# distros that still need Rust support: RHEL 6 (glibc 2.12, kernel 2.6.32) and
315
# SLES 11 SP4 (glibc 2.11, kernel 3.0).
@@ -14,8 +26,6 @@ RUN apt-get update && \
1426
apt-get install --allow-unauthenticated -y --no-install-recommends \
1527
automake \
1628
bzip2 \
17-
ca-certificates \
18-
curl \
1929
file \
2030
g++ \
2131
g++-multilib \
@@ -34,11 +44,6 @@ RUN apt-get update && \
3444
xz-utils \
3545
zlib1g-dev
3646

37-
# Install new Let's Encrypt root CA certificate and remove the expired one.
38-
COPY host-x86_64/shared/ISRG_Root_X1.crt /usr/local/share/ca-certificates/ISRG_Root_X1.crt
39-
RUN sed -i '/mozilla\/DST_Root_CA_X3\.crt/d' /etc/ca-certificates.conf
40-
RUN /usr/sbin/update-ca-certificates
41-
4247
ENV PATH=/rustroot/bin:$PATH
4348
ENV LD_LIBRARY_PATH=/rustroot/lib64:/rustroot/lib32:/rustroot/lib
4449
ENV PKG_CONFIG_PATH=/rustroot/lib/pkgconfig
@@ -50,6 +55,7 @@ COPY host-x86_64/dist-x86_64-linux/shared.sh /tmp/
5055
# static.rust-lang.org. This'll be used to link into libcurl below (and used
5156
# later as well), so build a copy of OpenSSL with dynamic libraries into our
5257
# generic root.
58+
COPY --from=0 /tmp/openssl.tar.gz /tmp/openssl.tar.gz
5359
COPY host-x86_64/dist-x86_64-linux/build-openssl.sh /tmp/
5460
RUN ./build-openssl.sh
5561

@@ -59,8 +65,13 @@ RUN ./build-openssl.sh
5965
#
6066
# Note that we also disable a bunch of optional features of curl that we don't
6167
# really need.
68+
COPY --from=0 /tmp/curl.tar.xz /tmp/curl.tar.xz
6269
COPY host-x86_64/dist-x86_64-linux/build-curl.sh /tmp/
63-
RUN ./build-curl.sh && apt-get remove -y curl
70+
RUN ./build-curl.sh
71+
72+
# Use up-to-date curl CA bundle
73+
COPY --from=0 /tmp/cacert.pem /tmp/cacert.pem
74+
ENV CURL_CA_BUNDLE /tmp/cacert.pem
6475

6576
# binutils < 2.22 has a bug where the 32-bit executables it generates
6677
# immediately segfault in Rust, so we need to install our own binutils.

‎src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile

+19-8
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,15 @@
1+
# We need recent curl, OpenSSL and CA certificates, so we can download further
2+
# dependencies in the debian:6 image. We use an ubuntu 20.04 image download
3+
# those.
4+
FROM ubuntu:20.04
5+
RUN apt-get update && \
6+
apt-get install -y --no-install-recommends \
7+
curl \
8+
ca-certificates
9+
WORKDIR /tmp
10+
COPY host-x86_64/dist-x86_64-linux/download-openssl-curl.sh /tmp/
11+
RUN ./download-openssl-curl.sh
12+
113
# We use Debian 6 (glibc 2.11, kernel 2.6.32) as a common base for other
214
# distros that still need Rust support: RHEL 6 (glibc 2.12, kernel 2.6.32) and
315
# SLES 11 SP4 (glibc 2.11, kernel 3.0).
@@ -14,8 +26,6 @@ RUN apt-get update && \
1426
apt-get install --allow-unauthenticated -y --no-install-recommends \
1527
automake \
1628
bzip2 \
17-
ca-certificates \
18-
curl \
1929
file \
2030
g++ \
2131
g++-multilib \
@@ -34,11 +44,6 @@ RUN apt-get update && \
3444
xz-utils \
3545
zlib1g-dev
3646

37-
# Install new Let's Encrypt root CA certificate and remove the expired one.
38-
COPY host-x86_64/shared/ISRG_Root_X1.crt /usr/local/share/ca-certificates/ISRG_Root_X1.crt
39-
RUN sed -i '/mozilla\/DST_Root_CA_X3\.crt/d' /etc/ca-certificates.conf
40-
RUN /usr/sbin/update-ca-certificates
41-
4247
ENV PATH=/rustroot/bin:$PATH
4348
ENV LD_LIBRARY_PATH=/rustroot/lib64:/rustroot/lib32:/rustroot/lib
4449
ENV PKG_CONFIG_PATH=/rustroot/lib/pkgconfig
@@ -50,6 +55,7 @@ COPY host-x86_64/dist-x86_64-linux/shared.sh /tmp/
5055
# static.rust-lang.org. This'll be used to link into libcurl below (and used
5156
# later as well), so build a copy of OpenSSL with dynamic libraries into our
5257
# generic root.
58+
COPY --from=0 /tmp/openssl.tar.gz /tmp/openssl.tar.gz
5359
COPY host-x86_64/dist-x86_64-linux/build-openssl.sh /tmp/
5460
RUN ./build-openssl.sh
5561

@@ -59,8 +65,13 @@ RUN ./build-openssl.sh
5965
#
6066
# Note that we also disable a bunch of optional features of curl that we don't
6167
# really need.
68+
COPY --from=0 /tmp/curl.tar.xz /tmp/curl.tar.xz
6269
COPY host-x86_64/dist-x86_64-linux/build-curl.sh /tmp/
63-
RUN ./build-curl.sh && apt-get remove -y curl
70+
RUN ./build-curl.sh
71+
72+
# Use up-to-date curl CA bundle
73+
COPY --from=0 /tmp/cacert.pem /tmp/cacert.pem
74+
ENV CURL_CA_BUNDLE /tmp/cacert.pem
6475

6576
# binutils < 2.22 has a bug where the 32-bit executables it generates
6677
# immediately segfault in Rust, so we need to install our own binutils.

‎src/ci/docker/host-x86_64/dist-x86_64-linux/build-curl.sh

+3-10
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,11 @@
33
set -ex
44
source shared.sh
55

6-
VERSION=7.66.0
7-
8-
# This needs to be downloaded directly from S3, it can't go through the CDN.
9-
# That's because the CDN is backed by CloudFront, which requires SNI and TLSv1
10-
# (without paying an absurd amount of money).
11-
curl https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/curl-$VERSION.tar.xz \
12-
| xz --decompress \
13-
| tar xf -
6+
tar xJf curl.tar.xz
147

158
mkdir curl-build
169
cd curl-build
17-
hide_output ../curl-$VERSION/configure \
10+
hide_output ../curl-*/configure \
1811
--prefix=/rustroot \
1912
--with-ssl=/rustroot \
2013
--disable-sspi \
@@ -35,4 +28,4 @@ hide_output make install
3528

3629
cd ..
3730
rm -rf curl-build
38-
rm -rf curl-$VERSION
31+
rm -rf curl-*

‎src/ci/docker/host-x86_64/dist-x86_64-linux/build-openssl.sh

+3-10
Original file line numberDiff line numberDiff line change
@@ -3,21 +3,14 @@
33
set -ex
44
source shared.sh
55

6-
VERSION=1.0.2k
6+
tar xzf openssl.tar.gz
77

8-
# This needs to be downloaded directly from S3, it can't go through the CDN.
9-
# That's because the CDN is backed by CloudFront, which requires SNI and TLSv1
10-
# (without paying an absurd amount of money).
11-
URL=https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/openssl-$VERSION.tar.gz
12-
13-
curl $URL | tar xzf -
14-
15-
cd openssl-$VERSION
8+
cd openssl-*
169
hide_output ./config --prefix=/rustroot shared -fPIC
1710
hide_output make -j$(nproc)
1811
hide_output make install
1912
cd ..
20-
rm -rf openssl-$VERSION
13+
rm -rf openssl-*
2114

2215
# Make the system cert collection available to the new install.
2316
ln -nsf /etc/pki/tls/cert.pem /rustroot/ssl/

‎src/ci/docker/host-x86_64/dist-x86_64-linux/download-openssl-curl.sh

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#!/usr/bin/env bash
2+
3+
set -ex
4+
5+
OPENSSL_VERSION=1.0.2k
6+
CURL_VERSION=7.66.0
7+
8+
curl -f https://ci-mirrors.rust-lang.org/rustc/openssl-$OPENSSL_VERSION.tar.gz -o openssl.tar.gz
9+
curl -f https://ci-mirrors.rust-lang.org/rustc/curl-$CURL_VERSION.tar.xz -o curl.tar.xz
10+
curl -f https://curl.se/ca/cacert.pem -o cacert.pem

‎src/ci/docker/host-x86_64/shared/ISRG_Root_X1.crt

-31
This file was deleted.

‎src/ci/docker/run.sh

+2-1
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,8 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then
5050
# Look for all source files involves in the COPY command
5151
copied_files=/tmp/.docker-copied-files.txt
5252
rm -f "$copied_files"
53-
for i in $(sed -n -e 's/^COPY \(.*\) .*$/\1/p' "$docker_dir/$image/Dockerfile"); do
53+
for i in $(sed -n -e '/^COPY --from=/! s/^COPY \(.*\) .*$/\1/p' \
54+
"$docker_dir/$image/Dockerfile"); do
5455
# List the file names
5556
find "$script_dir/$i" -type f >> $copied_files
5657
done

0 commit comments

Comments
 (0)