diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7eb41fec36aba..11b115cb2ff24 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -391,24 +391,28 @@ jobs:
             env:
               RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu --set llvm.allow-old-toolchain"
               SCRIPT: make ci-mingw-subset-1
+              NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
             os: windows-latest-xl
           - name: i686-mingw-2
             env:
               RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu --set llvm.allow-old-toolchain"
               SCRIPT: make ci-mingw-subset-2
+              NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
             os: windows-latest-xl
           - name: x86_64-mingw-1
             env:
               SCRIPT: make ci-mingw-subset-1
               RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-profiler --set llvm.allow-old-toolchain"
+              NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
             os: windows-latest-xl
           - name: x86_64-mingw-2
             env:
               SCRIPT: make ci-mingw-subset-2
               RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-profiler --set llvm.allow-old-toolchain"
+              NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
             os: windows-latest-xl
           - name: dist-x86_64-msvc
@@ -433,6 +437,7 @@ jobs:
           - name: dist-i686-mingw
             env:
               RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu --enable-full-tools --enable-profiler --set llvm.allow-old-toolchain"
+              NO_DOWNLOAD_CI_LLVM: 1
               SCRIPT: python x.py dist
               CUSTOM_MINGW: 1
               DIST_REQUIRE_ALL_TOOLS: 1
@@ -441,6 +446,7 @@ jobs:
             env:
               SCRIPT: python x.py dist
               RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler --set llvm.allow-old-toolchain"
+              NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
               DIST_REQUIRE_ALL_TOOLS: 1
             os: windows-latest-xl
diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs
index 56a0c1b2d9158..13e3049f2c81f 100644
--- a/src/bootstrap/config.rs
+++ b/src/bootstrap/config.rs
@@ -990,7 +990,7 @@ impl Config {
             config.llvm_from_ci = match llvm.download_ci_llvm {
                 Some(StringOrBool::String(s)) => {
                     assert!(s == "if-available", "unknown option `{}` for download-ci-llvm", s);
-                    crate::native::is_ci_llvm_available(&config)
+                    crate::native::is_ci_llvm_available(&config, llvm_assertions.unwrap_or(false))
                 }
                 Some(StringOrBool::Bool(b)) => b,
                 None => false,
diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs
index 6a33b264015f3..7ecf74d3068d0 100644
--- a/src/bootstrap/native.rs
+++ b/src/bootstrap/native.rs
@@ -148,7 +148,7 @@ pub(crate) fn detect_llvm_sha(config: &crate::config::Config) -> String {
 /// This checks both the build triple platform to confirm we're usable at all,
 /// and then verifies if the current HEAD matches the detected LLVM SHA head,
 /// in which case LLVM is indicated as not available.
-pub(crate) fn is_ci_llvm_available(config: &crate::config::Config) -> bool {
+pub(crate) fn is_ci_llvm_available(config: &crate::config::Config, asserts: bool) -> bool {
     // This is currently all tier 1 targets and tier 2 targets with host tools
     // (since others may not have CI artifacts)
     // https://doc.rust-lang.org/rustc/platform-support.html#tier-1
@@ -187,6 +187,12 @@ pub(crate) fn is_ci_llvm_available(config: &crate::config::Config) -> bool {
         return false;
     }
 
+    let triple = &*config.build.triple;
+    if (triple == "aarch64-unknown-linux-gnu" || triple.contains("i686")) && asserts {
+        // No alt builder for aarch64-unknown-linux-gnu today.
+        return false;
+    }
+
     if crate::util::CiEnv::is_ci() {
         let llvm_sha = detect_llvm_sha(config);
         let head_sha = output(config.git().arg("rev-parse").arg("HEAD"));
diff --git a/src/ci/docker/host-x86_64/arm-android/Dockerfile b/src/ci/docker/host-x86_64/arm-android/Dockerfile
index a2335687ab335..d0d367b39b493 100644
--- a/src/ci/docker/host-x86_64/arm-android/Dockerfile
+++ b/src/ci/docker/host-x86_64/arm-android/Dockerfile
@@ -29,6 +29,10 @@ ENV PATH=$PATH:/android/sdk/platform-tools
 
 ENV TARGETS=arm-linux-androideabi
 
+# We are intentionally allowing an old toolchain on this builder (and that's
+# incompatible with LLVM downloads today).
+ENV NO_DOWNLOAD_CI_LLVM 1
+
 ENV RUST_CONFIGURE_ARGS --arm-linux-androideabi-ndk=/android/ndk/arm-14 \
     --set llvm.allow-old-toolchain
 
diff --git a/src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile b/src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile
index 1f8d0a64ea48e..71ea13071bd1f 100644
--- a/src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile
+++ b/src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile
@@ -27,6 +27,10 @@ RUN mkdir -p /config
 RUN echo "[rust]" > /config/nopt-std-config.toml
 RUN echo "optimize = false" >> /config/nopt-std-config.toml
 
+# We are intentionally allowing an old toolchain on this builder (and that's
+# incompatible with LLVM downloads today).
+ENV NO_DOWNLOAD_CI_LLVM 1
+
 ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu --disable-optimize-tests \
     --set llvm.allow-old-toolchain
 ENV SCRIPT python3 ../x.py test --stage 0 --config /config/nopt-std-config.toml library/std \
diff --git a/src/ci/docker/host-x86_64/i686-gnu/Dockerfile b/src/ci/docker/host-x86_64/i686-gnu/Dockerfile
index 7bca0398dea7c..ebeab3dbdb799 100644
--- a/src/ci/docker/host-x86_64/i686-gnu/Dockerfile
+++ b/src/ci/docker/host-x86_64/i686-gnu/Dockerfile
@@ -23,6 +23,9 @@ RUN sh /scripts/sccache.sh
 COPY scripts/cmake.sh /scripts/
 RUN /scripts/cmake.sh
 
+# We are intentionally allowing an old toolchain on this builder (and that's
+# incompatible with LLVM downloads today).
+ENV NO_DOWNLOAD_CI_LLVM 1
 ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu \
     --set llvm.allow-old-toolchain
 # Exclude some tests that are unlikely to be platform specific, to speed up
diff --git a/src/ci/docker/host-x86_64/test-various/Dockerfile b/src/ci/docker/host-x86_64/test-various/Dockerfile
index 4d554a2852a09..b75e2f085cd3b 100644
--- a/src/ci/docker/host-x86_64/test-various/Dockerfile
+++ b/src/ci/docker/host-x86_64/test-various/Dockerfile
@@ -30,6 +30,10 @@ WORKDIR /
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 
+# We are disabling CI LLVM since this builder needs to build LLD, which is
+# currently unsupported when downloading pre-built LLVM.
+ENV NO_DOWNLOAD_CI_LLVM 1
+
 ENV RUST_CONFIGURE_ARGS \
   --musl-root-x86_64=/usr/local/x86_64-linux-musl \
   --set build.nodejs=/node-v15.14.0-linux-x64/bin/node \
diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile
index bfc6975c19dc9..321b3d6ace486 100644
--- a/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile
@@ -26,6 +26,10 @@ RUN sh /scripts/sccache.sh
 COPY scripts/cmake.sh /scripts/
 RUN /scripts/cmake.sh
 
+# We are intentionally allowing an old toolchain on this builder (and that's
+# incompatible with LLVM downloads today).
+ENV NO_DOWNLOAD_CI_LLVM 1
+
 ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu \
     --set llvm.allow-old-toolchain
 ENV RUST_CHECK_TARGET check-aux
diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile
index 13d440423b20b..739045248fea2 100644
--- a/src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile
@@ -31,6 +31,9 @@ RUN sh /scripts/sccache.sh
 ENV RUSTBUILD_FORCE_CLANG_BASED_TESTS 1
 ENV RUN_CHECK_WITH_PARALLEL_QUERIES 1
 
+# llvm.use-linker conflicts with downloading CI LLVM
+ENV NO_DOWNLOAD_CI_LLVM 1
+
 ENV RUST_CONFIGURE_ARGS \
       --build=x86_64-unknown-linux-gnu \
       --enable-debug \
diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile
index 4bb4042cd7e2f..be4def85f14c4 100644
--- a/src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile
@@ -22,6 +22,10 @@ RUN sh /scripts/sccache.sh
 COPY scripts/cmake.sh /scripts/
 RUN /scripts/cmake.sh
 
+# We are intentionally allowing an old toolchain on this builder (and that's
+# incompatible with LLVM downloads today).
+ENV NO_DOWNLOAD_CI_LLVM 1
+
 ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu --set rust.ignore-git=false \
     --set llvm.allow-old-toolchain
 ENV SCRIPT python3 ../x.py --stage 2 test distcheck
diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile
index 21891c1244d7c..23f2215c2d93c 100644
--- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile
@@ -26,7 +26,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 
-# using llvm-link-shared due to libffi issues -- see #34486
+# We are disabling CI LLVM since this builder is intentionally using a host
+# LLVM, rather than the typical src/llvm-project LLVM.
+ENV NO_DOWNLOAD_CI_LLVM 1
+
+# Using llvm-link-shared due to libffi issues -- see #34486
 ENV RUST_CONFIGURE_ARGS \
     --build=x86_64-unknown-linux-gnu \
     --llvm-root=/usr/lib/llvm-13 \
diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13/Dockerfile
index 5f3af2f47caec..8f6831bc54e63 100644
--- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13/Dockerfile
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13/Dockerfile
@@ -37,7 +37,11 @@ RUN apt-get update && \
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 
-# using llvm-link-shared due to libffi issues -- see #34486
+# We are disabling CI LLVM since this builder is intentionally using a host
+# LLVM, rather than the typical src/llvm-project LLVM.
+ENV NO_DOWNLOAD_CI_LLVM 1
+
+# Using llvm-link-shared due to libffi issues -- see #34486
 ENV RUST_CONFIGURE_ARGS \
       --build=x86_64-unknown-linux-gnu \
       --llvm-root=/usr/lib/llvm-13 \
diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile
index f442a477a7b1e..e2ec4f275001b 100644
--- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile
@@ -80,6 +80,10 @@ COPY host-x86_64/x86_64-gnu-tools/browser-ui-test.version /tmp/
 # the local version of the package is different than the one used by the CI.
 RUN npm install -g browser-ui-test@$(head -n 1 /tmp/browser-ui-test.version) --unsafe-perm=true
 
+# We are intentionally allowing an old toolchain on this builder (and that's
+# incompatible with LLVM downloads today).
+ENV NO_DOWNLOAD_CI_LLVM 1
+
 ENV RUST_CONFIGURE_ARGS \
   --set llvm.allow-old-toolchain \
   --build=x86_64-unknown-linux-gnu \
diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml
index 91204eb0b0572..a6bdd9f3186b1 100644
--- a/src/ci/github-actions/ci.yml
+++ b/src/ci/github-actions/ci.yml
@@ -600,6 +600,9 @@ jobs:
                 --build=i686-pc-windows-gnu
                 --set llvm.allow-old-toolchain
               SCRIPT: make ci-mingw-subset-1
+              # We are intentionally allowing an old toolchain on this builder (and that's
+              # incompatible with LLVM downloads today).
+              NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
             <<: *job-windows-xl
 
@@ -609,6 +612,9 @@ jobs:
                 --build=i686-pc-windows-gnu
                 --set llvm.allow-old-toolchain
               SCRIPT: make ci-mingw-subset-2
+              # We are intentionally allowing an old toolchain on this builder (and that's
+              # incompatible with LLVM downloads today).
+              NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
             <<: *job-windows-xl
 
@@ -619,6 +625,9 @@ jobs:
                 --build=x86_64-pc-windows-gnu
                 --enable-profiler
                 --set llvm.allow-old-toolchain
+              # We are intentionally allowing an old toolchain on this builder (and that's
+              # incompatible with LLVM downloads today).
+              NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
             <<: *job-windows-xl
 
@@ -629,6 +638,9 @@ jobs:
                 --build=x86_64-pc-windows-gnu
                 --enable-profiler
                 --set llvm.allow-old-toolchain
+              # We are intentionally allowing an old toolchain on this builder (and that's
+              # incompatible with LLVM downloads today).
+              NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
             <<: *job-windows-xl
 
@@ -678,6 +690,9 @@ jobs:
                 --enable-full-tools
                 --enable-profiler
                 --set llvm.allow-old-toolchain
+              # We are intentionally allowing an old toolchain on this builder (and that's
+              # incompatible with LLVM downloads today).
+              NO_DOWNLOAD_CI_LLVM: 1
               SCRIPT: python x.py dist
               CUSTOM_MINGW: 1
               DIST_REQUIRE_ALL_TOOLS: 1
@@ -691,6 +706,9 @@ jobs:
                 --enable-full-tools
                 --enable-profiler
                 --set llvm.allow-old-toolchain
+              # We are intentionally allowing an old toolchain on this builder (and that's
+              # incompatible with LLVM downloads today).
+              NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
               DIST_REQUIRE_ALL_TOOLS: 1
             <<: *job-windows-xl
diff --git a/src/ci/run.sh b/src/ci/run.sh
index 6545475d99641..091ccf58e2b81 100755
--- a/src/ci/run.sh
+++ b/src/ci/run.sh
@@ -102,6 +102,18 @@ else
   fi
 
   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir"
+
+  # We enable this for non-dist builders, since those aren't trying to produce
+  # fresh binaries. We currently don't entirely support distributing a fresh
+  # copy of the compiler (including llvm tools, etc.) if we haven't actually
+  # built LLVM, since not everything necessary is copied into the
+  # local-usage-only LLVM artifacts. If that changes, this could maybe be made
+  # true for all builds. In practice it's probably a good idea to keep building
+  # LLVM continuously on at least some builders to ensure it works, though.
+  # (And PGO is its own can of worms).
+  if [ "$NO_DOWNLOAD_CI_LLVM" = "" ]; then
+    RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set llvm.download-ci-llvm=if-available"
+  fi
 fi
 
 if [ "$RUST_RELEASE_CHANNEL" = "nightly" ] || [ "$DIST_REQUIRE_ALL_TOOLS" = "" ]; then
diff --git a/src/ci/scripts/install-clang.sh b/src/ci/scripts/install-clang.sh
index 0bc8a0389a8d4..02b72625d6eb5 100755
--- a/src/ci/scripts/install-clang.sh
+++ b/src/ci/scripts/install-clang.sh
@@ -61,6 +61,10 @@ elif isWindows && [[ ${CUSTOM_MINGW-0} -ne 1 ]]; then
     7z x -oclang-rust/ "LLVM-${LLVM_VERSION}-win64.exe"
     ciCommandSetEnv RUST_CONFIGURE_ARGS \
         "${RUST_CONFIGURE_ARGS} --set llvm.clang-cl=$(pwd)/clang-rust/bin/clang-cl.exe"
+
+    # Disable downloading CI LLVM on this builder;
+    # setting up clang-cl just above conflicts with the default if-available option.
+    ciCommandSetEnv NO_DOWNLOAD_CI_LLVM 1
 fi
 
 if isWindows; then