Fix bootstrap issue with git on MSYS + Experimental CI testing

.github/workflows/ci.yml

@@ -50,24 +50,61 @@ jobs:
     if: "github.event_name == 'pull_request'"
     continue-on-error: "${{ matrix.name == 'mingw-check-tidy' }}"
     strategy:
+      fail-fast: false
       matrix:
         include:
-          - name: mingw-check
-            os: ubuntu-20.04-4core-16gb
-            env: {}
-          - name: mingw-check-tidy
+          - name: i686-mingw
+            env:
+              RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu"
+              SCRIPT: make ci-mingw
+              NO_DOWNLOAD_CI_LLVM: 1
+              CUSTOM_MINGW: 1
+            os: windows-2019-8core-32gb
+          - name: x86_64-mingw
+            env:
+              SCRIPT: make ci-mingw
+              RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-profiler"
+              NO_DOWNLOAD_CI_LLVM: 1
+              CUSTOM_MINGW: 1
+            os: windows-2019-8core-32gb
+          - name: dist-i686-mingw
+            env:
+              RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu --enable-full-tools --enable-profiler"
+              NO_DOWNLOAD_CI_LLVM: 1
+              SCRIPT: python x.py dist bootstrap --include-default-paths
+              CUSTOM_MINGW: 1
+              DIST_REQUIRE_ALL_TOOLS: 1
+            os: windows-2019-8core-32gb
+          - name: dist-x86_64-mingw
+            env:
+              SCRIPT: python x.py dist bootstrap --include-default-paths
+              RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler"
+              NO_DOWNLOAD_CI_LLVM: 1
+              CUSTOM_MINGW: 1
+              DIST_REQUIRE_ALL_TOOLS: 1
+            os: windows-2019-8core-32gb
+          - name: x86_64-gnu
             os: ubuntu-20.04-4core-16gb
             env: {}
-          - name: x86_64-gnu-llvm-16
+          - name: x86_64-msvc
             env:
-              ENABLE_GCC_CODEGEN: "1"
-            os: ubuntu-20.04-16core-64gb
-          - name: x86_64-gnu-tools
-            os: ubuntu-20.04-16core-64gb
-            env: {}
+              RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-profiler"
+              SCRIPT: make ci-msvc
+            os: windows-2019-8core-32gb
+    defaults:
+      run:
+        shell: "${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}"
     timeout-minutes: 600
     runs-on: "${{ matrix.os }}"
     steps:
+      - if: "contains(matrix.os, 'windows')"
+        uses: msys2/setup-msys2@v2.22.0
+        with:
+          msystem: "${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}"
+          update: false
+          release: true
+          path-type: inherit
+          install: "make dos2unix diffutils\n"
       - name: disable git crlf conversion
         run: git config --global core.autocrlf false
       - name: checkout the source code
@@ -459,9 +496,20 @@ jobs:
               RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-extended --enable-profiler"
               SCRIPT: python x.py dist bootstrap --include-default-paths
             os: windows-2019-8core-32gb
+    defaults:
+      run:
+        shell: "${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}"
     timeout-minutes: 600
     runs-on: "${{ matrix.os }}"
     steps:
+      - if: "contains(matrix.os, 'windows')"
+        uses: msys2/setup-msys2@v2.22.0
+        with:
+          msystem: "${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}"
+          update: false
+          release: true
+          path-type: inherit
+          install: "make dos2unix diffutils\n"
       - name: disable git crlf conversion
         run: git config --global core.autocrlf false
       - name: checkout the source code
@@ -587,9 +635,20 @@ jobs:
             env:
               CODEGEN_BACKENDS: "llvm,cranelift"
             os: ubuntu-20.04-16core-64gb
+    defaults:
+      run:
+        shell: "${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}"
     timeout-minutes: 600
     runs-on: "${{ matrix.os }}"
     steps:
+      - if: "contains(matrix.os, 'windows')"
+        uses: msys2/setup-msys2@v2.22.0
+        with:
+          msystem: "${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}"
+          update: false
+          release: true
+          path-type: inherit
+          install: "make dos2unix diffutils\n"
       - name: disable git crlf conversion
         run: git config --global core.autocrlf false
       - name: checkout the source code

INSTALL.md

@@ -145,10 +145,9 @@ toolchain.
 
 1. Download the latest [MSYS2 installer][msys2] and go through the installer.
 
-2. Run `mingw32_shell.bat` or `mingw64_shell.bat` from the MSYS2 installation
-   directory (e.g. `C:\msys64`), depending on whether you want 32-bit or 64-bit
-   Rust. (As of the latest version of MSYS2 you have to run `msys2_shell.cmd
-   -mingw32` or `msys2_shell.cmd -mingw64` from the command line instead.)
+2. Start a MINGW64 or MINGW32 shell (depending on whether you want 32-bit
+   or 64-bit Rust) either from your start menu, or by running `mingw64.exe`
+   or `mingw32.exe` from your MSYS2 installation directory (e.g. `C:\msys64`).
 
 3. From this terminal, install the required tools:
 

src/bootstrap/src/core/config/config.rs

@@ -1227,22 +1227,31 @@ impl Config {
         // Infer the rest of the configuration.
 
         // Infer the source directory. This is non-trivial because we want to support a downloaded bootstrap binary,
-        // running on a completely machine from where it was compiled.
+        // running on a completely different machine from where it was compiled.
         let mut cmd = Command::new("git");
-        // NOTE: we cannot support running from outside the repository because the only path we have available
-        // is set at compile time, which can be wrong if bootstrap was downloaded from source.
+        // NOTE: we cannot support running from outside the repository because the only other path we have available
+        // is set at compile time, which can be wrong if bootstrap was downloaded rather than compiled locally.
         // We still support running outside the repository if we find we aren't in a git directory.
-        cmd.arg("rev-parse").arg("--show-toplevel");
+
+        // NOTE: We get a relative path from git to work around an issue on MSYS/mingw. If we used an absolute path,
+        // and end up using MSYS's git rather than git-for-windows, we would get a unix-y MSYS path. But as bootstrap
+        // has already been (kinda-cross-)compiled to Windows land, we require a normal Windows path.
+        cmd.arg("rev-parse").arg("--show-cdup");
         // Discard stderr because we expect this to fail when building from a tarball.
         let output = cmd
             .stderr(std::process::Stdio::null())
             .output()
             .ok()
             .and_then(|output| if output.status.success() { Some(output) } else { None });
         if let Some(output) = output {
-            let git_root = String::from_utf8(output.stdout).unwrap();
-            // We need to canonicalize this path to make sure it uses backslashes instead of forward slashes.
-            let git_root = PathBuf::from(git_root.trim()).canonicalize().unwrap();
+            let git_root_relative = String::from_utf8(output.stdout).unwrap();
+            // We need to canonicalize this path to make sure it uses backslashes instead of forward slashes,
+            // and to resolve any relative components.
+            let git_root = env::current_dir()
+                .unwrap()
+                .join(PathBuf::from(git_root_relative.trim()))
+                .canonicalize()
+                .unwrap();
             let s = git_root.to_str().unwrap();
 
             // Bootstrap is quite bad at handling /? in front of paths

src/ci/github-actions/ci.yml

@@ -111,10 +111,32 @@ x--expand-yaml-anchors--remove:
     if: success() && !env.SKIP_JOB
 
   - &base-ci-job
+    defaults:
+      run:
+        shell: ${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}
     timeout-minutes: 600
     runs-on: "${{ matrix.os }}"
-    env: *shared-ci-variables
+    env:
+      <<: *shared-ci-variables
     steps:
+      - if: contains(matrix.os, 'windows')
+        uses: msys2/setup-msys2@v2.22.0
+        with:
+          # i686 jobs use mingw32. x86_64 and cross-compile jobs use mingw64.
+          msystem: ${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}
+          # don't try to download updates for already installed packages
+          update: false
+          # don't try to use the msys that comes built-in to the github runner,
+          # so we can control what is installed (i.e. not python)
+          release: true
+          # Inherit the full path from the Windows environment, with MSYS2's */bin/
+          # dirs placed in front. This lets us run Windows-native Python etc.
+          path-type: inherit
+          install: >
+            make
+            dos2unix
+            diffutils
+
       - name: disable git crlf conversion
         run: git config --global core.autocrlf false
 
@@ -328,21 +350,67 @@ jobs:
     if: github.event_name == 'pull_request'
     continue-on-error: ${{ matrix.name == 'mingw-check-tidy' }}
     strategy:
+      fail-fast: false
       matrix:
         include:
-          - name: mingw-check
-            <<: *job-linux-4c
+          - name: i686-mingw
+            env:
+              RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
+              SCRIPT: make ci-mingw
+              # We are intentionally allowing an old toolchain on this builder (and that's
+              # incompatible with LLVM downloads today).
+              NO_DOWNLOAD_CI_LLVM: 1
+              CUSTOM_MINGW: 1
+            <<: *job-windows-8c
 
-          - name: mingw-check-tidy
-            <<: *job-linux-4c
+          - name: x86_64-mingw
+            env:
+              SCRIPT: make ci-mingw
+              RUST_CONFIGURE_ARGS: >-
+                --build=x86_64-pc-windows-gnu
+                --enable-profiler
+              # We are intentionally allowing an old toolchain on this builder (and that's
+              # incompatible with LLVM downloads today).
+              NO_DOWNLOAD_CI_LLVM: 1
+              CUSTOM_MINGW: 1
+            <<: *job-windows-8c
+
+          - name: dist-i686-mingw
+            env:
+              RUST_CONFIGURE_ARGS: >-
+                --build=i686-pc-windows-gnu
+                --enable-full-tools
+                --enable-profiler
+              # We are intentionally allowing an old toolchain on this builder (and that's
+              # incompatible with LLVM downloads today).
+              NO_DOWNLOAD_CI_LLVM: 1
+              SCRIPT: python x.py dist bootstrap --include-default-paths
+              CUSTOM_MINGW: 1
+              DIST_REQUIRE_ALL_TOOLS: 1
+            <<: *job-windows-8c
 
-          - name: x86_64-gnu-llvm-16
+          - name: dist-x86_64-mingw
             env:
-              ENABLE_GCC_CODEGEN: "1"
-            <<: *job-linux-16c
+              SCRIPT: python x.py dist bootstrap --include-default-paths
+              RUST_CONFIGURE_ARGS: >-
+                --build=x86_64-pc-windows-gnu
+                --enable-full-tools
+                --enable-profiler
+              # We are intentionally allowing an old toolchain on this builder (and that's
+              # incompatible with LLVM downloads today).
+              NO_DOWNLOAD_CI_LLVM: 1
+              CUSTOM_MINGW: 1
+              DIST_REQUIRE_ALL_TOOLS: 1
+            <<: *job-windows-8c
 
-          - name: x86_64-gnu-tools
-            <<: *job-linux-16c
+          - name: x86_64-gnu
+            <<: *job-linux-4c
+
+          - name: x86_64-msvc
+            env:
+              RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
+              SCRIPT: make ci-msvc
+            <<: *job-windows-8c
 
   auto:
     <<: *base-ci-job

src/ci/run.sh

@@ -76,7 +76,7 @@ RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set dist.compression-profile=balance
 # the LLVM build, as not to run out of memory.
 # This is an attempt to fix the spurious build error tracked by
 # https://github.com/rust-lang/rust/issues/108227.
-if isWindows && [[ ${CUSTOM_MINGW-0} -eq 1 ]]; then
+if isKnownToBeMingwBuild; then
     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set llvm.link-jobs=1"
 fi
 

src/ci/scripts/install-clang.sh

@@ -37,8 +37,7 @@ if isMacOS; then
     # Configure `AR` specifically so rustbuild doesn't try to infer it as
     # `clang-ar` by accident.
     ciCommandSetEnv AR "ar"
-elif isWindows && [[ ${CUSTOM_MINGW-0} -ne 1 ]]; then
-
+elif isWindows && ! isKnownToBeMingwBuild; then
     # If we're compiling for MSVC then we, like most other distribution builders,
     # switch to clang as the compiler. This'll allow us eventually to enable LTO
     # amongst LLVM and rustc. Note that we only do this on MSVC as I don't think

src/ci/scripts/install-mingw.sh

@@ -38,11 +38,12 @@ if isWindows; then
             ;;
     esac
 
-    if [[ "${CUSTOM_MINGW-0}" -ne 1 ]]; then
-        pacman -S --noconfirm --needed mingw-w64-$arch-toolchain mingw-w64-$arch-cmake \
-            mingw-w64-$arch-gcc \
-            mingw-w64-$arch-python # the python package is actually for python3
-        ciCommandAddPath "$(ciCheckoutPath)/msys2/mingw${bits}/bin"
+    if [[ "${CUSTOM_MINGW-0}" -eq 0 ]]; then
+        pacboy -S --noconfirm toolchain:p
+        # Maybe even:
+        # pacboy -S --noconfirm clang:p ...
+        # It kinda works, for the opposite CI jobs that gcc works for.
+        # The windows part of install-clang.sh has comments pertaining to this.
     else
         mingw_dir="mingw${bits}"
 

src/ci/scripts/install-msys2.sh

@@ -1,17 +1,12 @@
 #!/bin/bash
-# Download and install MSYS2, needed primarily for the test suite (run-make) but
-# also used by the MinGW toolchain for assembling things.
+# Clean up and prepare the MSYS2 installation. MSYS2 is needed primarily for
+# the test suite (run-make), but is also used by the MinGW toolchain for assembling things.
 
 set -euo pipefail
 IFS=$'\n\t'
 
 source "$(cd "$(dirname "$0")" && pwd)/../shared.sh"
-
 if isWindows; then
-    msys2Path="c:/msys64"
-    mkdir -p "${msys2Path}/home/${USERNAME}"
-    ciCommandAddPath "${msys2Path}/usr/bin"
-
     # Detect the native Python version installed on the agent. On GitHub
     # Actions, the C:\hostedtoolcache\windows\Python directory contains a
     # subdirectory for each installed Python version.
@@ -29,4 +24,33 @@ if isWindows; then
     fi
     ciCommandAddPath "C:\\hostedtoolcache\\windows\\Python\\${native_python_version}\\x64"
     ciCommandAddPath "C:\\hostedtoolcache\\windows\\Python\\${native_python_version}\\x64\\Scripts"
+
+    # Install pacboy for easily installing packages
+    pacman -S --noconfirm pactoys
+
+    # Delete these pre-installed tools so we can't accidentally use them, because we are using the
+    # MSYS2 setup action versions instead.
+    # Delete pre-installed version of MSYS2
+    rm -r "/c/msys64/"
+    # Delete Strawberry Perl, which contains a version of mingw
+    rm -r "/c/Strawberry/"
+    # Delete these other copies of mingw, I don't even know where they come from.
+    rm -r "/c/mingw64/"
+    rm -r "/c/mingw32/"
+
+    if isKnownToBeMingwBuild; then
+        # Use the mingw version of CMake for mingw builds.
+        # However, the MSVC build needs native CMake, as it fails with the mingw one.
+        # Delete native CMake
+        rm -r "/c/Program Files/CMake/"
+        # Install mingw-w64-$arch-cmake
+        pacboy -S --noconfirm cmake:p
+
+        # We use Git-for-Windows for MSVC builds, and MSYS2 Git for mingw builds,
+        # so that both are tested.
+        # Delete Windows-Git
+        rm -r "/c/Program Files/Git/"
+        # Install MSYS2 git
+        pacman -S --noconfirm git
+    fi
 fi

src/ci/shared.sh

@@ -52,6 +52,10 @@ function isLinux {
     [[ "${OSTYPE}" = "linux-gnu" ]]
 }
 
+function isKnownToBeMingwBuild {
+    isGitHubActions && [[ "${CI_JOB_NAME}" == *mingw ]]
+}
+
 function isCiBranch {
     if [[ $# -ne 1 ]]; then
         echo "usage: $0 <branch-name>"