Changes to CI related to mingw and MSYS

Author: majaha

.github/workflows/ci.yml

@@ -65,9 +65,20 @@ jobs:
           - name: x86_64-gnu-tools
             os: ubuntu-20.04-16core-64gb
             env: {}
+    defaults:
+      run:
+        shell: "${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}"
     timeout-minutes: 600
     runs-on: "${{ matrix.os }}"
     steps:
+      - if: "contains(matrix.os, 'windows')"
+        uses: msys2/setup-msys2@v2.22.0
+        with:
+          msystem: "${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}"
+          update: false
+          release: true
+          path-type: inherit
+          install: "make dos2unix diffutils\n"
       - name: disable git crlf conversion
         run: git config --global core.autocrlf false
       - name: checkout the source code
@@ -459,9 +470,20 @@ jobs:
               RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-extended --enable-profiler"
               SCRIPT: python x.py dist bootstrap --include-default-paths
             os: windows-2019-8core-32gb
+    defaults:
+      run:
+        shell: "${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}"
     timeout-minutes: 600
     runs-on: "${{ matrix.os }}"
     steps:
+      - if: "contains(matrix.os, 'windows')"
+        uses: msys2/setup-msys2@v2.22.0
+        with:
+          msystem: "${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}"
+          update: false
+          release: true
+          path-type: inherit
+          install: "make dos2unix diffutils\n"
       - name: disable git crlf conversion
         run: git config --global core.autocrlf false
       - name: checkout the source code
@@ -587,9 +609,20 @@ jobs:
             env:
               CODEGEN_BACKENDS: "llvm,cranelift"
             os: ubuntu-20.04-16core-64gb
+    defaults:
+      run:
+        shell: "${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}"
     timeout-minutes: 600
     runs-on: "${{ matrix.os }}"
     steps:
+      - if: "contains(matrix.os, 'windows')"
+        uses: msys2/setup-msys2@v2.22.0
+        with:
+          msystem: "${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}"
+          update: false
+          release: true
+          path-type: inherit
+          install: "make dos2unix diffutils\n"
       - name: disable git crlf conversion
         run: git config --global core.autocrlf false
       - name: checkout the source code

src/ci/github-actions/ci.yml

@@ -111,10 +111,31 @@ x--expand-yaml-anchors--remove:
     if: success() && !env.SKIP_JOB
 
   - &base-ci-job
+    defaults:
+      run:
+        shell: ${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}
     timeout-minutes: 600
     runs-on: "${{ matrix.os }}"
     env: *shared-ci-variables
     steps:
+      - if: contains(matrix.os, 'windows')
+        uses: msys2/setup-msys2@v2.22.0
+        with:
+          # i686 jobs use mingw32. x86_64 and cross-compile jobs use mingw64.
+          msystem: ${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}
+          # don't try to download updates for already installed packages
+          update: false
+          # don't try to use the msys that comes built-in to the github runner,
+          # so we can control what is installed (i.e. not python)
+          release: true
+          # Inherit the full path from the Windows environment, with MSYS2's */bin/
+          # dirs placed in front. This lets us run Windows-native Python etc.
+          path-type: inherit
+          install: >
+            make
+            dos2unix
+            diffutils
+
       - name: disable git crlf conversion
         run: git config --global core.autocrlf false
 

src/ci/run.sh

@@ -76,7 +76,7 @@ RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set dist.compression-profile=balance
 # the LLVM build, as not to run out of memory.
 # This is an attempt to fix the spurious build error tracked by
 # https://github.com/rust-lang/rust/issues/108227.
-if isWindows && [[ ${CUSTOM_MINGW-0} -eq 1 ]]; then
+if isKnownToBeMingwBuild; then
     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set llvm.link-jobs=1"
 fi
 

src/ci/scripts/install-clang.sh

@@ -37,8 +37,7 @@ if isMacOS; then
     # Configure `AR` specifically so rustbuild doesn't try to infer it as
     # `clang-ar` by accident.
     ciCommandSetEnv AR "ar"
-elif isWindows && [[ ${CUSTOM_MINGW-0} -ne 1 ]]; then
-
+elif isWindows && ! isKnownToBeMingwBuild; then
     # If we're compiling for MSVC then we, like most other distribution builders,
     # switch to clang as the compiler. This'll allow us eventually to enable LTO
     # amongst LLVM and rustc. Note that we only do this on MSVC as I don't think

src/ci/scripts/install-mingw.sh

@@ -38,11 +38,11 @@ if isWindows; then
             ;;
     esac
 
-    if [[ "${CUSTOM_MINGW-0}" -ne 1 ]]; then
-        pacman -S --noconfirm --needed mingw-w64-$arch-toolchain mingw-w64-$arch-cmake \
-            mingw-w64-$arch-gcc \
-            mingw-w64-$arch-python # the python package is actually for python3
-        ciCommandAddPath "$(ciCheckoutPath)/msys2/mingw${bits}/bin"
+    if [[ "${CUSTOM_MINGW:-0}" == 0 ]]; then
+        pacboy -S --noconfirm toolchain:p
+        # According to the comment in the Windows part of install-clang.sh, in the future we might
+        # want to do this instead:
+        # pacboy -S --noconfirm clang:p ...
     else
         mingw_dir="mingw${bits}"
 

src/ci/scripts/install-msys2.sh

@@ -1,17 +1,12 @@
 #!/bin/bash
-# Download and install MSYS2, needed primarily for the test suite (run-make) but
-# also used by the MinGW toolchain for assembling things.
+# Clean up and prepare the MSYS2 installation. MSYS2 is needed primarily for
+# the test suite (run-make), but is also used by the MinGW toolchain for assembling things.
 
 set -euo pipefail
 IFS=$'\n\t'
 
 source "$(cd "$(dirname "$0")" && pwd)/../shared.sh"
-
 if isWindows; then
-    msys2Path="c:/msys64"
-    mkdir -p "${msys2Path}/home/${USERNAME}"
-    ciCommandAddPath "${msys2Path}/usr/bin"
-
     # Detect the native Python version installed on the agent. On GitHub
     # Actions, the C:\hostedtoolcache\windows\Python directory contains a
     # subdirectory for each installed Python version.
@@ -29,4 +24,33 @@ if isWindows; then
     fi
     ciCommandAddPath "C:\\hostedtoolcache\\windows\\Python\\${native_python_version}\\x64"
     ciCommandAddPath "C:\\hostedtoolcache\\windows\\Python\\${native_python_version}\\x64\\Scripts"
+
+    # Install pacboy for easily installing packages
+    pacman -S --noconfirm pactoys
+
+    # Delete these pre-installed tools so we can't accidentally use them, because we are using the
+    # MSYS2 setup action versions instead.
+    # Delete pre-installed version of MSYS2
+    rm -r "/c/msys64/"
+    # Delete Strawberry Perl, which contains a version of mingw
+    rm -r "/c/Strawberry/"
+    # Delete these other copies of mingw, I don't even know where they come from.
+    rm -r "/c/mingw64/"
+    rm -r "/c/mingw32/"
+
+    if isKnownToBeMingwBuild; then
+        # Use the mingw version of CMake for mingw builds.
+        # However, the MSVC build needs native CMake, as it fails with the mingw one.
+        # Delete native CMake
+        rm -r "/c/Program Files/CMake/"
+        # Install mingw-w64-$arch-cmake
+        pacboy -S --noconfirm cmake:p
+
+        # We use Git-for-Windows for MSVC builds, and MSYS2 Git for mingw builds,
+        # so that both are tested.
+        # Delete Windows-Git
+        rm -r "/c/Program Files/Git/"
+        # Install MSYS2 git
+        pacman -S --noconfirm git
+    fi
 fi

src/ci/shared.sh

@@ -52,6 +52,10 @@ function isLinux {
     [[ "${OSTYPE}" = "linux-gnu" ]]
 }
 
+function isKnownToBeMingwBuild {
+    isGitHubActions && [[ "${CI_JOB_NAME}" == *mingw ]]
+}
+
 function isCiBranch {
     if [[ $# -ne 1 ]]; then
         echo "usage: $0 <branch-name>"