From 7cad771087450e3c690f59958df6f8a7f153c878 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 18 Feb 2021 15:14:58 +0100 Subject: [PATCH 1/6] ci: use the brand-new GitHub Action to download git-sdk-64-minimal In our continuous builds, Windows is the odd cookie that requires a complete development environment to be downloaded because it is not installed by default. Side note: technically, there _is_ a development environment: MSYS2. But it differs from Git for Windows' SDK in subtle points, enough so to prevent Git's test suite from running without failures. Traditionally, we support downloading this environment (which we nicknamed `git-sdk-64-minimal`) via a PowerShell scriptlet that accesses the build artifacts of a dedicated Azure Pipeline (which packages a tiny subset of the full Git for Windows SDK, containing just enough to build Git and run its test suite). This PowerShell script is unfortunately not very robust and sometimes due to network issues. Instead of doing all of this in Git's own `.github/workflows/`, let's offload this logic to the brand-new GitHub Action at https://github.com/marketplace/actions/setup-git-for-windows-sdk This Action not only downloads and extracts git-sdk-64-minimal _outside_ the worktree (making it no longer necessary to meddle with `.gitignore`), it also adds the `bash.exe` to the `PATH` and sets the environment variable `MSYSTEM` (an implementation detail that Git's workflow should never have needed to know about). This allows us to convert all those funny PowerShell tasks that wanted to call git-sdk-64-minimal's `bash.exe`: they all are now regular `bash` scriptlets. This finally lets us get rid of the funny quoting and escaping where we had to pay attention not only to quote and escape in the Bash scriptlets properly, but also to add a second level of escaping (with backslashes for double quotes and backticks for dollar signs) so that PowerShell would not do unintended things. Further, this Action uses a fast caching strategy native to GitHub Actions that is not only very fast, but should accelerate the download across CI runs: git-sdk-64-minimal is usually updated once per 24h, and needs to be cached only once within that period. With this we can drop the homerolled caching where we try to accelerate the test phase by uploading git-sdk-64-minimal as a workflow artifact after using it to build Git, and then download it as workflow artifact in the test phase. Even better: the `vs-test` job no longer needs to depend on the `windows-build` job. The only reason it depended on it was to ensure that the workflow artifact was available. Signed-off-by: Johannes Schindelin --- .github/workflows/main.yml | 103 +++++++------------------------------ 1 file changed, 19 insertions(+), 84 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 73856bafc9debd..c62766e7b1a670 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -82,43 +82,18 @@ jobs: runs-on: windows-latest steps: - uses: actions/checkout@v1 - - name: download git-sdk-64-minimal - shell: bash - run: | - ## Get artifact - urlbase=https://dev.azure.com/git-for-windows/git/_apis/build/builds - id=$(curl "$urlbase?definitions=22&statusFilter=completed&resultFilter=succeeded&\$top=1" | - jq -r ".value[] | .id") - download_url="$(curl "$urlbase/$id/artifacts" | - jq -r '.value[] | select(.name == "git-sdk-64-minimal").resource.downloadUrl')" - curl --connect-timeout 10 --retry 5 --retry-delay 0 --retry-max-time 240 \ - -o artifacts.zip "$download_url" - - ## Unzip and remove the artifact - unzip artifacts.zip - rm artifacts.zip + - uses: git-for-windows/setup-git-for-windows-sdk@v1 - name: build - shell: powershell + shell: bash env: HOME: ${{runner.workspace}} - MSYSTEM: MINGW64 NO_PERL: 1 - run: | - & .\git-sdk-64-minimal\usr\bin\bash.exe -lc @" - printf '%s\n' /git-sdk-64-minimal/ >>.git/info/exclude - - ci/make-test-artifacts.sh artifacts - "@ + run: ci/make-test-artifacts.sh artifacts - name: upload build artifacts uses: actions/upload-artifact@v1 with: name: windows-artifacts path: artifacts - - name: upload git-sdk-64-minimal - uses: actions/upload-artifact@v1 - with: - name: git-sdk-64-minimal - path: git-sdk-64-minimal windows-test: runs-on: windows-latest needs: [windows-build] @@ -136,25 +111,14 @@ jobs: - name: extract build artifacts shell: bash run: tar xf artifacts.tar.gz - - name: download git-sdk-64-minimal - uses: actions/download-artifact@v1 - with: - name: git-sdk-64-minimal - path: ${{github.workspace}}/git-sdk-64-minimal/ + - uses: git-for-windows/setup-git-for-windows-sdk@v1 - name: test - shell: powershell - run: | - & .\git-sdk-64-minimal\usr\bin\bash.exe -lc @" - # Let Git ignore the SDK - printf '%s\n' /git-sdk-64-minimal/ >>.git/info/exclude - - ci/run-test-slice.sh ${{matrix.nr}} 10 - "@ + shell: bash + run: ci/run-test-slice.sh ${{matrix.nr}} 10 - name: ci/print-test-failures.sh if: failure() - shell: powershell - run: | - & .\git-sdk-64-minimal\usr\bin\bash.exe -lc ci/print-test-failures.sh + shell: bash + run: ci/print-test-failures.sh - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' uses: actions/upload-artifact@v1 @@ -165,27 +129,12 @@ jobs: needs: ci-config if: needs.ci-config.outputs.enabled == 'yes' env: - MSYSTEM: MINGW64 NO_PERL: 1 GIT_CONFIG_PARAMETERS: "'user.name=CI' 'user.email=ci@git'" runs-on: windows-latest steps: - uses: actions/checkout@v1 - - name: download git-sdk-64-minimal - shell: bash - run: | - ## Get artifact - urlbase=https://dev.azure.com/git-for-windows/git/_apis/build/builds - id=$(curl "$urlbase?definitions=22&statusFilter=completed&resultFilter=succeeded&\$top=1" | - jq -r ".value[] | .id") - download_url="$(curl "$urlbase/$id/artifacts" | - jq -r '.value[] | select(.name == "git-sdk-64-minimal").resource.downloadUrl')" - curl --connect-timeout 10 --retry 5 --retry-delay 0 --retry-max-time 240 \ - -o artifacts.zip "$download_url" - - ## Unzip and remove the artifact - unzip artifacts.zip - rm artifacts.zip + - uses: git-for-windows/setup-git-for-windows-sdk@v1 - name: initialize vcpkg uses: actions/checkout@v2 with: @@ -211,19 +160,17 @@ jobs: shell: bash run: | cmake `pwd`/contrib/buildsystems/ -DCMAKE_PREFIX_PATH=`pwd`/compat/vcbuild/vcpkg/installed/x64-windows \ - -DMSGFMT_EXE=`pwd`/git-sdk-64-minimal/mingw64/bin/msgfmt.exe -DPERL_TESTS=OFF -DPYTHON_TESTS=OFF -DCURL_NO_CURL_CMAKE=ON + -DMSGFMT_EXE=C:/git-sdk-64-minimal/mingw64/bin/msgfmt.exe -DPERL_TESTS=OFF -DPYTHON_TESTS=OFF -DCURL_NO_CURL_CMAKE=ON - name: MSBuild run: msbuild git.sln -property:Configuration=Release -property:Platform=x64 -maxCpuCount:4 -property:PlatformToolset=v142 - name: bundle artifact tar - shell: powershell + shell: bash env: MSVC: 1 VCPKG_ROOT: ${{github.workspace}}\compat\vcbuild\vcpkg run: | - & git-sdk-64-minimal\usr\bin\bash.exe -lc @" - mkdir -p artifacts && - eval \"`$(make -n artifacts-tar INCLUDE_DLLS_IN_ARTIFACTS=YesPlease ARTIFACTS_DIRECTORY=artifacts 2>&1 | grep ^tar)\" - "@ + mkdir -p artifacts && + eval "$(make -n artifacts-tar INCLUDE_DLLS_IN_ARTIFACTS=YesPlease ARTIFACTS_DIRECTORY=artifacts 2>&1 | grep ^tar)" - name: upload build artifacts uses: actions/upload-artifact@v1 with: @@ -231,18 +178,14 @@ jobs: path: artifacts vs-test: runs-on: windows-latest - needs: [vs-build, windows-build] + needs: vs-build strategy: fail-fast: false matrix: nr: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] steps: - uses: actions/checkout@v1 - - name: download git-sdk-64-minimal - uses: actions/download-artifact@v1 - with: - name: git-sdk-64-minimal - path: ${{github.workspace}}/git-sdk-64-minimal/ + - uses: git-for-windows/setup-git-for-windows-sdk@v1 - name: download build artifacts uses: actions/download-artifact@v1 with: @@ -252,23 +195,15 @@ jobs: shell: bash run: tar xf artifacts.tar.gz - name: test - shell: powershell + shell: bash env: - MSYSTEM: MINGW64 NO_SVN_TESTS: 1 GIT_TEST_SKIP_REBASE_P: 1 - run: | - & .\git-sdk-64-minimal\usr\bin\bash.exe -lc @" - # Let Git ignore the SDK and the test-cache - printf '%s\n' /git-sdk-64-minimal/ /test-cache/ >>.git/info/exclude - - ci/run-test-slice.sh ${{matrix.nr}} 10 - "@ + run: ci/run-test-slice.sh ${{matrix.nr}} 10 - name: ci/print-test-failures.sh if: failure() - shell: powershell - run: | - & .\git-sdk-64-minimal\usr\bin\bash.exe -lc ci/print-test-failures.sh + shell: bash + run: ci/print-test-failures.sh - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' uses: actions/upload-artifact@v1 From de337510b98a6e1c81c71dc734d0796d3c77ba61 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 18 Feb 2021 15:35:08 +0100 Subject: [PATCH 2/6] ci (vs-build): use `cmd` to copy the DLLs, not `powershell` We use a `.bat` script to copy the DLLs in the `vs-build` job, and those type of scripts are native to CMD, not to PowerShell. Signed-off-by: Johannes Schindelin --- .github/workflows/main.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c62766e7b1a670..d430c4e0d20622 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -152,10 +152,8 @@ jobs: - name: add msbuild to PATH uses: microsoft/setup-msbuild@v1 - name: copy dlls to root - shell: powershell - run: | - & compat\vcbuild\vcpkg_copy_dlls.bat release - if (!$?) { exit(1) } + shell: cmd + run: compat\vcbuild\vcpkg_copy_dlls.bat release - name: generate Visual Studio solution shell: bash run: | From 940777d41d7037f2c76560911322050225f5875d Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 18 Feb 2021 15:58:45 +0100 Subject: [PATCH 3/6] ci: upgrade to using actions/{up,down}load-artifacts v2 The GitHub Actions to upload/download workflow artifacts saw a major upgrade since Git's GitHub workflow was established. Let's use it. Signed-off-by: Johannes Schindelin --- .github/workflows/main.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d430c4e0d20622..a399114c0f848c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -90,7 +90,7 @@ jobs: NO_PERL: 1 run: ci/make-test-artifacts.sh artifacts - name: upload build artifacts - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v2 with: name: windows-artifacts path: artifacts @@ -104,7 +104,7 @@ jobs: steps: - uses: actions/checkout@v1 - name: download build artifacts - uses: actions/download-artifact@v1 + uses: actions/download-artifact@v2 with: name: windows-artifacts path: ${{github.workspace}} @@ -121,7 +121,7 @@ jobs: run: ci/print-test-failures.sh - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v2 with: name: failed-tests-windows path: ${{env.FAILED_TEST_ARTIFACTS}} @@ -170,7 +170,7 @@ jobs: mkdir -p artifacts && eval "$(make -n artifacts-tar INCLUDE_DLLS_IN_ARTIFACTS=YesPlease ARTIFACTS_DIRECTORY=artifacts 2>&1 | grep ^tar)" - name: upload build artifacts - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v2 with: name: vs-artifacts path: artifacts @@ -185,7 +185,7 @@ jobs: - uses: actions/checkout@v1 - uses: git-for-windows/setup-git-for-windows-sdk@v1 - name: download build artifacts - uses: actions/download-artifact@v1 + uses: actions/download-artifact@v2 with: name: vs-artifacts path: ${{github.workspace}} @@ -204,7 +204,7 @@ jobs: run: ci/print-test-failures.sh - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v2 with: name: failed-tests-windows path: ${{env.FAILED_TEST_ARTIFACTS}} @@ -242,7 +242,7 @@ jobs: if: failure() - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v2 with: name: failed-tests-${{matrix.vector.jobname}} path: ${{env.FAILED_TEST_ARTIFACTS}} @@ -269,7 +269,7 @@ jobs: if: failure() - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v2 with: name: failed-tests-${{matrix.vector.jobname}} path: ${{env.FAILED_TEST_ARTIFACTS}} From 5103300d72dafa459df7b3569f57ca2121951189 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 18 Feb 2021 16:00:59 +0100 Subject: [PATCH 4/6] ci(windows): transfer the tracked files to the test jobs Git's test suite is excruciatingly slow on Windows, mainly due to the fact that it executes a lot of shell script code, and that's simply not native to Windows. To help with that, we established the pattern where the artifacts are first built in one job, and then multiple test jobs run in parallel using the artifacts built in the first job. We take pains in transferring only the build outputs, and letting `actions/checkout` fill in the rest of the files. One major downside of that strategy is that the test jobs might fail to check out the intended revision (e.g. because the branch has been updated while the build was running, as is frequently the case with the `seen` branch). Let's transfer also the files tracked by Git, and skip the checkout step in the test jobs. Signed-off-by: Johannes Schindelin --- .github/workflows/main.yml | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a399114c0f848c..0f7516c9ef3eaa 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -89,7 +89,9 @@ jobs: HOME: ${{runner.workspace}} NO_PERL: 1 run: ci/make-test-artifacts.sh artifacts - - name: upload build artifacts + - name: zip up tracked files + run: git archive -o artifacts/tracked.tar.gz HEAD + - name: upload tracked files and build artifacts uses: actions/upload-artifact@v2 with: name: windows-artifacts @@ -102,15 +104,14 @@ jobs: matrix: nr: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] steps: - - uses: actions/checkout@v1 - - name: download build artifacts + - name: download tracked files and build artifacts uses: actions/download-artifact@v2 with: name: windows-artifacts path: ${{github.workspace}} - - name: extract build artifacts + - name: extract tracked files and build artifacts shell: bash - run: tar xf artifacts.tar.gz + run: tar xf artifacts.tar.gz && tar xf tracked.tar.gz - uses: git-for-windows/setup-git-for-windows-sdk@v1 - name: test shell: bash @@ -169,7 +170,9 @@ jobs: run: | mkdir -p artifacts && eval "$(make -n artifacts-tar INCLUDE_DLLS_IN_ARTIFACTS=YesPlease ARTIFACTS_DIRECTORY=artifacts 2>&1 | grep ^tar)" - - name: upload build artifacts + - name: zip up tracked files + run: git archive -o artifacts/tracked.tar.gz HEAD + - name: upload tracked files and build artifacts uses: actions/upload-artifact@v2 with: name: vs-artifacts @@ -182,16 +185,15 @@ jobs: matrix: nr: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] steps: - - uses: actions/checkout@v1 - uses: git-for-windows/setup-git-for-windows-sdk@v1 - - name: download build artifacts + - name: download tracked files and build artifacts uses: actions/download-artifact@v2 with: name: vs-artifacts path: ${{github.workspace}} - - name: extract build artifacts + - name: extract tracked files and build artifacts shell: bash - run: tar xf artifacts.tar.gz + run: tar xf artifacts.tar.gz && tar xf tracked.tar.gz - name: test shell: bash env: From db8b05da4404222abcbd65a11bfa55ab3522b486 Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Thu, 11 Feb 2021 15:32:02 +0100 Subject: [PATCH 5/6] ci(vs-build): build with NO_GETTEXT We already build Git for Windows with `NO_GETTEXT` when compiling with GCC. Let's do the same with Visual C, too. Signed-off-by: Dennis Ameling Signed-off-by: Johannes Schindelin --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0f7516c9ef3eaa..3b40c677ab5b3f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -159,7 +159,7 @@ jobs: shell: bash run: | cmake `pwd`/contrib/buildsystems/ -DCMAKE_PREFIX_PATH=`pwd`/compat/vcbuild/vcpkg/installed/x64-windows \ - -DMSGFMT_EXE=C:/git-sdk-64-minimal/mingw64/bin/msgfmt.exe -DPERL_TESTS=OFF -DPYTHON_TESTS=OFF -DCURL_NO_CURL_CMAKE=ON + -DNO_GETTEXT=YesPlease -DPERL_TESTS=OFF -DPYTHON_TESTS=OFF -DCURL_NO_CURL_CMAKE=ON - name: MSBuild run: msbuild git.sln -property:Configuration=Release -property:Platform=x64 -maxCpuCount:4 -property:PlatformToolset=v142 - name: bundle artifact tar From 003dc8865679019980cbd800ba2fd00463a154b1 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 18 Feb 2021 16:15:49 +0100 Subject: [PATCH 6/6] ci: accelerate the checkout By upgrading from v1 to v2 of `actions/checkout`, we avoid fetching all the tags and the entire history: v2 only fetches one revision by default. This should make things a lot faster. Note that `actions/checkout@v2` seems to be incompatible with running in containers: https://github.com/actions/checkout/issues/151. Therefore, we stick with v1 there. Signed-off-by: Johannes Schindelin --- .github/workflows/main.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3b40c677ab5b3f..405204c78a7a6d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -81,7 +81,7 @@ jobs: if: needs.ci-config.outputs.enabled == 'yes' runs-on: windows-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - uses: git-for-windows/setup-git-for-windows-sdk@v1 - name: build shell: bash @@ -134,7 +134,7 @@ jobs: GIT_CONFIG_PARAMETERS: "'user.name=CI' 'user.email=ci@git'" runs-on: windows-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - uses: git-for-windows/setup-git-for-windows-sdk@v1 - name: initialize vcpkg uses: actions/checkout@v2 @@ -237,7 +237,7 @@ jobs: jobname: ${{matrix.vector.jobname}} runs-on: ${{matrix.vector.pool}} steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - run: ci/install-dependencies.sh - run: ci/run-build-and-tests.sh - run: ci/print-test-failures.sh @@ -282,7 +282,7 @@ jobs: jobname: StaticAnalysis runs-on: ubuntu-18.04 steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - run: ci/install-dependencies.sh - run: ci/run-static-analysis.sh documentation: @@ -292,6 +292,6 @@ jobs: jobname: Documentation runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - run: ci/install-dependencies.sh - run: ci/test-documentation.sh