Merge pull request #522 from swiftwasm/update-base-tag/main-swift-DEV… #1217
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build Toolchain | |
| on: | |
| push: | |
| branches: [main] | |
| paths-ignore: | |
| - 'docs/**' | |
| - '**.md' | |
| pull_request: | |
| branches: [main] | |
| paths-ignore: | |
| - 'docs/**' | |
| - '**.md' | |
| workflow_dispatch: | |
| inputs: | |
| scheme: | |
| description: 'Scheme to build' | |
| required: true | |
| default: 'main' | |
| jobs: | |
| start-runner: | |
| name: Start self-hosted EC2 runner | |
| runs-on: ubuntu-latest | |
| # Run only on main branches to avoid triggers by non-collaborator | |
| if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'check-self-hosted-ci') }} | |
| outputs: | |
| ubuntu20_04_aarch64-label: ${{ steps.start-ubuntu20_04_aarch64-runner.outputs.label }} | |
| ubuntu20_04_aarch64-ec2-instance-id: ${{ steps.start-ubuntu20_04_aarch64-runner.outputs.ec2-instance-id }} | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ secrets.AWS_REGION }} | |
| - name: Start EC2 runner (ubuntu20_04_aarch64) | |
| id: start-ubuntu20_04_aarch64-runner | |
| uses: machulav/ec2-github-runner@v2 | |
| with: | |
| mode: start | |
| github-token: ${{ secrets.SWIFTWASM_BUILDBOT_TOKEN }} | |
| ec2-image-id: ami-03803f6b50b56647a # swiftwasm-ci/ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-arm64-server-20221201 | |
| ec2-instance-type: c6g.xlarge | |
| subnet-id: subnet-327f4a13 | |
| security-group-id: sg-0429f5ec2bee9dc0c | |
| stop-runner: | |
| name: Stop self-hosted EC2 runner | |
| needs: [start-runner, build-toolchain] | |
| runs-on: ubuntu-latest | |
| # Required to stop the runner even if the error happened in the previous jobs | |
| if: ${{ always() && needs.start-runner.result == 'success' }} | |
| # The runner can be already stopped when re-running a part of failed jobs | |
| continue-on-error: true | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ secrets.AWS_REGION }} | |
| - name: Stop EC2 runner (ubuntu20_04_aarch64) | |
| uses: machulav/ec2-github-runner@v2 | |
| with: | |
| mode: stop | |
| github-token: ${{ secrets.SWIFTWASM_BUILDBOT_TOKEN }} | |
| label: ${{ needs.start-runner.outputs.ubuntu20_04_aarch64-label }} | |
| ec2-instance-id: ${{ needs.start-runner.outputs.ubuntu20_04_aarch64-ec2-instance-id }} | |
| build-matrix: | |
| name: Build matrix | |
| needs: [start-runner] | |
| runs-on: ubuntu-latest | |
| if: ${{ !cancelled() }} | |
| outputs: | |
| entries: ${{ steps.generate.outputs.entries }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dorny/paths-filter@v3 | |
| id: changes | |
| with: | |
| list-files: json | |
| filters: | | |
| changed: | |
| - "schemes/**" | |
| - "tools/build/**" | |
| - "tools/swift-sdk-generator" | |
| - "tools/git-swift-workspace" | |
| - name: Generate entries | |
| id: generate | |
| run: | | |
| START_RUNNER_JSON="${{ runner.temp }}/start-runner.json" | |
| cat <<EOF > "$START_RUNNER_JSON" | |
| ${{ toJson(needs.start-runner) }} | |
| EOF | |
| CHANGES_JSON="${{ runner.temp }}/changes.json" | |
| cat <<EOF > "$CHANGES_JSON" | |
| ${{ steps.changes.outputs.changed_files }} | |
| EOF | |
| if [[ "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]]; then | |
| echo "Toolchain build triggered by workflow_dispatch with following inputs:" | |
| cat "$GITHUB_EVENT_PATH" | |
| fi | |
| echo "entries=$(ruby .github/scripts/build-matrix.rb --runner $START_RUNNER_JSON --changes $CHANGES_JSON)" >> $GITHUB_OUTPUT | |
| build-toolchain: | |
| env: | |
| DEVELOPER_DIR: /Applications/Xcode_15.2.app/Contents/Developer/ | |
| # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#cache-segment-restore-timeout | |
| SEGMENT_DOWNLOAD_TIMEOUT_MINS: 30 | |
| needs: [build-matrix] | |
| if: ${{ !cancelled() && needs.build-matrix.outputs.entries != '[]' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJSON(needs.build-matrix.outputs.entries) }} | |
| name: ${{ matrix.job_name }} | |
| timeout-minutes: 0 | |
| runs-on: ${{ matrix.agent_query }} | |
| steps: | |
| - name: Free disk space | |
| if: ${{ matrix.free_disk_space && !startsWith(matrix.build_os, 'macos-') }} | |
| run: | | |
| df -h | |
| sudo rm -rf /opt/hostedtoolcache | |
| sudo rm -rf /usr/share/dotnet /usr/local/lib/android /usr/local/.ghcup | |
| sudo apt clean | |
| if docker info > /dev/null 2>&1; then | |
| docker rmi $(docker image ls -aq) | |
| fi | |
| df -h | |
| - name: Free disk space by removing unused Xcode | |
| if: ${{ matrix.free_disk_space && startsWith(matrix.build_os, 'macos-') }} | |
| run: | | |
| IN_USE_XCODE=$(xcode-select -p) | |
| echo "Removing all Xcode installations except $IN_USE_XCODE" | |
| for xcode in /Applications/Xcode*.app; do | |
| if [[ $IN_USE_XCODE != "$xcode"* ]]; then | |
| echo "Removing $xcode" | |
| rm -rf $xcode | |
| fi | |
| done | |
| - uses: actions/checkout@v4 | |
| with: | |
| path: swiftwasm-build | |
| submodules: true | |
| - name: Configure git user for git-am | |
| run: | | |
| # Linux self-hosted runners somehow clear HOME environment variable | |
| if [[ -z "$HOME" ]]; then | |
| export HOME="$RUNNER_TEMP/home/buildbot" | |
| mkdir -p "$HOME" | |
| echo "HOME=$HOME" >> $GITHUB_ENV | |
| fi | |
| git config --global user.name "github-actions[bot]" | |
| git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| - name: Prepare sccache timestamp | |
| id: cache_key | |
| run: | | |
| export SCCACHE_KEY_PREFIX="${{ matrix.target }}-${{ matrix.toolchain_channel }}-sccache-v15-" | |
| echo "SCCACHE_KEY_PREFIX=$SCCACHE_KEY_PREFIX" >> $GITHUB_OUTPUT | |
| echo "SCCACHE_KEY=$SCCACHE_KEY_PREFIX$(date +'%Y-%m-%d-%I-%M-%S')" >> $GITHUB_OUTPUT | |
| - name: Check Xcode version | |
| if: ${{ startsWith(matrix.build_os, 'macos-') }} | |
| run: | | |
| xcodebuild -version | |
| - name: Clean build directory | |
| if: ${{ matrix.clean_build_dir }} | |
| run: | | |
| rm -rf ${{ github.workspace }}/build \ | |
| ${{ github.workspace }}/swift \ | |
| ${{ github.workspace }}/swiftwasm-build/tools/swift-sdk-generator/Bundles \ | |
| ${{ github.workspace }}/build-cache \ | |
| ${{ github.workspace }}/build-sdk | |
| - uses: actions/cache/restore@v4 | |
| id: build-cache | |
| with: | |
| path: build-cache | |
| key: ${{ steps.cache_key.outputs.SCCACHE_KEY }} | |
| restore-keys: | | |
| ${{ steps.cache_key.outputs.SCCACHE_KEY_PREFIX }} | |
| - run: chmod -R 777 ./build-cache | |
| if: ${{ steps.build-cache.outputs.cache-hit }} | |
| - run: ./swiftwasm-build/tools/build/install-build-sdk.sh ${{ matrix.scheme }} | |
| - run: ./swiftwasm-build/tools/git-swift-workspace --skip-history --verbose --scheme ${{ matrix.scheme }} | |
| env: | |
| # icu uses git-lfs and it fails to checkout due to https://github.com/git-lfs/git-lfs/issues/5749 | |
| GIT_CLONE_PROTECTION_ACTIVE: false | |
| - name: Install Homebrew dependencies | |
| if: ${{ startsWith(matrix.build_os, 'macos-') }} | |
| run: | | |
| brew bundle --file swift/Brewfile | |
| brew install curl # For Foundation build | |
| # LLVM implicitly depends on libzstd if it's found in the system | |
| - name: Uninstall libzstd on macOS | |
| if: ${{ startsWith(matrix.build_os, 'macos-') }} | |
| run: rm -f /usr/local/lib/pkgconfig/libzstd.pc /usr/local/include/zstd.h | |
| # release-5.9 requires 3.11 or lower Python | |
| - uses: actions/setup-python@v5 | |
| if: ${{ startsWith(matrix.build_os, 'macos-') && (matrix.scheme == 'release-5.9' || matrix.scheme == 'release-5.10') && (matrix.agent_query == 'macos-13' || matrix.agent_query == 'macos-14') }} | |
| with: | |
| python-version: '3.11' | |
| - name: Build ${{ matrix.target }} installable archive on Docker container | |
| if: ${{ matrix.container != null }} | |
| timeout-minutes: 300 # Exit before 6 hours limit to allow cache upload | |
| run: | | |
| docker container rm --force swiftwasm-ci-buildbot | |
| docker volume rm --force oss-swift-package | |
| docker volume create oss-swift-package | |
| docker run --name swiftwasm-ci-buildbot \ | |
| --security-opt=no-new-privileges \ | |
| --cap-add=SYS_PTRACE \ | |
| --security-opt seccomp=unconfined \ | |
| -dit \ | |
| -e CI=$CI \ | |
| -e ONLY_SWIFT_SDK=${{ matrix.only_swift_sdk }} \ | |
| -w /home/build-user/ \ | |
| -v ${{ github.workspace }}:/source \ | |
| -v oss-swift-package:/home/build-user \ | |
| ${{ matrix.container }} | |
| docker exec swiftwasm-ci-buildbot /bin/bash -lc 'env; cp -r /source/* /home/build-user/; ./swiftwasm-build/tools/build/ci.sh ${{ matrix.scheme }}' | |
| - name: Extract installable archive from Docker container | |
| if: ${{ matrix.container != null && !matrix.only_swift_sdk }} | |
| run: | | |
| docker cp swiftwasm-ci-buildbot:/home/build-user/swift-wasm-${{ matrix.toolchain_channel }}-SNAPSHOT-${{ matrix.target }}.tar.gz . | |
| docker cp swiftwasm-ci-buildbot:/home/build-user/swift-wasm-${{ matrix.toolchain_channel }}-SNAPSHOT-${{ matrix.target }}.artifactbundle.zip . || true | |
| - name: Extract installable archive from Docker container (wasm32-unknown-wasi) | |
| if: ${{ matrix.container != null && matrix.only_swift_sdk }} | |
| run: | | |
| docker cp swiftwasm-ci-buildbot:/home/build-user/swift-wasm-${{ matrix.toolchain_channel }}-SNAPSHOT-wasm32-unknown-wasi.artifactbundle.zip . | |
| # release-6.0 doesn't have wasm32-unknown-wasip1-threads SDK support yet | |
| - name: Extract installable archive from Docker container (wasm32-unknown-wasip1-threads) | |
| if: ${{ matrix.container != null && matrix.only_swift_sdk && matrix.scheme != 'release-6.0' }} | |
| run: | | |
| docker cp swiftwasm-ci-buildbot:/home/build-user/swift-wasm-${{ matrix.toolchain_channel }}-SNAPSHOT-wasm32-unknown-wasip1-threads.artifactbundle.zip . | |
| - name: Extract build-cache from Docker container | |
| if: ${{ matrix.container != null && !cancelled() }} | |
| run: | | |
| rm -rf ${{ github.workspace }}/build-cache | |
| docker cp swiftwasm-ci-buildbot:/home/build-user/build-cache ${{ github.workspace }}/build-cache | |
| - name: Build ${{ matrix.target }} installable archive | |
| if: ${{ matrix.container == null }} | |
| timeout-minutes: 300 # Exit before 6 hours limit to allow cache upload | |
| env: | |
| SKIP_XCODE_VERSION_CHECK: 1 | |
| run: ./swiftwasm-build/tools/build/ci.sh ${{ matrix.scheme }} | |
| - uses: actions/cache/save@v4 | |
| if: ${{ !cancelled() }} | |
| with: | |
| path: build-cache | |
| key: ${{ steps.cache_key.outputs.SCCACHE_KEY }} | |
| - name: Upload ${{ matrix.target }} installable archive | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.target }}-${{ matrix.scheme }}-installable | |
| path: swift-wasm-${{ matrix.toolchain_channel }}-SNAPSHOT-${{ matrix.target }}.tar.gz | |
| - name: Upload ${{ matrix.target }} SDK artifact bundle | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ !matrix.only_swift_sdk }} | |
| with: | |
| name: ${{ matrix.target }}-${{ matrix.scheme }}-artifactbundle | |
| path: swift-wasm-${{ matrix.toolchain_channel }}-SNAPSHOT-${{ matrix.target }}.artifactbundle.zip | |
| if-no-files-found: ignore | |
| - name: Upload Swift SDK artifact bundle (wasm32-unknown-wasi) | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ matrix.only_swift_sdk }} | |
| with: | |
| name: ${{ matrix.scheme }}-wasm32-unknown-wasi-artifactbundle | |
| path: swift-wasm-${{ matrix.toolchain_channel }}-SNAPSHOT-wasm32-unknown-wasi.artifactbundle.zip | |
| - name: Upload Swift SDK artifact bundle (wasm32-unknown-wasip1-threads) | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ matrix.only_swift_sdk }} | |
| with: | |
| name: ${{ matrix.scheme }}-wasm32-unknown-wasip1-threads-artifactbundle | |
| path: swift-wasm-${{ matrix.toolchain_channel }}-SNAPSHOT-wasm32-unknown-wasip1-threads.artifactbundle.zip | |
| # Prepare for running tests | |
| - uses: bytecodealliance/actions/wasmtime/setup@v1 | |
| with: | |
| version: "v25.0.3" | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| # WORKAROUND: wasmkit-cli should be built from checked out source but we don't have | |
| # build script for it yet. So we create a symlink to wasmtime for now. | |
| # We should remove this step once we unify the build script with apple/swift's build-preset | |
| # based build script. | |
| - run: ln -sf "$(which wasmtime)" "$(dirname "$(which wasmtime)")/wasmkit-cli" | |
| if: ${{ matrix.run_stdlib_test && matrix.container == null }} | |
| - run: docker exec -u root swiftwasm-ci-buildbot bash -c 'ln -sf "$(which wasmtime)" "$(dirname "$(which wasmtime)")/wasmkit-cli"' | |
| if: ${{ matrix.run_stdlib_test && matrix.container != null }} | |
| - name: Run stdlib tests for wasi-wasm32 | |
| if: ${{ matrix.run_stdlib_test && matrix.container != null }} | |
| run: | | |
| docker exec swiftwasm-ci-buildbot /bin/bash -lc \ | |
| "./swiftwasm-build/schemes/${{ matrix.scheme }}/build/run-test.sh" | |
| # We need this step to save disk space for the next steps when building toolchain | |
| - name: Cleanup build directory | |
| if: ${{ matrix.run_stdlib_test && matrix.container != null && !matrix.only_swift_sdk }} | |
| run: docker exec swiftwasm-ci-buildbot /bin/bash -lc "rm -rf build" | |
| - name: Run stdlib tests for wasi-wasm32 | |
| if: ${{ matrix.run_stdlib_test && matrix.container == null }} | |
| run: ./swiftwasm-build/schemes/${{ matrix.scheme }}/build/run-test.sh | |
| - name: Pack test results | |
| if: ${{ matrix.run_full_test }} | |
| run: | | |
| tar cJf ./swift-test-results.tar.gz build/WebAssembly/swift-stdlib-wasi-wasm32/swift-test-results | |
| - name: Upload test results | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ matrix.run_full_test }} | |
| with: | |
| name: ${{ matrix.target }}-test-results | |
| path: ./swift-test-results.tar.gz | |
| - name: Run integration tests (Swift SDK) | |
| if: ${{ matrix.run_e2e_test && matrix.only_swift_sdk }} | |
| run: | | |
| docker exec swiftwasm-ci-buildbot /bin/bash -lc \ | |
| "./llvm-project/llvm/utils/lit/lit.py \ | |
| --param swift-sdk=wasm32-unknown-wasi \ | |
| --param swift-sdks-path=./swiftwasm-build/tools/swift-sdk-generator/Bundles \ | |
| --param scheme=${{ matrix.scheme }} ./swiftwasm-build/test -vv" | |
| - name: Cleanup docker volume | |
| if: ${{ matrix.container != null }} | |
| run: | | |
| docker stop swiftwasm-ci-buildbot | |
| docker rm swiftwasm-ci-buildbot | |
| docker volume rm --force oss-swift-package | |
| # Run e2e test | |
| - name: Prepare E2E test | |
| if: ${{ !matrix.only_swift_sdk }} | |
| run: | | |
| INSTALL_DIR=$(mktemp -d) | |
| tar xf swift-wasm-${{ matrix.toolchain_channel }}-SNAPSHOT-${{ matrix.target }}.tar.gz -C "$INSTALL_DIR" | |
| echo "TOOLCHAIN=$(find "$INSTALL_DIR" -name "swift-wasm-${{ matrix.toolchain_channel }}-*" -type d | head -n1)" >> $GITHUB_ENV | |
| - name: Build hello.wasm | |
| shell: bash | |
| if: ${{ matrix.build_hello_wasm && !matrix.only_swift_sdk }} | |
| run: | | |
| echo 'print("Hello, world!")' > hello.swift | |
| $TOOLCHAIN/usr/bin/swiftc \ | |
| -target wasm32-unknown-wasi \ | |
| -sdk $TOOLCHAIN/usr/share/wasi-sysroot \ | |
| -resource-dir $TOOLCHAIN/usr/lib/swift_static \ | |
| hello.swift -o hello.wasm && \ | |
| echo "Successfully linked hello.wasm" | |
| - name: Upload hello.wasm | |
| if: ${{ matrix.build_hello_wasm && !matrix.only_swift_sdk && matrix.scheme == 'main' }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.target }}-hello.wasm | |
| path: hello.wasm | |
| - name: Run integration tests | |
| if: ${{ matrix.run_e2e_test && !matrix.only_swift_sdk }} | |
| run: ${{ github.workspace }}/llvm-project/llvm/utils/lit/lit.py --param package-path=$TOOLCHAIN --param scheme=${{ matrix.scheme }} ${{ github.workspace }}/swiftwasm-build/test -vv |