diff --git a/.ci/Manifest.toml b/.ci/Manifest.toml index 51480d8..c3e6245 100644 --- a/.ci/Manifest.toml +++ b/.ci/Manifest.toml @@ -2,7 +2,6 @@ julia_version = "1.7.3" manifest_format = "2.0" -project_hash = "eb281049837da2b091ec667863c8824a57a70708" [[deps.ArgParse]] deps = ["Logging", "TextWrap"] @@ -38,7 +37,7 @@ uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" [[deps.BinaryBuilder]] deps = ["ArgParse", "BinaryBuilderBase", "Binutils_jll", "Dates", "Downloads", "GitHub", "HTTP", "JLD2", "JSON", "LibGit2", "Libdl", "Logging", "LoggingExtras", "ObjectFile", "OutputCollectors", "Patchelf_jll", "Pkg", "PkgLicenses", "REPL", "Random", "Registrator", "RegistryTools", "SHA", "Scratch", "Sockets", "TOML", "UUIDs", "ghr_jll"] -git-tree-sha1 = "83b84cf0cee886d6be4f398382da760fc7c898b6" +git-tree-sha1 = "5caeb4f25ead6a04742d9d3140c8d699ceea1fe3" repo-rev = "master" repo-url = "https://github.com/JuliaPackaging/BinaryBuilder.jl.git" uuid = "12aac903-9f7c-5d81-afc2-d9565ea332ae" @@ -46,11 +45,11 @@ version = "0.6.5" [[deps.BinaryBuilderBase]] deps = ["Bzip2_jll", "CodecZlib", "Downloads", "Gzip_jll", "HistoricalStdlibVersions", "InteractiveUtils", "JLLWrappers", "JSON", "LibGit2", "LibGit2_jll", "Libdl", "Logging", "OrderedCollections", "OutputCollectors", "Pkg", "Printf", "ProgressMeter", "REPL", "Random", "SHA", "Scratch", "SimpleBufferStream", "TOML", "Tar", "Tar_jll", "UUIDs", "XZ_jll", "Zstd_jll", "p7zip_jll", "pigz_jll", "unzip_jll"] -git-tree-sha1 = "d37350f8d8431a44261e29e37b4a4c1558ad121d" +git-tree-sha1 = "b0d610852483382d19de2d2db7df8c214aab3fd1" repo-rev = "master" repo-url = "https://github.com/JuliaPackaging/BinaryBuilderBase.jl.git" uuid = "7f725544-6523-48cd-82d1-3fa08ff4056e" -version = "1.41.0" +version = "1.42.0" [[deps.Binutils_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Zlib_jll", "Zstd_jll"] @@ -77,9 +76,9 @@ version = "0.7.8" [[deps.Compat]] deps = ["Dates", "LinearAlgebra", "TOML", "UUIDs"] -git-tree-sha1 = "0037835448781bb46feb39866934e243886d756a" +git-tree-sha1 = "9d8a54ce4b17aa5bdce0ea5c34bc5e7c340d16ad" uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "4.18.0" +version = "4.18.1" [[deps.CompilerSupportLibraries_jll]] deps = ["Artifacts", "Libdl"] @@ -126,9 +125,9 @@ version = "0.1.10" [[deps.FileIO]] deps = ["Pkg", "Requires", "UUIDs"] -git-tree-sha1 = "b66970a70db13f45b7e57fbda1736e1cf72174ea" +git-tree-sha1 = "d60eb76f37d7e5a40cc2e7c36974d864b82dc802" uuid = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" -version = "1.17.0" +version = "1.17.1" [[deps.FileWatching]] uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" @@ -141,9 +140,9 @@ version = "0.4.4" [[deps.GitHub]] deps = ["Base64", "Dates", "HTTP", "JSON", "MbedTLS", "Sockets", "SodiumSeal", "URIs"] -git-tree-sha1 = "2501485472c50b24874cc949f0fce9bd3c4a1738" +git-tree-sha1 = "12d0b1886e60c9d2a3d42ef1612bdfbedec68b42" uuid = "bc5e4493-9b4d-5f90-b8aa-2b2bcaad7a26" -version = "5.10.0" +version = "5.11.0" [[deps.Gzip_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] @@ -153,9 +152,9 @@ version = "1.14.0+0" [[deps.HTTP]] deps = ["Base64", "CodecZlib", "ConcurrentUtilities", "Dates", "ExceptionUnwrapping", "Logging", "LoggingExtras", "MbedTLS", "NetworkOptions", "OpenSSL", "PrecompileTools", "Random", "SimpleBufferStream", "Sockets", "URIs", "UUIDs"] -git-tree-sha1 = "ed5e9c58612c4e081aecdb6e1a479e18462e041e" +git-tree-sha1 = "5e6fe50ae7f23d171f44e311c2960294aaa0beb5" uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" -version = "1.10.17" +version = "1.10.19" [[deps.Hiccup]] deps = ["MacroTools", "Test"] @@ -252,9 +251,9 @@ uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" [[deps.LoggingExtras]] deps = ["Dates", "Logging"] -git-tree-sha1 = "f02b56007b064fbfddb4c9cd60161b6dd0f40df3" +git-tree-sha1 = "f00544d95982ea270145636c181ceda21c4e2575" uuid = "e6f89c97-d47a-5376-807f-9c37f3926c36" -version = "1.1.0" +version = "1.2.0" [[deps.Lz4_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] @@ -319,16 +318,16 @@ deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" [[deps.OpenSSL]] -deps = ["BitFlags", "Dates", "MozillaCACerts_jll", "OpenSSL_jll", "Sockets"] -git-tree-sha1 = "f1a7e086c677df53e064e0fdd2c9d0b0833e3f6e" +deps = ["BitFlags", "Dates", "MozillaCACerts_jll", "NetworkOptions", "OpenSSL_jll", "Sockets"] +git-tree-sha1 = "386b47442468acfb1add94bf2d85365dea10cbab" uuid = "4d8831e6-92b7-49fb-bdf8-b643e874388c" -version = "1.5.0" +version = "1.6.0" [[deps.OpenSSL_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "2ae7d4ddec2e13ad3bddf5c0796f7547cf682391" +git-tree-sha1 = "f19301ae653233bc88b1810ae908194f07f8db9d" uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "3.5.2+0" +version = "3.5.4+0" [[deps.OrderedCollections]] git-tree-sha1 = "05868e21324cede2207c6f0f466b4bfef6d5e7ee" @@ -417,9 +416,9 @@ version = "1.10.0" [[deps.RegistryTools]] deps = ["LibGit2", "Pkg", "SHA", "UUIDs"] -git-tree-sha1 = "c0ac28884fab9ae94ed8cf3448aa950afc2ff9c1" +git-tree-sha1 = "3087a02cf7baffe55fc5d26573bfd9ab8833c613" uuid = "d1eb7eb1-105f-429d-abf5-b0f65cb9e2c4" -version = "2.3.0" +version = "2.4.1" [[deps.Requires]] deps = ["UUIDs"] @@ -513,9 +512,9 @@ version = "0.3.0" [[deps.TimeZones]] deps = ["Artifacts", "Dates", "Downloads", "InlineStrings", "Mocking", "Printf", "RecipesBase", "Scratch", "TZJData", "Unicode", "p7zip_jll"] -git-tree-sha1 = "1f9a3f379a2ce2a213a0f606895567a08a1a2d08" +git-tree-sha1 = "06f4f1f3e8ff09e42e59b043a747332e88e01aba" uuid = "f269a46b-ccf7-5d73-abea-4c690281aa53" -version = "1.22.0" +version = "1.22.1" [[deps.TranscodingStreams]] git-tree-sha1 = "0c45878dcfdcfa8480052b6ab162cdd138781742" @@ -541,10 +540,10 @@ uuid = "ffd25f8a-64ca-5728-b0f7-c24cf3aae800" version = "5.8.1+0" [[deps.ZMQ]] -deps = ["FileWatching", "PrecompileTools", "Sockets", "ZeroMQ_jll"] -git-tree-sha1 = "2d060e1f014c07561817bf6f3c0eb66b309e04bd" +deps = ["FileWatching", "PrecompileTools", "Printf", "Sockets", "ZeroMQ_jll"] +git-tree-sha1 = "5f1c7008e2258c61af0eafef8c1f536b9fffbbd2" uuid = "c2297ded-f4af-51ae-bb23-16f91089e4e1" -version = "1.4.1" +version = "1.5.1" [[deps.ZeroMQ_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "libsodium_jll"] @@ -600,6 +599,6 @@ version = "4.7.2+0" [[deps.unzip_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "5d543463432c2e25027d88f5a314d40d44488391" +git-tree-sha1 = "0cd79af876ba6b5f93229d0e2c5455fcdd9930f7" uuid = "88f77b66-78eb-5ed0-bc16-ebba0796830d" -version = "6.0.2+0" +version = "6.0.3+0" diff --git a/.ci/Project.toml b/.ci/Project.toml index 2b9b041..431e84b 100644 --- a/.ci/Project.toml +++ b/.ci/Project.toml @@ -3,6 +3,7 @@ BinaryBuilder = "12aac903-9f7c-5d81-afc2-d9565ea332ae" BinaryBuilderBase = "7f725544-6523-48cd-82d1-3fa08ff4056e" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +GitHub = "bc5e4493-9b4d-5f90-b8aa-2b2bcaad7a26" HistoricalStdlibVersions = "6df8b67a-e8a0-4029-b4b7-ac196fe72102" JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" @@ -12,10 +13,15 @@ TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" ghr_jll = "07c12ed4-43bc-5495-8a2a-d5838ef8d533" squashfs_tools_jll = "eed32e3e-a7c5-5bf9-9121-5cf3ab653887" -[sources] -BinaryBuilder = {rev = "master", url = "https://github.com/JuliaPackaging/BinaryBuilder.jl.git"} -BinaryBuilderBase = {rev = "master", url = "https://github.com/JuliaPackaging/BinaryBuilderBase.jl.git"} - [compat] +GitHub = "5" HistoricalStdlibVersions = "=2.0.2" ghr_jll = "0.14.0" + +[sources.BinaryBuilder] +rev = "master" +url = "https://github.com/JuliaPackaging/BinaryBuilder.jl.git" + +[sources.BinaryBuilderBase] +rev = "master" +url = "https://github.com/JuliaPackaging/BinaryBuilderBase.jl.git" diff --git a/.ci/register_package.jl b/.ci/register_package.jl index 89be377..0147404 100644 --- a/.ci/register_package.jl +++ b/.ci/register_package.jl @@ -1,4 +1,5 @@ -using BinaryBuilder, BinaryBuilderBase, Downloads, Pkg +using BinaryBuilder, BinaryBuilder.Wizard, BinaryBuilderBase, Downloads, Pkg +using GitHub: authenticate, gh_get_json, DEFAULT_API # FIXME: Golang auto-upgrades to HTTP2, this can cause issue like https://github.com/google/go-github/issues/2113 ENV["GODEBUG"] = "http2client=0" @@ -30,7 +31,7 @@ dependencies = Dependency[dep for dep in merged["dependencies"] if !(isa(dep, Bu lazy_artifacts = merged["lazy_artifacts"] augment_platform_block = merged["augment_platform_block"] build_version = BinaryBuilder.get_next_wrapper_version(name, version) -repo = "JuliaBinaryWrappers/$(name)_jll.jl" +repo = "EnzymeAD/$(name)_jll.jl" code_dir = joinpath(Pkg.devdir(), "$(name)_jll") julia_compat = merged["julia_compat"] @@ -54,17 +55,6 @@ function mvdir(src, dest) end end -function download_cached_binaries(download_dir) - NAME = ENV["NAME"] - PROJECT = ENV["PROJECT"] - artifacts = "$(PROJECT)/products/$(NAME)*.tar.*" - cmd = `buildkite-agent artifact download $artifacts $download_dir` - if !success(pipeline(cmd; stderr)) - error("Download failed") - end - mvdir(joinpath(download_dir, PROJECT, "products"), download_dir) -end - function download_binaries_from_release(download_dir) function do_download(download_dir, info) url = info["url"] @@ -96,44 +86,58 @@ for json_obj in [merged, objs_unmerged...] json_obj["dependencies"] = Dependency[dep for dep in json_obj["dependencies"] if BinaryBuilderBase.is_runtime_dependency(dep)] end skip_build = get(ENV, "SKIP_BUILD", "false") == "true" -mktempdir() do download_dir - # Grab the binaries for our package - if skip_build - # We only want to update the wrappers, so download the tarballs from the - # latest build. - download_binaries_from_release(download_dir) - else - # We are going to publish the new binaries we've just baked, take them - # out of the cache while they're hot. - download_cached_binaries(download_dir) - end - # Push up the JLL package (pointing to as-of-yet missing tarballs) - tag = "$(name)-v$(build_version)" - upload_prefix = "https://github.com/$(repo)/releases/download/$(tag)" +# binaries will be automatically placed in the `products/` subdir by the GitHub +# Actions workflow. +download_dir = joinpath(pwd(), "products") +# Grab the binaries for our package +if skip_build + # We only want to update the wrappers, so download the tarballs from the + # latest build. + download_binaries_from_release(download_dir) +end + +# Push up the JLL package (pointing to as-of-yet missing tarballs) +tag = "$(name)-v$(build_version)" +upload_prefix = "https://github.com/$(repo)/releases/download/$(tag)" - # If we didn't rebuild the tarballs, save the original Artifacts.toml - artifacts_toml = skip_build ? read(joinpath(code_dir, "Artifacts.toml"), String) : "" - # This loop over the unmerged objects necessary in the event that we have multiple packages being built by a single build_tarballs.jl - for (i,json_obj) in enumerate(objs_unmerged) - from_scratch = (i == 1) - BinaryBuilder.rebuild_jll_package(json_obj; download_dir, upload_prefix, verbose, from_scratch) - end +# If we didn't rebuild the tarballs, save the original Artifacts.toml +artifacts_toml = skip_build ? read(joinpath(code_dir, "Artifacts.toml"), String) : "" +# This loop over the unmerged objects necessary in the event that we have multiple packages being built by a single build_tarballs.jl +for (i,json_obj) in enumerate(objs_unmerged) + from_scratch = (i == 1) + BinaryBuilder.rebuild_jll_package(json_obj; download_dir, upload_prefix, verbose, from_scratch) +end - # Restore Artifacts.toml - if skip_build - write(joinpath(code_dir, "Artifacts.toml"), artifacts_toml) - end +# Restore Artifacts.toml +if skip_build + write(joinpath(code_dir, "Artifacts.toml"), artifacts_toml) +end - # Push JLL package _before_ uploading to GitHub releases, so that this version of the code is what gets tagged - BinaryBuilder.push_jll_package(name, build_version) +# Push JLL package _before_ uploading to GitHub releases, so that this version of the code is what gets tagged +BinaryBuilder.push_jll_package(name, build_version; deploy_repo=repo) - if !skip_build - # Upload the tarballs to GitHub releases - BinaryBuilder.upload_to_github_releases(repo, tag, download_dir; verbose=verbose) - end +if !skip_build + # Upload the tarballs to GitHub releases + BinaryBuilder.upload_to_github_releases(repo, tag, download_dir; verbose=verbose) end # Sub off to Registrator to create a PR to General. Note: it's important to pass both # `augment_platform_block` and `lazy_artifacts` to build the right Project dictionary -BinaryBuilder.register_jll(name, build_version, dependencies, julia_compat; augment_platform_block, lazy_artifacts) +gh_auth = Wizard.github_auth(;allow_anonymous=false) +gh_username = "enzymead-bot[bot]" +registry_url = "https://github.com/JuliaRegistries/General" +registry_fork_org = "EnzymeAD" +registry_fork_url = "https://$(gh_username):$(gh_auth.token)@github.com/EnzymeAD/General" +gh_auth_pr = authenticate(ENV["JLBUILD_GITHUB_TOKEN"]) +BinaryBuilder.register_jll(name, build_version, dependencies, julia_compat; + deploy_repo=repo, + augment_platform_block, + lazy_artifacts, + gh_auth, + gh_username, + registry_url, + registry_fork_url, + registry_fork_org, + gh_auth_pr, + ) diff --git a/.github/workflows/build-reactant-reusable.yml b/.github/workflows/build-reactant-reusable.yml index 18cf8ec..d213522 100644 --- a/.github/workflows/build-reactant-reusable.yml +++ b/.github/workflows/build-reactant-reusable.yml @@ -2,6 +2,11 @@ name: "Build Reactant_jll" on: workflow_call: + secrets: + ENZYMEAD_BOT_ID: + required: true + ENZYMEAD_BOT_PRIVATE_KEY: + required: true inputs: julia_version: description: 'Julia version' @@ -206,16 +211,40 @@ jobs: register: name: Register runs-on: 'ubuntu-latest' - # We don't register the package when we're running CI for Enzyme-JAX. - if: ${{ inputs.enzyme_jax_commit == '' }} + # We only register on commits on `main` in the `EnzymeAD/ReactantBuilder` repo. + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'EnzymeAD/ReactantBuilder' }} needs: build-platforms steps: + - uses: actions/create-github-app-token@v2 + id: generate_token + with: + app-id: "${{ secrets.ENZYMEAD_BOT_ID }}" + private-key: "${{ secrets.ENZYMEAD_BOT_PRIVATE_KEY }}" + owner: EnzymeAD + repositories: | + Reactant_jll.jl + ReactantBuilder + General - uses: actions/checkout@v6 with: repository: 'EnzymeAD/ReactantBuilder' ref: ${{ inputs.reactantbuilder_ref == '' && github.sha || inputs.reactantbuilder_ref }} + - name: Set JULIA_PROJECT + # We have to use `${GITHUB_WORKSPACE}` instead of `github.workspace` because GitHub + # is terrible and the two don't match inside containers: + # https://github.com/actions/runner/issues/2058 + run: + echo "JULIA_PROJECT=${GITHUB_WORKSPACE}/.ci" >> "${GITHUB_ENV}" + - name: Set and create PRODUCTS_DIR + # We have to use `${GITHUB_WORKSPACE}` instead of `github.workspace` because GitHub + # is terrible and the two don't match inside containers: + # https://github.com/actions/runner/issues/2058 + run: | + PRODUCTS_DIR=${{ inputs.build_project }}/products + mkdir -p "${PRODUCTS_DIR}" + echo "PRODUCTS_DIR=${PRODUCTS_DIR}" >> "${GITHUB_ENV}" - uses: julia-actions/setup-julia@v2 with: version: '${{ inputs.julia_version }}' @@ -225,13 +254,41 @@ jobs: - uses: actions/download-artifact@v6 with: merge-multiple: true + path: ${{ env.PRODUCTS_DIR }} - name: Change Reactant commit run: | if [[ -n "${{ inputs.reactant_commit }}" ]]; then echo 'Changing Reactant commit to ${{ inputs.reactant_commit }}...' sed -i 's/reactant_commit = ".*"/reactant_commit = "'${{ inputs.reactant_commit }}'"/' R/Reactant/build_tarballs.jl fi - - name: Check + - name: Set up git + run: | + git config --global user.name "enzymead-bot[bot]" + git config --global user.email "238314553+enzymead-bot[bot]@users.noreply.github.com" + - name: Instantiate environment + timeout-minutes: 20 + shell: julia --color=yes {0} + run: | + using Pkg + Pkg.instantiate() + - name: Generate meta.json run: | - pwd - ls -lhrt + julia --color=yes --compile=min ./build_tarballs.jl --meta-json=$(basename ${{ inputs.build_project }}).meta.json + working-directory: ${{ inputs.build_project }} + env: + GITHUB_TOKEN: + - name: Register + run: | + julia --color=yes ${JULIA_PROJECT}/register_package.jl "$(basename ${{ inputs.build_project }}).meta.json" --verbose + working-directory: ${{ inputs.build_project }} + env: + GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }} + JLBUILD_GITHUB_TOKEN: ${{ secrets.JLBUILD_GITHUB_TOKEN }} + - name: Save Julia depot cache on cancel or failure + id: julia-cache-save + if: cancelled() || failure() + uses: actions/cache/save@v4 + with: + path: | + ${{ steps.julia-cache.outputs.cache-paths }} + key: ${{ steps.julia-cache.outputs.cache-key }} diff --git a/.github/workflows/build-reactant.yml b/.github/workflows/build-reactant.yml index fc097bb..71095ab 100644 --- a/.github/workflows/build-reactant.yml +++ b/.github/workflows/build-reactant.yml @@ -20,3 +20,4 @@ jobs: strategy: fail-fast: false uses: ./.github/workflows/build-reactant-reusable.yml + secrets: inherit diff --git a/H/HelloWorldC/build_tarballs.jl b/H/HelloWorldC2/build_tarballs.jl similarity index 96% rename from H/HelloWorldC/build_tarballs.jl rename to H/HelloWorldC2/build_tarballs.jl index 465df71..ad187f1 100644 --- a/H/HelloWorldC/build_tarballs.jl +++ b/H/HelloWorldC2/build_tarballs.jl @@ -1,7 +1,7 @@ using BinaryBuilder -name = "HelloWorldC" -version = v"1.4.1" +name = "HelloWorldC2" +version = v"1.0.0" # No sources, we're just building the testsuite sources = [ @@ -23,12 +23,11 @@ install_license /usr/share/licenses/MIT # These are the platforms we will build for by default, unless further # platforms are passed in on the command line -platforms = supported_platforms(; exclude=p->arch(p)=="riscv64") +platforms = supported_platforms(; exclude=p->arch(p)=="riscv64")[1:2] # The products that we will ensure are always built products = [ ExecutableProduct("hello_world", :hello_world), - # This ExecutableProduct is used in tests that change one of the paths ExecutableProduct("hello_world", :hello_world_doppelganger), ] diff --git a/H/HelloWorldC/bundled/CMakeLists.txt b/H/HelloWorldC2/bundled/CMakeLists.txt similarity index 100% rename from H/HelloWorldC/bundled/CMakeLists.txt rename to H/HelloWorldC2/bundled/CMakeLists.txt diff --git a/R/Reactant/build_tarballs.jl b/R/Reactant/build_tarballs.jl index a3effb2..fe1295b 100644 --- a/R/Reactant/build_tarballs.jl +++ b/R/Reactant/build_tarballs.jl @@ -3,11 +3,12 @@ using Base.BinaryPlatforms const YGGDRASIL_DIR = "../.." include(joinpath(YGGDRASIL_DIR, "fancy_toys.jl")) +include(joinpath(YGGDRASIL_DIR, "platforms", "macos_sdks.jl")) name = "Reactant" repo = "https://github.com/EnzymeAD/Reactant.jl.git" -reactant_commit = "e2f5c8c8c77f0f936d856486e4dd207161f81bf7" -version = v"0.0.257" +reactant_commit = "100135872d2a0793f7726de06bf46bc1871f4567" +version = v"0.0.265" sources = [ GitSource(repo, reactant_commit), @@ -46,23 +47,6 @@ if [[ -n "${ENZYME_JAX_COMMIT}" ]]; then sed -i.bak 's/ENZYMEXLA_COMMIT = ".*"/ENZYMEXLA_COMMIT = "'${ENZYME_JAX_COMMIT}'"/' WORKSPACE fi -if [[ "${target}" == *-apple-darwin* ]]; then - cd ${WORKSPACE}/srcdir - tar xf MacOSX*.sdk.tar.xz - cd - - # Compiling LLVM components within XLA requires macOS SDK 10.14 - # and then we use `std::reinterpret_pointer_cast` in ReactantExtra - # which requires macOS SDK 11.3. - # Install a newer SDK which supports C++20 - pushd $WORKSPACE/srcdir/MacOSX12.*.sdk - rm -rf /opt/${target}/${target}/sys-root/System - rm -rf /opt/${target}/${target}/sys-root/usr/* - cp -ra usr/* "/opt/${target}/${target}/sys-root/usr/." - cp -ra System "/opt/${target}/${target}/sys-root/." - popd - export MACOSX_DEPLOYMENT_TARGET=12.3 -fi - if [[ "${bb_full_target}" == *gpu+rocm* ]]; then cd ${WORKSPACE}/srcdir tar xzf therock-dist-linux-*.tar.gz @@ -463,6 +447,9 @@ if [[ "${bb_full_target}" == *gpu+rocm* ]]; then find bazel-bin find ${libdir} + install -Dvm 755 \ + $ROCM_PATH/lib/rocm_sysdeps/lib/librocm_sysdeps_dw.so* \ + -t ${libdir}/rocm_sysdeps/lib install -Dvm 755 \ $ROCM_PATH/lib/rocm_sysdeps/lib/librocm_sysdeps_numa.so* \ @@ -621,6 +608,13 @@ install -Dvm 755 bazel-bin/libReactantExtra.so "${libdir}/libReactantExtra.${dle install_license ../../LICENSE """ +# Compiling LLVM components within XLA requires macOS SDK 10.14 +# and then we use `std::reinterpret_pointer_cast` in ReactantExtra +# which requires macOS SDK 11.3. +# Install a newer SDK which supports C++20 +sources, script = require_macos_sdk("12.3", sources, script) + + # determine exactly which tarballs we should build builds = [] @@ -781,11 +775,6 @@ for gpu in ("none", "cuda", "rocm"), mode in ("opt", "dbg"), cuda_version in ("n HERMETIC_ROCM_VERSION=$(hermetic_rocm_version_map[rocm_version]) """ platform_sources = BinaryBuilder.AbstractSource[sources...] - if Sys.isapple(platform) - push!(platform_sources, - FileSource("https://github.com/realjf/MacOSX-SDKs/releases/download/v0.0.1/MacOSX12.3.sdk.tar.xz", - "a511c1cf1ebfe6fe3b8ec005374b9c05e89ac28b3d4eb468873f59800c02b030")) - end if arch(platform) == "aarch64" && gpu == "cuda" if hermetic_cuda_version_map[cuda_version] == "13.0.1" diff --git a/R/Reactant/platform_augmentation.jl b/R/Reactant/platform_augmentation.jl index cca2049..3dcacb9 100644 --- a/R/Reactant/platform_augmentation.jl +++ b/R/Reactant/platform_augmentation.jl @@ -29,7 +29,7 @@ else nothing end -const cuda_version_preference = if haskey(preferences, "gpu_version") +const gpu_version_preference = if haskey(preferences, "gpu_version") expected = ("none", "12.9", "13.0", "7.1") if isa(preferences["gpu_version"], String) && preferences["gpu_version"] in expected preferences["gpu_version"] @@ -118,11 +118,11 @@ function augment_platform!(platform::Platform) path = Libdl.dlpath(handle) Libdl.dlclose(handle) - if cuda_version_tag == "none" && current_cuda_version isa VersionNumber + if gpu_version_tag == "none" && current_cuda_version isa VersionNumber if v"12" <= current_cuda_version < v"13" - cuda_version_tag = "12.9" + gpu_version_tag = "12.9" elseif v"13.0" <= current_cuda_version < v"14" - cuda_version_tag = "13.0" + gpu_version_tag = "13.0" else @warn "CUDA version $(current_cuda_version) in $(path) not supported with this version of Reactant (min supported: 12)" end @@ -144,8 +144,8 @@ function augment_platform!(platform::Platform) #Libdl.dlclose(handle) gpu_version_tag = "7.1" - @debug "Adding include dependency on $(path)" - Base.include_dependency(path) + #@debug "Adding include dependency on $(path)" + #Base.include_dependency(path) gpu = "rocm" end diff --git a/platforms/macos_sdks.jl b/platforms/macos_sdks.jl new file mode 100644 index 0000000..60b160d --- /dev/null +++ b/platforms/macos_sdks.jl @@ -0,0 +1,128 @@ +# The natural way to handle installation of a new macOS SDK would be to +# add an ArchiveSource for the SDK and then moving it to the right place +# for the later build steps to find it. +# +# But this has the drawback of always extracting this (big) SDK, even on +# platforms that don't need it, which of course is most of them. Moreover, +# it floods the logs with the gargantuan list of files in these SDKs. +# +# Thus instead we use a FileSource. This way, the SDK is still always +# downloaded, but at least we can choose to only unpack it when we really +# need it. Plus we can directly unpack it in its final location; and +# suppress the full list of files being unpackaged +const macos_sdk_sources = Dict{String,FileSource}( + # this is the default SDK we use, so there is normally no need to request this; + # but we include it here to make this function also usable in the rootfs + "10.12" => + FileSource("https://github.com/phracker/MacOSX-SDKs/releases/download/10.15/MacOSX10.12.sdk.tar.xz", + "6852728af94399193599a55d00ae9c4a900925b6431534a3816496b354926774"), + "10.13" => + FileSource("https://github.com/phracker/MacOSX-SDKs/releases/download/10.15/MacOSX10.13.sdk.tar.xz", + "a3a077385205039a7c6f9e2c98ecdf2a720b2a819da715e03e0630c75782c1e4"), + "10.14" => + FileSource("https://github.com/phracker/MacOSX-SDKs/releases/download/10.15/MacOSX10.14.sdk.tar.xz", + "0f03869f72df8705b832910517b47dd5b79eb4e160512602f593ed243b28715f"), + "10.15" => + FileSource("https://github.com/phracker/MacOSX-SDKs/releases/download/10.15/MacOSX10.15.sdk.tar.xz", + "2408d07df7f324d3beea818585a6d990ba99587c218a3969f924dfcc4de93b62"), + "11.0" => + FileSource("https://github.com/phracker/MacOSX-SDKs/releases/download/11.0-11.1/MacOSX11.0.sdk.tar.xz", + "c8a9ff16196be43c35b699dd293f0c0563f1239c99caa6b3d53882e556a209bd"), + "11.1" => + FileSource("https://github.com/phracker/MacOSX-SDKs/releases/download/11.0-11.1/MacOSX11.1.sdk.tar.xz", + "9b86eab03176c56bb526de30daa50fa819937c54b280364784ce431885341bf6"), + "11.3" => + FileSource("https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX11.3.sdk.tar.xz", + "cd4f08a75577145b8f05245a2975f7c81401d75e9535dcffbb879ee1deefcbf4"), + "12.3" => + FileSource("https://github.com/realjf/MacOSX-SDKs/releases/download/v0.0.1/MacOSX12.3.sdk.tar.xz", + "a511c1cf1ebfe6fe3b8ec005374b9c05e89ac28b3d4eb468873f59800c02b030"), + "14.0" => + FileSource("https://github.com/roblabla/MacOSX-SDKs/releases/download/macosx14.0/MacOSX14.0.sdk.tar.xz", + "4a31565fd2644d1aec23da3829977f83632a20985561a2038e198681e7e7bf49"), + "14.5" => + FileSource("https://github.com/alexey-lysiuk/macos-sdk/releases/download/14.5/MacOSX14.5.tar.xz", + "f6acc6209db9d56b67fcaf91ec1defe48722e9eb13dc21fb91cfeceb1489e57e"), + "15.0" => + FileSource("https://github.com/joseluisq/MacOSX-SDKs/releases/download/15.0/MacOSX15.0.sdk.tar.xz", + "9df0293776fdc8a2060281faef929bf2fe1874c1f9368993e7a4ef87b1207f98"), +) + +""" + require_macos_sdk(version::String, sources::Vector, script::String; + deployment_target::String = version) + +Augment `sources` and `script` to ensure the macOS SDK with the indicated +`version` is available when building for macOS. Returns a new `sources` and +script value. + +Currently this only has an effect when building for macOS on Intel hardware, +as the ARM builder already uses a recent SDK. In the future this may also gain +support for installing newer SDK versions on ARM as well. +""" +function require_macos_sdk(version::String, sources::Vector, script::String; deployment_target::String = version) + return vcat(sources, get_macos_sdk_sources(version)), + get_macos_sdk_script(version; deployment_target) * script +end + +""" + get_macos_sdk_sources(version::String) + +Return a vector of sources to be appended to the `sources` list of a build recipe, +and `sdk_script` is a string to be inserted (usually at the start) of +the build script. + +If used together with `get_macos_sdk_script` this ensures that the macOS SDK +with the indicated `version` is available when building for macOS. + +Normally using `require_macos_sdk` should be preferred over this, so that the +SDK version is specified in a single place. But when necessary it is useful to +have this low-level alternative. +""" +function get_macos_sdk_sources(version::String) + haskey(macos_sdk_sources, version) || error("unsupported macOS SDK version $version") + + # return a vector just in case in the future we might need more than one source + return [ macos_sdk_sources[version] ] +end + +""" + get_macos_sdk_script(version::String) + +Return a a string to be inserted (usually at the start) of the build script +of a build recipe. + +If used together with `get_macos_sdk_sources` this ensures that the macOS SDK +with the indicated `version` is available when building for macOS. + +Normally using `require_macos_sdk` should be preferred over this, so that the +SDK version is specified in a single place. But when necessary it is useful to +have this low-level alternative. +""" +function get_macos_sdk_script(version::String; deployment_target::String = version) + # on ARM, the default macOS SDK we use is 11.1; so if the requested SDK + # version is older or equal to that, we can restrict to intel + arch = VersionNumber(version) <= v"11.1" ? "x86_64" : "*" + return """ + macos_sdk_version=$version + macosx_deployment_target=$deployment_target + """ * + raw""" + if [[ "${target}" == """*arch*raw"""-apple-darwin* ]]; then + echo "Extracting MacOSX${macos_sdk_version}.sdk.tar.xz (this may take a while)" + rm -rf /opt/${target}/${target}/sys-root/System + rm -rf /opt/${target}/${target}/sys-root/usr/include/libxml2/libxml + # extract the tarball into the sys-root so all compilers pick it up + # automatically, and use --warning=no-unknown-keyword to hide harmless + # warnings about unsupported pax header keywords like "SCHILY.fflags" + tar --extract \ + --file=${WORKSPACE}/srcdir/MacOSX${macos_sdk_version}.sdk.tar.xz \ + --directory="/opt/${target}/${target}/sys-root/." \ + --strip-components=1 \ + --warning=no-unknown-keyword \ + MacOSX${macos_sdk_version}.sdk/System \ + MacOSX${macos_sdk_version}.sdk/usr + export MACOSX_DEPLOYMENT_TARGET=${macosx_deployment_target} + fi + """ +end