diff --git a/docs/src/building.md b/docs/src/building.md index 93453aaa7..469eca01f 100644 --- a/docs/src/building.md +++ b/docs/src/building.md @@ -344,3 +344,45 @@ Examples of builders that depend on other binaries include: * [`Xorg_libX11`](https://github.com/JuliaPackaging/Yggdrasil/blob/eb3728a2303c98519338fe0be370ef299b807e19/X/Xorg_libX11/build_tarballs.jl#L36-L42) depends on `Xorg_libxcb_jll`, and `Xorg_xtrans_jll` at build- and run-time, and on `Xorg_xorgproto_jll` and `Xorg_util_macros_jll` only at build-time. + +# Building and testing JLL packages locally + +As a package developer, you may want to test JLL packages locally, or as a binary dependency +developer you may want to easily use custom binaries. Through a combination of `dev`'ing out +the JLL package and creating an `overrides` directory, it is easy to get complete control over +the local JLL package state. + +## Overriding a prebuilt JLL package's binaries + +After running `pkg> dev LibFoo_jll`, a local JLL package will be checked out to your depot's +`dev` directory (on most installations this is `~/.julia/dev`) and by default the JLL package +will make use of binaries within your depot's `artifacts` directory. If an `override` +directory is present within the JLL package directory, the JLL package will look within that +`override` directory for binaries, rather than in any artifact directory. Note that there is +no mixing and matching of binaries within a single JLL package; if an `override` directory is +present, all products defined within that JLL package must be found within the `override` +directory, none will be sourced from an artifact. Dependencies (e.g. found within another +JLL package) may still be loaded from their respective artifacts, so dependency JLLs must +themselves be `dev`'ed and have `override` directories created with files or symlinks +created within them. + +### Auto-populating the `override` directory + +To ease creation of an `override` directory, JLL packages contain a `dev_jll()` function, +that will ensure that a `~/.julia/dev/` package is `dev`'ed out, and will copy the +normal artifact contents into the appropriate `override` directory. This will result in no +functional difference from simply using the artifact directory, but provides a template of +files that can be replaced by custom-built binaries. + +Note that this feature is rolling out to new JLL packages as they are rebuilt; if a JLL +package does not have a `dev_jll()` function, [open an issue on Yggdrasil](https://github.com/JuliaPackaging/Yggdrasil/issues/new) +and a new JLL version will be generated to provide the function. + +## Building a custom JLL package locally + +When building a new version of a JLL package, if `--deploy` is passed to `build_tarballs.jl` +then a newly-built JLL package will be deployed to a GitHub repository. (Read the +documentation given by passing `--help` to a `build_tarballs.jl` script for more on +`--deploy` options). If `--deploy=local` is passed, the JLL package will still be built +in the `~/.julia/dev/` directory, but it will not be uploaded anywhere. This is useful +for local testing and validation that the built artifacts are working with your package. \ No newline at end of file diff --git a/docs/src/jll.md b/docs/src/jll.md index 35f142990..3520d3a07 100644 --- a/docs/src/jll.md +++ b/docs/src/jll.md @@ -209,3 +209,15 @@ defined for it are: * `data_txt_path`: this unexported variable is actually equal to `data_txt`, but is kept for consistency with all other product types. + + +## Overriding `dev`'ed JLL packages + +In the event that a user wishes to override the content within a JLL package with +their own binaries/libraries/files, the user may use the `dev_jll()` method provided +by JLL packages to check out a mutable copy of the package to their `~/.julia/dev` +directory. An `override` directory will be created within that package directory, +providing a convenient location for the user to copy in their own files over the +typically artifact-sourced ones. See the segment on "Building and testing JLL +packages locally" in the [Building Packages](./building.md) section of this +documentation for more information on this capability. \ No newline at end of file diff --git a/src/AutoBuild.jl b/src/AutoBuild.jl index 493faa836..2b514abc4 100644 --- a/src/AutoBuild.jl +++ b/src/AutoBuild.jl @@ -36,7 +36,9 @@ const BUILD_HELP = ( default, unless `` is set, in which case it should be set as `/_jll.jl`. Setting this option is equivalent to setting `--deploy-bin` - and `--deploy-jll`. + and `--deploy-jll`. If `` is set to "local" + then nothing will be uploaded, but JLL packages + will still be written out to `~/.julia/dev/`. --deploy-bin= Deploy just the built binaries @@ -141,6 +143,9 @@ function build_tarballs(ARGS, src_name, src_version, sources, script, if register && !deploy_jll error("Cannot register without deploying!") end + if register && deploy_jll_repo == "local" + error("Cannot register with a local deployment!") + end if deploy_bin || deploy_jll code_dir = joinpath(Pkg.devdir(), "$(src_name)_jll") @@ -183,13 +188,15 @@ function build_tarballs(ARGS, src_name, src_version, sources, script, # choose a version number that is greater than anything else existent. build_version = get_next_wrapper_version(src_name, src_version) if verbose - @info("Building and deploying version $(build_version) to $(deploy_repo)") + @info("Building and deploying version $(build_version) to $(deploy_jll_repo)") end tag = "$(src_name)-v$(build_version)" # We need to make sure that the JLL repo at least exists, so that we can deploy binaries to it # even if we're not planning to register things to it today. - init_jll_package(src_name, code_dir, deploy_jll_repo) + if deploy_jll_repo != "local" + init_jll_package(src_name, code_dir, deploy_jll_repo) + end end # Build the given platforms using the given sources @@ -240,7 +247,9 @@ function build_tarballs(ARGS, src_name, src_version, sources, script, dependencies, bin_path; verbose=verbose, extract_kwargs(kwargs, (:lazy_artifacts, :init_block))..., ) - push_jll_package(src_name, build_version; code_dir=code_dir, deploy_repo=deploy_repo) + if deploy_jll_repo != "local" + push_jll_package(src_name, build_version; code_dir=code_dir, deploy_repo=deploy_jll_repo) + end if register if verbose @info("Registering new wrapper code version $(build_version)...") @@ -251,7 +260,7 @@ function build_tarballs(ARGS, src_name, src_version, sources, script, end end - if deploy_bin + if deploy_bin && deploy_bin_repo != "local" # Upload the binaries if verbose @info("Deploying binaries to release $(tag) on $(deploy_bin_repo) via `ghr`...") @@ -1121,25 +1130,25 @@ function build_jll_package(src_name::String, end if !isempty(dependencies) - print(io, - """ - # Initialize PATH and LIBPATH environment variable listings. - # From the list of our dependencies, generate a tuple of all the PATH and LIBPATH lists, - # then append them to our own. - foreach(p -> append!(PATH_list, p), ($(join(["$(getname(dep)).PATH_list" for dep in dependencies], ", ")),)) - foreach(p -> append!(LIBPATH_list, p), ($(join(["$(getname(dep)).LIBPATH_list" for dep in dependencies], ", ")),)) - """) + print(io, """ + # Initialize PATH and LIBPATH environment variable listings. + # From the list of our dependencies, generate a tuple of all the PATH and LIBPATH lists, + # then append them to our own. + foreach(p -> append!(PATH_list, p), ($(join(["$(getname(dep)).PATH_list" for dep in dependencies], ", ")),)) + foreach(p -> append!(LIBPATH_list, p), ($(join(["$(getname(dep)).LIBPATH_list" for dep in dependencies], ", ")),)) + """) end print(io, """ # Inform that the wrapper is available for this platform - is_available() = true + wrapper_available = true \"\"\" Open all libraries \"\"\" function __init__() - global artifact_dir = abspath(artifact"$(src_name)") + # This either calls `@artifact_str()`, or returns a constant string if we're overridden. + global artifact_dir = find_artifact_dir() global PATH_list, LIBPATH_list """) @@ -1219,18 +1228,55 @@ function build_jll_package(src_name::String, using Pkg, Pkg.BinaryPlatforms, Pkg.Artifacts, Libdl import Base: UUID + wrapper_available = false \"\"\" is_available() Return whether the artifact is available for the current platform. \"\"\" - is_available() = false + is_available() = wrapper_available # We put these inter-JLL-package API values here so that they are always defined, even if there # is no underlying wrapper held within this JLL package. const PATH_list = String[] const LIBPATH_list = String[] + # We determine, here, at compile-time, whether our JLL package has been dev'ed and overridden + override_dir = joinpath(dirname(@__DIR__), "override") + if isdir(override_dir) + function find_artifact_dir() + return override_dir + end + else + function find_artifact_dir() + return artifact"$(src_name)" + end + + \"\"\" + dev_jll() + + Check this package out to the dev package directory (usually ~/.julia/dev), + copying the artifact over to a local `override` directory, allowing package + developers to experiment with a locally-built binary. + \"\"\" + function dev_jll() + # First, `dev` out the package, but don't effect the current project + mktempdir() do temp_env + Pkg.activate(temp_env) do + Pkg.develop("$(src_name)_jll") + end + end + # Create the override directory + override_dir = joinpath(Pkg.devdir(), "$(src_name)_jll", "override") + # Copy the current artifact contents into that directory + if !isdir(override_dir) + cp(artifact"$(src_name)", override_dir) + end + # Force recompilation of that package, just in case it wasn't dev'ed before + touch(joinpath(Pkg.devdir(), "$(src_name)_jll", "src", "$(src_name)_jll.jl")) + @info("$(src_name)_ll dev'ed out to $(joinpath(Pkg.devdir(), "$(src_name)_jll")) with pre-populated override directory") + end + end """ if Set(platforms) == Set([AnyPlatform()]) # We know directly the wrapper we want to include @@ -1385,6 +1431,11 @@ function build_jll_package(src_name::String, open(joinpath(code_dir, "Project.toml"), "w") do io Pkg.TOML.print(io, project) end + + # Add a `.gitignore` + open(joinpath(code_dir, ".gitignore"), "w") do io + println(io, "override/") + end end function push_jll_package(name, build_version;