Improve local development workflow

docs/src/building.md

@@ -344,3 +344,45 @@ Examples of builders that depend on other binaries include:
 * [`Xorg_libX11`](https://github.com/JuliaPackaging/Yggdrasil/blob/eb3728a2303c98519338fe0be370ef299b807e19/X/Xorg_libX11/build_tarballs.jl#L36-L42)
   depends on `Xorg_libxcb_jll`, and `Xorg_xtrans_jll` at build- and run-time,
   and on `Xorg_xorgproto_jll` and `Xorg_util_macros_jll` only at build-time.
+
+# Building and testing JLL packages locally
+
+As a package developer, you may want to test JLL packages locally, or as a binary dependency
+developer you may want to easily use custom binaries.  Through a combination of `dev`'ing out
+the JLL package and creating an `overrides` directory, it is easy to get complete control over
+the local JLL package state.
+
+## Overriding a prebuilt JLL package's binaries
+
+After running `pkg> dev LibFoo_jll`, a local JLL package will be checked out to your depot's
+`dev` directory (on most installations this is `~/.julia/dev`) and by default the JLL package
+will make use of binaries within your depot's `artifacts` directory.  If an `override`
+directory is present within the JLL package directory, the JLL package will look within that
+`override` directory for binaries, rather than in any artifact directory.  Note that there is
+no mixing and matching of binaries within a single JLL package; if an `override` directory is
+present, all products defined within that JLL package must be found within the `override`
+directory, none will be sourced from an artifact.  Dependencies (e.g. found within another
+JLL package) may still be loaded from their respective artifacts, so dependency JLLs must
+themselves be `dev`'ed and have `override` directories created with files or symlinks
+created within them.
+
+### Auto-populating the `override` directory
+
+To ease creation of an `override` directory, JLL packages contain a `dev_jll()` function,
+that will ensure that a `~/.julia/dev/<jll name>` package is `dev`'ed out, and will copy the
+normal artifact contents into the appropriate `override` directory.  This will result in no
+functional difference from simply using the artifact directory, but provides a template of
+files that can be replaced by custom-built binaries.
+
+Note that this feature is rolling out to new JLL packages as they are rebuilt; if a JLL
+package does not have a `dev_jll()` function, [open an issue on Yggdrasil](https://github.com/JuliaPackaging/Yggdrasil/issues/new)
+and a new JLL version will be generated to provide the function.
+
+## Building a custom JLL package locally
+
+When building a new version of a JLL package, if `--deploy` is passed to `build_tarballs.jl`
+then a newly-built JLL package will be deployed to a GitHub repository.  (Read the
+documentation given by passing `--help` to a `build_tarballs.jl` script for more on
+`--deploy` options).  If `--deploy=local` is passed, the JLL package will still be built
+in the `~/.julia/dev/` directory, but it will not be uploaded anywhere.  This is useful
+for local testing and validation that the built artifacts are working with your package.

docs/src/jll.md

@@ -209,3 +209,15 @@ defined for it are:
 
 * `data_txt_path`: this unexported variable is actually equal to `data_txt`, but
   is kept for consistency with all other product types.
+
+
+## Overriding `dev`'ed JLL packages
+
+In the event that a user wishes to override the content within a JLL package with
+their own binaries/libraries/files, the user may use the `dev_jll()` method provided
+by JLL packages to check out a mutable copy of the package to their `~/.julia/dev`
+directory.  An `override` directory will be created within that package directory,
+providing a convenient location for the user to copy in their own files over the
+typically artifact-sourced ones.  See the segment on "Building and testing JLL
+packages locally" in the [Building Packages](./building.md) section of this
+documentation for more information on this capability.

src/AutoBuild.jl

@@ -36,7 +36,9 @@ const BUILD_HELP = (
                             default, unless `<repo>` is set, in which case it
                             should be set as `<owner>/<name>_jll.jl`.  Setting
                             this option is equivalent to setting `--deploy-bin`
-                            and `--deploy-jll`.
+                            and `--deploy-jll`.  If `<repo>` is set to "local"
+                            then nothing will be uploaded, but JLL packages
+                            will still be written out to `~/.julia/dev/`.
 
         --deploy-bin=<repo> Deploy just the built binaries
 
@@ -141,6 +143,9 @@ function build_tarballs(ARGS, src_name, src_version, sources, script,
     if register && !deploy_jll
         error("Cannot register without deploying!")
     end
+    if register && deploy_jll_repo == "local"
+        error("Cannot register with a local deployment!")
+    end
 
     if deploy_bin || deploy_jll
         code_dir = joinpath(Pkg.devdir(), "$(src_name)_jll")
@@ -183,13 +188,15 @@ function build_tarballs(ARGS, src_name, src_version, sources, script,
         # choose a version number that is greater than anything else existent.
         build_version = get_next_wrapper_version(src_name, src_version)
         if verbose
-            @info("Building and deploying version $(build_version) to $(deploy_repo)")
+            @info("Building and deploying version $(build_version) to $(deploy_jll_repo)")
         end
         tag = "$(src_name)-v$(build_version)"
 
         # We need to make sure that the JLL repo at least exists, so that we can deploy binaries to it
         # even if we're not planning to register things to it today.
-        init_jll_package(src_name, code_dir, deploy_jll_repo)
+        if deploy_jll_repo != "local"
+            init_jll_package(src_name, code_dir, deploy_jll_repo)
+        end
     end
 
     # Build the given platforms using the given sources
@@ -240,7 +247,9 @@ function build_tarballs(ARGS, src_name, src_version, sources, script,
                           dependencies, bin_path; verbose=verbose,
                           extract_kwargs(kwargs, (:lazy_artifacts, :init_block))...,
                           )
-        push_jll_package(src_name, build_version; code_dir=code_dir, deploy_repo=deploy_repo)
+        if deploy_jll_repo != "local"
+            push_jll_package(src_name, build_version; code_dir=code_dir, deploy_repo=deploy_jll_repo)
+        end
         if register
             if verbose
                 @info("Registering new wrapper code version $(build_version)...")
@@ -251,7 +260,7 @@ function build_tarballs(ARGS, src_name, src_version, sources, script,
         end
     end
 
-    if deploy_bin
+    if deploy_bin && deploy_bin_repo != "local"
         # Upload the binaries
         if verbose
             @info("Deploying binaries to release $(tag) on $(deploy_bin_repo) via `ghr`...")
@@ -1121,25 +1130,25 @@ function build_jll_package(src_name::String,
             end
 
             if !isempty(dependencies)
-                print(io,
-                      """
-                      # Initialize PATH and LIBPATH environment variable listings.
-                      # From the list of our dependencies, generate a tuple of all the PATH and LIBPATH lists,
-                      # then append them to our own.
-                      foreach(p -> append!(PATH_list, p), ($(join(["$(getname(dep)).PATH_list" for dep in dependencies], ", ")),))
-                      foreach(p -> append!(LIBPATH_list, p), ($(join(["$(getname(dep)).LIBPATH_list" for dep in dependencies], ", ")),))
-                      """)
+                print(io, """
+                # Initialize PATH and LIBPATH environment variable listings.
+                # From the list of our dependencies, generate a tuple of all the PATH and LIBPATH lists,
+                # then append them to our own.
+                foreach(p -> append!(PATH_list, p), ($(join(["$(getname(dep)).PATH_list" for dep in dependencies], ", ")),))
+                foreach(p -> append!(LIBPATH_list, p), ($(join(["$(getname(dep)).LIBPATH_list" for dep in dependencies], ", ")),))
+                """)
             end
 
             print(io, """
             # Inform that the wrapper is available for this platform
-            is_available() = true
+            wrapper_available = true
 
             \"\"\"
             Open all libraries
             \"\"\"
             function __init__()
-                global artifact_dir = abspath(artifact"$(src_name)")
+                # This either calls `@artifact_str()`, or returns a constant string if we're overridden.
+                global artifact_dir = find_artifact_dir()
 
                 global PATH_list, LIBPATH_list
             """)
@@ -1219,18 +1228,55 @@ function build_jll_package(src_name::String,
         using Pkg, Pkg.BinaryPlatforms, Pkg.Artifacts, Libdl
         import Base: UUID
 
+        wrapper_available = false
         \"\"\"
             is_available()
 
         Return whether the artifact is available for the current platform.
         \"\"\"
-        is_available() = false
+        is_available() = wrapper_available
 
         # We put these inter-JLL-package API values here so that they are always defined, even if there
         # is no underlying wrapper held within this JLL package.
         const PATH_list = String[]
         const LIBPATH_list = String[]
 
+        # We determine, here, at compile-time, whether our JLL package has been dev'ed and overridden
+        override_dir = joinpath(dirname(@__DIR__), "override")
+        if isdir(override_dir)
+            function find_artifact_dir()
+                return override_dir
+            end
+        else
+            function find_artifact_dir()
+                return artifact"$(src_name)"
+            end
+
+            \"\"\"
+                dev_jll()
+
+            Check this package out to the dev package directory (usually ~/.julia/dev),
+            copying the artifact over to a local `override` directory, allowing package
+            developers to experiment with a locally-built binary.
+            \"\"\"
+            function dev_jll()
+                # First, `dev` out the package, but don't effect the current project
+                mktempdir() do temp_env
+                    Pkg.activate(temp_env) do
+                        Pkg.develop("$(src_name)_jll")
+                    end
+                end
+                # Create the override directory
+                override_dir = joinpath(Pkg.devdir(), "$(src_name)_jll", "override")
+                # Copy the current artifact contents into that directory
+                if !isdir(override_dir)
+                    cp(artifact"$(src_name)", override_dir)
+                end
+                # Force recompilation of that package, just in case it wasn't dev'ed before
+                touch(joinpath(Pkg.devdir(), "$(src_name)_jll", "src", "$(src_name)_jll.jl"))
+                @info("$(src_name)_ll dev'ed out to $(joinpath(Pkg.devdir(), "$(src_name)_jll")) with pre-populated override directory")
+            end
+        end
         """
     if Set(platforms) == Set([AnyPlatform()])
         # We know directly the wrapper we want to include
@@ -1385,6 +1431,11 @@ function build_jll_package(src_name::String,
     open(joinpath(code_dir, "Project.toml"), "w") do io
         Pkg.TOML.print(io, project)
     end
+
+    # Add a `.gitignore`
+    open(joinpath(code_dir, ".gitignore"), "w") do io
+        println(io, "override/")
+    end
 end
 
 function push_jll_package(name, build_version;