Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Setup dependencies that match the GCC ABI of the compiler #604

Merged
merged 8 commits into from
Jan 18, 2020
20 changes: 18 additions & 2 deletions src/AutoBuild.jl
Original file line number Diff line number Diff line change
Expand Up @@ -629,25 +629,41 @@ function autobuild(dir::AbstractString,
build_path = joinpath(dir, "build", triplet(platform))
mkpath(build_path)

shards = choose_shards(platform; extract_kwargs(kwargs, (:preferred_gcc_version,:preferred_llvm_version,:bootstrap_list,:compilers))...)
# We want to get dependencies that have exactly the same GCC ABI as the
# choosen compiler, otherwise we risk, e.g., to build in an environment
# with libgfortran3 a dependency built with libgfortran5.
# `concrete_platform` is needed only to setup the dependencies and the
# runner. We _don't_ want the platform passed to `audit()` or
# `package()` to be more specific than it is.
concrete_platform = platform
gccboostrap_shard_idx = findfirst(x -> x.name == "GCCBootstrap" && x.target.arch == platform.arch && x.target.libc == platform.libc, shards)
if !isnothing(gccboostrap_shard_idx)
libgfortran_version = preferred_libgfortran_version(platform, shards[gccboostrap_shard_idx])
cxxstring_abi = preferred_cxxstring_abi(platform, shards[gccboostrap_shard_idx])
concrete_platform = replace_cxxstring_abi(replace_libgfortran_version(platform, libgfortran_version), cxxstring_abi)
end

prefix = setup_workspace(
build_path,
src_paths,
src_hashes;
verbose=verbose,
)
artifact_paths = setup_dependencies(prefix, dependencies, platform)
artifact_paths = setup_dependencies(prefix, dependencies, concrete_platform)

# Create a runner to work inside this workspace with the nonce built-in
ur = preferred_runner()(
prefix.path;
cwd = "/workspace/srcdir",
platform = platform,
platform = concrete_platform,
verbose = verbose,
workspaces = [
joinpath(prefix, "metadir") => "/meta",
],
compiler_wrapper_dir = joinpath(prefix, "compiler_wrappers"),
src_name = src_name,
shards = shards,
extract_kwargs(kwargs, (:preferred_gcc_version,:preferred_llvm_version,:compilers,:allow_unsafe_flags))...,
)

Expand Down
7 changes: 5 additions & 2 deletions src/DockerRunner.jl
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ function DockerRunner(workspace_root::String;
verbose::Bool = false,
compiler_wrapper_path::String = mktempdir(),
src_name::AbstractString = "",
shards = nothing,
kwargs...)
global use_ccache

Expand All @@ -97,8 +98,10 @@ function DockerRunner(workspace_root::String;
push!(workspaces, "binarybuilder_ccache" => "/root/.ccache")
end

# Choose the shards we're going to mount
shards = choose_shards(platform; extract_kwargs(kwargs, (:preferred_gcc_version,:preferred_llvm_version,:bootstrap_list,:compilers))...)
if isnothing(shards)
# Choose the shards we're going to mount
shards = choose_shards(platform; extract_kwargs(kwargs, (:preferred_gcc_version,:preferred_llvm_version,:bootstrap_list,:compilers))...)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we move this into the kwarg initialization above? I'd rather avoid the "default value of nothing means initialize it halfway through the method" pattern if we can, and I don't think there's a hidden reason why choose_shards() must be run after the stuff above it.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We don't know kwargs when we initialise shards, do we?

end

# Import docker image
import_docker_image(shards[1], workspace_root; verbose=verbose)
Expand Down
140 changes: 129 additions & 11 deletions src/Rootfs.jl
Original file line number Diff line number Diff line change
Expand Up @@ -319,15 +319,78 @@ function select_closest_version(preferred::VersionNumber, versions::Vector{Versi
return versions[closest_idx]
end

const available_gcc_builds = [v"4.8.5", v"5.2.0", v"6.1.0", v"7.1.0", v"8.1.0", v"9.1.0"]
const available_llvm_builds = [v"6.0.1", v"7.1.0", v"8.0.1", v"9.0.1"]
abstract type CompilerBuild end

struct GCCBuild <: CompilerBuild
version::VersionNumber
abi::CompilerABI
end
GCCBuild(v::VersionNumber) = GCCBuild(v, CompilerABI())

struct LLVMBuild <: CompilerBuild
version::VersionNumber
abi::CompilerABI
end
LLVMBuild(v::VersionNumber) = LLVMBuild(v, CompilerABI())

getversion(c::CompilerBuild) = c.version
getabi(c::CompilerBuild) = c.abi

const available_gcc_builds = [GCCBuild(v"4.8.5", CompilerABI(libgfortran_version = v"3", libstdcxx_version = v"3.4.19", cxxstring_abi = :cxx03)),
GCCBuild(v"5.2.0", CompilerABI(libgfortran_version = v"3", libstdcxx_version = v"3.4.21", cxxstring_abi = :cxx11)),
GCCBuild(v"6.1.0", CompilerABI(libgfortran_version = v"3", libstdcxx_version = v"3.4.22", cxxstring_abi = :cxx11)),
GCCBuild(v"7.1.0", CompilerABI(libgfortran_version = v"4", libstdcxx_version = v"3.4.23", cxxstring_abi = :cxx11)),
GCCBuild(v"8.1.0", CompilerABI(libgfortran_version = v"5", libstdcxx_version = v"3.4.25", cxxstring_abi = :cxx11)),
GCCBuild(v"9.1.0", CompilerABI(libgfortran_version = v"5", libstdcxx_version = v"3.4.26", cxxstring_abi = :cxx11))]
const available_llvm_builds = [LLVMBuild(v"6.0.1"),
LLVMBuild(v"7.1.0"),
LLVMBuild(v"8.0.1"),
LLVMBuild(v"9.0.1")]

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can see us eventually adding Rust, Go, etc... things here. Nice.

"""
gcc_version(cabi::CompilerABI, GCC_builds::Vector{GCCBuild})

Returns the closest matching GCC version number for the given CompilerABI
giordano marked this conversation as resolved.
Show resolved Hide resolved
representing a particular platofrm, from the given set of options. If no match
is found, returns an empty list. This method assumes that `cabi` represents a
platform that binaries will be run on, and thus versions are always rounded
down; e.g. if the platform supports a `libstdc++` version that corresponds to
`GCC 5.1.0`, but the only GCC versions available to be picked from are `4.8.5`
and `5.2.0`, it will return `4.8.5`, as binaries compiled with that version
will run on this platform, whereas binaries compiled with `5.2.0` may not.
"""
function gcc_version(cabi::CompilerABI, GCC_builds::Vector{GCCBuild})
# First, filter by libgfortran version.
if cabi.libgfortran_version !== nothing
GCC_builds = filter(b -> getabi(b).libgfortran_version == cabi.libgfortran_version, GCC_builds)
end

# Next, filter by libstdc++ GLIBCXX symbol version. Note that this
# mapping is conservative; it is often the case that we return a
# version that is slightly lower than what is actually installed on
# a system. See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html
# for the whole list, as well as many other interesting factoids.
if cabi.libstdcxx_version !== nothing
GCC_builds = filter(b -> getabi(b).libstdcxx_version <= cabi.libstdcxx_version, GCC_builds)
end

# Finally, enforce cxxstring_abi guidelines. It is possible to build
# :cxx03 binaries on GCC 5+, (although increasingly rare) so the only
# filtering we do is that if the platform is explicitly :cxx11, we
# disallow running on < GCC 5.
if cabi.cxxstring_abi !== nothing && cabi.cxxstring_abi === :cxx11
GCC_builds = filter(b -> getversion(b) >= v"5", GCC_builds)
end

return getversion.(GCC_builds)
end

function select_gcc_version(p::Platform,
GCC_builds::Vector{VersionNumber} = available_gcc_builds,
preferred_gcc_version::VersionNumber = GCC_builds[1],
GCC_builds::Vector{GCCBuild} = available_gcc_builds,
preferred_gcc_version::VersionNumber = getversion(GCC_builds[1]),
)
# Determine which GCC build we're going to match with this CompilerABI:
GCC_builds = Pkg.BinaryPlatforms.gcc_version(compiler_abi(p), GCC_builds)
GCC_builds = gcc_version(compiler_abi(p), GCC_builds)

if isempty(GCC_builds)
error("Impossible CompilerABI constraints $(cabi)!")
Expand All @@ -350,19 +413,22 @@ function choose_shards(p::Platform;
compilers::Vector{Symbol} = [:c],
rootfs_build::VersionNumber=v"2020.01.07",
ps_build::VersionNumber=v"2020.01.15",
GCC_builds::Vector{VersionNumber}=available_gcc_builds,
LLVM_builds::Vector{VersionNumber}=available_llvm_builds,
GCC_builds::Vector{GCCBuild}=available_gcc_builds,
LLVM_builds::Vector{LLVMBuild}=available_llvm_builds,
Rust_build::VersionNumber=v"1.18.3",
Go_build::VersionNumber=v"1.13",
archive_type::Symbol = (use_squashfs ? :squashfs : :unpacked),
bootstrap_list::Vector{Symbol} = bootstrap_list,
# We prefer the oldest GCC version by default
preferred_gcc_version::VersionNumber = GCC_builds[1],
preferred_llvm_version::VersionNumber = LLVM_builds[end],
# Because GCC has lots of compatibility issues, we always default to
# the earliest version possible.
preferred_gcc_version::VersionNumber = getversion(GCC_builds[1]),
# Because LLVM doesn't have compatibilty issues, we always default
# to the newest version possible.
preferred_llvm_version::VersionNumber = getversion(LLVM_builds[end]),
giordano marked this conversation as resolved.
Show resolved Hide resolved
)

GCC_build = select_gcc_version(p, GCC_builds, preferred_gcc_version)
LLVM_build = select_closest_version(preferred_llvm_version, LLVM_builds)
LLVM_build = select_closest_version(preferred_llvm_version, getversion.(LLVM_builds))
# Our host platform is x86_64-linux-musl
host_platform = Linux(:x86_64; libc=:musl)

Expand Down Expand Up @@ -531,6 +597,58 @@ function expand_cxxstring_abis(ps::Vector{<:Platform})
return Platform[p for p in Iterators.flatten(expand_cxxstring_abis.(ps))]
end

"""
preferred_libgfortran_version(platform::Platform, shard::CompilerShard)

Return the libgfortran version preferred by the given platform or GCCBootstrap shard.
"""
function preferred_libgfortran_version(platform::Platform, shard::CompilerShard)
# Some input validation
if shard.name != "GCCBootstrap"
error("Shard must be `GCCBootstrap`")
end
if shard.target.arch != platform.arch || shard.target.libc != platform.libc
error("Incompatible platform and shard target")
end

if compiler_abi(platform).libgfortran_version != nothing
# Here we can't use `shard.target` because the shard always has the
# target as ABI-agnostic, thus we have also to ask for the platform.
return compiler_abi(platform).libgfortran_version
elseif shard.version < v"7"
giordano marked this conversation as resolved.
Show resolved Hide resolved
return v"3"
elseif v"7" <= shard.version < v"8"
return v"4"
else
return v"5"
end
end

"""
preferred_cxxstring_abi(platform::Platform, shard::CompilerShard)

Return the C++ string ABI preferred by the given platform or GCCBootstrap shard.
"""
function preferred_cxxstring_abi(platform::Platform, shard::CompilerShard)
# Some input validation
if shard.name != "GCCBootstrap"
error("Shard must be `GCCBootstrap`")
end
if shard.target.arch != platform.arch || shard.target.libc != platform.libc
error("Incompatible platform and shard target")
end

if compiler_abi(platform).cxxstring_abi != nothing
# Here we can't use `shard.target` because the shard always has the
# target as ABI-agnostic, thus we have also to ask for the platform.
return compiler_abi(platform).cxxstring_abi
elseif shard.version < v"5"
return :cxx03
else
return :cxx11
end
end

"""
download_all_shards(; verbose::Bool=false)

Expand Down
41 changes: 37 additions & 4 deletions src/Runner.jl
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,11 @@ exeext(p::Union{Linux,FreeBSD,MacOS}) = ""
exeext(p::Platform) = error("Unknown exeext for platform $(p)")

"""
generate_compiler_wrappers(p::Platform, bin_path::AbstractString)
generate_compiler_wrappers!(platform::Platform; bin_path::AbstractString,
host_platform::Platform = Linux(:x86_64; libc=:musl),
rust_platform::Platform = Linux(:x86_64; libc=:glibc),
compilers::Vector{Symbol} = [:c],
allow_unsafe_flags::Bool = false)

We generate a set of compiler wrapper scripts within our build environment to force all
build systems to honor the necessary sets of compiler flags to build for our systems.
Expand Down Expand Up @@ -61,6 +65,32 @@ function generate_compiler_wrappers!(platform::Platform; bin_path::AbstractStrin
host_target = aatriplet(host_platform)
rust_target = aatriplet(rust_platform)


# If the ABI-agnostic triplets of the target and the host platform are the
# same, then we have to be very careful: we can't have distinct wrappers, so
# we have to be sure that their ABIs are consistent, in particular that
# we're correctly writing the wrappers for the target platform. In
# particular, what we care about with regard to the wrappers is the C++
# string ABI. We have the following situations:
# * they are equal: this is fine
# * they're different and the host has a prefernce for the C++ string ABI:
# we can't deal with this situation as the host wrappers will be always
# overwritten, then error out
# * in all other cases we don't do anything here but later below we'll let
# the host wrappers to be overwritten by the wrappers for the target
if target == host_target
target_cxxabi = compiler_abi(platform).cxxstring_abi
host_cxxabi = compiler_abi(host_platform).cxxstring_abi
if target_cxxabi !== host_cxxabi
if host_cxxabi !== nothing
# This is a very unlikely situation as ideally the host
# shouldn't have particular preferences for the ABI, thus in
# practice we shouldn't never reach this.
error("Incompatible C++ string ABIs between the host and target platforms")
end
end
end

# If we should use ccache, prepend this to every compiler invocation
ccache = use_ccache ? "ccache" : ""

Expand All @@ -73,7 +103,7 @@ function generate_compiler_wrappers!(platform::Platform; bin_path::AbstractStrin
unsafe_flags = String[])
write(io, """
#!/bin/bash
# This compiler wrapper script brought into existence by `generate_compiler_wrappers()`
# This compiler wrapper script brought into existence by `generate_compiler_wrappers!()`

if [ "x\${SUPER_VERBOSE}" = "x" ]; then
vrun() { "\$@"; }
Expand Down Expand Up @@ -308,8 +338,11 @@ function generate_compiler_wrappers!(platform::Platform; bin_path::AbstractStrin
chmod(joinpath(bin_path, fname), 0o775)
end

## Generate compiler wrappers for both our target and our host
for p in unique((platform, host_platform))
## Generate compiler wrappers for both our host and our target. In
## particular, we write the wrapper for the target after those for the host,
## to override host-specific ABI in case this is incompatible with that of
## the target
for p in unique((host_platform, platform))
t = aatriplet(p)

# Generate `:c` compilers
Expand Down
7 changes: 5 additions & 2 deletions src/UserNSRunner.jl
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ function UserNSRunner(workspace_root::String;
verbose::Bool = false,
compiler_wrapper_path::String = mktempdir(),
src_name::AbstractString = "",
shards = nothing,
kwargs...)
global use_ccache, use_squashfs, runner_override

Expand Down Expand Up @@ -55,8 +56,10 @@ function UserNSRunner(workspace_root::String;
push!(workspaces, ccache_dir() => "/root/.ccache")
end

# Choose the shards we're going to mount
shards = choose_shards(platform; extract_kwargs(kwargs, (:preferred_gcc_version,:preferred_llvm_version,:bootstrap_list,:compilers))...)
if isnothing(shards)
# Choose the shards we're going to mount
shards = choose_shards(platform; extract_kwargs(kwargs, (:preferred_gcc_version,:preferred_llvm_version,:bootstrap_list,:compilers))...)
end
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same here; let's try moving this initialization into the kwargs specification above.


# Construct sandbox command to look at the location it'll be mounted under
mpath = mount_path(shards[1], workspace_root)
Expand Down
4 changes: 2 additions & 2 deletions src/wizard/deploy.jl
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,11 @@ function print_build_tarballs(io::IO, state::WizardState)
push!(kwargs_vec, "compilers = [$(join(map(x -> ":$(x)", state.compilers), ", "))]")
end
# Default GCC version is the oldest one
if state.preferred_gcc_version != available_gcc_builds[1]
if state.preferred_gcc_version != getversion(available_gcc_builds[1])
push!(kwargs_vec, "preferred_gcc_version = v\"$(state.preferred_gcc_version)\"")
end
# Default LLVM version is the latest one
if state.preferred_llvm_version != available_llvm_builds[end]
if state.preferred_llvm_version != getversion(available_llvm_builds[end])
push!(kwargs_vec, "preferred_llvm_version = v\"$(state.preferred_llvm_version)\"")
end
kwargs = ""
Expand Down
8 changes: 4 additions & 4 deletions src/wizard/obtain_source.jl
Original file line number Diff line number Diff line change
Expand Up @@ -404,13 +404,13 @@ function step2(state::WizardState)
get_name_and_version(state)
if yn_prompt(state, "Do you want to customize the set of compilers?", :n) == :y
get_compilers(state)
get_preferred_version(state, "GCC", available_gcc_builds)
get_preferred_version(state, "LLVM", available_llvm_builds)
get_preferred_version(state, "GCC", getversion.(available_gcc_builds))
get_preferred_version(state, "LLVM", getversion.(available_llvm_builds))
else
state.compilers = [:c]
# Default GCC version is the oldest one
state.preferred_gcc_version = available_gcc_builds[1]
state.preferred_gcc_version = getversion(available_gcc_builds[1])
# Default LLVM version is the latest one
state.preferred_llvm_version = available_llvm_builds[end]
state.preferred_llvm_version = getversion(available_llvm_builds[end])
end
end
Loading