Upgrade to Julia 1.6+

.travis.yml

@@ -3,8 +3,6 @@ language: julia
 os:
   - linux
 julia:
-  - 1.4
-  - 1.5
   - nightly
 git:
   depth: 999999
@@ -33,28 +31,28 @@ addons:
       - gzip
 
 jobs:
-  allow_failures:
-    - julia: nightly
+  #allow_failures:
+  #  - julia: nightly
   include:
     # Add a job that uses the privileged builder with squashfs shards
-    - julia: 1.4
+    - julia: nightly
       env:
         - BINARYBUILDER_RUNNER=privileged
         - BINARYBUILDER_USE_SQUASHFS=true
 
     # Add a job that uses the unprivileged builder with unpacked shards
-    - julia: 1.4
+    - julia: nightly
       env:
         - BINARYBUILDER_RUNNER=unprivileged
         - BINARYBUILDER_USE_SQUASHFS=false
 
     # Add a job that uses the docker builder with unpacked shards
-    - julia: 1.4
+    - julia: nightly
       env:
         - BINARYBUILDER_RUNNER=docker
         - BINARYBUILDER_USE_SQUASHFS=false
     - stage: "Documentation"
-      julia: 1
+      julia: nightly
       os: linux
       script:
         - julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd()));

Manifest.toml

@@ -1,5 +1,13 @@
 # This file is machine-generated - editing it directly is not advised
 
+[[ArgTools]]
+git-tree-sha1 = "bdf73eec6a88885256f282d48eafcad25d7de494"
+uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
+version = "1.1.1"
+
+[[Artifacts]]
+uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
+
 [[Base64]]
 uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 
@@ -17,15 +25,27 @@ uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
 deps = ["Random", "Serialization", "Sockets"]
 uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 
+[[Downloads]]
+deps = ["LibCURL"]
+uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
+
 [[InteractiveUtils]]
 deps = ["Markdown"]
 uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 
 [[JSON]]
 deps = ["Dates", "Mmap", "Parsers", "Unicode"]
-git-tree-sha1 = "b34d7cef7b337321e97d22242c3c2b91f476748e"
+git-tree-sha1 = "81690084b6198a2e1da36fcfda16eeca9f9f24e4"
 uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
-version = "0.21.0"
+version = "0.21.1"
+
+[[LibCURL]]
+deps = ["LibCURL_jll", "MozillaCACerts_jll"]
+uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"
+
+[[LibCURL_jll]]
+deps = ["Libdl"]
+uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"
 
 [[LibGit2]]
 deps = ["Printf"]
@@ -44,27 +64,30 @@ uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
 [[Mmap]]
 uuid = "a63ad114-7e13-5084-954f-fe012c677804"
 
+[[MozillaCACerts_jll]]
+uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
+
 [[OutputCollectors]]
 git-tree-sha1 = "d86c19b7fa8ad6a4dc8ec2c726642cc6291b2941"
 uuid = "6c11c7d4-943b-4e2b-80de-f2cfc2930a8c"
 version = "0.1.0"
 
 [[Parsers]]
 deps = ["Dates", "Test"]
-git-tree-sha1 = "eb3e09940c0d7ae01b01d9291ebad7b081c844d3"
+git-tree-sha1 = "8077624b3c450b15c087944363606a6ba12f925e"
 uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
-version = "1.0.5"
+version = "1.0.10"
 
 [[Pkg]]
-deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"]
+deps = ["Artifacts", "Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "TOML", "UUIDs"]
 uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 
 [[Printf]]
 deps = ["Unicode"]
 uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 
 [[REPL]]
-deps = ["InteractiveUtils", "Markdown", "Sockets"]
+deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
 uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
 
 [[Random]]
@@ -77,9 +100,24 @@ uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 [[Serialization]]
 uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 
+[[SimpleBufferStream]]
+git-tree-sha1 = "874e8867b33a00e784c8a7e4b60afe9e037b74e1"
+uuid = "777ac1f9-54b0-4bf8-805c-2214025038e7"
+version = "1.1.0"
+
 [[Sockets]]
 uuid = "6462fe0b-24de-5631-8697-dd941f90decc"
 
+[[TOML]]
+deps = ["Dates"]
+uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
+
+[[Tar]]
+deps = ["ArgTools", "Logging", "SHA"]
+git-tree-sha1 = "83da81284c449b31c85dd4b0b3a3e1930e4fa195"
+uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
+version = "1.7.0"
+
 [[Test]]
 deps = ["Distributed", "InteractiveUtils", "Logging", "Random"]
 uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
@@ -99,6 +137,12 @@ uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
 
 [[Zlib_jll]]
 deps = ["Libdl", "Pkg"]
-git-tree-sha1 = "a2e0d558f6031002e380a90613b199e37a8565bf"
+git-tree-sha1 = "fdd89e5ab270ea0f2a0174bd9093e557d06d4bfa"
 uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
-version = "1.2.11+10"
+version = "1.2.11+16"
+
+[[p7zip_jll]]
+deps = ["Libdl", "Pkg"]
+git-tree-sha1 = "e7fe234ab9d67b1c01e44b25b637b2ff60312ac1"
+uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
+version = "16.2.0+2"

Project.toml

@@ -1,10 +1,12 @@
 name = "BinaryBuilderBase"
 uuid = "7f725544-6523-48cd-82d1-3fa08ff4056e"
 authors = ["Elliot Saba <staticfloat@gmail.com>"]
-version = "0.4.1"
+version = "0.5.0"
 
 [deps]
+Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
 CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193"
+Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433"
 Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
@@ -13,13 +15,17 @@ OutputCollectors = "6c11c7d4-943b-4e2b-80de-f2cfc2930a8c"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+SimpleBufferStream = "777ac1f9-54b0-4bf8-805c-2214025038e7"
+Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
+p7zip_jll = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
 
 [compat]
 CodecZlib = "0.5, 0.6, 0.7"
 JSON = "0.21"
 OutputCollectors = "0.1"
-julia = "1.4"
+Tar = "1.7"
+julia = "1.6"
 
 [extras]
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

src/ArchiveUtils.jl

@@ -0,0 +1,132 @@
+using Downloads, Tar, p7zip_jll, SimpleBufferStream, SHA
+
+export unpack, list_tarball_files, verify, download_verify
+
+function detect_compressor(header::Vector)
+    compressor_magic_bytes = Dict(
+        "gzip" => [0x1f, 0x8b],
+        "xz" => [0xfd, 0x37, 0x7a, 0x58, 0x5A, 0x00],
+        "zstd" => [0x28, 0xB5, 0x2F, 0xFD],
+        "bzip2" => [0x42, 0x5a, 0x68],
+    )
+    for (compressor, magic) in compressor_magic_bytes
+        lm = length(magic)
+        if length(header) >= lm && header[1:lm] == magic
+            return compressor
+        end
+    end
+    return nothing
+end
+
+function decompress(path::AbstractString)
+    # Read the first few bytes of data to classify it:
+    compressor = open(path) do io
+        detect_compressor(read(io, 6))
+    end
+    if compressor === nothing
+        error("Called decompress() on uncompressed file")
+    end
+
+    p7zip() do p7z
+        # Launch p7zip as our decompressor engine, clueing it in to the compressor
+        p = open(`$p7z e $(path) -so -t$(compressor)`; read=true)
+        return p.out
+    end
+end
+
+# Many functions don't like `PipeEndpoint`, so we interface with a BufferStream
+function buff_wrap(io::IO; blocksize = 2*1024*1024)
+    buff = BufferStream(blocksize)
+    @async begin
+        while !eof(io)
+            write(buff, readavailable(io))
+        end
+        close(buff)
+    end
+    return buff
+end
+
+
+# Reimplement Pkg.PlatformEngines verbs using Tar and Downloader
+function unpack(tarball_path::AbstractString, dest::AbstractString)
+    mkpath(dest)
+
+    @sync begin
+        Tar.extract(decompress(tarball_path), dest)
+    end
+end
+
+function list_tarball_files(tarball_path::AbstractString)
+    @sync begin
+        # Decompress to a file, since `Tar.list()` can't feed off of a BufferStream yet
+        mktemp() do path, _
+            function remove_dotslash(path)
+                if startswith(path, "./")
+                    return string(path[3:end])
+                end
+                return path
+            end
+            return String[remove_dotslash(h.path) for h in Tar.list(buff_wrap(decompress(tarball_path)))]
+        end
+    end
+end
+
+function verify(path::AbstractString, hash::AbstractString; hash_path::AbstractString="$(path).sha256")
+    # Check hash string format
+    if !occursin(r"^[0-9a-f]{64}$"i, hash)
+        msg = "Hash value must be 64 hexadecimal characters (256 bits), "
+        if !isascii(hash)
+            msg *= "given hash value is non-ASCII"
+        elseif occursin(r"^[0-9a-f]*$"i, hash)
+            msg *= "given hash value has the wrong length ($(length(hash)))"
+        else
+            msg *= "given hash value contains non-hexadecimal characters"
+        end
+        msg *= ": $(repr(hash))"
+        error(msg)
+    end
+    hash = lowercase(hash)
+
+
+    # First, it must exist
+    if isfile(hash_path)
+        # Next, it must contain the same hash as what we're verifying against
+        if read(hash_path, String) == hash
+            # Next, it must be no older than the actual path
+            if stat(hash_path).mtime >= stat(path).mtime
+                return true
+            end
+        end
+    end
+
+    calc_hash = open(path) do file
+        bytes2hex(sha256(file))
+    end
+    @assert occursin(r"^[0-9a-f]{64}$", calc_hash)
+
+    if calc_hash != hash
+        msg  = "Hash Mismatch!\n"
+        msg *= "  Expected sha256:   $hash\n"
+        msg *= "  Calculated sha256: $calc_hash"
+        @error(msg)
+        return false
+    end
+
+    # Try to save a hash cache if everything worked out fine
+    try
+        open(hash_path, "w") do file
+            write(file, hash)
+        end
+    catch e
+        if isa(e, InterruptException)
+            rethrow(e)
+        end
+    end
+
+    return true
+end
+
+function download_verify(url, hash, path)
+    Downloads.download(url, path)
+    verify(path, hash)
+end

src/BinaryBuilderBase.jl

@@ -1,28 +1,31 @@
 module BinaryBuilderBase
 
-using Pkg, Pkg.BinaryPlatforms, Pkg.PlatformEngines, Pkg.Artifacts, Random, Libdl
+using Pkg, Pkg.Artifacts, Random, Libdl
+using Base.BinaryPlatforms
+using Downloads
 using JSON, OutputCollectors
 
-# Re-export useful stuff from Pkg:
-export platform_key_abi, platform_dlext, valid_dl_path, arch, libc, compiler_abi,
-       libgfortran_version, libstdcxx_version, cxxstring_abi, parse_dl_name_version,
+# Re-export useful stuff from Base.BinaryPlatforms:
+export HostPlatform, platform_dlext, valid_dl_path, arch, libc,
+       libgfortran_version, libstdcxx_version, cxxstring_abi,
        detect_libgfortran_version, detect_libstdcxx_version, detect_cxxstring_abi,
        call_abi, wordsize, triplet, select_platform, platforms_match,
-       CompilerABI, Platform, UnknownPlatform, Linux, MacOS, Windows, FreeBSD
+       Platform, AnyPlatform
 
 export AbstractSource, AbstractDependency, SetupSource, PatchSource,
     resolve_jlls, coerce_dependency, coerce_source, Runner,
     generate_compiler_wrappers!, preferred_runner, CompilerShard, UserNSRunner,
     DockerRunner, choose_shards, exeext, preferred_libgfortran_version,
     preferred_cxxstring_abi, gcc_version, available_gcc_builds, getversion,
-    getpkg, replace_libgfortran_version, replace_cxxstring_abi, aatriplet,
+    getpkg, aatriplet,
     nbits, proc_family, storage_dir, extract_kwargs, extract_fields,
     download_source, setup_workspace, setup_dependencies, update_registry,
     getname, cleanup_dependencies, compress_dir, prepare_for_deletion,
     run_interactive, sourcify, dependencify, with_logfile, get_concrete_platform
 
 include("compat.jl")
 
+include("ArchiveUtils.jl")
 include("Sources.jl")
 include("Dependencies.jl")
 include("Prefix.jl")
@@ -70,9 +73,6 @@ function __init__()
     global runner_override, use_squashfs, allow_ecryptfs
     global use_ccache, storage_cache
 
-    # Pkg does this lazily; do it explicitly here.
-    Pkg.PlatformEngines.probe_platform_engines!()
-
     # Allow the user to override the default value for `storage_dir`
     storage_cache = get(ENV, "BINARYBUILDER_STORAGE_DIR",
                         abspath(joinpath(@__DIR__, "..", "deps")))