From 5be1f3209520b982ef5cdc16c5c32f1267ac6121 Mon Sep 17 00:00:00 2001 From: Kevin Cox Date: Sat, 5 Aug 2017 15:38:48 +0100 Subject: [PATCH] Update cargo builder to fetch registry dynamically. The biggest benefit is that we no longer have to update the registry package. This means that just about any cargo package can be built by nix. No longer does `cargo update` need to be feared because it will update to packages newer then what is available in nixpkgs. This works very similarly to the old technique except instead of using cargo's internal cache and stripping out as much as possible it uses the new custom registry support to generate a minimal registry with exactly what we need. Like this existing method this will download crates multiple times for different packages. This is a shame because the Cargo.lock hash hashes inside of it but Nix doesn't really support this. This also uses the new --frozen and --locked flags which is nice. Currently cargo-local-registry only provides binaries for Linux and macOS 64-bit. This can be solved by building it for the other architectures and uploading it somewhere (like the NixOS cache). This also has the downside that it requires a change to everyone's deps hash. And if the old one is used because it was cached it will fail to build as it will attempt to use the old version. --- pkgs/applications/misc/alacritty/default.nix | 2 +- .../rust/cargo-local-registry.nix | 33 +++ pkgs/build-support/rust/default.nix | 61 ++--- pkgs/build-support/rust/fetch-cargo-deps | 209 ------------------ pkgs/build-support/rust/fetchcargo.nix | 34 ++- pkgs/development/compilers/rust/default.nix | 2 +- pkgs/top-level/all-packages.nix | 4 +- pkgs/top-level/rust-packages.nix | 46 ---- 8 files changed, 79 insertions(+), 312 deletions(-) create mode 100644 pkgs/build-support/rust/cargo-local-registry.nix delete mode 100755 pkgs/build-support/rust/fetch-cargo-deps delete mode 100644 pkgs/top-level/rust-packages.nix diff --git a/pkgs/applications/misc/alacritty/default.nix b/pkgs/applications/misc/alacritty/default.nix index 50af18ca1fca8..a68d6c086ceb1 100644 --- a/pkgs/applications/misc/alacritty/default.nix +++ b/pkgs/applications/misc/alacritty/default.nix @@ -38,7 +38,7 @@ buildRustPackage rec { sha256 = "0h5hrb2g0fpc6xn94hmvxjj21cqbj4vgqkznvd64jl84qbyh1xjl"; }; - depsSha256 = "1pbb0swgpsbd6x3avxz6fv3q31dg801li47jibz721a4n9c0rssx"; + depsSha256 = "broken see https://github.com/alexcrichton/cargo-local-registry/issues/9"; buildInputs = [ cmake diff --git a/pkgs/build-support/rust/cargo-local-registry.nix b/pkgs/build-support/rust/cargo-local-registry.nix new file mode 100644 index 0000000000000..4748d6a9b14b3 --- /dev/null +++ b/pkgs/build-support/rust/cargo-local-registry.nix @@ -0,0 +1,33 @@ +{ fetchurl, stdenv }: + +let + version = "0.1.4"; + platform = + if stdenv.system == "x86_64-linux" + then "x86_64-unknown-linux-musl" + else if stdenv.system == "x86_64-darwin" + then "x86_64-apple-darwin" + else throw "missing bootstrap url for platform ${stdenv.system}"; + + # fetch hashes by patching print-hashes.sh to not use the "$DATE" variable + # then running `print-hashes.sh 1.16.0` + hash = + if stdenv.system == "x86_64-linux" + then "7716524846297abc6e8a7b26c57f56b1e0c2b261a6d0583e78f2a8fd60b33695" + else if stdenv.system == "x86_64-darwin" + then "439ff0303bc17fd8587196f0658ae4430850a906e67648f9fde047c9c7c75998" + else throw "missing bootstrap hash for platform ${stdenv.system}"; +in stdenv.mkDerivation { + name = "cargo-local-registry-${version}"; + + src = fetchurl { + url = "https://github.com/alexcrichton/cargo-local-registry/releases/download/${version}/cargo-local-registry-${version}-${platform}.tar.gz"; + sha256 = hash; + }; + + phases = "unpackPhase installPhase"; + + installPhase = '' + install -Dm755 cargo-local-registry $out/bin/cargo-local-registry + ''; +} diff --git a/pkgs/build-support/rust/default.nix b/pkgs/build-support/rust/default.nix index 36130289fbaa5..9716e41ffaf56 100644 --- a/pkgs/build-support/rust/default.nix +++ b/pkgs/build-support/rust/default.nix @@ -1,10 +1,6 @@ -{ stdenv, callPackage, path, cacert, git, rust, rustRegistry }: +{ fetchurl, stdenv, callPackage, path, cacert, git, rust }: -let - rustRegistry' = rustRegistry; -in { name, depsSha256 -, rustRegistry ? rustRegistry' , src ? null , srcs ? null , sourceRoot ? null @@ -19,7 +15,7 @@ let lib = stdenv.lib; fetchDeps = import ./fetchcargo.nix { - inherit stdenv cacert git rust rustRegistry; + inherit fetchurl stdenv cacert git rust; }; cargoDeps = fetchDeps { @@ -28,7 +24,7 @@ let }; in stdenv.mkDerivation (args // { - inherit cargoDeps rustRegistry; + inherit cargoDeps; patchRegistryDeps = ./patch-registry-deps; @@ -43,49 +39,22 @@ in stdenv.mkDerivation (args // { postUnpack = '' eval "$cargoDepsHook" - echo "Using cargo deps from $cargoDeps" - - cp -a "$cargoDeps" deps - chmod +w deps -R - - # It's OK to use /dev/null as the URL because by the time we do this, cargo - # won't attempt to update the registry anymore, so the URL is more or less - # irrelevant + mkdir .cargo + cat >.cargo/config <<-EOF + [source.crates-io] + registry = 'https://github.com/rust-lang/crates.io-index' + replace-with = 'local-registry' - cat < deps/config - [registry] - index = "file:///dev/null" + [source.local-registry] + local-registry = '$cargoDeps' EOF - export CARGO_HOME="$(realpath deps)" + export CARGO_HOME="$(pwd)/deps" export RUST_LOG=${logLevel} export SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt - # Let's find out which $indexHash cargo uses for file:///dev/null - (cd $sourceRoot && cargo fetch &>/dev/null) || true - cd deps - indexHash="$(basename $(echo registry/index/*))" - - echo "Using indexHash '$indexHash'" - - rm -rf -- "registry/cache/$indexHash" \ - "registry/index/$indexHash" - - mv registry/cache/HASH "registry/cache/$indexHash" - - echo "Using rust registry from $rustRegistry" - ln -s "$rustRegistry" "registry/index/$indexHash" - - # Retrieved the Cargo.lock file which we saved during the fetch - cd .. - mv deps/Cargo.lock $sourceRoot/ - - ( - cd $sourceRoot - - cargo fetch - cargo clean - ) + # Unpack crates. This is only needed for $patchRegistryDeps + cargo fetch --frozen --verbose --manifest-path "$sourceRoot/Cargo.toml" '' + (args.postUnpack or ""); prePatch = '' @@ -93,7 +62,7 @@ in stdenv.mkDerivation (args // { ( set -euo pipefail - cd $NIX_BUILD_TOP/deps/registry/src/* + cd "$CARGO_HOME/registry/src/"* for script in $patchRegistryDeps/*; do # Run in a subshell so that directory changes and shell options don't @@ -107,7 +76,7 @@ in stdenv.mkDerivation (args // { buildPhase = with builtins; args.buildPhase or '' runHook preBuild echo "Running cargo build --release ${concatStringsSep " " cargoBuildFlags}" - cargo build --release ${concatStringsSep " " cargoBuildFlags} + cargo build --release --frozen ${concatStringsSep " " cargoBuildFlags} runHook postBuild ''; diff --git a/pkgs/build-support/rust/fetch-cargo-deps b/pkgs/build-support/rust/fetch-cargo-deps deleted file mode 100755 index 3c7e034364f46..0000000000000 --- a/pkgs/build-support/rust/fetch-cargo-deps +++ /dev/null @@ -1,209 +0,0 @@ -# copied from libgit2 source code 'repo-template.h' -makeGitTemplate() { - local target="$1" - mkdir -p -m777 "$target/info" "$target/pack" "$target/objects" "$target/refs" - mkdir -p -m777 "$target/refs/heads" "$target/refs/tags" "$target/objects/info" "$target/objects/pack" - cat <<'EOF' > "$target/description" -Unnamed repository; edit this file 'description' to name the repository. -EOF - chmod 666 "$target/description" - cat <<'EOF' > "$target/info/exclude" -# File patterns to ignore; see `git help ignore` for more information. -# Lines that start with '#' are comments. -EOF -} - -fetchCargoDeps() { - src=$(realpath $1) - out=$(realpath $2) - - echo "Fetching $src to $out" - - mkdir $out - - # Configure git template dir to make libgit2 more deterministic - # - # Without a template dir, libgit2 defaults to /usr/share/git-core/templates, - # which can vary between systems if sandboxed builds aren't used. - # - # Note: we explictly set --tmpdir for mktemp here to make it more friendly - # for nix-shell users, where $TMPDIR is not necessarily set to NIX_BUILD_TOP - echo "Setting up git templatedir" - export GIT_TEMPLATE_DIR="$(mktemp -d --tmpdir=$NIX_BUILD_TOP git-template.XXX)" - makeGitTemplate "$GIT_TEMPLATE_DIR" - export XDG_CONFIG_HOME="$(mktemp -d --tmpdir=$NIX_BUILD_TOP home.XXX)" - mkdir -p $XDG_CONFIG_HOME/git - cat < $XDG_CONFIG_HOME/git/config -[init] - templatedir = $GIT_TEMPLATE_DIR -EOF - - # Configure cargo to fetch from a local copy of the crates.io registry - - echo "Using rust registry from $rustRegistry" - - cat < $out/config -[registry] -index = "file://$rustRegistry" -EOF - - export CARGO_HOME=$out - cd $src - - if [[ ! -f Cargo.lock ]]; then - echo - echo "ERROR: The Cargo.lock file doesn't exist" - echo - echo "Cargo.lock is needed to make sure that depsSha256 doesn't change" - echo "when the registry is updated." - echo - - exit 1 - fi - - # We need to do the following string replacement so that 'cargo fetch' - # doesn't ignore the versions specified in Cargo.lock - substituteInPlace Cargo.lock \ - --replace "registry+https://github.com/rust-lang/crates.io-index" \ - "registry+file://$rustRegistry" - - # Do any possible 'cargo update -p --precise ' ad-hoc updates - eval "$cargoUpdateHook" - - # Do the fetch - cargo fetch --verbose - - # Now that we have fetched everything, let's make the output deterministic - - # Cargo uses the following directory structure for fetched data, where - # $indexHash is a hash of the registry index URL: - # - # - # /config: - # - # Cargo config file. We'll delete this because it's not deterministic, - # and instead recreate it just before running 'cargo build'. - # - # /registry/cache/$indexHash/: - # - # This is where tarballs of registry package dependencies are kept - # We'll need to keep this, but make sure $indexHash is a fixed name. - # - # /registry/index/$indexHash/: - # - # A copy of the registry index is kept here. We can delete this, and - # instead, just before running 'cargo build', we'll symlink this - # directory to our static copy of the registry in the Nix store. - # - # /registry/src/$indexHash/{pkgName-pkgVersion}/: - # - # Here cargo keeps extracted sources of the cached tarballs. - # We'll just delete this because cargo will re-populate them from the - # tarballs. - # - # /git/db/{domain-hash}/: - # - # Here cargo keeps the `.git` directories of git dependencies. - # We'll need to keep these, but make them deterministic. - # - # /git/checkouts/{domain-hash}/{branchName}/: - # - # Here cargo keeps checked-out sources of the git dependencies. - # We can delete this, because cargo will re-populate them from the above - # `.git` directories. - # - # Let's start - - # Remove cargo config file, which points to the ever-changing registry - rm $out/config - - # Save the Cargo.lock file into the output, so that we don't have to do another - # 'cargo update' during the build (which would try to access the network) for - # any ad-hoc package updates (through $cargoUpdateHook). - # - # We need to replace the rustRegistry URL with something deterministic. - # Since the URL won't actually be accessed anymore, it's fine to use /dev/null. - - substituteInPlace Cargo.lock \ - --replace "registry+file://$rustRegistry" \ - "registry+file:///dev/null" - mv Cargo.lock $out/ - - - # Let's replace $indexHash with something more deterministic - mv $out/registry/cache/* $out/registry/cache/HASH - - # The registry index changes all the time, so it's not deterministic - # We'll symlink it before running 'cargo build' - rm -rf $out/registry/index/* - - # Make git DBs deterministic - # TODO: test with git submodules - [[ ! -d $out/git/checkouts ]] || (cd $out/git/checkouts && for name in *; do - revs="" - cd "$out/git/checkouts/$name" - while read dir; do - # extract substring: [dir = "./xxx/yyy/.git"] => [branch = "xxx/yyy"] - branch="${dir:2:$((${#dir}-7))}" - - cd "$out/git/checkouts/$name/$branch" - rev="$(git rev-parse HEAD)" - revs="$rev $revs" - done < <(find . -type d -name .git -print) - - echo "List of revs to keep for git db $name: $revs" - - ( - # The following code was adapted from nix-prefetch-git - - cd "$out/git/db/$name" - - export GIT_DIR=. - - # Remove all remote branches - git branch -r | while read branch; do - git branch -rD "$branch" >&2 - done - - # Remove all tags - git tag | while read tag; do - git tag -d "$tag" >&2 - done - - # Remove all local branches - branchrefs=() - eval "$(git for-each-ref --shell --format='branchrefs+=(%(refname))' refs/heads/)" - - for branchref in "${branchrefs[@]}"; do - git update-ref -d "$branchref" >&2 - done - - # Create ad-hoc branches for the revs we need - echo "$revs" | while read -d " " rev; do - echo "Creating git branch b_$rev $rev" - git branch b_$rev $rev - done - - # Remove files that have timestamps or otherwise have non-deterministic - # properties. - rm -rf logs/ hooks/ index FETCH_HEAD ORIG_HEAD refs/remotes/origin/HEAD config - - # Do a full repack. Must run single-threaded, or else we lose determinism. - git config pack.threads 1 - git repack -A -d -f - rm -f config - - # Garbage collect unreferenced objects. - git gc --prune=all - ) - done) - - # Remove unneeded outputs - [[ ! -d $out/registry/src ]] || rm -rf $out/registry/src - [[ ! -d $out/git/checkouts ]] || rm -rf $out/git/checkouts - - # XXX: provide some debugging output to see find out why we are seeing - # sporadic hash mismatches - find $out ! -type f - find $out -type f -exec sha256sum {} + -} diff --git a/pkgs/build-support/rust/fetchcargo.nix b/pkgs/build-support/rust/fetchcargo.nix index 0c9625e51405f..7a44b82a87b2f 100644 --- a/pkgs/build-support/rust/fetchcargo.nix +++ b/pkgs/build-support/rust/fetchcargo.nix @@ -1,19 +1,41 @@ -{ stdenv, cacert, git, rust, rustRegistry }: +{ fetchurl, stdenv, cacert, git, rust }: { name ? "cargo-deps", src, srcs, sourceRoot, sha256, cargoUpdateHook ? "" }: -stdenv.mkDerivation { +let + cargoLocalRegistry = import ./cargo-local-registry.nix { + inherit fetchurl stdenv; + }; + +in stdenv.mkDerivation { name = "${name}-fetch"; - buildInputs = [ rust.cargo rust.rustc git ]; - inherit src srcs sourceRoot rustRegistry cargoUpdateHook; + buildInputs = [ cacert cargoLocalRegistry git rust.cargo ]; + inherit src srcs sourceRoot; phases = "unpackPhase installPhase"; installPhase = '' - source ${./fetch-cargo-deps} + if [[ ! -f Cargo.lock ]]; then + echo + echo "ERROR: The Cargo.lock file doesn't exist" + echo + echo "Cargo.lock is needed to make sure that depsSha256 doesn't change" + echo "when the registry is updated." + echo + + exit 1 + fi export SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt + export CARGO_HOME="$(mktemp -d cargo-home.XXX)" + + cargo local-registry --sync Cargo.lock --git "$out" - fetchCargoDeps . "$out" + # cargo-local-registry isn't reproducible. + # https://github.com/alexcrichton/cargo-local-registry/issues/8 + for entry in "$out/index"/*/*/*; do + sort "$entry" >sorted + mv sorted "$entry" + done ''; outputHashAlgo = "sha256"; diff --git a/pkgs/development/compilers/rust/default.nix b/pkgs/development/compilers/rust/default.nix index 0d7e26e06f471..686d9878a7933 100644 --- a/pkgs/development/compilers/rust/default.nix +++ b/pkgs/development/compilers/rust/default.nix @@ -29,7 +29,7 @@ rec { version = "0.18.0"; srcRev = "fe7b0cdcf5ca7aab81630706ce40b70f6aa2e666"; srcSha = "164iywv1l3v87b0pznf5kkzxigd6w19myv9d7ka4c65zgrk9n9px"; - depsSha256 = "1mrgd8ib48vxxbhkvsqqq4p19sc6b74x3cd8p6lhhlm6plrajrvm"; + depsSha256 = "1ydlsxnklanmg0zskcxg9cfw0j0fvgncjvccirzml3h4cnl8ji5j"; inherit rustc; # the rustc that will be wrapped by cargo inherit rustPlatform; # used to build cargo diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 2f655579d6c43..de66a41b655e1 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -1052,7 +1052,7 @@ with pkgs; glock = callPackage ../development/tools/glock { }; glslviewer = callPackage ../development/tools/glslviewer { }; - + gmic = callPackage ../tools/graphics/gmic { }; goa = callPackage ../development/tools/goa { }; @@ -5905,8 +5905,6 @@ with pkgs; inherit (darwin) apple_sdk; }; - rustRegistry = callPackage ./rust-packages.nix { }; - rust = rustStable; rustStable = callPackage ../development/compilers/rust { inherit (llvmPackages_4) llvm; diff --git a/pkgs/top-level/rust-packages.nix b/pkgs/top-level/rust-packages.nix deleted file mode 100644 index 1caef26322d7b..0000000000000 --- a/pkgs/top-level/rust-packages.nix +++ /dev/null @@ -1,46 +0,0 @@ -# This file defines the source of Rust / cargo's crates registry -# -# buildRustPackage will automatically download dependencies from the registry -# version that we define here. If you're having problems downloading / finding -# a Rust library, try updating this to a newer commit. - -{ stdenv, fetchFromGitHub, git }: - -stdenv.mkDerivation { - name = "rustRegistry-2017-07-23"; - - src = fetchFromGitHub { - owner = "rust-lang"; - repo = "crates.io-index"; - rev = "ed8e6a6761278861db046073cc69d6a5e7dd8c15"; - sha256 = "1v72m0h31xcay2m64n2wil5wqnl8z4n4adxxpdllcpgj3pj5jai6"; - }; - phases = [ "unpackPhase" "installPhase" ]; - installPhase = '' - # For some reason, cargo doesn't like fetchgit's git repositories, not even - # if we set leaveDotGit to true, set the fetchgit branch to 'master' and clone - # the repository (tested with registry rev - # 965b634156cc5c6f10c7a458392bfd6f27436e7e), failing with the message: - # - # "Target OID for the reference doesn't exist on the repository" - # - # So we'll just have to create a new git repository from scratch with the - # contents downloaded with fetchgit... - - mkdir -p $out - - cp -r ./* $out/ - - cd $out - - git="${git}/bin/git" - - $git init - $git config --local user.email "example@example.com" - $git config --local user.name "example" - $git add . - $git commit --quiet -m 'Rust registry commit' - - touch $out/touch . "$out/.cargo-index-lock" - ''; -}