From 91a4e55c066b32249b4e1ed5949aeb06b0dff6ef Mon Sep 17 00:00:00 2001 From: Philipp Mildenberger Date: Fri, 12 May 2023 00:00:21 +0200 Subject: [PATCH 1/3] Symlink cargo build artifacts instead of copying them --- lib/cargoTarpaulin.nix | 2 +- lib/setupHooks/inheritCargoArtifacts.nix | 2 + lib/setupHooks/inheritCargoArtifactsHook.sh | 42 +++++++++++++++------ 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/lib/cargoTarpaulin.nix b/lib/cargoTarpaulin.nix index 5ac007d9..1b094f82 100644 --- a/lib/cargoTarpaulin.nix +++ b/lib/cargoTarpaulin.nix @@ -3,7 +3,7 @@ }: { cargoExtraArgs ? "" -, cargoTarpaulinExtraArgs ? "--skip-clean --out Xml --output-dir $out" +, cargoTarpaulinExtraArgs ? "--out Xml --output-dir $out" , ... }@origArgs: let diff --git a/lib/setupHooks/inheritCargoArtifacts.nix b/lib/setupHooks/inheritCargoArtifacts.nix index afb3d67e..aebdfa86 100644 --- a/lib/setupHooks/inheritCargoArtifacts.nix +++ b/lib/setupHooks/inheritCargoArtifacts.nix @@ -1,8 +1,10 @@ { makeSetupHook +, rsync }: makeSetupHook { name = "inheritCargoArtifactsHook"; + propagatedBuildInputs = [ rsync ]; } ./inheritCargoArtifactsHook.sh diff --git a/lib/setupHooks/inheritCargoArtifactsHook.sh b/lib/setupHooks/inheritCargoArtifactsHook.sh index 2456ef1c..fce3e3ea 100644 --- a/lib/setupHooks/inheritCargoArtifactsHook.sh +++ b/lib/setupHooks/inheritCargoArtifactsHook.sh @@ -24,18 +24,36 @@ inheritCargoArtifacts() { elif [ -d "${preparedArtifacts}" ]; then echo "copying cargo artifacts from ${preparedArtifacts} to ${cargoTargetDir}" - # NB: rustc doesn't like it when artifacts are either symlinks or hardlinks to the store - # (it tries to truncate files instead of unlinking and recreating them) - # so we're forced to do a full copy here :( - # - # Notes: - # - --no-target-directory to avoid nesting (i.e. `./target/target`) - # - preserve timestamps to avoid rebuilding - # - no-preserve ownership (root) so we can make the files writable - cp -r "${preparedArtifacts}" \ - --no-target-directory "${cargoTargetDir}" \ - --preserve=timestamps \ - --no-preserve=ownership + # copy target dir but ignore crate build artifacts + rsync -r --chmod=Du=rwx,Dg=rx,Do=rx --exclude "release/build" --exclude "release/deps" --exclude "*/release/build" --exclude "*/release/deps" "${preparedArtifacts}/" "${cargoTargetDir}/" + + link_build_artifacts() { + local artifacts="$1" + local target="$2" + + if [ -d "${artifacts}/release/deps" ]; then + mkdir -p "${target}/release/deps" + for dep in $(ls "${artifacts}/release/deps"); do + ln -fs "${artifacts}/release/deps/$dep" "${target}/release/deps/$dep" + done + fi + + if [ -d "${artifacts}/release/build" ]; then + mkdir -p "${target}/release/build" + for build in $(ls "${artifacts}/release/build"); do + ln -fs "${artifacts}/release/build/$build" "${target}/release/build/$build" + done + fi + } + + # symlink crate build artifacts + link_build_artifacts "${preparedArtifacts}" "${cargoTargetDir}" + + # for each build target as well + # all other directories are ignored in `link_build_artifacts` + for target in $(ls "${preparedArtifacts}"); do + link_build_artifacts "${preparedArtifacts}/$target" "${cargoTargetDir}/$target" + done # Keep existing permissions (e.g. exectuable), but also make things writable # since the store is read-only and cargo would otherwise choke From c41599656db25662504c561f5c2b381e29621095 Mon Sep 17 00:00:00 2001 From: Philipp Mildenberger Date: Thu, 18 May 2023 23:41:37 +0200 Subject: [PATCH 2/3] Simplify build-artifacts symlinking and support all build profiles instead of just 'release' --- lib/setupHooks/inheritCargoArtifactsHook.sh | 39 +++++---------------- 1 file changed, 9 insertions(+), 30 deletions(-) diff --git a/lib/setupHooks/inheritCargoArtifactsHook.sh b/lib/setupHooks/inheritCargoArtifactsHook.sh index fce3e3ea..a55a3109 100644 --- a/lib/setupHooks/inheritCargoArtifactsHook.sh +++ b/lib/setupHooks/inheritCargoArtifactsHook.sh @@ -24,36 +24,15 @@ inheritCargoArtifacts() { elif [ -d "${preparedArtifacts}" ]; then echo "copying cargo artifacts from ${preparedArtifacts} to ${cargoTargetDir}" - # copy target dir but ignore crate build artifacts - rsync -r --chmod=Du=rwx,Dg=rx,Do=rx --exclude "release/build" --exclude "release/deps" --exclude "*/release/build" --exclude "*/release/deps" "${preparedArtifacts}/" "${cargoTargetDir}/" - - link_build_artifacts() { - local artifacts="$1" - local target="$2" - - if [ -d "${artifacts}/release/deps" ]; then - mkdir -p "${target}/release/deps" - for dep in $(ls "${artifacts}/release/deps"); do - ln -fs "${artifacts}/release/deps/$dep" "${target}/release/deps/$dep" - done - fi - - if [ -d "${artifacts}/release/build" ]; then - mkdir -p "${target}/release/build" - for build in $(ls "${artifacts}/release/build"); do - ln -fs "${artifacts}/release/build/$build" "${target}/release/build/$build" - done - fi - } - - # symlink crate build artifacts - link_build_artifacts "${preparedArtifacts}" "${cargoTargetDir}" - - # for each build target as well - # all other directories are ignored in `link_build_artifacts` - for target in $(ls "${preparedArtifacts}"); do - link_build_artifacts "${preparedArtifacts}/$target" "${cargoTargetDir}/$target" - done + # copy target dir but ignore content-addressed build artifacts + rsync -r --chmod=Du=rwx --exclude "*/build/*" --exclude "*/deps/*" --exclude "*/*/build/*" --exclude "*/*/deps/*" "${preparedArtifacts}/" "${cargoTargetDir}/" + + # symlink all remaining content-addressed artifacts + pushd "${cargoTargetDir}" + for d in $(ls -d */{deps,build} */*/{deps,build}); do + ls "${preparedArtifacts}/${d}" | xargs -P 100 -I '##{}##' ln -fs "${preparedArtifacts}/${d}/##{}##" "${d}/##{}##" + done + popd # Keep existing permissions (e.g. exectuable), but also make things writable # since the store is read-only and cargo would otherwise choke From 638ab25a5247dfe1f25b87b6a9ef87b62344a019 Mon Sep 17 00:00:00 2001 From: Ivan Petkov Date: Tue, 6 Jun 2023 20:45:01 -0700 Subject: [PATCH 3/3] Fix build --- lib/mkCargoDerivation.nix | 2 ++ lib/setupHooks/inheritCargoArtifacts.nix | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/mkCargoDerivation.nix b/lib/mkCargoDerivation.nix index 76e35cb6..929acb45 100644 --- a/lib/mkCargoDerivation.nix +++ b/lib/mkCargoDerivation.nix @@ -5,6 +5,7 @@ , crateNameFromCargoToml , inheritCargoArtifactsHook , installCargoArtifactsHook +, rsync , stdenv , vendorCargoDeps , zstd @@ -59,6 +60,7 @@ chosenStdenv.mkDerivation (cleanedArgs // { configureCargoVendoredDepsHook inheritCargoArtifactsHook installCargoArtifactsHook + rsync zstd ]; diff --git a/lib/setupHooks/inheritCargoArtifacts.nix b/lib/setupHooks/inheritCargoArtifacts.nix index aebdfa86..d4a8a8fe 100644 --- a/lib/setupHooks/inheritCargoArtifacts.nix +++ b/lib/setupHooks/inheritCargoArtifacts.nix @@ -5,6 +5,5 @@ makeSetupHook { name = "inheritCargoArtifactsHook"; - propagatedBuildInputs = [ rsync ]; } ./inheritCargoArtifactsHook.sh