From 3235a3de3adde91539a3bba8d1ed54042ed46dc7 Mon Sep 17 00:00:00 2001 From: Jeremy Fitzhardinge Date: Mon, 25 Nov 2019 16:32:04 -0800 Subject: [PATCH 1/5] vendor: implement --explicit-version Implement --explicit-version from standalone cargo-vendor. This helps with vendoring performance as it avoids redundantly deleting and re-copying already vendored packages. For example, when re-vendoring cargo's dependencies it makes a big improvement on wallclock time. For initial vendoring it makes no difference, but re-vendoring (ie, when most or all dependencies haven't changed) without explicit versions is actually slightly slower (5.8s -> 6s), but with explicit versions it goes from 5.8s -> 1.6s. Timings: Without explicit versions, initial vendor real 0m5.810s user 0m0.924s sys 0m2.491s Re-vendor: real 0m6.083s user 0m0.937s sys 0m2.654s With explicit versions, initial vendor: real 0m5.810s user 0m0.937s sys 0m2.461s Re-vendor: real 0m1.567s user 0m0.578s sys 0m0.967s The summaries of syscalls executed shows why: Revendoring without explicit versions: ``` % time seconds usecs/call calls errors syscall ------ ----------- ----------- --------- --------- ---------------- 25.17 1.104699 18 59432 1065 openat 19.86 0.871574 21 41156 13825 unlink 13.64 0.598739 2 210510 lstat 9.02 0.395948 29 13208 copy_file_range 8.00 0.351242 11 30245 read 6.36 0.279005 3 72487 4476 statx 5.35 0.235027 6 37219 write 4.02 0.176267 3 58368 close ``` with explicit versions: ``` 29.38 0.419068 15 27798 13825 unlink 25.52 0.364021 1 209586 lstat 20.67 0.294788 16 17967 1032 openat 10.42 0.148586 4 35646 write 3.53 0.050350 3 13825 chmod 3.14 0.044786 2 16701 1622 statx 2.19 0.031171 1 16936 close 1.86 0.026538 24 1078 rmdir ``` Specifically, there are a lot fewer opens, copy_file_ranges, and unlinks. --- src/bin/cargo/commands/vendor.rs | 15 +++++------ src/cargo/ops/vendor.rs | 3 ++- tests/testsuite/vendor.rs | 45 ++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 9 deletions(-) diff --git a/src/bin/cargo/commands/vendor.rs b/src/bin/cargo/commands/vendor.rs index da0e5838562..48cc54308a6 100644 --- a/src/bin/cargo/commands/vendor.rs +++ b/src/bin/cargo/commands/vendor.rs @@ -27,6 +27,12 @@ pub fn cli() -> App { .help("Respect `[source]` config in `.cargo/config`") .multiple(true), ) + .arg( + Arg::with_name("explicit-version") + .short("-x") + .long("explicit-version") + .help("Always include version in subdir name"), + ) .arg( Arg::with_name("no-merge-sources") .long("no-merge-sources") @@ -42,12 +48,6 @@ pub fn cli() -> App { .long("only-git-deps") .hidden(true), ) - .arg( - Arg::with_name("explicit-version") - .short("-x") - .long("explicit-version") - .hidden(true), - ) .arg( Arg::with_name("disallow-duplicates") .long("disallow-duplicates") @@ -85,8 +85,6 @@ pub fn exec(config: &mut Config, args: &ArgMatches<'_>) -> CliResult { Some("--relative-path") } else if args.is_present("only-git-deps") { Some("--only-git-deps") - } else if args.is_present("explicit-version") { - Some("--explicit-version") } else if args.is_present("disallow-duplicates") { Some("--disallow-duplicates") } else { @@ -116,6 +114,7 @@ https://github.com/rust-lang/cargo/issues/new &ops::VendorOptions { no_delete: args.is_present("no-delete"), destination: &path, + explicit_version: args.is_present("explicit-version"), extra: args .values_of_os("tomls") .unwrap_or_default() diff --git a/src/cargo/ops/vendor.rs b/src/cargo/ops/vendor.rs index b452ba282ad..87916967aab 100644 --- a/src/cargo/ops/vendor.rs +++ b/src/cargo/ops/vendor.rs @@ -14,6 +14,7 @@ use std::path::{Path, PathBuf}; pub struct VendorOptions<'a> { pub no_delete: bool, + pub explicit_version: bool, pub destination: &'a Path, pub extra: Vec, } @@ -186,7 +187,7 @@ fn sync( .parent() .expect("manifest_path should point to a file"); let max_version = *versions[&id.name()].iter().rev().next().unwrap().0; - let dir_has_version_suffix = id.version() != max_version; + let dir_has_version_suffix = opts.explicit_version || id.version() != max_version; let dst_name = if dir_has_version_suffix { // Eg vendor/futures-0.1.13 format!("{}-{}", id.name(), id.version()) diff --git a/tests/testsuite/vendor.rs b/tests/testsuite/vendor.rs index 4e448602f04..9da98b908b7 100644 --- a/tests/testsuite/vendor.rs +++ b/tests/testsuite/vendor.rs @@ -88,6 +88,51 @@ fn two_versions() { p.cargo("build").run(); } +#[cargo_test] +fn two_explicit_versions() { + let p = project() + .file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.1.0" + + [dependencies] + bitflags = "0.8.0" + bar = { path = "bar" } + "#, + ) + .file("src/lib.rs", "") + .file( + "bar/Cargo.toml", + r#" + [package] + name = "bar" + version = "0.1.0" + + [dependencies] + bitflags = "0.7.0" + "#, + ) + .file("bar/src/lib.rs", "") + .build(); + + Package::new("bitflags", "0.7.0").publish(); + Package::new("bitflags", "0.8.0").publish(); + + p.cargo("vendor --respect-source-config --explicit-version") + .run(); + + let lock = p.read_file("vendor/bitflags-0.8.0/Cargo.toml"); + assert!(lock.contains("version = \"0.8.0\"")); + let lock = p.read_file("vendor/bitflags-0.7.0/Cargo.toml"); + assert!(lock.contains("version = \"0.7.0\"")); + + add_vendor_config(&p); + p.cargo("build").run(); +} + #[cargo_test] fn help() { let p = project().build(); From 596cb798ec32e6dad7bf75079b43cd404bbbd90d Mon Sep 17 00:00:00 2001 From: Jeremy Fitzhardinge Date: Wed, 27 Nov 2019 00:50:08 -0800 Subject: [PATCH 2/5] vendor: add --explicit-version to docs --- src/etc/man/cargo-vendor.1 | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/etc/man/cargo-vendor.1 b/src/etc/man/cargo-vendor.1 index b136da4d890..3860057f551 100644 --- a/src/etc/man/cargo-vendor.1 +++ b/src/etc/man/cargo-vendor.1 @@ -57,6 +57,14 @@ Don\(cqt delete the "vendor" directory when vendoring, but rather keep all existing contents of the vendor directory .RE .sp +\fB\-\-explicit\-version\fP +.RS 4 +Normally versions are only added to disambiguate multiple versions of the same package. +This option causes all directories in the "vendor" directory to be explicitly versioned, +which makes it easier to track the history of vendored packages over time, and can help +with the performance of re-vendoring when only a subset of the packages have changed. +.RE +.sp \fB\-\-respect\-source\-config\fP .RS 4 Instead of ignoring \fB[source]\fP configuration by default in \fB.cargo/config\fP @@ -237,4 +245,4 @@ cargo vendor \-s ../path/to/Cargo.toml .RE .SH "SEE ALSO" .sp -\fBcargo\fP(1) \ No newline at end of file +\fBcargo\fP(1) From 78f45b95dc4d05a81920f2c90403b260a194ba09 Mon Sep 17 00:00:00 2001 From: Jeremy Fitzhardinge Date: Mon, 2 Dec 2019 23:40:10 -0800 Subject: [PATCH 3/5] Minor punctuation fixes for manpage. --- src/etc/man/cargo-vendor.1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/etc/man/cargo-vendor.1 b/src/etc/man/cargo-vendor.1 index 3860057f551..1b22ed7013d 100644 --- a/src/etc/man/cargo-vendor.1 +++ b/src/etc/man/cargo-vendor.1 @@ -54,7 +54,7 @@ vendored and synced to the output. \fB\-\-no\-delete\fP .RS 4 Don\(cqt delete the "vendor" directory when vendoring, but rather keep all -existing contents of the vendor directory +existing contents of the vendor directory. .RE .sp \fB\-\-explicit\-version\fP @@ -68,7 +68,7 @@ with the performance of re-vendoring when only a subset of the packages have cha \fB\-\-respect\-source\-config\fP .RS 4 Instead of ignoring \fB[source]\fP configuration by default in \fB.cargo/config\fP -read it and use it when downloading crates from crates.io, for example +read it and use it when downloading crates from crates.io, for example. .RE .SS "Manifest Options" .sp From fd80795503237caa3458db02098b990be3198e96 Mon Sep 17 00:00:00 2001 From: Jeremy Fitzhardinge Date: Wed, 4 Dec 2019 15:30:13 -0800 Subject: [PATCH 4/5] Convert --explicit-version -> --versioned-dirs --- src/bin/cargo/commands/vendor.rs | 7 +++---- src/cargo/ops/vendor.rs | 4 ++-- src/etc/man/cargo-vendor.1 | 4 ++-- tests/testsuite/vendor.rs | 2 +- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/bin/cargo/commands/vendor.rs b/src/bin/cargo/commands/vendor.rs index 48cc54308a6..311258af5b7 100644 --- a/src/bin/cargo/commands/vendor.rs +++ b/src/bin/cargo/commands/vendor.rs @@ -28,9 +28,8 @@ pub fn cli() -> App { .multiple(true), ) .arg( - Arg::with_name("explicit-version") - .short("-x") - .long("explicit-version") + Arg::with_name("versioned-dirs") + .long("versioned-dirs") .help("Always include version in subdir name"), ) .arg( @@ -114,7 +113,7 @@ https://github.com/rust-lang/cargo/issues/new &ops::VendorOptions { no_delete: args.is_present("no-delete"), destination: &path, - explicit_version: args.is_present("explicit-version"), + versioned_dirs: args.is_present("versioned-dirs"), extra: args .values_of_os("tomls") .unwrap_or_default() diff --git a/src/cargo/ops/vendor.rs b/src/cargo/ops/vendor.rs index 87916967aab..5397d162d66 100644 --- a/src/cargo/ops/vendor.rs +++ b/src/cargo/ops/vendor.rs @@ -14,7 +14,7 @@ use std::path::{Path, PathBuf}; pub struct VendorOptions<'a> { pub no_delete: bool, - pub explicit_version: bool, + pub versioned_dirs: bool, pub destination: &'a Path, pub extra: Vec, } @@ -187,7 +187,7 @@ fn sync( .parent() .expect("manifest_path should point to a file"); let max_version = *versions[&id.name()].iter().rev().next().unwrap().0; - let dir_has_version_suffix = opts.explicit_version || id.version() != max_version; + let dir_has_version_suffix = opts.versioned_dirs || id.version() != max_version; let dst_name = if dir_has_version_suffix { // Eg vendor/futures-0.1.13 format!("{}-{}", id.name(), id.version()) diff --git a/src/etc/man/cargo-vendor.1 b/src/etc/man/cargo-vendor.1 index 1b22ed7013d..98bdf4bc548 100644 --- a/src/etc/man/cargo-vendor.1 +++ b/src/etc/man/cargo-vendor.1 @@ -57,10 +57,10 @@ Don\(cqt delete the "vendor" directory when vendoring, but rather keep all existing contents of the vendor directory. .RE .sp -\fB\-\-explicit\-version\fP +\fB\-\-versioned\-dirs\fP .RS 4 Normally versions are only added to disambiguate multiple versions of the same package. -This option causes all directories in the "vendor" directory to be explicitly versioned, +This option causes all directories in the "vendor" directory to be versioned, which makes it easier to track the history of vendored packages over time, and can help with the performance of re-vendoring when only a subset of the packages have changed. .RE diff --git a/tests/testsuite/vendor.rs b/tests/testsuite/vendor.rs index 9da98b908b7..3f3fb1ff0d1 100644 --- a/tests/testsuite/vendor.rs +++ b/tests/testsuite/vendor.rs @@ -121,7 +121,7 @@ fn two_explicit_versions() { Package::new("bitflags", "0.7.0").publish(); Package::new("bitflags", "0.8.0").publish(); - p.cargo("vendor --respect-source-config --explicit-version") + p.cargo("vendor --respect-source-config --versioned-dirs") .run(); let lock = p.read_file("vendor/bitflags-0.8.0/Cargo.toml"); From 2214cf124db7722b6848c2e60dec5bb1c071b1df Mon Sep 17 00:00:00 2001 From: Jeremy Fitzhardinge Date: Mon, 9 Dec 2019 16:46:25 -0800 Subject: [PATCH 5/5] Document --version-dirs in cargo-vendor.adoc and re-generate all doc formats. --- src/doc/man/cargo-vendor.adoc | 7 +++++++ src/doc/man/generated/cargo-vendor.html | 8 ++++++++ src/etc/man/cargo-vendor.1 | 25 +++++++++++++------------ 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/src/doc/man/cargo-vendor.adoc b/src/doc/man/cargo-vendor.adoc index d5606a8529b..068d23391f8 100644 --- a/src/doc/man/cargo-vendor.adoc +++ b/src/doc/man/cargo-vendor.adoc @@ -38,6 +38,13 @@ to use the vendored sources, which you will need to add to `.cargo/config`. Instead of ignoring `[source]` configuration by default in `.cargo/config` read it and use it when downloading crates from crates.io, for example +*--versioned-dirs*:: + Normally versions are only added to disambiguate multiple versions of the + same package. This option causes all directories in the "vendor" directory + to be versioned, which makes it easier to track the history of vendored + packages over time, and can help with the performance of re-vendoring when + only a subset of the packages have changed. + === Manifest Options include::options-manifest-path.adoc[] diff --git a/src/doc/man/generated/cargo-vendor.html b/src/doc/man/generated/cargo-vendor.html index 0834a0ccce8..134b86a77bd 100644 --- a/src/doc/man/generated/cargo-vendor.html +++ b/src/doc/man/generated/cargo-vendor.html @@ -49,6 +49,14 @@

Owner Options

Instead of ignoring [source] configuration by default in .cargo/config read it and use it when downloading crates from crates.io, for example

+
--versioned-dirs
+
+

Normally versions are only added to disambiguate multiple versions of the +same package. This option causes all directories in the "vendor" directory +to be versioned, which makes it easier to track the history of vendored +packages over time, and can help with the performance of re-vendoring when +only a subset of the packages have changed.

+
diff --git a/src/etc/man/cargo-vendor.1 b/src/etc/man/cargo-vendor.1 index 98bdf4bc548..de260c8d2b2 100644 --- a/src/etc/man/cargo-vendor.1 +++ b/src/etc/man/cargo-vendor.1 @@ -2,12 +2,12 @@ .\" Title: cargo-vendor .\" Author: [see the "AUTHOR(S)" section] .\" Generator: Asciidoctor 2.0.10 -.\" Date: 2019-09-08 +.\" Date: 2019-12-09 .\" Manual: \ \& .\" Source: \ \& .\" Language: English .\" -.TH "CARGO\-VENDOR" "1" "2019-09-08" "\ \&" "\ \&" +.TH "CARGO\-VENDOR" "1" "2019-12-09" "\ \&" "\ \&" .ie \n(.g .ds Aq \(aq .el .ds Aq ' .ss \n[.ss] 0 @@ -54,21 +54,22 @@ vendored and synced to the output. \fB\-\-no\-delete\fP .RS 4 Don\(cqt delete the "vendor" directory when vendoring, but rather keep all -existing contents of the vendor directory. +existing contents of the vendor directory .RE .sp -\fB\-\-versioned\-dirs\fP +\fB\-\-respect\-source\-config\fP .RS 4 -Normally versions are only added to disambiguate multiple versions of the same package. -This option causes all directories in the "vendor" directory to be versioned, -which makes it easier to track the history of vendored packages over time, and can help -with the performance of re-vendoring when only a subset of the packages have changed. +Instead of ignoring \fB[source]\fP configuration by default in \fB.cargo/config\fP +read it and use it when downloading crates from crates.io, for example .RE .sp -\fB\-\-respect\-source\-config\fP +\fB\-\-versioned\-dirs\fP .RS 4 -Instead of ignoring \fB[source]\fP configuration by default in \fB.cargo/config\fP -read it and use it when downloading crates from crates.io, for example. +Normally versions are only added to disambiguate multiple versions of the +same package. This option causes all directories in the "vendor" directory +to be versioned, which makes it easier to track the history of vendored +packages over time, and can help with the performance of re\-vendoring when +only a subset of the packages have changed. .RE .SS "Manifest Options" .sp @@ -245,4 +246,4 @@ cargo vendor \-s ../path/to/Cargo.toml .RE .SH "SEE ALSO" .sp -\fBcargo\fP(1) +\fBcargo\fP(1) \ No newline at end of file