feature: faster self-install without rate limit issue (#1963)

* Add new hidden option `--self-install` Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Fix typo Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Optimize: Only call `LazyJobserverClient::new` when necessary Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * `--self-install` should include a path to a binary On windows, we cannot just copy an executable with process running from it, so better to copy it to a temporary location. Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Impl manifest update and basic API of `self_install` Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Add dep atomic-file-install to cargo-binstall Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Impl `self_install` Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Make `--self-install` a boolean flag Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Accept no duration in `MainExit::new` Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Impl self-install mode in main_impl.rs Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Use `--self-install` mode in install-from-binstall-release.sh Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Use `--self-install` in install-from-binstall-release.ps1 Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Fix import in mod entry Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Fix call of `self_install` in `main_impl` Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Fix `--self-install` clap doc Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Fix `entry::self_install` Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Apply suggestions from code review Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Fix parsing semver number in entry.rs Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Fix compilation in entry.rs Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * fix entry.rs Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Fix fmt in bin_util.rs Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Fix fmt in entry.rs Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Use --self-install if supported in unix install script Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Use --self-install if available in powershell install script Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Fix install-from-binstall-release.ps1 Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Fi install-from-binstall-release.ps1 Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Fix install-from-binstall-release.ps1 Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Create self-install.sh Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Add e2e-test-self-install to justfile Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Fix args::parse() for self-install mode Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Fix args parsing: Do no require positional arg if --self-install is present Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> --------- Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>
cargo-bins · Nov 9, 2024 · c9d5e6d · c9d5e6d
1 parent b1137ec
commit c9d5e6d
Show file tree

Hide file tree

Showing 9 changed files with 89 additions and 15 deletions.
diff --git a/crates/bin/Cargo.toml b/crates/bin/Cargo.toml
@@ -22,6 +22,7 @@ pkg-fmt = "zip"
 pkg-fmt = "zip"
 
 [dependencies]
+atomic-file-install = { version = "1.0.5", path = "../atomic-file-install" }
 binstalk = { path = "../binstalk", version = "0.28.11", default-features = false }
 binstalk-manifests = { path = "../binstalk-manifests", version = "0.15.8" }
 clap = { version = "4.5.3", features = ["derive", "env"] }

diff --git a/crates/bin/src/args.rs b/crates/bin/src/args.rs
@@ -52,7 +52,7 @@ pub struct Args {
     #[clap(
         help_heading = "Package selection",
         value_name = "crate[@version]",
-        required_unless_present_any = ["version", "help"],
+        required_unless_present_any = ["version", "self_install", "help"],
     )]
     pub(crate) crate_names: Vec<CrateName>,
 
@@ -404,6 +404,9 @@ pub struct Args {
     /// This would override the `log_level`.
     #[clap(help_heading = "Meta", short, long, conflicts_with("verbose"))]
     pub(crate) quiet: bool,
+
+    #[clap(long, hide(true))]
+    pub(crate) self_install: bool,
 }
 
 #[derive(Debug, Clone)]
@@ -515,6 +518,10 @@ pub fn parse() -> (Args, PkgOverride) {
     // Load options
     let mut opts = Args::parse_from(args);
 
+    if opts.self_install {
+        return (opts, Default::default());
+    }
+
     if opts.log_level.is_none() {
         if let Some(log) = env::var("BINSTALL_LOG_LEVEL")
             .ok()

diff --git a/crates/bin/src/bin_util.rs b/crates/bin/src/bin_util.rs
@@ -36,13 +36,12 @@ impl Termination for MainExit {
 }
 
 impl MainExit {
-    pub fn new(res: Result<()>, done: Duration) -> Self {
-        res.map(|()| MainExit::Success(Some(done)))
-            .unwrap_or_else(|err| {
-                err.downcast::<BinstallError>()
-                    .map(MainExit::Error)
-                    .unwrap_or_else(MainExit::Report)
-            })
+    pub fn new(res: Result<()>, done: Option<Duration>) -> Self {
+        res.map(|()| MainExit::Success(done)).unwrap_or_else(|err| {
+            err.downcast::<BinstallError>()
+                .map(MainExit::Error)
+                .unwrap_or_else(MainExit::Report)
+        })
     }
 }
 

diff --git a/crates/bin/src/entry.rs b/crates/bin/src/entry.rs
@@ -5,6 +5,7 @@ use std::{
     time::Duration,
 };
 
+use atomic_file_install::atomic_install;
 use binstalk::{
     errors::{BinstallError, CrateContextError},
     fetchers::{Fetcher, GhCrateMeta, QuickInstall, SignaturePolicy},
@@ -20,16 +21,20 @@ use binstalk::{
         resolve::{CrateName, Resolution, ResolutionFetch, VersionReqExt},
         CargoTomlFetchOverride, Options, Resolver,
     },
+    TARGET,
 };
 use binstalk_manifests::{
     cargo_config::Config,
     cargo_toml_binstall::{PkgOverride, Strategy},
+    crate_info::{CrateInfo, CrateSource},
     crates_manifests::Manifests,
 };
+use compact_str::CompactString;
 use file_format::FileFormat;
 use home::cargo_home;
 use log::LevelFilter;
 use miette::{miette, Report, Result, WrapErr};
+use semver::Version;
 use tokio::task::block_in_place;
 use tracing::{debug, error, info, warn};
 
@@ -582,3 +587,44 @@ fn do_install_fetches_continue_on_failure(
         Ok(())
     })
 }
+
+pub fn self_install(args: Args) -> Result<()> {
+    // Load .cargo/config.toml
+    let cargo_home = cargo_home().map_err(BinstallError::from)?;
+    let mut config = Config::load_from_path(cargo_home.join("config.toml"))?;
+
+    // Compute paths
+    let cargo_root = args.root;
+    let (install_path, manifests, _) = compute_paths_and_load_manifests(
+        cargo_root.clone(),
+        args.install_path,
+        args.no_track,
+        cargo_home,
+        &mut config,
+    )?;
+
+    let mut dest = install_path.join("cargo-binstall");
+    if cfg!(windows) {
+        assert!(dest.set_extension("exe"));
+    }
+
+    atomic_install(&env::current_exe().map_err(BinstallError::from)?, &dest)
+        .map_err(BinstallError::from)?;
+
+    if let Some(manifests) = manifests {
+        manifests.update(vec![CrateInfo {
+            name: CompactString::const_new("cargo-binstall"),
+            version_req: CompactString::const_new("*"),
+            current_version: Version::new(
+                env!("CARGO_PKG_VERSION_MAJOR").parse().unwrap(),
+                env!("CARGO_PKG_VERSION_MINOR").parse().unwrap(),
+                env!("CARGO_PKG_VERSION_PATCH").parse().unwrap(),
+            ),
+            source: CrateSource::cratesio_registry(),
+            target: CompactString::const_new(TARGET),
+            bins: vec![CompactString::const_new("cargo-binstall")],
+        }])?;
+    }
+
+    Ok(())
+}
diff --git a/crates/bin/src/main_impl.rs b/crates/bin/src/main_impl.rs
@@ -12,9 +12,6 @@ use crate::{
 };
 
 pub fn do_main() -> impl Termination {
-    // This must be the very first thing to happen
-    let jobserver_client = LazyJobserverClient::new();
-
     let (args, cli_overrides) = args::parse();
 
     if args.version {
@@ -46,6 +43,8 @@ rustc-llvm-version: {rustc_llvm_version}"#
             println!("{cargo_binstall_version}");
         }
         MainExit::Success(None)
+    } else if args.self_install {
+        MainExit::new(entry::self_install(args), None)
     } else {
         logging(
             args.log_level.unwrap_or(LevelFilter::Info),
@@ -54,12 +53,14 @@ rustc-llvm-version: {rustc_llvm_version}"#
 
         let start = Instant::now();
 
+        let jobserver_client = LazyJobserverClient::new();
+
         let result =
             run_tokio_main(|| entry::install_crates(args, cli_overrides, jobserver_client));
 
         let done = start.elapsed();
         debug!("run time: {done:?}");
 
-        MainExit::new(result, done)
+        MainExit::new(result, Some(done))
     }
 }
diff --git a/e2e-tests/self-install.sh b/e2e-tests/self-install.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+unset CARGO_INSTALL_ROOT
+
+CARGO_HOME=$(mktemp -d 2>/dev/null || mktemp -d -t 'cargo-home')
+export CARGO_HOME
+export PATH="$CARGO_HOME/bin:$PATH"
+
+"./$1" --self-install
+
+cargo binstall --help
+cargo install --list
diff --git a/install-from-binstall-release.ps1 b/install-from-binstall-release.ps1
@@ -19,7 +19,12 @@ $url = "$base_url$arch-pc-windows-msvc.zip"
 Invoke-WebRequest $url -OutFile $tmpdir\cargo-binstall.zip
 Expand-Archive -Force $tmpdir\cargo-binstall.zip $tmpdir\cargo-binstall
 Write-Host ""
-Invoke-Expression "$tmpdir\cargo-binstall\cargo-binstall.exe -y --force cargo-binstall"
+
+$ps = Start-Process -PassThru -Wait "$tmpdir\cargo-binstall\cargo-binstall.exe" "--self-install"
+if ($ps.ExitCode -ne 0) {
+    Invoke-Expression "$tmpdir\cargo-binstall\cargo-binstall.exe -y --force cargo-binstall"
+}
+
 Remove-Item -Force $tmpdir\cargo-binstall.zip
 Remove-Item -Recurse -Force $tmpdir\cargo-binstall
 $cargo_home = if ($Env:CARGO_HOME -ne $null) { $Env:CARGO_HOME } else { "$HOME\.cargo" }

diff --git a/install-from-binstall-release.sh b/install-from-binstall-release.sh
@@ -36,7 +36,7 @@ else
     exit 1
 fi
 
-./cargo-binstall -y --force cargo-binstall
+./cargo-binstall --self-install || ./cargo-binstall -y --force cargo-binstall
 
 CARGO_HOME="${CARGO_HOME:-$HOME/.cargo}"
 

diff --git a/justfile b/justfile
@@ -244,6 +244,7 @@ e2e-test-registries: (e2e-test "registries")
 e2e-test-signing: (e2e-test "signing")
 e2e-test-continue-on-failure: (e2e-test "continue-on-failure")
 e2e-test-private-github-repo: (e2e-test "private-github-repo")
+e2e-test-self-install: (e2e-test "self-install")
 
 # WinTLS (Windows in CI) does not have TLS 1.3 support
 [windows]
@@ -252,7 +253,7 @@ e2e-test-tls: (e2e-test "tls" "1.2")
 [macos]
 e2e-test-tls: (e2e-test "tls" "1.2") (e2e-test "tls" "1.3")
 
-e2e-tests: e2e-test-live e2e-test-manifest-path e2e-test-git e2e-test-other-repos e2e-test-strategies e2e-test-version-syntax e2e-test-upgrade e2e-test-tls e2e-test-self-upgrade-no-symlink e2e-test-uninstall e2e-test-subcrate e2e-test-no-track e2e-test-registries e2e-test-signing e2e-test-continue-on-failure e2e-test-private-github-repo
+e2e-tests: e2e-test-live e2e-test-manifest-path e2e-test-git e2e-test-other-repos e2e-test-strategies e2e-test-version-syntax e2e-test-upgrade e2e-test-tls e2e-test-self-upgrade-no-symlink e2e-test-uninstall e2e-test-subcrate e2e-test-no-track e2e-test-registries e2e-test-signing e2e-test-continue-on-failure e2e-test-private-github-repo e2e-test-self-install
 
 unit-tests: print-env
     cargo test --no-run --target {{target}}
-Original file line number
+Diff line change
@@ Expand Up / @@ -36,7 +36,7 @@ else @@
         exit 1
     fi
-    ./cargo-binstall -y --force cargo-binstall
+    ./cargo-binstall --self-install || ./cargo-binstall -y --force cargo-binstall
     CARGO_HOME="${CARGO_HOME:-$HOME/.cargo}"
@@ Expand Down @@