Merge branch 'adjustments-for-cargo'

Author: Byron

.github/workflows/ci.yml

@@ -120,7 +120,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: dtolnay/rust-toolchain@master
         with:
-          toolchain: stable
+          toolchain: 1.65 # clippy is broken for us in 1.66
           components: clippy,rustfmt
       - name: Run cargo clippy
         run: cargo clippy --all --tests

Cargo.lock

@@ -95,7 +95,7 @@ check: ## Build all code in suitable configurations
 			   && cargo check
 	cd git-features && cargo check --all-features \
 			   && cargo check --features parallel \
-			   && cargo check --features parallel,fs-walkdir-parallel \
+			   && cargo check --features fs-walkdir-parallel \
 			   && cargo check --features rustsha1 \
 			   && cargo check --features fast-sha1 \
 			   && cargo check --features progress \

Makefile

@@ -32,7 +32,7 @@ mod ignore {
 
     #[test]
     fn from_git_dir() -> crate::Result {
-        let dir = git_testtools::scripted_fixture_repo_read_only("make_global_and_external_and_dir_ignores.sh")?;
+        let dir = git_testtools::scripted_fixture_read_only("make_global_and_external_and_dir_ignores.sh")?;
         let repo_dir = dir.join("repo");
         let git_dir = repo_dir.join(".git");
         let baseline = std::fs::read(git_dir.parent().unwrap().join("git-check-ignore.baseline"))?;

git-attributes/tests/match_group/mod.rs

@@ -60,9 +60,9 @@ pub fn check_common(cg: &Graph, expected: &HashMap<String, RefInfo, impl BuildHa
     );
 }
 
-use git_testtools::scripted_fixture_repo_read_only;
+use git_testtools::scripted_fixture_read_only;
 pub fn make_readonly_repo(script_path: &str) -> std::path::PathBuf {
-    scripted_fixture_repo_read_only(script_path).expect("script succeeds all the time")
+    scripted_fixture_read_only(script_path).expect("script succeeds all the time")
 }
 
 pub fn hex_to_id(hex: &[u8]) -> git_hash::ObjectId {

git-commitgraph/tests/commitgraph.rs

@@ -21,7 +21,7 @@ fn from_environment_overrides() {
 #[test]
 #[serial]
 fn from_git_dir() -> crate::Result {
-    let worktree_dir = git_testtools::scripted_fixture_repo_read_only("make_config_repo.sh")?;
+    let worktree_dir = git_testtools::scripted_fixture_read_only("make_config_repo.sh")?;
     let git_dir = worktree_dir.join(".git");
     let worktree_dir = worktree_dir.canonicalize()?;
     let _env = Env::new()
@@ -84,7 +84,7 @@ fn from_git_dir() -> crate::Result {
 #[test]
 #[serial]
 fn from_git_dir_with_worktree_extension() -> crate::Result {
-    let git_dir = git_testtools::scripted_fixture_repo_read_only("config_with_worktree_extension.sh")?
+    let git_dir = git_testtools::scripted_fixture_read_only("config_with_worktree_extension.sh")?
         .join("main-worktree")
         .join(".git");
     let config = git_config::File::from_git_dir(git_dir)?;

git-config/tests/file/init/comfort.rs

@@ -13,7 +13,7 @@ struct Sample {
 
 static BASELINE: Lazy<HashMap<String, Sample>> = Lazy::new(|| {
     (|| -> Result<_> {
-        let base = git_testtools::scripted_fixture_repo_read_only("generate_git_date_baseline.sh")?;
+        let base = git_testtools::scripted_fixture_read_only("generate_git_date_baseline.sh")?;
         let mut map = HashMap::new();
         let file = std::fs::read(base.join("baseline.git"))?;
         let baseline = std::str::from_utf8(&file).expect("valid utf");

git-date/tests/time/baseline.rs

@@ -11,7 +11,7 @@ mod changes {
 
         fn db(args: impl IntoIterator<Item = &'static str>) -> crate::Result<git_odb::Handle> {
             git_odb::at(
-                git_testtools::scripted_fixture_repo_read_only_with_args("make_diff_repo.sh", args)?
+                git_testtools::scripted_fixture_read_only_with_args("make_diff_repo.sh", args)?
                     .join(".git")
                     .join("objects"),
             )

git-diff/tests/tree/mod.rs

@@ -7,7 +7,7 @@ fn verify_on_exfat() -> crate::Result<()> {
 
     use git_discover::repository::Kind;
 
-    let fixtures = git_testtools::scripted_fixture_repo_read_only("make_exfat_repo_darwin.sh")?;
+    let fixtures = git_testtools::scripted_fixture_read_only("make_exfat_repo_darwin.sh")?;
     let mount_point = tempfile::tempdir()?;
 
     let _cleanup = {
@@ -54,7 +54,7 @@ fn missing_configuration_file_is_not_a_dealbreaker_in_bare_repo() -> crate::Resu
 fn missing_configuration_file_is_not_a_dealbreaker_in_nonbare_repo() -> crate::Result {
     for name in ["worktree-no-config-after-init/.git", "worktree-no-config/.git"] {
         let repo = repo_path()?.join(name);
-        let kind = git_discover::is_git(&repo)?;
+        let kind = git_discover::is_git(repo)?;
         assert_eq!(kind, git_discover::repository::Kind::WorkTree { linked_git_dir: None });
     }
     Ok(())

git-discover/tests/is_git/mod.rs

@@ -6,7 +6,7 @@ use serial_test::serial;
 #[test]
 #[serial]
 fn upwards_with_relative_directories_and_optional_ceiling() -> git_testtools::Result {
-    let repo = git_testtools::scripted_fixture_repo_read_only("make_basic_repo.sh")?;
+    let repo = git_testtools::scripted_fixture_read_only("make_basic_repo.sh")?;
 
     std::env::set_current_dir(repo.join("subdir"))?;
     let cwd = std::env::current_dir()?;

git-discover/tests/isolated.rs

@@ -19,7 +19,7 @@ fn git_dir_candidate_within_ceiling_allows_discovery() -> crate::Result {
     let work_dir = repo_path()?;
     let dir = work_dir.join("some/very/deeply/nested/subdir");
     let (repo_path, _trust) = git_discover::upwards_opts(
-        &dir,
+        dir,
         Options {
             ceiling_dirs: vec![work_dir.clone()],
             ..Default::default()
@@ -37,7 +37,7 @@ fn ceiling_dir_limits_are_respected_and_prevent_discovery() -> crate::Result {
     let dir = work_dir.join("some/very/deeply/nested/subdir");
 
     let err = git_discover::upwards_opts(
-        &dir,
+        dir,
         Options {
             ceiling_dirs: vec![work_dir.join("some/../some")],
             ..Default::default()
@@ -57,7 +57,7 @@ fn no_matching_ceiling_dir_error_can_be_suppressed() -> crate::Result {
     let work_dir = repo_path()?;
     let dir = work_dir.join("some/very/deeply/nested/subdir");
     let (repo_path, _trust) = git_discover::upwards_opts(
-        &dir,
+        dir,
         Options {
             match_ceiling_dir_or_error: false,
             ceiling_dirs: vec![
@@ -80,7 +80,7 @@ fn more_restrictive_ceiling_dirs_overrule_less_restrictive_ones() -> crate::Resu
     let work_dir = repo_path()?;
     let dir = work_dir.join("some/very/deeply/nested/subdir");
     let err = git_discover::upwards_opts(
-        &dir,
+        dir,
         Options {
             ceiling_dirs: vec![work_dir.clone(), work_dir.join("some")],
             ..Default::default()
@@ -100,7 +100,7 @@ fn ceiling_dirs_are_not_processed_differently_than_the_git_dir_candidate() -> cr
     let work_dir = repo_path()?;
     let dir = work_dir.join("some/very/deeply/nested/subdir/../../../../../..");
     let (repo_path, _trust) = git_discover::upwards_opts(
-        &dir,
+        dir,
         Options {
             match_ceiling_dir_or_error: false,
             ceiling_dirs: vec![Path::new("./some").into()],
@@ -123,7 +123,7 @@ fn no_matching_ceiling_dirs_errors_by_default() -> crate::Result {
     let relative_work_dir = repo_path()?;
     let dir = relative_work_dir.join("some");
     let res = git_discover::upwards_opts(
-        &dir,
+        dir,
         Options {
             ceiling_dirs: vec!["/something/somewhere".into()],
             ..Default::default()

git-discover/tests/upwards/ceiling_dirs.rs

@@ -38,7 +38,7 @@ fn from_bare_git_dir_without_config_file() -> crate::Result {
 fn from_inside_bare_git_dir() -> crate::Result {
     let git_dir = repo_path()?.join("bare.git");
     let dir = git_dir.join("objects");
-    let (path, trust) = git_discover::upwards(&dir)?;
+    let (path, trust) = git_discover::upwards(dir)?;
     assert_eq!(
         path.as_ref(),
         git_dir,
@@ -89,7 +89,7 @@ fn from_working_dir_no_config() -> crate::Result {
 fn from_nested_dir() -> crate::Result {
     let working_dir = repo_path()?;
     let dir = working_dir.join("some/very/deeply/nested/subdir");
-    let (path, trust) = git_discover::upwards(&dir)?;
+    let (path, trust) = git_discover::upwards(dir)?;
     assert_eq!(path.kind(), Kind::WorkTree { linked_git_dir: None });
     assert_eq!(path.as_ref(), working_dir, "a working tree dir yields the git dir");
     assert_eq!(trust, expected_trust());
@@ -105,7 +105,7 @@ fn from_dir_with_dot_dot() -> crate::Result {
     // exploring ancestors.)
     let working_dir = repo_path()?;
     let dir = working_dir.join("some/very/deeply/nested/subdir/../../../../../..");
-    let (path, trust) = git_discover::upwards(&dir)?;
+    let (path, trust) = git_discover::upwards(dir)?;
     assert_ne!(
         path.as_ref().canonicalize()?,
         working_dir.canonicalize()?,
@@ -140,7 +140,7 @@ fn from_dir_with_dot_dot() -> crate::Result {
 fn from_nested_dir_inside_a_git_dir() -> crate::Result {
     let working_dir = repo_path()?;
     let dir = working_dir.join(".git").join("objects");
-    let (path, trust) = git_discover::upwards(&dir)?;
+    let (path, trust) = git_discover::upwards(dir)?;
     assert_eq!(path.kind(), Kind::WorkTree { linked_git_dir: None });
     assert_eq!(path.as_ref(), working_dir, "we find .git directories on the way");
     assert_eq!(trust, expected_trust());
@@ -226,7 +226,7 @@ fn cross_fs() -> crate::Result {
         return Ok(());
     }
 
-    let top_level_repo = git_testtools::scripted_fixture_repo_writable("make_basic_repo.sh")?;
+    let top_level_repo = git_testtools::scripted_fixture_writable("make_basic_repo.sh")?;
 
     let _cleanup = {
         // Create an empty dmg file
@@ -269,7 +269,7 @@ fn cross_fs() -> crate::Result {
     ));
 
     let (repo_path, _trust) = git_discover::upwards_opts(
-        &top_level_repo.path().join("remote"),
+        top_level_repo.path().join("remote"),
         Options {
             cross_fs: true,
             ..Default::default()
@@ -307,7 +307,7 @@ fn do_not_shorten_absolute_paths() -> crate::Result {
 mod submodules {
     #[test]
     fn by_their_worktree_checkout() -> crate::Result {
-        let dir = git_testtools::scripted_fixture_repo_read_only("make_submodules.sh")?;
+        let dir = git_testtools::scripted_fixture_read_only("make_submodules.sh")?;
         let parent = dir.join("with-submodules");
         let modules = parent.join(".git").join("modules");
         for module in ["m1", "dir/m1"] {
@@ -322,7 +322,7 @@ mod submodules {
                 submodule_m1_gitdir
             );
 
-            let (path, _trust) = git_discover::upwards(&submodule_m1_workdir.join("subdir"))?;
+            let (path, _trust) = git_discover::upwards(submodule_m1_workdir.join("subdir"))?;
             assert!(
                 matches!(path, git_discover::repository::Path::LinkedWorkTree{ref work_dir, ref git_dir} if work_dir == &submodule_m1_workdir && git_dir == &submodule_m1_gitdir),
                 "{:?} should match {:?} {:?}",
@@ -336,7 +336,7 @@ mod submodules {
 
     #[test]
     fn by_their_module_git_dir() -> crate::Result {
-        let dir = git_testtools::scripted_fixture_repo_read_only("make_submodules.sh")?;
+        let dir = git_testtools::scripted_fixture_read_only("make_submodules.sh")?;
         let modules = dir.join("with-submodules").join(".git").join("modules");
         for module in ["m1", "dir/m1"] {
             let submodule_m1_gitdir = modules.join(module);
@@ -353,5 +353,5 @@ mod submodules {
 }
 
 pub(crate) fn repo_path() -> crate::Result<PathBuf> {
-    git_testtools::scripted_fixture_repo_read_only("make_basic_repo.sh")
+    git_testtools::scripted_fixture_read_only("make_basic_repo.sh")
 }

git-discover/tests/upwards/mod.rs

@@ -19,8 +19,7 @@ default = []
 progress = ["prodash"]
 
 ## If set, walkdir iterators will be multi-threaded.
-## This feature has [certain side-effects](https://github.com/starship/starship/issues/4251) of rayon threadpool configuration with `jwalk`.
-fs-walkdir-parallel = ["parallel", "jwalk" ]
+fs-walkdir-parallel = [ "num_cpus", "jwalk" ]
 
 ## Use scoped threads and channels to parallelize common workloads on multiple objects. If enabled, it is used everywhere
 ## where it makes sense.
@@ -107,7 +106,7 @@ crossbeam-channel = { version = "0.5.0", optional = true }
 num_cpus = { version = "1.13.0", optional = true }
 parking_lot = { version = "0.12.0", default-features = false, optional = true }
 
-jwalk = { version = "0.6.0", optional = true }
+jwalk = { version = "0.8.1", optional = true }
 ## Makes facilities of the `walkdir` crate partially available.
 ## In conjunction with the **parallel** feature, directory walking will be parallel instead behind a compatible interface.
 walkdir = { version = "2.3.2", optional = true } # used when parallel is off

git-features/Cargo.toml

@@ -6,29 +6,62 @@
 //! For information on how to use the [`WalkDir`] type, have a look at
 //! * [`jwalk::WalkDir`](https://docs.rs/jwalk/0.5.1/jwalk/type.WalkDir.html) if `parallel` feature is enabled
 //! * [walkdir::WalkDir](https://docs.rs/walkdir/2.3.1/walkdir/struct.WalkDir.html) otherwise
-#[cfg(feature = "fs-walkdir-parallel")]
+
+#[cfg(any(feature = "walkdir", feature = "fs-walkdir-parallel"))]
+mod shared {
+    /// The desired level of parallelism.
+    pub enum Parallelism {
+        /// Do not parallelize at all by making a serial traversal on the current thread.
+        Serial,
+        /// Create a new thread pool for each traversal with up to 16 threads or the amount of logical cores of the machine.
+        ThreadPoolPerTraversal {
+            /// The base name of the threads we create as part of the thread-pool.
+            thread_name: &'static str,
+        },
+    }
+}
+
 ///
+#[cfg(feature = "fs-walkdir-parallel")]
 pub mod walkdir {
-    use std::path::Path;
-
+    pub use super::shared::Parallelism;
     pub use jwalk::{DirEntry as DirEntryGeneric, DirEntryIter as DirEntryIterGeneric, Error, WalkDir};
+    use std::path::Path;
 
     /// An alias for an uncustomized directory entry to match the one of the non-parallel version offered by `walkdir`.
     pub type DirEntry = DirEntryGeneric<((), ())>;
 
-    /// Instantiate a new directory iterator which will not skip hidden files.
-    pub fn walkdir_new(root: impl AsRef<Path>) -> WalkDir {
-        WalkDir::new(root)
-            .skip_hidden(false)
-            .parallelism(jwalk::Parallelism::Serial)
+    impl From<Parallelism> for jwalk::Parallelism {
+        fn from(v: Parallelism) -> Self {
+            match v {
+                Parallelism::Serial => jwalk::Parallelism::Serial,
+                Parallelism::ThreadPoolPerTraversal { thread_name } => {
+                    let pool = jwalk::rayon::ThreadPoolBuilder::new()
+                        .num_threads(num_cpus::get().min(16))
+                        .stack_size(128 * 1024)
+                        .thread_name(move |idx| format!("{thread_name} {idx}"))
+                        .build()
+                        .expect("we only set options that can't cause a build failure");
+                    jwalk::Parallelism::RayonExistingPool {
+                        pool: pool.into(),
+                        busy_timeout: None,
+                    }
+                }
+            }
+        }
+    }
+
+    /// Instantiate a new directory iterator which will not skip hidden files, with the given level of `parallelism`.
+    pub fn walkdir_new(root: impl AsRef<Path>, parallelism: Parallelism) -> WalkDir {
+        WalkDir::new(root).skip_hidden(false).parallelism(parallelism.into())
     }
 
     /// Instantiate a new directory iterator which will not skip hidden files and is sorted
-    pub fn walkdir_sorted_new(root: impl AsRef<Path>) -> WalkDir {
+    pub fn walkdir_sorted_new(root: impl AsRef<Path>, parallelism: Parallelism) -> WalkDir {
         WalkDir::new(root)
             .skip_hidden(false)
             .sort(true)
-            .parallelism(jwalk::Parallelism::Serial)
+            .parallelism(parallelism.into())
     }
 
     /// The Iterator yielding directory items
@@ -38,17 +71,18 @@ pub mod walkdir {
 #[cfg(all(feature = "walkdir", not(feature = "fs-walkdir-parallel")))]
 ///
 pub mod walkdir {
+    pub use super::shared::Parallelism;
     use std::path::Path;
 
     pub use walkdir::{DirEntry, Error, WalkDir};
 
-    /// Instantiate a new directory iterator which will not skip hidden files.
-    pub fn walkdir_new(root: impl AsRef<Path>) -> WalkDir {
+    /// Instantiate a new directory iterator which will not skip hidden files, with the given level of `parallelism`.
+    pub fn walkdir_new(root: impl AsRef<Path>, _: Parallelism) -> WalkDir {
         WalkDir::new(root)
     }
 
-    /// Instantiate a new directory iterator which will not skip hidden files and is sorted
-    pub fn walkdir_sorted_new(root: impl AsRef<Path>) -> WalkDir {
+    /// Instantiate a new directory iterator which will not skip hidden files and is sorted, with the given level of `parallelism`.
+    pub fn walkdir_sorted_new(root: impl AsRef<Path>, _: Parallelism) -> WalkDir {
         WalkDir::new(root).sort_by_file_name()
     }
 

git-features/src/fs.rs

@@ -88,9 +88,7 @@ pub fn optimize_chunk_size_and_thread_limit(
         .map(|num_items| {
             let desired_chunks_per_thread_at_least = 2;
             let items = num_items;
-            let chunk_size = (items / (available_threads * desired_chunks_per_thread_at_least))
-                .max(1)
-                .min(upper);
+            let chunk_size = (items / (available_threads * desired_chunks_per_thread_at_least)).clamp(1, upper);
             let num_chunks = items / chunk_size;
             let thread_limit = if num_chunks <= available_threads {
                 (num_chunks / desired_chunks_per_thread_at_least).max(1)

git-features/src/parallel/mod.rs

@@ -50,7 +50,7 @@ where
 {
     fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
         let bytes_read = self.inner.read(buf)?;
-        self.progress.inc_by(bytes_read as usize);
+        self.progress.inc_by(bytes_read);
         Ok(bytes_read)
     }
 }

git-features/src/progress.rs

@@ -43,7 +43,7 @@ impl<'a> Baseline<'a> {
 
 #[test]
 fn compare_baseline_with_ours() {
-    let dir = git_testtools::scripted_fixture_repo_read_only("make_baseline.sh").unwrap();
+    let dir = git_testtools::scripted_fixture_read_only("make_baseline.sh").unwrap();
     let (mut total_matches, mut total_correct, mut panics) = (0, 0, 0);
     let mut mismatches = Vec::new();
     for (input_file, expected_matches, case) in &[