Discovery: data consistency and resource usage

Cargo.toml

@@ -132,6 +132,7 @@ members = [
     "experiments/object-access",
     "experiments/diffing",
     "experiments/traversal",
+    "experiments/odb-redesign",
 
     "cargo-smart-release",
 

cargo-smart-release/src/command/release/manifest.rs

@@ -11,12 +11,11 @@ use git_repository::lock::File;
 use semver::{Version, VersionReq};
 
 use super::{cargo, git, Context, Oid, Options};
-use crate::utils::version_req_unset_or_default;
 use crate::{
     changelog,
     changelog::{write::Linkables, Section},
     traverse::Dependency,
-    utils::{names_and_versions, try_to_published_crate_and_new_version, will},
+    utils::{names_and_versions, try_to_published_crate_and_new_version, version_req_unset_or_default, will},
     version, ChangeLog,
 };
 

experiments/object-access/Cargo.toml

@@ -12,3 +12,4 @@ anyhow = "1"
 git-repository = { version ="^0.12.0", path = "../../git-repository", features = ["unstable"] }
 git2 = "0.13"
 rayon = "1.5.0"
+parking_lot = { version = "0.11.0", default-features = false }

experiments/object-access/src/main.rs

@@ -1,4 +1,4 @@
-use std::{path::Path, time::Instant};
+use std::{path::Path, sync::Arc, time::Instant};
 
 use anyhow::anyhow;
 use git_repository::{hash::ObjectId, odb, prelude::*, Repository};
@@ -21,6 +21,7 @@ fn main() -> anyhow::Result<()> {
     };
 
     let objs_per_sec = |elapsed: std::time::Duration| hashes.len() as f32 / elapsed.as_secs_f32();
+
     let start = Instant::now();
     do_gitoxide_in_parallel(&hashes, &repo, || odb::pack::cache::Never, AccessMode::ObjectExists)?;
     let elapsed = start.elapsed();
@@ -32,36 +33,65 @@ fn main() -> anyhow::Result<()> {
     );
 
     let start = Instant::now();
-    let bytes = do_gitoxide_in_parallel(
+    let bytes = do_gitoxide_in_parallel_through_arc(
         &hashes,
-        &repo,
-        || odb::pack::cache::lru::MemoryCappedHashmap::new(GITOXIDE_CACHED_OBJECT_DATA_PER_THREAD_IN_BYTES),
+        &repo.odb.dbs[0].loose.path,
+        odb::pack::cache::Never::default,
         AccessMode::ObjectData,
     )?;
     let elapsed = start.elapsed();
     println!(
-        "parallel gitoxide (cache = {:.0}MB): confirmed {} bytes in {:?} ({:0.0} objects/s)",
-        GITOXIDE_CACHED_OBJECT_DATA_PER_THREAD_IN_BYTES as f32 / (1024 * 1024) as f32,
+        "parallel gitoxide (uncached, Arc): confirmed {} bytes in {:?} ({:0.0} objects/s)",
         bytes,
         elapsed,
         objs_per_sec(elapsed)
     );
 
     let start = Instant::now();
-    let bytes = do_gitoxide_in_parallel(
+    do_gitoxide_in_parallel_through_arc(
         &hashes,
-        &repo,
-        odb::pack::cache::lru::StaticLinkedList::<GITOXIDE_STATIC_CACHE_SIZE>::default,
+        &repo.odb.dbs[0].loose.path,
+        odb::pack::cache::Never::default,
+        AccessMode::ObjectExists,
+    )?;
+    let elapsed = start.elapsed();
+    println!(
+        "parallel gitoxide (Arc): confirmed {} objects exists in {:?} ({:0.0} objects/s)",
+        hashes.len(),
+        elapsed,
+        objs_per_sec(elapsed)
+    );
+
+    let start = Instant::now();
+    let bytes = do_gitoxide_in_parallel_through_arc_rw_lock(
+        &hashes,
+        &repo.odb.dbs[0].loose.path,
+        odb::pack::cache::Never::default,
         AccessMode::ObjectData,
     )?;
     let elapsed = start.elapsed();
     println!(
-        "parallel gitoxide (small static cache): confirmed {} bytes in {:?} ({:0.0} objects/s)",
+        "parallel gitoxide (uncached, Arc, Lock): confirmed {} bytes in {:?} ({:0.0} objects/s)",
         bytes,
         elapsed,
         objs_per_sec(elapsed)
     );
 
+    let start = Instant::now();
+    do_gitoxide_in_parallel_through_arc_rw_lock(
+        &hashes,
+        &repo.odb.dbs[0].loose.path,
+        odb::pack::cache::Never::default,
+        AccessMode::ObjectExists,
+    )?;
+    let elapsed = start.elapsed();
+    println!(
+        "parallel gitoxide (Arc, Lock): confirmed {} objects exists in {:?} ({:0.0} objects/s)",
+        hashes.len(),
+        elapsed,
+        objs_per_sec(elapsed)
+    );
+
     let start = Instant::now();
     let bytes = do_gitoxide_in_parallel(&hashes, &repo, odb::pack::cache::Never::default, AccessMode::ObjectData)?;
     let elapsed = start.elapsed();
@@ -72,6 +102,37 @@ fn main() -> anyhow::Result<()> {
         objs_per_sec(elapsed)
     );
 
+    let start = Instant::now();
+    let bytes = do_gitoxide_in_parallel(
+        &hashes,
+        &repo,
+        || odb::pack::cache::lru::MemoryCappedHashmap::new(GITOXIDE_CACHED_OBJECT_DATA_PER_THREAD_IN_BYTES),
+        AccessMode::ObjectData,
+    )?;
+    let elapsed = start.elapsed();
+    println!(
+        "parallel gitoxide (cache = {:.0}MB): confirmed {} bytes in {:?} ({:0.0} objects/s)",
+        GITOXIDE_CACHED_OBJECT_DATA_PER_THREAD_IN_BYTES as f32 / (1024 * 1024) as f32,
+        bytes,
+        elapsed,
+        objs_per_sec(elapsed)
+    );
+
+    let start = Instant::now();
+    let bytes = do_gitoxide_in_parallel(
+        &hashes,
+        &repo,
+        odb::pack::cache::lru::StaticLinkedList::<GITOXIDE_STATIC_CACHE_SIZE>::default,
+        AccessMode::ObjectData,
+    )?;
+    let elapsed = start.elapsed();
+    println!(
+        "parallel gitoxide (small static cache): confirmed {} bytes in {:?} ({:0.0} objects/s)",
+        bytes,
+        elapsed,
+        objs_per_sec(elapsed)
+    );
+
     let start = Instant::now();
     let bytes = do_parallel_git2(hashes.as_slice(), repo.git_dir())?;
     let elapsed = start.elapsed();
@@ -213,3 +274,73 @@ where
 
     Ok(bytes.load(std::sync::atomic::Ordering::Acquire))
 }
+
+fn do_gitoxide_in_parallel_through_arc<C>(
+    hashes: &[ObjectId],
+    repo: &Path,
+    new_cache: impl Fn() -> C + Send + Clone,
+    mode: AccessMode,
+) -> anyhow::Result<u64>
+where
+    C: odb::pack::cache::DecodeEntry,
+{
+    let bytes = std::sync::atomic::AtomicU64::default();
+    let odb = Arc::new(git_repository::odb::linked::Store::at(repo)?);
+
+    git_repository::parallel::in_parallel(
+        hashes.chunks(1000),
+        None,
+        move |_| (Vec::new(), new_cache(), odb.clone()),
+        |hashes, (buf, cache, odb)| {
+            for hash in hashes {
+                match mode {
+                    AccessMode::ObjectData => {
+                        let obj = odb.find(hash, buf, cache)?;
+                        bytes.fetch_add(obj.data.len() as u64, std::sync::atomic::Ordering::Relaxed);
+                    }
+                    AccessMode::ObjectExists => {
+                        assert!(odb.contains(hash), "each traversed object exists");
+                    }
+                }
+            }
+            Ok(())
+        },
+        git_repository::parallel::reduce::IdentityWithResult::<(), anyhow::Error>::default(),
+    )?;
+    Ok(bytes.load(std::sync::atomic::Ordering::Acquire))
+}
+
+fn do_gitoxide_in_parallel_through_arc_rw_lock<C>(
+    hashes: &[ObjectId],
+    repo: &Path,
+    new_cache: impl Fn() -> C + Send + Clone,
+    mode: AccessMode,
+) -> anyhow::Result<u64>
+where
+    C: odb::pack::cache::DecodeEntry,
+{
+    let bytes = std::sync::atomic::AtomicU64::default();
+    let odb = Arc::new(parking_lot::RwLock::new(git_repository::odb::linked::Store::at(repo)?));
+
+    git_repository::parallel::in_parallel(
+        hashes.chunks(1000),
+        None,
+        move |_| (Vec::new(), new_cache(), odb.clone()),
+        |hashes, (buf, cache, odb)| {
+            for hash in hashes {
+                match mode {
+                    AccessMode::ObjectData => {
+                        let obj = odb.read().find(hash, buf, cache)?;
+                        bytes.fetch_add(obj.data.len() as u64, std::sync::atomic::Ordering::Relaxed);
+                    }
+                    AccessMode::ObjectExists => {
+                        assert!(odb.read().contains(hash), "each traversed object exists");
+                    }
+                }
+            }
+            Ok(())
+        },
+        git_repository::parallel::reduce::IdentityWithResult::<(), anyhow::Error>::default(),
+    )?;
+    Ok(bytes.load(std::sync::atomic::Ordering::Acquire))
+}

experiments/odb-redesign/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "odb-redesign"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[features]
+default = ["thread-safe"]
+thread-safe = []
+
+[dependencies]
+git-pack = { path = "../../git-pack", version = "*" }
+git-odb = { path = "../../git-odb", version = "*" }
+git-hash = { path = "../../git-hash", version ="^0.8.0" }
+git-ref = { path = "../../git-ref", version ="^0.9.1" }
+parking_lot = { version = "0.11.0", default-features = false }
+thiserror = "1.0.30"
+anyhow = "1.0.47"