Introduce the Env::nested_read_txn from an RwTxn

.gitmodules

@@ -1,7 +1,7 @@
 [submodule "lmdb-master-sys/lmdb"]
 	path = lmdb-master-sys/lmdb
-	url = https://git.openldap.org/openldap/openldap
-	branch = mdb.master
+	url = https://github.com/meilisearch/lmdb.git
+	branch = allow-nested-rtxn-from-wtxn
 [submodule "lmdb-master3-sys/lmdb"]
 	path = lmdb-master3-sys/lmdb
 	url = https://git.openldap.org/openldap/openldap

examples/nested-rtxns.rs

@@ -0,0 +1,78 @@
+use heed::types::*;
+use heed::{Database, EnvOpenOptions};
+use rand::prelude::*;
+use rayon::prelude::*;
+use roaring::RoaringBitmap;
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let dir = tempfile::tempdir()?;
+    let env = unsafe {
+        EnvOpenOptions::new()
+            .read_txn_without_tls()
+            .map_size(2 * 1024 * 1024 * 1024) // 2 GiB
+            .open(dir.path())?
+    };
+
+    // opening a write transaction
+    let mut wtxn = env.write_txn()?;
+    // we will open the default unnamed database
+    let db: Database<U32<byteorder::BigEndian>, Bytes> = env.create_database(&mut wtxn, None)?;
+
+    let mut buffer = Vec::new();
+    for i in 0..1000 {
+        let mut rng = StdRng::seed_from_u64(i as u64);
+        let max = rng.random_range(10_000..=100_000);
+        let roaring = RoaringBitmap::from_sorted_iter(0..max)?;
+        buffer.clear();
+        roaring.serialize_into(&mut buffer)?;
+        db.put(&mut wtxn, &i, &buffer)?;
+    }
+
+    // opening multiple read-only transactions
+    // to check if those values are now available
+    // without committing beforehand
+    let rtxns = (0..1000).map(|_| env.nested_read_txn(&wtxn)).collect::<heed::Result<Vec<_>>>()?;
+
+    rtxns.into_par_iter().enumerate().for_each(|(i, rtxn)| {
+        let mut rng = StdRng::seed_from_u64(i as u64);
+        let max = rng.random_range(10_000..=100_000);
+        let roaring = RoaringBitmap::from_sorted_iter(0..max).unwrap();
+
+        let mut buffer = Vec::new();
+        roaring.serialize_into(&mut buffer).unwrap();
+
+        let i = i as u32;
+        let ret = db.get(&rtxn, &i).unwrap();
+        assert_eq!(ret, Some(&buffer[..]));
+    });
+
+    for i in 1000..10_000 {
+        let mut rng = StdRng::seed_from_u64(i as u64);
+        let max = rng.random_range(10_000..=100_000);
+        let roaring = RoaringBitmap::from_sorted_iter(0..max)?;
+        buffer.clear();
+        roaring.serialize_into(&mut buffer)?;
+        db.put(&mut wtxn, &i, &buffer)?;
+    }
+
+    // opening multiple read-only transactions
+    // to check if those values are now available
+    // without committing beforehand
+    let rtxns =
+        (1000..10_000).map(|_| env.nested_read_txn(&wtxn)).collect::<heed::Result<Vec<_>>>()?;
+
+    rtxns.into_par_iter().enumerate().for_each(|(i, rtxn)| {
+        let mut rng = StdRng::seed_from_u64(i as u64);
+        let max = rng.random_range(10_000..=100_000);
+        let roaring = RoaringBitmap::from_sorted_iter(0..max).unwrap();
+
+        let mut buffer = Vec::new();
+        roaring.serialize_into(&mut buffer).unwrap();
+
+        let i = i as u32;
+        let ret = db.get(&rtxn, &i).unwrap();
+        assert_eq!(ret, Some(&buffer[..]));
+    });
+
+    Ok(())
+}

heed/Cargo.toml

@@ -16,6 +16,7 @@ include = [
     "../examples/custom-comparator.rs",
     "../examples/multi-env.rs",
     "../examples/nested.rs",
+    "../examples/nested-rtxns.rs",
     "../examples/rmp-serde.rs",
 ]
 
@@ -33,6 +34,9 @@ synchronoise = "1.0.1"
 
 [dev-dependencies]
 memchr = "2.7.4"
+rand = "0.9.0"
+rayon = "1.10.0"
+roaring = "0.10.10"
 serde = { version = "1.0.217", features = ["derive"] }
 tempfile = "3.15.0"
 
@@ -135,6 +139,10 @@ path = "../examples/custom-comparator.rs"
 name = "multi-env"
 path = "../examples/multi-env.rs"
 
+[[example]]
+name = "nested-rtxns"
+path = "../examples/nested-rtxns.rs"
+
 [[example]]
 name = "nested"
 path = "../examples/nested.rs"

heed/src/envs/env.rs

@@ -21,8 +21,8 @@ use crate::mdb::lmdb_flags::AllDatabaseFlags;
 #[allow(unused)] // for cargo auto doc links
 use crate::EnvOpenOptions;
 use crate::{
-    CompactionOption, Database, DatabaseOpenOptions, EnvFlags, Error, Result, RoTxn, RwTxn,
-    Unspecified,
+    assert_eq_env_txn, CompactionOption, Database, DatabaseOpenOptions, EnvFlags, Error, Result,
+    RoTxn, RwTxn, Unspecified, WithoutTls,
 };
 
 /// An environment handle constructed by using [`EnvOpenOptions::open`].
@@ -326,6 +326,8 @@ impl<T> Env<T> {
     /// A parent transaction and its cursors may not issue any other operations than _commit_ and
     /// _abort_ while it has active child transactions.
     pub fn nested_write_txn<'p>(&'p self, parent: &'p mut RwTxn) -> Result<RwTxn<'p>> {
+        assert_eq_env_txn!(self, parent);
+
         RwTxn::nested(self, parent)
     }
 
@@ -528,6 +530,57 @@ impl<T> Env<T> {
     }
 }
 
+impl Env<WithoutTls> {
+    /// Create a nested transaction with read only access for use with the environment.
+    ///
+    /// The new transaction will be a nested transaction, with the transaction indicated by parent
+    /// as its parent. Transactions may be nested to any level.
+    ///
+    /// ```
+    /// use std::fs;
+    /// use std::path::Path;
+    /// use heed::{EnvOpenOptions, Database};
+    /// use heed::types::*;
+    ///
+    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
+    /// let dir = tempfile::tempdir()?;
+    /// let env = unsafe {
+    ///     EnvOpenOptions::new()
+    ///         .read_txn_without_tls()
+    ///         .map_size(2 * 1024 * 1024) // 2 MiB
+    ///         .open(dir.path())?
+    /// };
+    ///
+    /// // we will open the default unnamed database
+    /// let mut wtxn = env.write_txn()?;
+    /// let db: Database<U32<byteorder::BigEndian>, U32<byteorder::BigEndian>> = env.create_database(&mut wtxn, None)?;
+    ///
+    /// // opening a write transaction
+    /// for i in 0..1000 {
+    ///     db.put(&mut wtxn, &i, &i)?;
+    /// }
+    ///
+    /// // opening multiple read-only transactions
+    /// // to check if those values are now available
+    /// // without committing beforehand
+    /// let rtxns = (0..1000).map(|_| env.nested_read_txn(&wtxn)).collect::<heed::Result<Vec<_>>>()?;
+    ///
+    /// for (i, rtxn) in rtxns.iter().enumerate() {
+    ///     let i = i as u32;
+    ///     let ret = db.get(&rtxn, &i)?;
+    ///     assert_eq!(ret, Some(i));
+    /// }
+    ///
+    /// # Ok(()) }
+    /// ```
+    #[cfg(not(master3))]
+    pub fn nested_read_txn<'p>(&'p self, parent: &'p RwTxn) -> Result<RoTxn<'p, WithoutTls>> {
+        assert_eq_env_txn!(self, parent);
+
+        RoTxn::<WithoutTls>::nested(self, parent)
+    }
+}
+
 impl<T> Clone for Env<T> {
     fn clone(&self) -> Self {
         Env { inner: self.inner.clone(), _tls_marker: PhantomData }

heed/src/txn.rs

@@ -100,6 +100,28 @@ impl<'e, T> RoTxn<'e, T> {
         self.inner.txn.unwrap()
     }
 
+    #[cfg(not(master3))]
+    pub(crate) fn nested<'p>(
+        env: &'p Env<WithoutTls>,
+        parent: &'p RwTxn,
+    ) -> Result<RoTxn<'p, WithoutTls>> {
+        let mut txn: *mut ffi::MDB_txn = ptr::null_mut();
+        let parent_ptr: *mut ffi::MDB_txn = unsafe { parent.txn.inner.txn.unwrap().as_mut() };
+
+        unsafe {
+            // Note that we open a write transaction here and this is the (current)
+            // ugly way to trick LMDB and let me create multiple write txn.
+            mdb_result(ffi::mdb_txn_begin(env.env_mut_ptr().as_mut(), parent_ptr, 0, &mut txn))?
+        };
+
+        // Note that we wrap the write txn into a RoTxn so it's
+        // safe as the user cannot do any modification with it.
+        Ok(RoTxn {
+            inner: RoTxnInner { txn: NonNull::new(txn), env: Cow::Borrowed(&env.inner) },
+            _tls_marker: PhantomData,
+        })
+    }
+
     pub(crate) fn env_mut_ptr(&self) -> NonNull<ffi::MDB_env> {
         self.inner.env.env_mut_ptr()
     }

lmdb-master-sys/build.rs

@@ -135,6 +135,9 @@ fn main() {
         .flag_if_supported("-Wbad-function-cast")
         .flag_if_supported("-Wuninitialized");
 
+    #[cfg(windows)]
+    builder.flag("/experimental:c11atomics");
+
     if cfg!(feature = "posix-sem") {
         builder.define("MDB_USE_POSIX_SEM", None);
     }