diff --git a/Cargo.toml b/Cargo.toml index 495a2ecd..6dd16711 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,7 @@ members = [ [dependencies] bitflags = "1" libc = "0.2" -lmdb-sys = "0.8.0" +lmdb-sys = { path = "lmdb-sys" } [dev-dependencies] rand = "0.4" diff --git a/lmdb-sys/build.rs b/lmdb-sys/build.rs index b1bd80e5..c77f3c82 100644 --- a/lmdb-sys/build.rs +++ b/lmdb-sys/build.rs @@ -16,6 +16,20 @@ fn main() { if target.contains("android") { build.define("ANDROID", "1"); } + if target.contains("windows") { + // LMDB on Windows has an issue with ´off_t´ being defined as ´long´ 32-bit signed + // which caused a max size of the database of 2 GB which we work + // around by redefining ´off_t´ to 64-bit + // + // was discussed here and allgedly fixed in January 2016 but fix doesn't work as + // Windows doesn't support _FILE_OFFSET_BITS + // https://www.openldap.org/lists/openldap-bugs/201605/msg00015.htm + // https://github.com/LMDB/lmdb/commit/20dec1f69bf4860202c764ce92b1fbbe3d11a065 + build.define("_OFF_T_DEFINED", "1"); + build.define("off_t", "__int64"); + build.define("_off_t", "__int64"); + } + build .file(lmdb.join("mdb.c")) .file(lmdb.join("midl.c")) diff --git a/lmdb-sys/lmdb b/lmdb-sys/lmdb index 60d50020..ef836457 160000 --- a/lmdb-sys/lmdb +++ b/lmdb-sys/lmdb @@ -1 +1 @@ -Subproject commit 60d500206a108b2c64ca7e36b0113b2cd3711b98 +Subproject commit ef8364575f43e5e7e48e39b16503b6cfaf976b53 diff --git a/src/lib.rs b/src/lib.rs index 3d2d406e..938c5cd5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -120,4 +120,58 @@ mod test_utils { tx.commit().expect("tx.commit") } } + + // verify that the map file is sparsly allocated + // this used to fail on earlier versions of LMDB on Windows, before ITS#8324 + #[test] + fn verify_sparse() { + const HEIGHT_KEY: [u8; 1] = [0]; + + let dir = TempDir::new("test").unwrap(); + + { + let env = { + let mut builder = Environment::new(); + builder.set_map_size(1_000_000_000); + builder.open(dir.path()).expect("open lmdb env") + }; + let db = env.open_db(None).unwrap(); + + for height in 0..1000 { + let mut value = [0u8; 8]; + LittleEndian::write_u64(&mut value, height); + let mut tx = env.begin_rw_txn().expect("begin_rw_txn"); + tx.put(db, &HEIGHT_KEY, &value, WriteFlags::empty()) + .expect("tx.put"); + tx.commit().expect("tx.commit") + } + } + + let size = std::fs::metadata(dir.path().join("data.mdb")) + .expect("get file size") + .len(); + assert!(size < 1_000_000); + } + + + // Verify that one can create a database larger than 2 GB + #[test] + fn verify_2gb_plus() { + let dir = TempDir::new("test").unwrap(); + let env = Environment::new() + .set_map_size(4_000_000_000) + .open(dir.path()).expect("open lmdb env"); + let db = env.open_db(None).unwrap(); + + let data: Vec = (0..1_000_000).into_iter().map(|i| i as u8).collect(); + + // try to write 3 GB of data + let mut tx = env.begin_rw_txn().expect("begin_rw_txn"); + for i in 0..3000 { + let key = &data[i..(i+32)]; + tx.put(db, &key, &data, WriteFlags::empty()).expect("tx.put"); + } + tx.commit().expect("tx.commit") + } + }