Fix support for larger than 2 GB databases on Windows

Cargo.toml

@@ -28,7 +28,7 @@ members = [
 [dependencies]
 bitflags = "1"
 libc = "0.2"
-lmdb-sys = "0.8.0"
+lmdb-sys = { path = "lmdb-sys" }
 
 [dev-dependencies]
 rand = "0.4"

lmdb-sys/build.rs

@@ -16,6 +16,20 @@ fn main() {
  if target.contains("android") {
  build.define("ANDROID", "1");
  }
+ if target.contains("windows") {
+ // LMDB on Windows has an issue with ´off_t´ being defined as ´long´ 32-bit signed
+ // which caused a max size of the database of 2 GB which we work 
+ // around by redefining ´off_t´ to 64-bit
+ // 
+ // was discussed here and allgedly fixed in January 2016 but fix doesn't work as 
+ // Windows doesn't support _FILE_OFFSET_BITS
+ // https://www.openldap.org/lists/openldap-bugs/201605/msg00015.htm
+ // https://github.com/LMDB/lmdb/commit/20dec1f69bf4860202c764ce92b1fbbe3d11a065
+ build.define("_OFF_T_DEFINED", "1");
+ build.define("off_t", "__int64");
+ build.define("_off_t", "__int64");
+ }
+
  build
  .file(lmdb.join("mdb.c"))
  .file(lmdb.join("midl.c"))

src/lib.rs

@@ -120,4 +120,58 @@ mod test_utils {
  tx.commit().expect("tx.commit")
  }
  }
+
+ // verify that the map file is sparsly allocated
+ // this used to fail on earlier versions of LMDB on Windows, before ITS#8324
+ #[test]
+ fn verify_sparse() {
+ const HEIGHT_KEY: [u8; 1] = [0];
+
+ let dir = TempDir::new("test").unwrap();
+
+ {
+ let env = {
+ let mut builder = Environment::new();
+ builder.set_map_size(1_000_000_000);
+ builder.open(dir.path()).expect("open lmdb env")
+ };
+ let db = env.open_db(None).unwrap();
+
+ for height in 0..1000 {
+ let mut value = [0u8; 8];
+ LittleEndian::write_u64(&mut value, height);
+ let mut tx = env.begin_rw_txn().expect("begin_rw_txn");
+ tx.put(db, &HEIGHT_KEY, &value, WriteFlags::empty())
+ .expect("tx.put");
+ tx.commit().expect("tx.commit")
+ }
+ }
+
+ let size = std::fs::metadata(dir.path().join("data.mdb"))
+ .expect("get file size")
+ .len();
+ assert!(size < 1_000_000);
+ }
+
+
+ // Verify that one can create a database larger than 2 GB
+ #[test]
+ fn verify_2gb_plus() {
+ let dir = TempDir::new("test").unwrap();
+ let env = Environment::new()
+ .set_map_size(4_000_000_000)
+ .open(dir.path()).expect("open lmdb env");
+ let db = env.open_db(None).unwrap();
+
+ let data: Vec<u8> = (0..1_000_000).into_iter().map(|i| i as u8).collect();
+
+ // try to write 3 GB of data
+ let mut tx = env.begin_rw_txn().expect("begin_rw_txn");
+ for i in 0..3000 {
+ let key = &data[i..(i+32)];
+ tx.put(db, &key, &data, WriteFlags::empty()).expect("tx.put");
+ }
+ tx.commit().expect("tx.commit")
+ }
+
 }