Use mmap IO for value tables (#214)

* Experimental mmap IO for tables

* Eliminate extra copy on read

* Reclaim overlay mem

* Tweak reclaim params

* Bump version

* Style

* Style

* Loom workaround

* Loom workaround

* fmt

* Destroy old map first

* Really destroy old map first

* Reserve address space for the file mapping

* aarch64 CI test

* aarch64 CI test

* Align file size

* aarch64 CI test fix

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "parity-db"
-version = "0.4.9"
+version = "0.4.10"
 authors = ["Parity Technologies <admin@parity.io>"]
 edition = "2021"
 license = "MIT OR Apache-2.0"

admin/src/lib.rs

@@ -114,12 +114,12 @@ pub fn run() -> Result<(), String> {
 
 			let mut db_options = options.clone();
 			if args.compress {
-				for mut c in &mut db_options.columns {
+				for c in &mut db_options.columns {
 					c.compression = parity_db::CompressionType::Lz4;
 				}
 			}
 			if args.uniform {
-				for mut c in &mut db_options.columns {
+				for c in &mut db_options.columns {
 					c.uniform = true;
 				}
 			}

src/column.rs

@@ -27,7 +27,7 @@ use std::{
 	},
 };
 
-const MIN_INDEX_BITS: u8 = 16;
+pub const MIN_INDEX_BITS: u8 = 16;
 // Measured in index entries
 const MAX_REINDEX_BATCH: usize = 8192;
 

src/file.rs

@@ -4,12 +4,19 @@
 //! Utilities for db file.
 
 use crate::{
-	error::{try_io, Error, Result},
-	parking_lot::{RwLock, RwLockUpgradableReadGuard, RwLockWriteGuard},
+	error::{try_io, Result},
+	parking_lot::RwLock,
 	table::TableId,
 };
 use std::sync::atomic::{AtomicU64, Ordering};
 
+#[cfg(not(test))]
+const RESERVE_ADDRESS_SPACE: usize = 1024 * 1024 * 1024; // 1 Gb
+
+// Use different value for tests to work around docker limits on the test machine.
+#[cfg(test)]
+const RESERVE_ADDRESS_SPACE: usize = 64 * 1024 * 1024; // 64 Mb
+
 #[cfg(target_os = "linux")]
 fn disable_read_ahead(file: &std::fs::File) -> std::io::Result<()> {
 	use std::os::unix::io::AsRawFd;
@@ -36,42 +43,34 @@ fn disable_read_ahead(_file: &std::fs::File) -> std::io::Result<()> {
 	Ok(())
 }
 
-// `File::sync_data` uses F_FULLSYNC fcntl on MacOS. It it supposed to be
-// the safest way to make sure data is fully persisted. However starting from
-// MacOS 11.0 it severely degrades parallel write performance, even when writing to
-// other files. Regular `fsync` is good enough for our use case.
-// SSDs used in modern macs seem to be able to flush data even on unexpected power loss.
-// We performed some testing with power shutdowns and kernel panics on both mac hardware
-// and VMs and in all cases `fsync` was enough to prevent data corruption.
-#[cfg(target_os = "macos")]
-fn fsync(file: &std::fs::File) -> std::io::Result<()> {
-	use std::os::unix::io::AsRawFd;
-	if unsafe { libc::fsync(file.as_raw_fd()) } != 0 {
-		Err(std::io::Error::last_os_error())
-	} else {
-		Ok(())
+#[cfg(unix)]
+pub fn madvise_random(map: &mut memmap2::MmapMut) {
+	unsafe {
+		libc::madvise(map.as_mut_ptr() as _, map.len(), libc::MADV_RANDOM);
 	}
 }
 
-#[cfg(not(target_os = "macos"))]
-fn fsync(file: &std::fs::File) -> std::io::Result<()> {
-	file.sync_data()
-}
+#[cfg(not(unix))]
+pub fn madvise_random(_id: TableId, _map: &mut memmap2::MmapMut) {}
 
 const GROW_SIZE_BYTES: u64 = 256 * 1024;
 
 #[derive(Debug)]
 pub struct TableFile {
-	pub file: RwLock<Option<std::fs::File>>,
+	pub map: RwLock<Option<(memmap2::MmapMut, std::fs::File)>>,
 	pub path: std::path::PathBuf,
 	pub capacity: AtomicU64,
 	pub id: TableId,
 }
 
+fn map_len(file_len: u64) -> usize {
+	file_len as usize + RESERVE_ADDRESS_SPACE
+}
+
 impl TableFile {
 	pub fn open(filepath: std::path::PathBuf, entry_size: u16, id: TableId) -> Result<Self> {
 		let mut capacity = 0u64;
-		let file = if std::fs::metadata(&filepath).is_ok() {
+		let map = if std::fs::metadata(&filepath).is_ok() {
 			let file = try_io!(std::fs::OpenOptions::new()
 				.read(true)
 				.write(true)
@@ -81,17 +80,20 @@ impl TableFile {
 			if len == 0 {
 				// Preallocate.
 				capacity += GROW_SIZE_BYTES / entry_size as u64;
-				try_io!(file.set_len(capacity * entry_size as u64));
+				try_io!(file.set_len(GROW_SIZE_BYTES));
 			} else {
 				capacity = len / entry_size as u64;
 			}
-			Some(file)
+			let mut map =
+				try_io!(unsafe { memmap2::MmapOptions::new().len(map_len(len)).map_mut(&file) });
+			madvise_random(&mut map);
+			Some((map, file))
 		} else {
 			None
 		};
 		Ok(TableFile {
 			path: filepath,
-			file: RwLock::new(file),
+			map: RwLock::new(map),
 			capacity: AtomicU64::new(capacity),
 			id,
 		})
@@ -108,111 +110,87 @@ impl TableFile {
 		Ok(file)
 	}
 
-	#[cfg(unix)]
 	pub fn read_at(&self, buf: &mut [u8], offset: u64) -> Result<()> {
-		use std::os::unix::fs::FileExt;
-		try_io!(self
-			.file
-			.read()
-			.as_ref()
-			.ok_or_else(|| Error::Corruption("File does not exist.".into()))?
-			.read_exact_at(buf, offset));
-		Ok(())
-	}
-
-	#[cfg(unix)]
-	pub fn write_at(&self, buf: &[u8], offset: u64) -> Result<()> {
-		use std::os::unix::fs::FileExt;
-		try_io!(self.file.read().as_ref().unwrap().write_all_at(buf, offset));
+		let offset = offset as usize;
+		let map = self.map.read();
+		let (map, _) = map.as_ref().unwrap();
+		buf.copy_from_slice(&map[offset..offset + buf.len()]);
 		Ok(())
 	}
 
-	#[cfg(windows)]
-	pub fn read_at(&self, mut buf: &mut [u8], mut offset: u64) -> Result<()> {
-		use crate::error::Error;
-		use std::{io, os::windows::fs::FileExt};
-
-		let file = self.file.read();
-		let file = file.as_ref().ok_or_else(|| Error::Corruption("File does not exist.".into()))?;
-
-		while !buf.is_empty() {
-			match file.seek_read(buf, offset) {
-				Ok(0) => break,
-				Ok(n) => {
-					buf = &mut buf[n..];
-					offset += n as u64;
-				},
-				Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {
-					// Try again
-				},
-				Err(e) => return Err(Error::Io(e)),
-			}
-		}
-
-		if !buf.is_empty() {
-			Err(Error::Io(io::Error::new(
-				io::ErrorKind::UnexpectedEof,
-				"failed to fill whole buffer",
-			)))
-		} else {
-			Ok(())
-		}
+	pub fn slice_at(&self, offset: u64, len: usize) -> &[u8] {
+		let offset = offset as usize;
+		let map = self.map.read();
+		let (map, _) = map.as_ref().unwrap();
+		let data: &[u8] = unsafe {
+			let ptr = map.as_ptr().add(offset);
+			std::slice::from_raw_parts(ptr, len)
+		};
+		data
 	}
 
-	#[cfg(windows)]
-	pub fn write_at(&self, mut buf: &[u8], mut offset: u64) -> Result<()> {
-		use crate::error::Error;
-		use std::{io, os::windows::fs::FileExt};
-
-		let file = self.file.read();
-		let file = file.as_ref().unwrap();
-
-		while !buf.is_empty() {
-			match file.seek_write(buf, offset) {
-				Ok(0) =>
-					return Err(Error::Io(io::Error::new(
-						io::ErrorKind::WriteZero,
-						"failed to write whole buffer",
-					))),
-				Ok(n) => {
-					buf = &buf[n..];
-					offset += n as u64;
-				},
-				Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {
-					// Try again
-				},
-				Err(e) => return Err(Error::Io(e)),
-			}
-		}
-
+	pub fn write_at(&self, buf: &[u8], offset: u64) -> Result<()> {
+		let map = self.map.read();
+		let (map, _) = map.as_ref().unwrap();
+		let offset = offset as usize;
+
+		// Nasty mutable pointer cast. We do ensure that all chunks that are being written are
+		// accessed through the overlay in other threads.
+		let ptr: *mut u8 = map.as_ptr() as *mut u8;
+		let data: &mut [u8] = unsafe {
+			let ptr = ptr.add(offset);
+			std::slice::from_raw_parts_mut(ptr, buf.len())
+		};
+		data.copy_from_slice(buf);
 		Ok(())
 	}
 
 	pub fn grow(&self, entry_size: u16) -> Result<()> {
-		let mut capacity = self.capacity.load(Ordering::Relaxed);
-		capacity += GROW_SIZE_BYTES / entry_size as u64;
-
+		let mut map_and_file = self.map.write();
+		let new_len = match map_and_file.as_mut() {
+			None => {
+				let file = self.create_file()?;
+				let len = GROW_SIZE_BYTES;
+				try_io!(file.set_len(len));
+				let mut map = try_io!(unsafe {
+					memmap2::MmapOptions::new().len(RESERVE_ADDRESS_SPACE).map_mut(&file)
+				});
+				madvise_random(&mut map);
+				*map_and_file = Some((map, file));
+				len
+			},
+			Some((map, file)) => {
+				let new_len = try_io!(file.metadata()).len() + GROW_SIZE_BYTES;
+				try_io!(file.set_len(new_len));
+				if map.len() < new_len as usize {
+					let mut new_map = try_io!(unsafe {
+						memmap2::MmapOptions::new().len(map_len(new_len)).map_mut(&*file)
+					});
+					madvise_random(&mut new_map);
+					let old_map = std::mem::replace(map, new_map);
+					try_io!(old_map.flush());
+					// Leak the old mapping as there might be concurrent readers.
+					std::mem::forget(old_map);
+				}
+				new_len
+			},
+		};
+		let capacity = new_len / entry_size as u64;
 		self.capacity.store(capacity, Ordering::Relaxed);
-		let mut file = self.file.upgradable_read();
-		if file.is_none() {
-			let mut wfile = RwLockUpgradableReadGuard::upgrade(file);
-			*wfile = Some(self.create_file()?);
-			file = RwLockWriteGuard::downgrade_to_upgradable(wfile);
-		}
-		try_io!(file.as_ref().unwrap().set_len(capacity * entry_size as u64));
 		Ok(())
 	}
 
 	pub fn flush(&self) -> Result<()> {
-		if let Some(file) = self.file.read().as_ref() {
-			try_io!(fsync(file));
+		if let Some((map, _)) = self.map.read().as_ref() {
+			try_io!(map.flush());
 		}
 		Ok(())
 	}
 
 	pub fn remove(&self) -> Result<()> {
-		let mut file = self.file.write();
-		if let Some(file) = file.take() {
+		let mut map = self.map.write();
+		if let Some((map, file)) = map.take() {
+			drop(map);
 			drop(file);
 			try_io!(std::fs::remove_file(&self.path));
 		}

src/index.rs

@@ -2,9 +2,10 @@
 // This file is dual-licensed as Apache-2.0 or MIT.
 
 use crate::{
-	column::ColId,
+	column::{ColId, MIN_INDEX_BITS},
 	display::hex,
 	error::{try_io, Error, Result},
+	file::madvise_random,
 	log::{LogQuery, LogReader, LogWriter},
 	parking_lot::{RwLock, RwLockUpgradableReadGuard, RwLockWriteGuard},
 	stats::{self, ColumnStats},
@@ -142,20 +143,6 @@ fn file_size(index_bits: u8) -> u64 {
 	total_entries(index_bits) * 8 + META_SIZE as u64
 }
 
-#[cfg(unix)]
-fn madvise_random(id: TableId, map: &mut memmap2::MmapMut) {
-	unsafe {
-		libc::madvise(
-			map.as_mut_ptr() as _,
-			file_size(id.index_bits()) as usize,
-			libc::MADV_RANDOM,
-		);
-	}
-}
-
-#[cfg(not(unix))]
-fn madvise_random(_id: TableId, _map: &mut memmap2::MmapMut) {}
-
 #[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
 pub struct TableId(u16);
 
@@ -195,6 +182,20 @@ impl TableId {
 	pub fn total_entries(&self) -> u64 {
 		total_entries(self.index_bits())
 	}
+
+	pub fn log_index(&self) -> usize {
+		self.col() as usize * (64 - MIN_INDEX_BITS) as usize + self.index_bits() as usize
+	}
+
+	pub fn from_log_index(i: usize) -> Self {
+		let col = i / (64 - MIN_INDEX_BITS) as usize;
+		let bits = i % (64 - MIN_INDEX_BITS) as usize;
+		TableId::new(col as ColId, bits as u8)
+	}
+
+	pub const fn max_log_indicies(num_columns: usize) -> usize {
+		(64 - MIN_INDEX_BITS) as usize * num_columns
+	}
 }
 
 impl std::fmt::Display for TableId {
@@ -216,7 +217,7 @@ impl IndexTable {
 
 		try_io!(file.set_len(file_size(id.index_bits())));
 		let mut map = try_io!(unsafe { memmap2::MmapMut::map_mut(&file) });
-		madvise_random(id, &mut map);
+		madvise_random(&mut map);
 		log::debug!(target: "parity-db", "Opened existing index {}", id);
 		Ok(Some(IndexTable { id, path, map: RwLock::new(Some(map)) }))
 	}
@@ -564,7 +565,7 @@ impl IndexTable {
 			log::debug!(target: "parity-db", "Created new index {}", self.id);
 			try_io!(file.set_len(file_size(self.id.index_bits())));
 			let mut mmap = try_io!(unsafe { memmap2::MmapMut::map_mut(&file) });
-			madvise_random(self.id, &mut mmap);
+			madvise_random(&mut mmap);
 			*wmap = Some(mmap);
 			map = RwLockWriteGuard::downgrade_to_upgradable(wmap);
 		}
@@ -641,7 +642,6 @@ mod test {
 	use super::*;
 	use rand::{Rng, SeedableRng};
 	use std::path::PathBuf;
-
 	#[cfg(feature = "bench")]
 	use test::Bencher;
 	#[cfg(feature = "bench")]