Merge pull request #937 from divergentdave/david_miri_support

Miri support

Cargo.toml

@@ -34,6 +34,7 @@ measure_allocs = []
 pretty_backtrace = ["color-backtrace"]
 io_uring = ["rio"]
 docs = []
+miri_optimizations = []
 mutex = []
 
 [dependencies]

scripts/cross_compile.sh

@@ -17,6 +17,8 @@ for target in $targets; do
  cargo check --target $target
 done
 
+RUSTFLAGS="--cfg miri" cargo check
+
 rustup toolchain install 1.39.0
 cargo clean
 rm Cargo.lock

src/config.rs

@@ -455,8 +455,12 @@ impl Config {
  .as_nanos()
  << 48;
 
+ #[cfg(not(miri))]
  let pid = u128::from(std::process::id());
 
+ #[cfg(miri)]
+ let pid = 0;
+
  let salt = (pid << 16) + now + seed;
 
  if cfg!(target_os = "linux") {
@@ -573,7 +577,7 @@ impl Config {
  }
 
  fn try_lock(&self, file: File) -> Result<File> {
- #[cfg(any(windows, target_os = "linux", target_os = "macos"))]
+ #[cfg(all(not(miri), any(windows, target_os = "linux", target_os = "macos")))]
  {
  use fs2::FileExt;
 

src/context.rs

@@ -12,15 +12,15 @@ pub struct Context {
  /// When the last high-level reference is dropped, it
  /// should trigger all background threads to clean
  /// up synchronously.
- #[cfg(any(
+ #[cfg(all(not(miri), any(
  windows,
  target_os = "linux",
  target_os = "macos",
  target_os = "dragonfly",
  target_os = "freebsd",
  target_os = "openbsd",
  target_os = "netbsd",
- ))]
+ )))]
  pub(crate) flusher: Arc<Mutex<Option<flusher::Flusher>>>,
  #[doc(hidden)]
  pub pagecache: Arc<PageCache>,
@@ -36,15 +36,15 @@ impl std::ops::Deref for Context {
 
 impl Drop for Context {
  fn drop(&mut self) {
- #[cfg(any(
+ #[cfg(all(not(miri), any(
  windows,
  target_os = "linux",
  target_os = "macos",
  target_os = "dragonfly",
  target_os = "freebsd",
  target_os = "openbsd",
  target_os = "netbsd",
- ))]
+ )))]
  {
  if let Some(flusher) = self.flusher.lock().take() {
  drop(flusher)
@@ -77,15 +77,15 @@ impl Context {
  Ok(Self {
  config,
  pagecache,
- #[cfg(any(
+ #[cfg(all(not(miri), any(
  windows,
  target_os = "linux",
  target_os = "macos",
  target_os = "dragonfly",
  target_os = "freebsd",
  target_os = "openbsd",
  target_os = "netbsd",
- ))]
+ )))]
  flusher: Arc::new(parking_lot::Mutex::new(None)),
  })
  }

src/db.rs

@@ -69,15 +69,15 @@ impl Db {
 
  let context = Context::start(config)?;
 
- #[cfg(any(
+ #[cfg(all(not(miri), any(
  windows,
  target_os = "linux",
  target_os = "macos",
  target_os = "dragonfly",
  target_os = "freebsd",
  target_os = "openbsd",
  target_os = "netbsd",
- ))]
+ )))]
  {
  let flusher_pagecache = context.pagecache.clone();
  let flusher = context.flush_every_ms.map(move |fem| {

src/debug_delay.rs

@@ -37,6 +37,12 @@ pub fn debug_delay() {
  static LOCAL_DELAYS: std::cell::RefCell<usize> = std::cell::RefCell::new(0)
  );
 
+ if cfg!(feature = "miri_optimizations") {
+ // Each interaction with LOCAL_DELAYS adds more stacked borrows
+ // tracking information, and Miri is single-threaded anyway.
+ return;
+ }
+
  let global_delays = GLOBAL_DELAYS.fetch_add(1, Relaxed);
  let local_delays = LOCAL_DELAYS.with(|ld| {
  let mut ld = ld.borrow_mut();

src/fastcmp.rs

@@ -46,7 +46,9 @@ mod qc {
  prop_cmp_matches(pair[0], pair[1]);
  }
  }
+
  quickcheck::quickcheck! {
+ #[cfg_attr(miri, ignore)]
  fn qc_fastcmp(l: Vec<u8>, r: Vec<u8>) -> bool {
  prop_cmp_matches(&l, &r)
  }

src/histogram.rs

@@ -45,11 +45,32 @@ pub struct Histogram {
 }
 
 impl Default for Histogram {
+ #[allow(unsafe_code)]
  fn default() -> Histogram {
- let mut vals = Vec::with_capacity(BUCKETS);
- vals.resize_with(BUCKETS, Default::default);
+ #[cfg(not(feature = "miri_optimizations"))]
+ {
+ let mut vals = Vec::with_capacity(BUCKETS);
+ vals.resize_with(BUCKETS, Default::default);
+
+ Histogram { vals, sum: AtomicUsize::new(0), count: AtomicUsize::new(0) }
+ }
 
- Histogram { vals, sum: AtomicUsize::new(0), count: AtomicUsize::new(0) }
+ #[cfg(feature = "miri_optimizations")]
+ {
+ // Avoid calling Vec::resize_with with a large length because its
+ // internals cause stacked borrows tracking information to add an
+ // item for each element of the vector.
+ let mut vals = std::mem::ManuallyDrop::new(vec![0_usize; BUCKETS]);
+ let ptr: *mut usize = vals.as_mut_ptr();
+ let len = vals.len();
+ let capacity = vals.capacity();
+
+ let vals: Vec<AtomicUsize> = unsafe {
+ Vec::from_raw_parts(ptr as *mut AtomicUsize, len, capacity)
+ };
+
+ Histogram { vals, sum: AtomicUsize::new(0), count: AtomicUsize::new(0) }
+ }
  }
 }
 

src/lib.rs

@@ -211,15 +211,15 @@ pub mod fail;
 #[cfg(feature = "docs")]
 pub mod doc;
 
-#[cfg(not(any(
+#[cfg(any(miri, not(any(
  windows,
  target_os = "linux",
  target_os = "macos",
  target_os = "dragonfly",
  target_os = "freebsd",
  target_os = "openbsd",
  target_os = "netbsd",
-)))]
+))))]
 mod threadpool {
  use super::OneShot;
 
@@ -235,26 +235,26 @@ mod threadpool {
  }
 }
 
-#[cfg(any(
+#[cfg(all(not(miri), any(
  windows,
  target_os = "linux",
  target_os = "macos",
  target_os = "dragonfly",
  target_os = "freebsd",
  target_os = "openbsd",
  target_os = "netbsd",
-))]
+)))]
 mod threadpool;
 
-#[cfg(any(
+#[cfg(all(not(miri), any(
  windows,
  target_os = "linux",
  target_os = "macos",
  target_os = "dragonfly",
  target_os = "freebsd",
  target_os = "openbsd",
  target_os = "netbsd",
-))]
+)))]
 mod flusher;
 
 #[cfg(feature = "event_log")]

src/lru.rs

@@ -157,11 +157,12 @@ impl<'a> Iterator for CacheAccessIter<'a> {
 
  debug_delay();
  if self.current_offset >= MAX_QUEUE_ITEMS {
- let to_drop = unsafe { Box::from_raw(self.current_block) };
+ let to_drop_ptr = self.current_block;
  debug_delay();
  self.current_block = current_block.next.load(Ordering::Acquire);
  self.current_offset = 0;
  debug_delay();
+ let to_drop = unsafe { Box::from_raw(to_drop_ptr) };
  self.guard.defer(|| to_drop);
  continue;
  }
@@ -347,3 +348,14 @@ impl Shard {
 fn safe_usize(value: PageId) -> usize {
  usize::try_from(value).unwrap()
 }
+
+#[test]
+fn lru_smoke_test() {
+ use crate::pin;
+
+ let lru = Lru::new(256);
+ for i in 0..1000 {
+ let guard = pin();
+ lru.accessed(i, 16, &guard);
+ }
+}

src/pagecache/mod.rs

@@ -10,11 +10,11 @@ mod disk_pointer;
 mod iobuf;
 mod iterator;
 mod pagetable;
-#[cfg(all(not(unix), not(windows)))]
+#[cfg(any(all(not(unix), not(windows)), miri))]
 mod parallel_io_polyfill;
-#[cfg(unix)]
+#[cfg(all(unix, not(miri)))]
 mod parallel_io_unix;
-#[cfg(windows)]
+#[cfg(all(windows, not(miri)))]
 mod parallel_io_windows;
 mod reservation;
 mod segment;
@@ -24,13 +24,13 @@ use std::{collections::BinaryHeap, ops::Deref};
 
 use crate::*;
 
-#[cfg(all(not(unix), not(windows)))]
+#[cfg(any(all(not(unix), not(windows)), miri))]
 use parallel_io_polyfill::{pread_exact, pread_exact_or_eof, pwrite_all};
 
-#[cfg(unix)]
+#[cfg(all(unix, not(miri)))]
 use parallel_io_unix::{pread_exact, pread_exact_or_eof, pwrite_all};
 
-#[cfg(windows)]
+#[cfg(all(windows, not(miri)))]
 use parallel_io_windows::{pread_exact, pread_exact_or_eof, pwrite_all};
 
 use self::{
@@ -841,6 +841,15 @@ impl PageCache {
  /// move a page. Returns Ok(false) if there were no pages
  /// to GC. Returns an Err if we encountered an IO problem
  /// while performing this GC.
+ #[cfg(all(not(miri), any(
+ windows,
+ target_os = "linux",
+ target_os = "macos",
+ target_os = "dragonfly",
+ target_os = "freebsd",
+ target_os = "openbsd",
+ target_os = "netbsd",
+ )))]
  pub(crate) fn attempt_gc(&self) -> Result<bool> {
  let guard = pin();
  let cc = concurrency_control::read();
@@ -898,6 +907,15 @@ impl PageCache {
 
  #[doc(hidden)]
  #[cfg(feature = "failpoints")]
+ #[cfg(all(not(miri), any(
+ windows,
+ target_os = "linux",
+ target_os = "macos",
+ target_os = "dragonfly",
+ target_os = "freebsd",
+ target_os = "openbsd",
+ target_os = "netbsd",
+ )))]
  pub(crate) fn set_failpoint(&self, e: Error) {
  if let Error::FailPoint = e {
  self.config.set_global_error(e);
@@ -1382,7 +1400,16 @@ impl PageCache {
 
  // it's possible the blob file was removed lazily
  // in the background and no longer exists
- size += blob_file.metadata().map(|m| m.len()).unwrap_or(0);
+ #[cfg(not(miri))]
+ {
+ size += blob_file.metadata().map(|m| m.len()).unwrap_or(0);
+ }
+
+ // workaround to avoid missing `dirfd` shim
+ #[cfg(miri)]
+ {
+ size += std::fs::metadata(blob_file.path()).map(|m| m.len()).unwrap_or(0);
+ }
  }
 
  Ok(size)

src/serialization.rs

@@ -1043,50 +1043,62 @@ mod qc {
  }
 
  quickcheck::quickcheck! {
+ #[cfg_attr(miri, ignore)]
  fn bool(item: bool) -> bool {
  prop_serialize(item)
  }
 
+ #[cfg_attr(miri, ignore)]
  fn u8(item: u8) -> bool {
  prop_serialize(item)
  }
 
+ #[cfg_attr(miri, ignore)]
  fn i64(item: SpreadI64) -> bool {
  prop_serialize(item.0)
  }
 
+ #[cfg_attr(miri, ignore)]
  fn u64(item: SpreadU64) -> bool {
  prop_serialize(item.0)
  }
 
+ #[cfg_attr(miri, ignore)]
  fn disk_ptr(item: DiskPtr) -> bool {
  prop_serialize(item)
  }
 
+ #[cfg_attr(miri, ignore)]
  fn page_state(item: PageState) -> bool {
  prop_serialize(item)
  }
 
+ #[cfg_attr(miri, ignore)]
  fn meta(item: Meta) -> bool {
  prop_serialize(item)
  }
 
+ #[cfg_attr(miri, ignore)]
  fn snapshot(item: Snapshot) -> bool {
  prop_serialize(item)
  }
 
+ #[cfg_attr(miri, ignore)]
  fn node(item: Node) -> bool {
  prop_serialize(item)
  }
 
+ #[cfg_attr(miri, ignore)]
  fn data(item: Data) -> bool {
  prop_serialize(item)
  }
 
+ #[cfg_attr(miri, ignore)]
  fn link(item: Link) -> bool {
  prop_serialize(item)
  }
 
+ #[cfg_attr(miri, ignore)]
  fn msg_header(item: MessageHeader) -> bool {
  prop_serialize(item)
  }