diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 193fb7d1d..529a79aad 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,7 +5,7 @@ on: - trying pull_request: branches: - - master + - main name: Continuous integration @@ -18,7 +18,7 @@ jobs: - stable - beta - nightly - - 1.56.1 + - 1.65.0 env: RUSTFLAGS: "-C target-cpu=native -C opt-level=3" ROARINGRS_BENCH_OFFLINE: "true" diff --git a/Cargo.toml b/Cargo.toml index d4448a4d1..99fdbbf2b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,7 +25,7 @@ serde = { version = "1.0.139", optional = true } simd = [] [dev-dependencies] -proptest = "1.0.0" +proptest = "1.2.0" serde_json = "1.0.85" bincode = "1.3.3" diff --git a/README.md b/README.md index dc692f8b2..2d0c259aa 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,8 @@ Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you shall be dual licensed as above, without any additional terms or conditions. -[github-actions-badge]: https://img.shields.io/github/workflow/status/RoaringBitmap/roaring-rs/Continuous%20integration.svg?style=flat-square +[github-actions-badge]: +https://github.com/RoaringBitmap/roaring-rs/actions/workflows/test.yml/badge.svg [github-actions]: https://github.com/RoaringBitmap/roaring-rs/actions [release-badge]: https://img.shields.io/github/release/RoaringBitmap/roaring-rs.svg?style=flat-square [cargo]: https://crates.io/crates/roaring diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index 7e5d26e05..cdfafeda8 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -11,7 +11,7 @@ roaring = { path = ".." } [dev-dependencies] once_cell = "1.9" -git2 = { version = "0.13", default-features = false, features = ["https", "vendored-openssl"] } +git2 = { version = "0.17", default-features = false, features = ["https", "vendored-openssl"] } zip = { version = "0.5", default-features = false, features = ["deflate"] } indicatif = "0.16" criterion = { version = "0.3", features = ["html_reports"] } diff --git a/bors.toml b/bors.toml deleted file mode 100644 index 6a1baf1e7..000000000 --- a/bors.toml +++ /dev/null @@ -1,3 +0,0 @@ -status = ["ci (stable)"] -# 4 hours timeout -timeout-sec = 14400 diff --git a/src/bitmap/arbitrary.rs b/src/bitmap/arbitrary.rs index c25a73b16..f1dda728c 100644 --- a/src/bitmap/arbitrary.rs +++ b/src/bitmap/arbitrary.rs @@ -173,7 +173,7 @@ mod test { fn containers(n: usize) (keys in ArrayStore::sampled(..=n, ..=n), stores in vec(Store::arbitrary(), n)) -> RoaringBitmap { - let containers = keys.into_iter().zip(stores.into_iter()).map(|(key, store)| { + let containers = keys.into_iter().zip(stores).map(|(key, store)| { let mut container = Container { key, store }; container.ensure_correct_store(); container diff --git a/src/bitmap/container.rs b/src/bitmap/container.rs index e6d0cf84a..dee6ee1f2 100644 --- a/src/bitmap/container.rs +++ b/src/bitmap/container.rs @@ -6,7 +6,7 @@ use std::ops::{ use super::store::{self, Store}; use super::util; -const ARRAY_LIMIT: u64 = 4096; +pub const ARRAY_LIMIT: u64 = 4096; #[derive(PartialEq, Clone)] pub struct Container { @@ -94,6 +94,36 @@ impl Container { result } + pub fn remove_front(&mut self, n: u64) { + match &self.store { + Store::Bitmap(bits) => { + if bits.len() - n <= ARRAY_LIMIT { + let mut replace_array = Vec::with_capacity((bits.len() - n) as usize); + replace_array.extend(bits.iter().skip(n as usize)); + self.store = Store::Array(store::ArrayStore::from_vec_unchecked(replace_array)); + } else { + self.store.remove_front(n) + } + } + Store::Array(_) => self.store.remove_front(n), + }; + } + + pub fn remove_back(&mut self, n: u64) { + match &self.store { + Store::Bitmap(bits) => { + if bits.len() - n <= ARRAY_LIMIT { + let mut replace_array = Vec::with_capacity((bits.len() - n) as usize); + replace_array.extend(bits.iter().take((bits.len() - n) as usize)); + self.store = Store::Array(store::ArrayStore::from_vec_unchecked(replace_array)); + } else { + self.store.remove_back(n) + } + } + Store::Array(_) => self.store.remove_back(n), + }; + } + pub fn contains(&self, index: u16) -> bool { self.store.contains(index) } diff --git a/src/bitmap/inherent.rs b/src/bitmap/inherent.rs index 60e9e6190..2670609c1 100644 --- a/src/bitmap/inherent.rs +++ b/src/bitmap/inherent.rs @@ -578,6 +578,76 @@ impl RoaringBitmap { None } + + /// Removes the specified number of elements from the top. + /// + /// # Examples + /// + /// ```rust + /// use roaring::RoaringBitmap; + /// + /// let mut rb = RoaringBitmap::from_iter([1, 5, 7, 9]); + /// rb.remove_front(2); + /// assert_eq!(rb, RoaringBitmap::from_iter([7, 9])); + /// + /// let mut rb = RoaringBitmap::from_iter([1, 3, 7, 9]); + /// rb.remove_front(2); + /// assert_eq!(rb, RoaringBitmap::from_iter([7, 9])); + pub fn remove_front(&mut self, mut n: u64) { + // remove containers up to the front of the target + let position = self.containers.iter().position(|container| { + let container_len = container.len(); + if container_len <= n { + n -= container_len; + false + } else { + true + } + }); + let position = position.unwrap_or(self.containers.len()); + if position > 0 { + self.containers.drain(..position); + } + // remove data in containers if there are still targets for deletion + if n > 0 && !self.containers.is_empty() { + // container immediately before should have been deleted, so the target is 0 index + self.containers[0].remove_front(n); + } + } + + /// Removes the specified number of elements from the tail. + /// + /// # Examples + /// + /// ```rust + /// use roaring::RoaringBitmap; + /// + /// let mut rb = RoaringBitmap::from_iter([1, 5, 7, 9]); + /// rb.remove_back(2); + /// assert_eq!(rb, RoaringBitmap::from_iter([1, 5])); + /// rb.remove_back(1); + /// assert_eq!(rb, RoaringBitmap::from_iter([1])); + pub fn remove_back(&mut self, mut n: u64) { + // remove containers up to the back of the target + let position = self.containers.iter().rposition(|container| { + let container_len = container.len(); + if container_len <= n { + n -= container_len; + false + } else { + true + } + }); + // It is checked at the beginning of the function, so it is usually never an Err + if let Some(position) = position { + self.containers.drain(position + 1..); + if n > 0 && !self.containers.is_empty() { + self.containers[position].remove_back(n); + } + } else { + self.containers.clear(); + } + } } impl Default for RoaringBitmap { @@ -728,4 +798,129 @@ mod tests { assert_eq!(bitmap.containers.len(), 1); assert_eq!(bitmap.containers[0].key, 1); } + + #[test] + fn remove_front_for_vec() { + let mut bitmap = RoaringBitmap::from_iter([1, 2, 3, 7, 9, 11]); + bitmap.remove_front(3); + assert_eq!(bitmap.len(), 3); + assert_eq!(bitmap, RoaringBitmap::from_iter([7, 9, 11])); + + bitmap = RoaringBitmap::from_iter([1, 2, 5, 7, 9, 11]); + bitmap.remove_front(3); + assert_eq!(bitmap.len(), 3); + assert_eq!(bitmap, RoaringBitmap::from_iter([7, 9, 11])); + + bitmap = RoaringBitmap::from_iter([1, 3]); + bitmap.remove_front(2); + assert_eq!(bitmap.len(), 0); + + bitmap = RoaringBitmap::from_iter([1, 2, 3, 7, 9, 11]); + bitmap.remove_front(0); + assert_eq!(bitmap.len(), 6); + assert_eq!(bitmap, RoaringBitmap::from_iter([1, 2, 3, 7, 9, 11])); + + bitmap = RoaringBitmap::new(); + bitmap.insert_range(0..(1_u32 << 16) + 5); + bitmap.remove_front(65537); + assert_eq!(bitmap.len(), 4); + assert_eq!(bitmap, RoaringBitmap::from_iter([65537, 65538, 65539, 65540])); + + bitmap = RoaringBitmap::from_iter([1, 2, 5, 7, 9, 11]); + bitmap.remove_front(7); + assert_eq!(bitmap, RoaringBitmap::default()); + } + + #[test] + fn remove_front_for_bit() { + let mut bitmap = RoaringBitmap::new(); + bitmap.insert_range(0..4098); + bitmap.remove_front(4095); + assert_eq!(bitmap.len(), 3); + // removed bit to vec + assert_eq!(bitmap, RoaringBitmap::from_iter([4095, 4096, 4097])); + + bitmap = RoaringBitmap::new(); + bitmap.insert_range(0..6000); + bitmap.remove_front(999); + assert_eq!(bitmap.len(), 5001); + + bitmap = RoaringBitmap::new(); + bitmap.insert_range(0..8000); + bitmap.remove_front(10); + assert_eq!(bitmap.len(), 7990); + + bitmap = RoaringBitmap::new(); + bitmap.insert_range(0..200000); + bitmap.remove_front(2000); + assert_eq!(bitmap.len(), 198000); + assert_eq!(bitmap, RoaringBitmap::from_iter(2000..200000)); + + bitmap = RoaringBitmap::new(); + bitmap.insert_range(0..2); + bitmap.insert_range(4..7); + bitmap.insert_range(1000..6000); + bitmap.remove_front(30); + assert_eq!(bitmap.len(), 4975); + + bitmap = RoaringBitmap::new(); + bitmap.insert_range(0..65535); + bitmap.remove_front(0); + assert_eq!(bitmap.len(), 65535); + } + + #[test] + fn remove_back_for_bit() { + let mut bitmap = RoaringBitmap::new(); + bitmap.insert_range(0..5000); + bitmap.remove_back(1000); + assert_eq!(bitmap.len(), 4000); + + bitmap = RoaringBitmap::new(); + bitmap.insert_range(0..6000); + bitmap.remove_back(1000); + assert_eq!(bitmap.len(), 5000); + + bitmap = RoaringBitmap::new(); + bitmap.insert_range(0..200000); + bitmap.remove_back(196000); + assert_eq!(bitmap.len(), 4000); + + bitmap = RoaringBitmap::new(); + bitmap.insert_range(0..200000); + bitmap.remove_back(2000); + assert_eq!(bitmap.len(), 198000); + assert_eq!(bitmap, RoaringBitmap::from_iter(0..198000)); + + bitmap = RoaringBitmap::new(); + bitmap.insert_range(0..65535); + bitmap.remove_back(0); + assert_eq!(bitmap.len(), 65535); + } + + #[test] + fn remove_back_for_vec() { + let mut bitmap = RoaringBitmap::from_iter([1, 2, 3, 7, 9, 11]); + bitmap.remove_back(2); + assert_eq!(bitmap, RoaringBitmap::from_iter([1, 2, 3, 7])); + + bitmap = RoaringBitmap::from_iter([1, 2, 3, 7, 9, 11]); + bitmap.remove_back(6); + assert_eq!(bitmap.len(), 0); + + bitmap = RoaringBitmap::from_iter([1, 2, 3, 7, 9, 11]); + bitmap.remove_back(0); + assert_eq!(bitmap.len(), 6); + assert_eq!(bitmap, RoaringBitmap::from_iter([1, 2, 3, 7, 9, 11])); + + bitmap = RoaringBitmap::new(); + bitmap.insert_range(0..(1_u32 << 16) + 5); + bitmap.remove_back(65537); + assert_eq!(bitmap.len(), 4); + assert_eq!(bitmap, RoaringBitmap::from_iter([0, 1, 2, 3])); + + let mut bitmap = RoaringBitmap::from_iter([1, 2, 3]); + bitmap.remove_back(4); + assert_eq!(bitmap, RoaringBitmap::default()); + } } diff --git a/src/bitmap/iter.rs b/src/bitmap/iter.rs index cc8aa6564..35eddc820 100644 --- a/src/bitmap/iter.rs +++ b/src/bitmap/iter.rs @@ -133,7 +133,7 @@ impl IntoIterator for RoaringBitmap { impl From<[u32; N]> for RoaringBitmap { fn from(arr: [u32; N]) -> Self { - RoaringBitmap::from_iter(arr.into_iter()) + RoaringBitmap::from_iter(arr) } } diff --git a/src/bitmap/multiops.rs b/src/bitmap/multiops.rs index fc721c5ee..cb337bdc9 100644 --- a/src/bitmap/multiops.rs +++ b/src/bitmap/multiops.rs @@ -125,7 +125,7 @@ fn try_multi_and_owned( let mut start = start.into_iter(); if let Some(mut lhs) = start.next() { - for rhs in start.into_iter().map(Ok).chain(iter) { + for rhs in start.map(Ok).chain(iter) { if lhs.is_empty() { return Ok(lhs); } @@ -151,7 +151,7 @@ fn try_multi_and_ref<'a, E>( let mut start = start.into_iter(); if let Some(mut lhs) = start.next().cloned() { - for rhs in start.into_iter().map(Ok).chain(iter) { + for rhs in start.map(Ok).chain(iter) { if lhs.is_empty() { return Ok(lhs); } @@ -225,7 +225,7 @@ fn try_multi_or_owned( return Ok(RoaringBitmap::new()); }; - for bitmap in start.into_iter().map(Ok).chain(iter) { + for bitmap in start.map(Ok).chain(iter) { merge_container_owned(&mut containers, bitmap?.containers, BitOrAssign::bitor_assign); } @@ -323,7 +323,7 @@ fn try_multi_or_ref<'a, E: 'a>( }; // Phase 2: Operate on the remaining containers - for bitmap in start.into_iter().map(Ok).chain(iter) { + for bitmap in start.map(Ok).chain(iter) { merge_container_ref(&mut containers, &bitmap?.containers, |a, b| *a |= b); } diff --git a/src/bitmap/proptests.rs b/src/bitmap/proptests.rs index db1ef72c7..54b9c07d5 100644 --- a/src/bitmap/proptests.rs +++ b/src/bitmap/proptests.rs @@ -399,6 +399,7 @@ mod test { fn the_empty_set_is_the_identity_for_union(a in RoaringBitmap::arbitrary()) { prop_assert_eq!(&(&a | &empty_set()), &a); + #[allow(clippy::redundant_clone)] { // op_assign_ref let mut x = a.clone(); x |= &empty_set(); @@ -419,6 +420,7 @@ mod test { fn the_empty_set_is_the_identity_for_symmetric_difference(a in RoaringBitmap::arbitrary()) { prop_assert_eq!(&(&a ^ &empty_set()), &a); + #[allow(clippy::redundant_clone)] { // op_assign_ref let mut x = a.clone(); x ^= &empty_set(); diff --git a/src/bitmap/serialization.rs b/src/bitmap/serialization.rs index c2871c4ab..6ae84df41 100644 --- a/src/bitmap/serialization.rs +++ b/src/bitmap/serialization.rs @@ -3,15 +3,19 @@ use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; use std::convert::{Infallible, TryFrom}; use std::error::Error; use std::io; +use std::ops::RangeInclusive; -use super::container::Container; -use crate::bitmap::store::{ArrayStore, BitmapStore, Store}; +use crate::bitmap::container::{Container, ARRAY_LIMIT}; +use crate::bitmap::store::{ArrayStore, BitmapStore, Store, BITMAP_LENGTH}; use crate::RoaringBitmap; const SERIAL_COOKIE_NO_RUNCONTAINER: u32 = 12346; const SERIAL_COOKIE: u16 = 12347; -// TODO: Need this once run containers are supported -// const NO_OFFSET_THRESHOLD: u8 = 4; +const NO_OFFSET_THRESHOLD: usize = 4; + +// Sizes of header structures +const DESCRIPTION_BYTES: usize = 4; +const OFFSET_BYTES: usize = 4; impl RoaringBitmap { /// Return the size in bytes of the serialized output. @@ -163,49 +167,81 @@ impl RoaringBitmap { B: Fn(u64, Box<[u64; 1024]>) -> Result, BErr: Error + Send + Sync + 'static, { - let (size, has_offsets) = { + // First read the cookie to determine which version of the format we are reading + let (size, has_offsets, has_run_containers) = { let cookie = reader.read_u32::()?; if cookie == SERIAL_COOKIE_NO_RUNCONTAINER { - (reader.read_u32::()? as usize, true) + (reader.read_u32::()? as usize, true, false) } else if (cookie as u16) == SERIAL_COOKIE { - return Err(io::Error::new(io::ErrorKind::Other, "run containers are unsupported")); + let size = ((cookie >> 16) + 1) as usize; + (size, size >= NO_OFFSET_THRESHOLD, true) } else { return Err(io::Error::new(io::ErrorKind::Other, "unknown cookie value")); } }; + // Read the run container bitmap if necessary + let run_container_bitmap = if has_run_containers { + let mut bitmap = vec![0u8; (size + 7) / 8]; + reader.read_exact(&mut bitmap)?; + Some(bitmap) + } else { + None + }; + if size > u16::MAX as usize + 1 { return Err(io::Error::new(io::ErrorKind::Other, "size is greater than supported")); } - let mut description_bytes = vec![0u8; size * 4]; + // Read the container descriptions + let mut description_bytes = vec![0u8; size * DESCRIPTION_BYTES]; reader.read_exact(&mut description_bytes)?; let mut description_bytes = &description_bytes[..]; if has_offsets { - let mut offsets = vec![0u8; size * 4]; + let mut offsets = vec![0u8; size * OFFSET_BYTES]; reader.read_exact(&mut offsets)?; drop(offsets); // Not useful when deserializing into memory } let mut containers = Vec::with_capacity(size); - for _ in 0..size { + // Read each container + for i in 0..size { let key = description_bytes.read_u16::()?; - let len = u64::from(description_bytes.read_u16::()?) + 1; + let cardinality = u64::from(description_bytes.read_u16::()?) + 1; + + // If the run container bitmap is present, check if this container is a run container + let is_run_container = + run_container_bitmap.as_ref().map_or(false, |bm| bm[i / 8] & (1 << (i % 8)) != 0); + + let store = if is_run_container { + let runs = reader.read_u16::()?; + let mut intervals = vec![[0, 0]; runs as usize]; + reader.read_exact(cast_slice_mut(&mut intervals))?; + intervals.iter_mut().for_each(|[s, len]| { + *s = u16::from_le(*s); + *len = u16::from_le(*len); + }); - let store = if len <= 4096 { - let mut values = vec![0; len as usize]; + let cardinality = intervals.iter().map(|[_, len]| *len as usize).sum(); + let mut store = Store::with_capacity(cardinality); + intervals.into_iter().for_each(|[s, len]| { + store.insert_range(RangeInclusive::new(s, s + len)); + }); + store + } else if cardinality <= ARRAY_LIMIT { + let mut values = vec![0; cardinality as usize]; reader.read_exact(cast_slice_mut(&mut values))?; values.iter_mut().for_each(|n| *n = u16::from_le(*n)); let array = a(values).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; Store::Array(array) } else { - let mut values = Box::new([0; 1024]); + let mut values = Box::new([0; BITMAP_LENGTH]); reader.read_exact(cast_slice_mut(&mut values[..]))?; values.iter_mut().for_each(|n| *n = u64::from_le(*n)); - let bitmap = - b(len, values).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + let bitmap = b(cardinality, values) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; Store::Bitmap(bitmap) }; diff --git a/src/bitmap/store/array_store/mod.rs b/src/bitmap/store/array_store/mod.rs index ca6ee2069..5895eb37b 100644 --- a/src/bitmap/store/array_store/mod.rs +++ b/src/bitmap/store/array_store/mod.rs @@ -21,6 +21,10 @@ impl ArrayStore { ArrayStore { vec: vec![] } } + pub fn with_capacity(capacity: usize) -> ArrayStore { + ArrayStore { vec: Vec::with_capacity(capacity) } + } + /// /// Create a new SortedU16Vec from a given vec /// It is up to the caller to ensure the vec is sorted and deduplicated @@ -106,6 +110,15 @@ impl ArrayStore { (pos_end - pos_start) as u64 } + pub fn remove_front(&mut self, n: u64) { + self.vec.rotate_left(n as usize); + self.vec.truncate(self.vec.len() - n as usize); + } + + pub fn remove_back(&mut self, n: u64) { + self.vec.truncate(self.vec.len() - n as usize); + } + pub fn contains(&self, index: u16) -> bool { self.vec.binary_search(&index).is_ok() } @@ -558,4 +571,18 @@ mod tests { assert_eq!(into_vec(store), want); } + + #[test] + fn test_bitmap_remove_front() { + let mut store = Store::Array(ArrayStore::from_vec_unchecked(vec![1, 2, 130, 500])); + store.remove_front(3); + assert_eq!(into_vec(store), vec![500]); + } + + #[test] + fn test_bitmap_remove_back() { + let mut store = Store::Array(ArrayStore::from_vec_unchecked(vec![1, 2, 130, 500])); + store.remove_back(2); + assert_eq!(into_vec(store), vec![1, 2]); + } } diff --git a/src/bitmap/store/bitmap_store.rs b/src/bitmap/store/bitmap_store.rs index 55a53c57a..683666436 100644 --- a/src/bitmap/store/bitmap_store.rs +++ b/src/bitmap/store/bitmap_store.rs @@ -302,6 +302,57 @@ impl BitmapStore { pub fn as_array(&self) -> &[u64; BITMAP_LENGTH] { &self.bits } + + pub fn clear(&mut self) { + self.bits.fill(0); + self.len = 0; + } + + /// Set N bits that are currently 1 bit from the lower bit to 0. + pub fn remove_front(&mut self, mut clear_bits: u64) { + if self.len() < clear_bits { + self.clear(); + return; + } + self.len -= clear_bits; + for word in self.bits.iter_mut() { + let count = word.count_ones() as u64; + if clear_bits < count { + for _ in 0..clear_bits { + *word = *word & (*word - 1); + } + return; + } + *word = 0; + clear_bits -= count; + if clear_bits == 0 { + return; + } + } + } + + /// Set N bits that are currently 1 bit from the lower bit to 0. + pub fn remove_back(&mut self, mut clear_bits: u64) { + if self.len() < clear_bits { + self.clear(); + return; + } + self.len -= clear_bits; + for word in self.bits.iter_mut().rev() { + let count = word.count_ones() as u64; + if clear_bits < count { + for _ in 0..clear_bits { + *word &= !(1 << (63 - word.leading_zeros())); + } + return; + } + *word = 0; + clear_bits -= count; + if clear_bits == 0 { + return; + } + } + } } // this can be done in 3 instructions on x86-64 with bmi2 with: tzcnt(pdep(1 << rank, value)) @@ -490,3 +541,38 @@ impl BitXorAssign<&ArrayStore> for BitmapStore { self.len = len as u64; } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_bitmap_remove_front() { + let mut store = BitmapStore::new(); + let range = RangeInclusive::new(1, 3); + store.insert_range(range); + let range_second = RangeInclusive::new(5, 65535); + // store.bits[0] = 0b1111111111111111111111111111111111111111111111111111111111101110 + store.insert_range(range_second); + store.remove_front(2); + assert_eq!( + store.bits[0], + 0b1111111111111111111111111111111111111111111111111111111111101000 + ); + } + + #[test] + fn test_bitmap_remove_back() { + let mut store = BitmapStore::new(); + let range = RangeInclusive::new(1, 3); + store.insert_range(range); + let range_second = RangeInclusive::new(5, 65535); + // store.bits[1023] = 0b1111111111111111111111111111111111111111111111111111111111111111 + store.insert_range(range_second); + store.remove_back(2); + assert_eq!( + store.bits[1023], + 0b11111111111111111111111111111111111111111111111111111111111111 + ); + } +} diff --git a/src/bitmap/store/mod.rs b/src/bitmap/store/mod.rs index 0ebc150ba..0b8017e64 100644 --- a/src/bitmap/store/mod.rs +++ b/src/bitmap/store/mod.rs @@ -7,12 +7,14 @@ use std::ops::{ }; use std::{slice, vec}; -use self::bitmap_store::BITMAP_LENGTH; +pub use self::bitmap_store::BITMAP_LENGTH; use self::Store::{Array, Bitmap}; pub use self::array_store::ArrayStore; pub use self::bitmap_store::{BitmapIter, BitmapStore}; +use crate::bitmap::container::ARRAY_LIMIT; + #[derive(Clone)] pub enum Store { Array(ArrayStore), @@ -31,6 +33,14 @@ impl Store { Store::Array(ArrayStore::new()) } + pub fn with_capacity(capacity: usize) -> Store { + if capacity <= ARRAY_LIMIT as usize { + Store::Array(ArrayStore::with_capacity(capacity)) + } else { + Store::Bitmap(BitmapStore::new()) + } + } + pub fn full() -> Store { Store::Bitmap(BitmapStore::full()) } @@ -96,6 +106,20 @@ impl Store { } } + pub fn remove_front(&mut self, index: u64) { + match self { + Array(vec) => vec.remove_front(index), + Bitmap(bits) => bits.remove_front(index), + } + } + + pub fn remove_back(&mut self, index: u64) { + match self { + Array(vec) => vec.remove_back(index), + Bitmap(bits) => bits.remove_back(index), + } + } + pub fn contains(&self, index: u16) -> bool { match self { Array(vec) => vec.contains(index), @@ -196,7 +220,7 @@ impl BitOr<&Store> for &Store { fn bitor(self, rhs: &Store) -> Store { match (self, rhs) { - (&Array(ref vec1), &Array(ref vec2)) => Array(BitOr::bitor(vec1, vec2)), + (Array(vec1), Array(vec2)) => Array(BitOr::bitor(vec1, vec2)), (&Bitmap(..), &Array(..)) => { let mut lhs = self.clone(); BitOrAssign::bitor_assign(&mut lhs, rhs); @@ -239,17 +263,17 @@ impl BitOrAssign for Store { impl BitOrAssign<&Store> for Store { fn bitor_assign(&mut self, rhs: &Store) { match (self, rhs) { - (&mut Array(ref mut vec1), &Array(ref vec2)) => { + (&mut Array(ref mut vec1), Array(vec2)) => { let this = mem::take(vec1); *vec1 = BitOr::bitor(&this, vec2); } - (&mut Bitmap(ref mut bits1), &Array(ref vec2)) => { + (&mut Bitmap(ref mut bits1), Array(vec2)) => { BitOrAssign::bitor_assign(bits1, vec2); } - (&mut Bitmap(ref mut bits1), &Bitmap(ref bits2)) => { + (&mut Bitmap(ref mut bits1), Bitmap(bits2)) => { BitOrAssign::bitor_assign(bits1, bits2); } - (this @ &mut Array(..), &Bitmap(ref bits2)) => { + (this @ &mut Array(..), Bitmap(bits2)) => { let mut lhs: Store = Bitmap(bits2.clone()); BitOrAssign::bitor_assign(&mut lhs, &*this); *this = lhs; @@ -263,7 +287,7 @@ impl BitAnd<&Store> for &Store { fn bitand(self, rhs: &Store) -> Store { match (self, rhs) { - (&Array(ref vec1), &Array(ref vec2)) => Array(BitAnd::bitand(vec1, vec2)), + (Array(vec1), Array(vec2)) => Array(BitAnd::bitand(vec1, vec2)), (&Bitmap(..), &Array(..)) => { let mut rhs = rhs.clone(); BitAndAssign::bitand_assign(&mut rhs, self); @@ -306,7 +330,7 @@ impl BitAndAssign<&Store> for Store { #[allow(clippy::suspicious_op_assign_impl)] fn bitand_assign(&mut self, rhs: &Store) { match (self, rhs) { - (&mut Array(ref mut vec1), &Array(ref vec2)) => { + (&mut Array(ref mut vec1), Array(vec2)) => { let (mut lhs, rhs) = if vec2.len() < vec1.len() { (vec2.clone(), &*vec1) } else { @@ -316,10 +340,10 @@ impl BitAndAssign<&Store> for Store { BitAndAssign::bitand_assign(&mut lhs, rhs); *vec1 = lhs; } - (&mut Bitmap(ref mut bits1), &Bitmap(ref bits2)) => { + (&mut Bitmap(ref mut bits1), Bitmap(bits2)) => { BitAndAssign::bitand_assign(bits1, bits2); } - (&mut Array(ref mut vec1), &Bitmap(ref bits2)) => { + (&mut Array(ref mut vec1), Bitmap(bits2)) => { BitAndAssign::bitand_assign(vec1, bits2); } (this @ &mut Bitmap(..), &Array(..)) => { @@ -336,7 +360,7 @@ impl Sub<&Store> for &Store { fn sub(self, rhs: &Store) -> Store { match (self, rhs) { - (&Array(ref vec1), &Array(ref vec2)) => Array(Sub::sub(vec1, vec2)), + (Array(vec1), Array(vec2)) => Array(Sub::sub(vec1, vec2)), _ => { let mut lhs = self.clone(); SubAssign::sub_assign(&mut lhs, rhs); @@ -349,16 +373,16 @@ impl Sub<&Store> for &Store { impl SubAssign<&Store> for Store { fn sub_assign(&mut self, rhs: &Store) { match (self, rhs) { - (&mut Array(ref mut vec1), &Array(ref vec2)) => { + (&mut Array(ref mut vec1), Array(vec2)) => { SubAssign::sub_assign(vec1, vec2); } - (&mut Bitmap(ref mut bits1), &Array(ref vec2)) => { + (&mut Bitmap(ref mut bits1), Array(vec2)) => { SubAssign::sub_assign(bits1, vec2); } - (&mut Bitmap(ref mut bits1), &Bitmap(ref bits2)) => { + (&mut Bitmap(ref mut bits1), Bitmap(bits2)) => { SubAssign::sub_assign(bits1, bits2); } - (&mut Array(ref mut vec1), &Bitmap(ref bits2)) => { + (&mut Array(ref mut vec1), Bitmap(bits2)) => { SubAssign::sub_assign(vec1, bits2); } } @@ -370,7 +394,7 @@ impl BitXor<&Store> for &Store { fn bitxor(self, rhs: &Store) -> Store { match (self, rhs) { - (&Array(ref vec1), &Array(ref vec2)) => Array(BitXor::bitxor(vec1, vec2)), + (Array(vec1), Array(vec2)) => Array(BitXor::bitxor(vec1, vec2)), (&Array(..), &Bitmap(..)) => { let mut lhs = rhs.clone(); BitXorAssign::bitxor_assign(&mut lhs, self); @@ -408,17 +432,17 @@ impl BitXorAssign for Store { impl BitXorAssign<&Store> for Store { fn bitxor_assign(&mut self, rhs: &Store) { match (self, rhs) { - (&mut Array(ref mut vec1), &Array(ref vec2)) => { + (&mut Array(ref mut vec1), Array(vec2)) => { let this = mem::take(vec1); *vec1 = BitXor::bitxor(&this, vec2); } - (&mut Bitmap(ref mut bits1), &Array(ref vec2)) => { + (&mut Bitmap(ref mut bits1), Array(vec2)) => { BitXorAssign::bitxor_assign(bits1, vec2); } - (&mut Bitmap(ref mut bits1), &Bitmap(ref bits2)) => { + (&mut Bitmap(ref mut bits1), Bitmap(bits2)) => { BitXorAssign::bitxor_assign(bits1, bits2); } - (this @ &mut Array(..), &Bitmap(ref bits2)) => { + (this @ &mut Array(..), Bitmap(bits2)) => { let mut lhs: Store = Bitmap(bits2.clone()); BitXorAssign::bitxor_assign(&mut lhs, &*this); *this = lhs; diff --git a/src/treemap/iter.rs b/src/treemap/iter.rs index 6f13a323f..285a0c5ce 100644 --- a/src/treemap/iter.rs +++ b/src/treemap/iter.rs @@ -232,7 +232,7 @@ impl IntoIterator for RoaringTreemap { impl From<[u64; N]> for RoaringTreemap { fn from(arr: [u64; N]) -> Self { - RoaringTreemap::from_iter(arr.into_iter()) + RoaringTreemap::from_iter(arr) } } diff --git a/tests/bitmapwithruns.bin b/tests/bitmapwithruns.bin new file mode 100644 index 000000000..5ed243753 Binary files /dev/null and b/tests/bitmapwithruns.bin differ diff --git a/tests/clone.rs b/tests/clone.rs index 0dfb2593e..9c485b44a 100644 --- a/tests/clone.rs +++ b/tests/clone.rs @@ -2,6 +2,7 @@ extern crate roaring; use roaring::RoaringBitmap; #[test] +#[allow(clippy::redundant_clone)] fn array() { let original = (0..2000).collect::(); let clone = original.clone(); @@ -10,6 +11,7 @@ fn array() { } #[test] +#[allow(clippy::redundant_clone)] fn bitmap() { let original = (0..6000).collect::(); let clone = original.clone(); @@ -18,6 +20,7 @@ fn bitmap() { } #[test] +#[allow(clippy::redundant_clone)] fn arrays() { let original = (0..2000) .chain(1_000_000..1_002_000) @@ -29,6 +32,7 @@ fn arrays() { } #[test] +#[allow(clippy::redundant_clone)] fn bitmaps() { let original = (0..6000) .chain(1_000_000..1_012_000) diff --git a/tests/iter.rs b/tests/iter.rs index f15ad787d..14746ea9b 100644 --- a/tests/iter.rs +++ b/tests/iter.rs @@ -64,7 +64,7 @@ proptest! { fn iter(values in btree_set(any::(), ..=10_000)) { let bitmap = RoaringBitmap::from_sorted_iter(values.iter().cloned()).unwrap(); // Iterator::eq != PartialEq::eq - cannot use assert_eq macro - assert!(values.into_iter().eq(bitmap.into_iter())); + assert!(values.into_iter().eq(bitmap)); } } @@ -99,7 +99,7 @@ fn from_iter() { // with u32 as well as &u32 elements. let vals = vec![1, 5, 10000]; let a = RoaringBitmap::from_iter(vals.iter()); - let b = RoaringBitmap::from_iter(vals.into_iter()); + let b = RoaringBitmap::from_iter(vals); assert_eq!(a, b); } @@ -131,7 +131,7 @@ where #[test] fn outside_in_iterator() { let values = 0..10; - assert!(outside_in(values).eq(vec![0, 9, 1, 8, 2, 7, 3, 6, 4, 5].into_iter())); + assert!(outside_in(values).eq(vec![0, 9, 1, 8, 2, 7, 3, 6, 4, 5])); } #[test] diff --git a/tests/serialization.rs b/tests/serialization.rs index 0b31b0606..42efdd439 100644 --- a/tests/serialization.rs +++ b/tests/serialization.rs @@ -4,6 +4,7 @@ use roaring::RoaringBitmap; // Test data from https://github.com/RoaringBitmap/RoaringFormatSpec/tree/master/testdata static BITMAP_WITHOUT_RUNS: &[u8] = include_bytes!("bitmapwithoutruns.bin"); +static BITMAP_WITH_RUNS: &[u8] = include_bytes!("bitmapwithruns.bin"); fn test_data_bitmap() -> RoaringBitmap { (0..100) @@ -21,10 +22,18 @@ fn serialize_and_deserialize(bitmap: &RoaringBitmap) -> RoaringBitmap { } #[test] -fn test_deserialize_from_provided_data() { +fn test_deserialize_without_runs_from_provided_data() { assert_eq!(RoaringBitmap::deserialize_from(BITMAP_WITHOUT_RUNS).unwrap(), test_data_bitmap()); } +#[test] +fn test_deserialize_with_runs_from_provided_data() { + assert_eq!( + RoaringBitmap::deserialize_from(&mut &BITMAP_WITH_RUNS[..]).unwrap(), + test_data_bitmap() + ); +} + #[test] fn test_serialize_into_provided_data() { let bitmap = test_data_bitmap(); diff --git a/tests/treemap_clone.rs b/tests/treemap_clone.rs index deb93f7b0..ddeb84984 100644 --- a/tests/treemap_clone.rs +++ b/tests/treemap_clone.rs @@ -2,6 +2,7 @@ extern crate roaring; use roaring::RoaringTreemap; #[test] +#[allow(clippy::redundant_clone)] fn array() { let original = (0..2000).collect::(); let clone = original.clone(); @@ -10,6 +11,7 @@ fn array() { } #[test] +#[allow(clippy::redundant_clone)] fn bitmap() { let original = (0..6000).collect::(); let clone = original.clone(); @@ -18,6 +20,7 @@ fn bitmap() { } #[test] +#[allow(clippy::redundant_clone)] fn arrays() { let original = ((0..2000).chain(1_000_000..1_002_000).chain(2_000_000..2_001_000)) .collect::(); @@ -27,6 +30,7 @@ fn arrays() { } #[test] +#[allow(clippy::redundant_clone)] fn bitmaps() { let original = ((0..6000).chain(1_000_000..1_012_000).chain(2_000_000..2_010_000)) .collect::(); diff --git a/tests/treemap_iter.rs b/tests/treemap_iter.rs index 6966da21e..ddde2efc3 100644 --- a/tests/treemap_iter.rs +++ b/tests/treemap_iter.rs @@ -74,7 +74,7 @@ proptest! { fn iter(values in btree_set(any::(), ..=10_000)) { let bitmap = RoaringTreemap::from_sorted_iter(values.iter().cloned()).unwrap(); - assert!(values.into_iter().eq(bitmap.into_iter())); + assert!(values.into_iter().eq(bitmap)); } } @@ -104,7 +104,7 @@ fn from_iter() { // with u64 as well as &u64 elements. let vals = vec![1, 5, 1_000_000_000_000_000]; let a = RoaringTreemap::from_iter(vals.iter()); - let b = RoaringTreemap::from_iter(vals.into_iter()); + let b = RoaringTreemap::from_iter(vals); assert_eq!(a, b); }