From fdcf1db333acbd4b55a218bb7fac53a8ff00a3ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9A=AB=EF=B8=8F?= Date: Fri, 30 Aug 2024 18:49:37 +0200 Subject: [PATCH 1/7] add anybytes; use `PackedSlice`s for RSVec --- Cargo.toml | 2 ++ src/bit_vec/fast_rs_vec/mod.rs | 27 ++++++++++++++++----------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 23f68f6..65d35dd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,8 @@ exclude = [ ] [dependencies] +anybytes = "0.5.0" +zerocopy = "0.7.35" serde = { version = "1.0", optional = true, features = ["derive"] } [dev-dependencies] diff --git a/src/bit_vec/fast_rs_vec/mod.rs b/src/bit_vec/fast_rs_vec/mod.rs index c495c72..ffd5e46 100644 --- a/src/bit_vec/fast_rs_vec/mod.rs +++ b/src/bit_vec/fast_rs_vec/mod.rs @@ -3,6 +3,7 @@ use std::mem::size_of; +use anybytes::PackedSlice; #[cfg(all( feature = "simd", target_arch = "x86_64", @@ -13,6 +14,7 @@ use std::mem::size_of; ))] pub use bitset::*; pub use iter::*; +use zerocopy::{AsBytes, FromBytes, FromZeroes}; use crate::util::impl_vector_iterator; use crate::BitVec; @@ -41,8 +43,9 @@ const SELECT_BLOCK_SIZE: usize = 1 << 13; /// beginning from the last super-block boundary. This means the first block in a super-block /// always stores the number zero, which serves as a sentinel value to avoid special-casing the /// first block in a super-block (which would be a performance hit due branch prediction failures). -#[derive(Clone, Copy, Debug)] +#[derive(FromZeroes, FromBytes, AsBytes, Clone, Copy, Debug)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[repr(C)] struct BlockDescriptor { zeros: u16, } @@ -50,8 +53,9 @@ struct BlockDescriptor { /// Meta-data for a super-block. The `zeros` field stores the number of zeros up to this super-block. /// This allows the `BlockDescriptor` to store the number of zeros in a much smaller /// space. The `zeros` field is the number of zeros up to the super-block. -#[derive(Clone, Copy, Debug)] +#[derive(FromZeroes, FromBytes, AsBytes, Clone, Copy, Debug)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[repr(C)] struct SuperBlockDescriptor { zeros: usize, } @@ -59,8 +63,9 @@ struct SuperBlockDescriptor { /// Meta-data for the select query. Each entry i in the select vector contains the indices to find /// the i * `SELECT_BLOCK_SIZE`'th 0- and 1-bit in the bitvector. Those indices may be very far apart. /// The indices do not point into the bit-vector, but into the super-block vector. -#[derive(Clone, Debug)] +#[derive(FromZeroes, FromBytes, AsBytes, Clone, Debug)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[repr(C)] struct SelectSuperBlockDescriptor { index_0: usize, index_1: usize, @@ -85,13 +90,13 @@ struct SelectSuperBlockDescriptor { #[derive(Clone, Debug)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub struct RsVec { - data: Vec, len: usize, - blocks: Vec, - super_blocks: Vec, - select_blocks: Vec, pub(crate) rank0: usize, pub(crate) rank1: usize, + data: PackedSlice, + blocks: PackedSlice, + super_blocks: PackedSlice, + select_blocks: PackedSlice, } impl RsVec { @@ -212,11 +217,11 @@ impl RsVec { } RsVec { - data: vec.data, + data: vec.data.into(), len: vec.len, - blocks, - super_blocks, - select_blocks, + blocks: blocks.into(), + super_blocks: super_blocks.into(), + select_blocks: select_blocks.into(), // the last block may contain padding zeros, which should not be counted rank0: total_zeros + current_zeros - ((WORD_SIZE - (vec.len % WORD_SIZE)) % WORD_SIZE), rank1: vec.len From 9a4a2a105b5670da0bdbdb5c3588cb8e2229f71c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9A=AB=EF=B8=8F?= Date: Mon, 23 Sep 2024 12:35:11 +0200 Subject: [PATCH 2/7] upgrade anybytes dependency --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 65d35dd..c13fd3c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ exclude = [ ] [dependencies] -anybytes = "0.5.0" +anybytes = "0.6.0" zerocopy = "0.7.35" serde = { version = "1.0", optional = true, features = ["derive"] } From fd03b8ee7253510904d25981caf17a8b8c24a64c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9A=AB=EF=B8=8F?= Date: Tue, 24 Sep 2024 15:19:36 +0200 Subject: [PATCH 3/7] move zerocopy behind feature gate --- .github/workflows/rust.yml | 16 +++++++++++----- Cargo.toml | 8 +++++--- src/bit_vec/fast_rs_vec/mod.rs | 30 +++++++++++++++++++++++------- src/lib.rs | 3 +++ 4 files changed, 42 insertions(+), 15 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index e6d705f..df7e481 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -11,13 +11,19 @@ env: RUSTFLAGS: -C target-cpu=native jobs: - build: - + test_serde: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Build + run: cargo build --verbose --features simd serde + - name: Run tests + run: cargo test --verbose --features simd serde + test_zerocopy: runs-on: ubuntu-latest - steps: - uses: actions/checkout@v4 - name: Build - run: cargo build --verbose --all-features + run: cargo build --verbose --features simd zerocopy - name: Run tests - run: cargo test --verbose --all-features + run: cargo test --verbose --features simd zerocopy diff --git a/Cargo.toml b/Cargo.toml index c13fd3c..515bad4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,9 +16,9 @@ exclude = [ ] [dependencies] -anybytes = "0.6.0" -zerocopy = "0.7.35" +anybytes = { version = "0.6.0", optional = true } serde = { version = "1.0", optional = true, features = ["derive"] } +zerocopy = { version = "0.7.35", optional = true } [dev-dependencies] # benchmarking @@ -26,8 +26,10 @@ criterion = { version = "0.5.1", features = ["html_reports"] } rand = { version = "0.8", features = ["alloc"] } [features] -simd = [] docsrs = [] # special feature for docs.rs to enable doc_auto_cfg on nightly +simd = [] +serde = ["dep:serde"] +zerocopy = ["dep:anybytes", "dep:zerocopy"] [[bench]] name = "rank" diff --git a/src/bit_vec/fast_rs_vec/mod.rs b/src/bit_vec/fast_rs_vec/mod.rs index ffd5e46..d9f7b51 100644 --- a/src/bit_vec/fast_rs_vec/mod.rs +++ b/src/bit_vec/fast_rs_vec/mod.rs @@ -3,7 +3,12 @@ use std::mem::size_of; +#[cfg(feature = "zerocopy")] use anybytes::PackedSlice; + +#[cfg(feature = "zerocopy")] +use zerocopy::{AsBytes, FromBytes, FromZeroes}; + #[cfg(all( feature = "simd", target_arch = "x86_64", @@ -14,7 +19,6 @@ use anybytes::PackedSlice; ))] pub use bitset::*; pub use iter::*; -use zerocopy::{AsBytes, FromBytes, FromZeroes}; use crate::util::impl_vector_iterator; use crate::BitVec; @@ -43,9 +47,9 @@ const SELECT_BLOCK_SIZE: usize = 1 << 13; /// beginning from the last super-block boundary. This means the first block in a super-block /// always stores the number zero, which serves as a sentinel value to avoid special-casing the /// first block in a super-block (which would be a performance hit due branch prediction failures). -#[derive(FromZeroes, FromBytes, AsBytes, Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug)] +#[cfg_attr(feature = "zerocopy", repr(C), derive(zerocopy::FromZeroes, zerocopy::FromBytes, zerocopy::AsBytes))] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[repr(C)] struct BlockDescriptor { zeros: u16, } @@ -53,9 +57,9 @@ struct BlockDescriptor { /// Meta-data for a super-block. The `zeros` field stores the number of zeros up to this super-block. /// This allows the `BlockDescriptor` to store the number of zeros in a much smaller /// space. The `zeros` field is the number of zeros up to the super-block. -#[derive(FromZeroes, FromBytes, AsBytes, Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug)] +#[cfg_attr(feature = "zerocopy", repr(C), derive(zerocopy::FromZeroes, zerocopy::FromBytes, zerocopy::AsBytes))] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[repr(C)] struct SuperBlockDescriptor { zeros: usize, } @@ -63,9 +67,9 @@ struct SuperBlockDescriptor { /// Meta-data for the select query. Each entry i in the select vector contains the indices to find /// the i * `SELECT_BLOCK_SIZE`'th 0- and 1-bit in the bitvector. Those indices may be very far apart. /// The indices do not point into the bit-vector, but into the super-block vector. -#[derive(FromZeroes, FromBytes, AsBytes, Clone, Debug)] +#[derive(Clone, Debug)] +#[cfg_attr(feature = "zerocopy", repr(C), derive(zerocopy::FromZeroes, zerocopy::FromBytes, zerocopy::AsBytes))] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[repr(C)] struct SelectSuperBlockDescriptor { index_0: usize, index_1: usize, @@ -93,9 +97,21 @@ pub struct RsVec { len: usize, pub(crate) rank0: usize, pub(crate) rank1: usize, + #[cfg(not(feature = "zerocopy"))] + data: Vec, + #[cfg(not(feature = "zerocopy"))] + blocks: Vec, + #[cfg(not(feature = "zerocopy"))] + super_blocks: Vec, + #[cfg(not(feature = "zerocopy"))] + select_blocks: Vec, + #[cfg(feature = "zerocopy")] data: PackedSlice, + #[cfg(feature = "zerocopy")] blocks: PackedSlice, + #[cfg(feature = "zerocopy")] super_blocks: PackedSlice, + #[cfg(feature = "zerocopy")] select_blocks: PackedSlice, } diff --git a/src/lib.rs b/src/lib.rs index 5d3cfb0..bdf2f27 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -60,6 +60,9 @@ //! - `serde` (disabled by default): Enables serialization and deserialization support for all //! data structures in this crate using the `serde` crate. +#[cfg(all(feature = "serde", feature = "zerocopy"))] +compile_error!("`serde` and `zerocopy` are mutually excusive features"); + pub use bit_vec::fast_rs_vec::RsVec; pub use bit_vec::BitVec; pub use elias_fano::EliasFanoVec; From db53f161b5fa6201375e7a805acb6b1cbf66cd10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9A=AB=EF=B8=8F?= Date: Tue, 24 Sep 2024 15:21:50 +0200 Subject: [PATCH 4/7] fix feature arg for CI --- .github/workflows/rust.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index df7e481..70c22eb 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -16,14 +16,14 @@ jobs: steps: - uses: actions/checkout@v4 - name: Build - run: cargo build --verbose --features simd serde + run: cargo build --verbose --features "simd serde" - name: Run tests - run: cargo test --verbose --features simd serde + run: cargo test --verbose --features "simd serde" test_zerocopy: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Build - run: cargo build --verbose --features simd zerocopy + run: cargo build --verbose --features "simd zerocopy" - name: Run tests - run: cargo test --verbose --features simd zerocopy + run: cargo test --verbose --features "simd zerocopy" From 87835c70665e8fedf225fff643d50543208a7638 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9A=AB=EF=B8=8F?= Date: Tue, 24 Sep 2024 15:30:06 +0200 Subject: [PATCH 5/7] explicitly enable zerocopy derive --- Cargo.toml | 2 +- src/bit_vec/fast_rs_vec/mod.rs | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 515bad4..2bceb64 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,7 @@ exclude = [ [dependencies] anybytes = { version = "0.6.0", optional = true } serde = { version = "1.0", optional = true, features = ["derive"] } -zerocopy = { version = "0.7.35", optional = true } +zerocopy = { version = "0.7.35", optional = true, features = ["derive"] } [dev-dependencies] # benchmarking diff --git a/src/bit_vec/fast_rs_vec/mod.rs b/src/bit_vec/fast_rs_vec/mod.rs index d9f7b51..4291654 100644 --- a/src/bit_vec/fast_rs_vec/mod.rs +++ b/src/bit_vec/fast_rs_vec/mod.rs @@ -6,9 +6,6 @@ use std::mem::size_of; #[cfg(feature = "zerocopy")] use anybytes::PackedSlice; -#[cfg(feature = "zerocopy")] -use zerocopy::{AsBytes, FromBytes, FromZeroes}; - #[cfg(all( feature = "simd", target_arch = "x86_64", From 1d2d11cda23ca2a5c3def60920bf70ca08710fcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9A=AB=EF=B8=8F?= Date: Fri, 27 Sep 2024 08:29:24 +0200 Subject: [PATCH 6/7] upgrade anybytes dependency --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 2bceb64..1608666 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ exclude = [ ] [dependencies] -anybytes = { version = "0.6.0", optional = true } +anybytes = { version = "0.9.0-alpha", optional = true } serde = { version = "1.0", optional = true, features = ["derive"] } zerocopy = { version = "0.7.35", optional = true, features = ["derive"] } From 1e46caba6995a9c4660e0c555f9edd0ba9d88d62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9A=AB=EF=B8=8F?= Date: Sun, 15 Dec 2024 22:30:36 +0100 Subject: [PATCH 7/7] upgrade anybytes dependency --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index fec87c1..c655f35 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ exclude = [ ] [dependencies] -anybytes = { version = "0.9.0-alpha", optional = true } +anybytes = { version = "0.11.0", optional = true } serde = { version = "1.0", optional = true, features = ["derive"] } zerocopy = { version = "0.7.35", optional = true, features = ["derive"] }