From 6f8f7c9f81a0eeb27d8bbd2364326a2f4ad4c4fa Mon Sep 17 00:00:00 2001 From: MiyakoMeow <110924386+MiyakoMeow@users.noreply.github.com> Date: Thu, 19 Jun 2025 06:37:30 +0800 Subject: [PATCH 1/7] add(feature): bincode --- Cargo.lock | 33 ++++++++++ Cargo.toml | 31 ++++++---- src/array_bincode.rs | 141 +++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 75 ++++++++--------------- 4 files changed, 219 insertions(+), 61 deletions(-) create mode 100644 src/array_bincode.rs diff --git a/Cargo.lock b/Cargo.lock index 057a36d03..abdfc85e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -47,6 +47,26 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bincode" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" +dependencies = [ + "bincode_derive", + "serde", + "unty", +] + +[[package]] +name = "bincode_derive" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" +dependencies = [ + "virtue", +] + [[package]] name = "bitflags" version = "2.9.1" @@ -460,6 +480,7 @@ name = "ndarray" version = "0.16.1" dependencies = [ "approx", + "bincode", "cblas-sys", "defmac", "itertools", @@ -1129,6 +1150,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unty" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" + [[package]] name = "ureq" version = "2.10.1" @@ -1167,6 +1194,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "virtue" +version = "0.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" diff --git a/Cargo.toml b/Cargo.toml index 14226986e..2b49dbe47 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,10 +4,7 @@ name = "ndarray" version = "0.16.1" edition = "2021" rust-version = "1.64" -authors = [ - "Ulrik Sverdrup \"bluss\"", - "Jim Turner" -] +authors = ["Ulrik Sverdrup \"bluss\"", "Jim Turner"] license = "MIT OR Apache-2.0" readme = "README-crates.io.md" @@ -39,11 +36,20 @@ rayon = { version = "1.10.0", optional = true } cblas-sys = { workspace = true, optional = true } libc = { version = "0.2.82", optional = true } -matrixmultiply = { version = "0.3.2", default-features = false, features=["cgemm"] } +matrixmultiply = { version = "0.3.2", default-features = false, features = [ + "cgemm", +] } -serde = { version = "1.0", optional = true, default-features = false, features = ["alloc"] } +serde = { version = "1.0", optional = true, default-features = false, features = [ + "alloc", +] } rawpointer = { version = "0.2" } +bincode = { version = "2.0", optional = true, default-features = false, features = [ + "alloc", + "derive", +] } + [dev-dependencies] defmac = "0.2" quickcheck = { workspace = true } @@ -60,6 +66,8 @@ blas = ["dep:cblas-sys", "dep:libc"] serde = ["dep:serde"] +bincode = ["dep:bincode"] + std = ["num-traits/std", "matrixmultiply/std"] rayon = ["dep:rayon", "std"] @@ -70,13 +78,10 @@ portable-atomic-critical-section = ["portable-atomic/critical-section"] [target.'cfg(not(target_has_atomic = "ptr"))'.dependencies] portable-atomic = { version = "1.6.0" } -portable-atomic-util = { version = "0.2.0", features = [ "alloc" ] } +portable-atomic-util = { version = "0.2.0", features = ["alloc"] } [workspace] -members = [ - "ndarray-rand", - "crates/*", -] +members = ["ndarray-rand", "crates/*"] default-members = [ ".", "ndarray-rand", @@ -98,7 +103,9 @@ approx = { version = "0.5", default-features = false } quickcheck = { version = "1.0", default-features = false } rand = { version = "0.9.0", features = ["small_rng"] } rand_distr = { version = "0.5.0" } -itertools = { version = "0.13.0", default-features = false, features = ["use_std"] } +itertools = { version = "0.13.0", default-features = false, features = [ + "use_std", +] } cblas-sys = { version = "0.1.4", default-features = false } [profile.bench] diff --git a/src/array_bincode.rs b/src/array_bincode.rs new file mode 100644 index 000000000..02a51a126 --- /dev/null +++ b/src/array_bincode.rs @@ -0,0 +1,141 @@ +// Copyright 2014-2025 bluss and ndarray developers. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. +use bincode::{ + de::{BorrowDecoder, Decoder}, + enc::Encoder, + error::{DecodeError, EncodeError}, + BorrowDecode, Decode, Encode, +}; + +use alloc::format; +#[cfg(not(feature = "std"))] +use alloc::vec::Vec; + +use crate::{imp_prelude::*, IntoDimension, IxDynImpl, ShapeError}; + +use super::arraytraits::ARRAY_FORMAT_VERSION; +use crate::iterators::Iter; + +/// Verifies that the version of the deserialized array matches the current +/// `ARRAY_FORMAT_VERSION`. +pub fn verify_version(v: u8) -> Result<(), DecodeError> { + if v != ARRAY_FORMAT_VERSION { + let err_msg = format!("unknown array version: {}", v); + Err(DecodeError::OtherString(err_msg)) + } else { + Ok(()) + } +} + +/// **Requires crate feature `"bincode"`** +impl Encode for Dim +where + I: Encode, +{ + fn encode(&self, encoder: &mut E) -> Result<(), EncodeError> { + Encode::encode(&self.ix(), encoder) + } +} + +/// **Requires crate feature `"bincode"`** +impl Decode for Dim +where + I: Decode, +{ + fn decode>(decoder: &mut D) -> Result { + Ok(Dim::new(Decode::decode(decoder)?)) + } +} + +/// **Requires crate feature `"bincode"`** +impl<'de, Context, I> BorrowDecode<'de, Context> for Dim +where + I: BorrowDecode<'de, Context>, +{ + fn borrow_decode>(decoder: &mut D) -> Result { + Ok(Dim::new(BorrowDecode::borrow_decode(decoder)?)) + } +} + +/// **Requires crate feature `"bincode"`** +impl Encode for IxDyn { + fn encode(&self, encoder: &mut E) -> Result<(), EncodeError> { + let ix: &IxDynImpl = self.ix(); + Encode::encode(&ix.len(), encoder)?; + for ix in ix.into_iter() { + Encode::encode(ix, encoder)?; + } + Ok(()) + } +} + +/// **Requires crate feature `"bincode"`** +impl Decode for IxDynImpl { + fn decode>(decoder: &mut D) -> Result { + let len: usize = Decode::decode(decoder)?; + let vals = (0..len) + .map(|_| Decode::decode(decoder)) + .collect::, DecodeError>>()?; + Ok(IxDynImpl::from(vals)) + } +} + +/// **Requires crate feature `"bincode"`** +impl<'de, Context> bincode::BorrowDecode<'de, Context> for IxDynImpl { + fn borrow_decode>(decoder: &mut D) -> Result { + let len: usize = BorrowDecode::borrow_decode(decoder)?; + let vals = (0..len) + .map(|_| BorrowDecode::borrow_decode(decoder)) + .collect::, DecodeError>>()?; + Ok(IxDynImpl::from(vals)) + } +} + +/// **Requires crate feature `"serde"`** +impl Encode for ArrayBase +where + A: Encode, + D: Dimension + Encode, + S: Data, +{ + fn encode(&self, encoder: &mut E) -> Result<(), EncodeError> { + Encode::encode(&ARRAY_FORMAT_VERSION, encoder)?; + Encode::encode(&self.raw_dim(), encoder)?; + let iter = self.iter(); + Encode::encode(&iter.len(), encoder)?; + for elt in iter.clone() { + Encode::encode(elt, encoder)?; + } + Ok(()) + } +} + +/// **Requires crate feature `"bincode"`** +impl Decode for ArrayBase +where + A: Decode, + D: Dimension + Decode, + S: DataOwned, +{ + fn decode>(decoder: &mut De) -> Result { + let data_version: u8 = Decode::decode(decoder)?; + (data_version == ARRAY_FORMAT_VERSION) + .then_some(()) + .ok_or(DecodeError::Other("ARRAY_FORMAT_VERSION not match!"))?; + let dim: D = Decode::decode(decoder)?; + let data_len: usize = Decode::decode(decoder)?; + let data = (0..data_len) + .map(|_| Decode::decode(decoder)) + .collect::, DecodeError>>()?; + let expected_size = dim.size(); + ArrayBase::from_shape_vec(dim, data).map_err(|_err: ShapeError| DecodeError::ArrayLengthMismatch { + required: expected_size, + found: data_len, + }) + } +} diff --git a/src/lib.rs b/src/lib.rs index 3efb378ce..e1be1973c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -175,6 +175,8 @@ mod itertools; mod argument_traits; #[cfg(feature = "serde")] mod array_serde; +#[cfg(feature = "bincode")] +mod array_bincode; mod arrayformat; mod arraytraits; pub use crate::argument_traits::AssignElem; @@ -224,24 +226,12 @@ pub use crate::zip::{FoldWhile, IntoNdProducer, NdProducer, Zip}; pub use crate::layout::Layout; /// Implementation's prelude. Common types used everywhere. -mod imp_prelude -{ +mod imp_prelude { pub use crate::dimension::DimensionExt; pub use crate::prelude::*; pub use crate::ArcArray; pub use crate::{ - CowRepr, - Data, - DataMut, - DataOwned, - DataShared, - Ix, - Ixs, - RawData, - RawDataMut, - RawViewRepr, - RemoveAxis, - ViewRepr, + CowRepr, Data, DataMut, DataOwned, DataShared, Ix, Ixs, RawData, RawDataMut, RawViewRepr, RemoveAxis, ViewRepr, }; } @@ -1292,7 +1282,8 @@ pub type Ixs = isize; // // [`.offset()`]: https://doc.rust-lang.org/stable/std/primitive.pointer.html#method.offset-1 pub struct ArrayBase::Elem> -where S: RawData +where + S: RawData, { /// Data buffer / ownership information. (If owned, contains the data /// buffer; if borrowed, contains the lifetime and mutability.) @@ -1401,8 +1392,7 @@ where S: RawData // alter the offset of the pointer. This is allowed, as it does not // cause a pointer deref. #[derive(Debug)] -pub struct LayoutRef -{ +pub struct LayoutRef { /// A non-null pointer into the buffer held by `data`; may point anywhere /// in its range. If `S: Data`, this pointer must be aligned. ptr: std::ptr::NonNull, @@ -1643,10 +1633,8 @@ pub use data_repr::OwnedRepr; #[derive(Debug)] pub struct OwnedArcRepr(Arc>); -impl Clone for OwnedArcRepr -{ - fn clone(&self) -> Self - { +impl Clone for OwnedArcRepr { + fn clone(&self) -> Self { OwnedArcRepr(self.0.clone()) } } @@ -1657,16 +1645,13 @@ impl Clone for OwnedArcRepr /// [`RawArrayView`] / [`RawArrayViewMut`] for the array type!* #[derive(Copy, Clone)] // This is just a marker type, to carry the mutability and element type. -pub struct RawViewRepr -{ +pub struct RawViewRepr { ptr: PhantomData, } -impl RawViewRepr -{ +impl RawViewRepr { #[inline(always)] - const fn new() -> Self - { + const fn new() -> Self { RawViewRepr { ptr: PhantomData } } } @@ -1677,16 +1662,13 @@ impl RawViewRepr /// [`ArrayView`] / [`ArrayViewMut`] for the array type!* #[derive(Copy, Clone)] // This is just a marker type, to carry the lifetime parameter. -pub struct ViewRepr -{ +pub struct ViewRepr { life: PhantomData, } -impl ViewRepr -{ +impl ViewRepr { #[inline(always)] - const fn new() -> Self - { + const fn new() -> Self { ViewRepr { life: PhantomData } } } @@ -1695,19 +1677,16 @@ impl ViewRepr /// /// *Don't use this type directly—use the type alias /// [`CowArray`] for the array type!* -pub enum CowRepr<'a, A> -{ +pub enum CowRepr<'a, A> { /// Borrowed data. View(ViewRepr<&'a A>), /// Owned data. Owned(OwnedRepr), } -impl CowRepr<'_, A> -{ +impl CowRepr<'_, A> { /// Returns `true` iff the data is the `View` variant. - pub fn is_view(&self) -> bool - { + pub fn is_view(&self) -> bool { match self { CowRepr::View(_) => true, CowRepr::Owned(_) => false, @@ -1715,8 +1694,7 @@ impl CowRepr<'_, A> } /// Returns `true` iff the data is the `Owned` variant. - pub fn is_owned(&self) -> bool - { + pub fn is_owned(&self) -> bool { match self { CowRepr::View(_) => false, CowRepr::Owned(_) => true, @@ -1738,11 +1716,11 @@ mod impl_owned_array; mod impl_special_element_types; /// Private Methods -impl ArrayRef -{ +impl ArrayRef { #[inline] fn broadcast_unwrap(&self, dim: E) -> ArrayView<'_, A, E> - where E: Dimension + where + E: Dimension, { #[cold] #[inline(never)] @@ -1764,7 +1742,8 @@ impl ArrayRef // (Checked in debug assertions). #[inline] fn broadcast_assume(&self, dim: E) -> ArrayView<'_, A, E> - where E: Dimension + where + E: Dimension, { let dim = dim.into_dimension(); debug_assert_eq!(self.shape(), dim.slice()); @@ -1781,8 +1760,7 @@ where D: Dimension, { /// Remove array axis `axis` and return the result. - fn try_remove_axis(self, axis: Axis) -> ArrayBase - { + fn try_remove_axis(self, axis: Axis) -> ArrayBase { let d = self.layout.dim.try_remove_axis(axis); let s = self.layout.strides.try_remove_axis(axis); // safe because new dimension, strides allow access to a subset of old data @@ -1822,8 +1800,7 @@ mod impl_cow; mod impl_arc_array; /// Returns `true` if the pointer is aligned. -pub(crate) fn is_aligned(ptr: *const T) -> bool -{ +pub(crate) fn is_aligned(ptr: *const T) -> bool { (ptr as usize) % ::std::mem::align_of::() == 0 } From 13b2daf06004a1b9caa34a7b6b39d45fd62aff31 Mon Sep 17 00:00:00 2001 From: MiyakoMeow <110924386+MiyakoMeow@users.noreply.github.com> Date: Sat, 21 Jun 2025 05:52:25 +0800 Subject: [PATCH 2/7] add(feature): bincode, almost finish --- Cargo.lock | 43 ++++++ Cargo.toml | 2 + crates/bincode-store-tests/Cargo.toml | 23 +++ crates/bincode-store-tests/src/lib.rs | 1 + crates/bincode-store-tests/tests/serialize.rs | 78 ++++++++++ src/array_bincode.rs | 140 ++++++++++++++++++ src/arraytraits.rs | 4 +- src/lib.rs | 2 + 8 files changed, 291 insertions(+), 2 deletions(-) create mode 100644 crates/bincode-store-tests/Cargo.toml create mode 100644 crates/bincode-store-tests/src/lib.rs create mode 100644 crates/bincode-store-tests/tests/serialize.rs create mode 100644 src/array_bincode.rs diff --git a/Cargo.lock b/Cargo.lock index 057a36d03..c2f9a31c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -47,6 +47,36 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bincode" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" +dependencies = [ + "bincode_derive", + "serde", + "unty", +] + +[[package]] +name = "bincode-store-tests" +version = "0.1.0" +dependencies = [ + "bincode", + "ndarray", + "rmp", + "rmp-serde", +] + +[[package]] +name = "bincode_derive" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" +dependencies = [ + "virtue", +] + [[package]] name = "bitflags" version = "2.9.1" @@ -460,6 +490,7 @@ name = "ndarray" version = "0.16.1" dependencies = [ "approx", + "bincode", "cblas-sys", "defmac", "itertools", @@ -1129,6 +1160,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unty" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" + [[package]] name = "ureq" version = "2.10.1" @@ -1167,6 +1204,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "virtue" +version = "0.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" diff --git a/Cargo.toml b/Cargo.toml index 14226986e..434ecb295 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,8 @@ matrixmultiply = { version = "0.3.2", default-features = false, features=["cgemm serde = { version = "1.0", optional = true, default-features = false, features = ["alloc"] } rawpointer = { version = "0.2" } +bincode = { version = "2.0", optional = true, default-features = false, features = ["alloc", "derive"] } + [dev-dependencies] defmac = "0.2" quickcheck = { workspace = true } diff --git a/crates/bincode-store-tests/Cargo.toml b/crates/bincode-store-tests/Cargo.toml new file mode 100644 index 000000000..4f86faaf6 --- /dev/null +++ b/crates/bincode-store-tests/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "bincode-store-tests" +version = "0.1.0" +authors = ["MiyakoMeow"] +publish = false +edition = "2018" + +[lib] +test = false +doc = false +doctest = false + +[dependencies] +ndarray = { workspace = true, features = ["bincode"] } + +bincode = { version = "2" } + +[dev-dependencies] +# >=0.8.11 to avoid rmp-serde security vulnerability +# <0.8.14 to allows MSRV 1.64.0 +rmp = { version = ">=0.8.11,<0.8.14" } +# Old version to work with Rust 1.64+ +rmp-serde = { version = ">=1.1.1" } diff --git a/crates/bincode-store-tests/src/lib.rs b/crates/bincode-store-tests/src/lib.rs new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/crates/bincode-store-tests/src/lib.rs @@ -0,0 +1 @@ + diff --git a/crates/bincode-store-tests/tests/serialize.rs b/crates/bincode-store-tests/tests/serialize.rs new file mode 100644 index 000000000..b2f11103e --- /dev/null +++ b/crates/bincode-store-tests/tests/serialize.rs @@ -0,0 +1,78 @@ +extern crate ndarray; + +use ndarray::{arr0, arr1, arr2, s, ArcArray, ArcArray2, ArrayD, IxDyn}; + +#[test] +fn store_many_dim_excrate() { + { + let a = arr0::(2.72); + let store_bytes = bincode::encode_to_vec(a, bincode::config::standard()).unwrap(); + println!("Bincode encode {:?} => {:?}", a, store_bytes); + let res = bincode::decode_from_slice(&store_bytes, bincode::config::standard()); + println!("{:?}", res); + assert_eq!(a, res.unwrap()); + } + + { + let a = arr1::(&[2.72, 1., 2.]); + let store_bytes = bincode::encode_to_vec(a, bincode::config::standard()).unwrap(); + println!("Bincode encode {:?} => {:?}", a, store_bytes); + let res = bincode::decode_from_slice(&store_bytes, bincode::config::standard()); + println!("{:?}", res); + assert_eq!(a, res.unwrap()); + } + + { + let a = arr2(&[[3., 1., 2.2], [3.1, 4., 7.]]); + let store_bytes = bincode::encode_to_vec(a, bincode::config::standard()).unwrap(); + println!("Bincode encode {:?} => {:?}", a, store_bytes); + let res = bincode::decode_from_slice(&store_bytes, bincode::config::standard()); + println!("{:?}", res); + assert_eq!(a, res.unwrap()); + } + + { + // Test a sliced array. + let mut a = ArcArray::from_iter(0..32) + .into_shape_with_order((2, 2, 2, 4)) + .unwrap(); + a.slice_collapse(s![..;-1, .., .., ..2]); + let store_bytes = bincode::encode_to_vec(a, bincode::config::standard()).unwrap(); + println!("Bincode encode {:?} => {:?}", a, store_bytes); + let res = bincode::decode_from_slice(&store_bytes, bincode::config::standard()); + println!("{:?}", res); + assert_eq!(a, res.unwrap()); + } +} + +#[test] +fn serial_ixdyn_serde() { + { + let a = arr0::(2.72).into_dyn(); + let store_bytes = bincode::encode_to_vec(a, bincode::config::standard()).unwrap(); + println!("Bincode encode {:?} => {:?}", a, store_bytes); + let res = bincode::decode_from_slice(&store_bytes, bincode::config::standard()); + println!("{:?}", res); + assert_eq!(a, res.unwrap()); + } + + { + let a = arr1::(&[2.72, 1., 2.]).into_dyn(); + let store_bytes = bincode::encode_to_vec(a, bincode::config::standard()).unwrap(); + println!("Bincode encode {:?} => {:?}", a, store_bytes); + let res = bincode::decode_from_slice(&store_bytes, bincode::config::standard()); + println!("{:?}", res); + assert_eq!(a, res.unwrap()); + } + + { + let a = arr2(&[[3., 1., 2.2], [3.1, 4., 7.]]) + .into_shape_with_order(IxDyn(&[3, 1, 1, 1, 2, 1])) + .unwrap(); + let store_bytes = bincode::encode_to_vec(a, bincode::config::standard()).unwrap(); + println!("Bincode encode {:?} => {:?}", a, store_bytes); + let res = bincode::decode_from_slice(&store_bytes, bincode::config::standard()); + println!("{:?}", res); + assert_eq!(a, res.unwrap()); + } +} diff --git a/src/array_bincode.rs b/src/array_bincode.rs new file mode 100644 index 000000000..257d72f19 --- /dev/null +++ b/src/array_bincode.rs @@ -0,0 +1,140 @@ +// Copyright 2014-2025 MiyakoMeow and ndarray developers. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. +use bincode::{ + de::{BorrowDecoder, Decoder}, + enc::Encoder, + error::{DecodeError, EncodeError}, + BorrowDecode, Decode, Encode, +}; + +use alloc::format; +#[cfg(not(feature = "std"))] +use alloc::vec::Vec; + +use crate::{imp_prelude::*, IxDynImpl, ShapeError}; + +use super::arraytraits::ARRAY_FORMAT_VERSION; + +/// Verifies that the version of the deserialized array matches the current +/// `ARRAY_FORMAT_VERSION`. +pub fn verify_version(v: u8) -> Result<(), DecodeError> { + if v != ARRAY_FORMAT_VERSION { + let err_msg = format!("unknown array version: {}", v); + Err(DecodeError::OtherString(err_msg)) + } else { + Ok(()) + } +} + +/// **Requires crate feature `"bincode"`** +impl Encode for Dim +where + I: Encode, +{ + fn encode(&self, encoder: &mut E) -> Result<(), EncodeError> { + Encode::encode(&self.ix(), encoder) + } +} + +/// **Requires crate feature `"bincode"`** +impl Decode for Dim +where + I: Decode, +{ + fn decode>(decoder: &mut D) -> Result { + Ok(Dim::new(Decode::decode(decoder)?)) + } +} + +/// **Requires crate feature `"bincode"`** +impl<'de, Context, I> BorrowDecode<'de, Context> for Dim +where + I: BorrowDecode<'de, Context>, +{ + fn borrow_decode>(decoder: &mut D) -> Result { + Ok(Dim::new(BorrowDecode::borrow_decode(decoder)?)) + } +} + +/// **Requires crate feature `"bincode"`** +impl Encode for IxDyn { + fn encode(&self, encoder: &mut E) -> Result<(), EncodeError> { + let ix: &IxDynImpl = self.ix(); + Encode::encode(&ix.len(), encoder)?; + for ix in ix.into_iter() { + Encode::encode(ix, encoder)?; + } + Ok(()) + } +} + +/// **Requires crate feature `"bincode"`** +impl Decode for IxDynImpl { + fn decode>(decoder: &mut D) -> Result { + let len: usize = Decode::decode(decoder)?; + let vals = (0..len) + .map(|_| Decode::decode(decoder)) + .collect::, DecodeError>>()?; + Ok(IxDynImpl::from(vals)) + } +} + +/// **Requires crate feature `"bincode"`** +impl<'de, Context> bincode::BorrowDecode<'de, Context> for IxDynImpl { + fn borrow_decode>(decoder: &mut D) -> Result { + let len: usize = BorrowDecode::borrow_decode(decoder)?; + let vals = (0..len) + .map(|_| BorrowDecode::borrow_decode(decoder)) + .collect::, DecodeError>>()?; + Ok(IxDynImpl::from(vals)) + } +} + +/// **Requires crate feature `"serde"`** +impl Encode for ArrayBase +where + A: Encode, + D: Dimension + Encode, + S: Data, +{ + fn encode(&self, encoder: &mut E) -> Result<(), EncodeError> { + Encode::encode(&ARRAY_FORMAT_VERSION, encoder)?; + Encode::encode(&self.raw_dim(), encoder)?; + let iter = self.iter(); + Encode::encode(&iter.len(), encoder)?; + for elt in iter.clone() { + Encode::encode(elt, encoder)?; + } + Ok(()) + } +} + +/// **Requires crate feature `"bincode"`** +impl Decode for ArrayBase +where + A: Decode, + D: Dimension + Decode, + S: DataOwned, +{ + fn decode>(decoder: &mut De) -> Result { + let data_version: u8 = Decode::decode(decoder)?; + (data_version == ARRAY_FORMAT_VERSION) + .then_some(()) + .ok_or(DecodeError::Other("ARRAY_FORMAT_VERSION not match!"))?; + let dim: D = Decode::decode(decoder)?; + let data_len: usize = Decode::decode(decoder)?; + let data = (0..data_len) + .map(|_| Decode::decode(decoder)) + .collect::, DecodeError>>()?; + let expected_size = dim.size(); + ArrayBase::from_shape_vec(dim, data).map_err(|_err: ShapeError| DecodeError::ArrayLengthMismatch { + required: expected_size, + found: data_len, + }) + } +} diff --git a/src/arraytraits.rs b/src/arraytraits.rs index a34b1985e..7920bd48d 100644 --- a/src/arraytraits.rs +++ b/src/arraytraits.rs @@ -432,8 +432,8 @@ where { } -#[cfg(feature = "serde")] -#[cfg_attr(docsrs, doc(cfg(feature = "serde")))] +#[cfg(any(feature = "serde", feature = "bincode"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "serde", feature = "bincode"))))] // Use version number so we can add a packed format later. pub const ARRAY_FORMAT_VERSION: u8 = 1u8; diff --git a/src/lib.rs b/src/lib.rs index 3efb378ce..a1fe1fa2e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -175,6 +175,8 @@ mod itertools; mod argument_traits; #[cfg(feature = "serde")] mod array_serde; +#[cfg(feature = "bincode")] +mod array_bincode; mod arrayformat; mod arraytraits; pub use crate::argument_traits::AssignElem; From 07d38538be6ffe85fab1aa4da897500a763c04ac Mon Sep 17 00:00:00 2001 From: MiyakoMeow <110924386+MiyakoMeow@users.noreply.github.com> Date: Sat, 21 Jun 2025 05:54:48 +0800 Subject: [PATCH 3/7] Revert "add(feature): bincode" This reverts commit 6f8f7c9f81a0eeb27d8bbd2364326a2f4ad4c4fa. --- Cargo.lock | 33 ---------- Cargo.toml | 31 ++++------ src/array_bincode.rs | 141 ------------------------------------------- src/lib.rs | 75 +++++++++++++++-------- 4 files changed, 61 insertions(+), 219 deletions(-) delete mode 100644 src/array_bincode.rs diff --git a/Cargo.lock b/Cargo.lock index abdfc85e4..057a36d03 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -47,26 +47,6 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" -[[package]] -name = "bincode" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" -dependencies = [ - "bincode_derive", - "serde", - "unty", -] - -[[package]] -name = "bincode_derive" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" -dependencies = [ - "virtue", -] - [[package]] name = "bitflags" version = "2.9.1" @@ -480,7 +460,6 @@ name = "ndarray" version = "0.16.1" dependencies = [ "approx", - "bincode", "cblas-sys", "defmac", "itertools", @@ -1150,12 +1129,6 @@ dependencies = [ "tinyvec", ] -[[package]] -name = "unty" -version = "0.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" - [[package]] name = "ureq" version = "2.10.1" @@ -1194,12 +1167,6 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" -[[package]] -name = "virtue" -version = "0.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" diff --git a/Cargo.toml b/Cargo.toml index 2b49dbe47..14226986e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,10 @@ name = "ndarray" version = "0.16.1" edition = "2021" rust-version = "1.64" -authors = ["Ulrik Sverdrup \"bluss\"", "Jim Turner"] +authors = [ + "Ulrik Sverdrup \"bluss\"", + "Jim Turner" +] license = "MIT OR Apache-2.0" readme = "README-crates.io.md" @@ -36,20 +39,11 @@ rayon = { version = "1.10.0", optional = true } cblas-sys = { workspace = true, optional = true } libc = { version = "0.2.82", optional = true } -matrixmultiply = { version = "0.3.2", default-features = false, features = [ - "cgemm", -] } +matrixmultiply = { version = "0.3.2", default-features = false, features=["cgemm"] } -serde = { version = "1.0", optional = true, default-features = false, features = [ - "alloc", -] } +serde = { version = "1.0", optional = true, default-features = false, features = ["alloc"] } rawpointer = { version = "0.2" } -bincode = { version = "2.0", optional = true, default-features = false, features = [ - "alloc", - "derive", -] } - [dev-dependencies] defmac = "0.2" quickcheck = { workspace = true } @@ -66,8 +60,6 @@ blas = ["dep:cblas-sys", "dep:libc"] serde = ["dep:serde"] -bincode = ["dep:bincode"] - std = ["num-traits/std", "matrixmultiply/std"] rayon = ["dep:rayon", "std"] @@ -78,10 +70,13 @@ portable-atomic-critical-section = ["portable-atomic/critical-section"] [target.'cfg(not(target_has_atomic = "ptr"))'.dependencies] portable-atomic = { version = "1.6.0" } -portable-atomic-util = { version = "0.2.0", features = ["alloc"] } +portable-atomic-util = { version = "0.2.0", features = [ "alloc" ] } [workspace] -members = ["ndarray-rand", "crates/*"] +members = [ + "ndarray-rand", + "crates/*", +] default-members = [ ".", "ndarray-rand", @@ -103,9 +98,7 @@ approx = { version = "0.5", default-features = false } quickcheck = { version = "1.0", default-features = false } rand = { version = "0.9.0", features = ["small_rng"] } rand_distr = { version = "0.5.0" } -itertools = { version = "0.13.0", default-features = false, features = [ - "use_std", -] } +itertools = { version = "0.13.0", default-features = false, features = ["use_std"] } cblas-sys = { version = "0.1.4", default-features = false } [profile.bench] diff --git a/src/array_bincode.rs b/src/array_bincode.rs deleted file mode 100644 index 02a51a126..000000000 --- a/src/array_bincode.rs +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2014-2025 bluss and ndarray developers. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. -use bincode::{ - de::{BorrowDecoder, Decoder}, - enc::Encoder, - error::{DecodeError, EncodeError}, - BorrowDecode, Decode, Encode, -}; - -use alloc::format; -#[cfg(not(feature = "std"))] -use alloc::vec::Vec; - -use crate::{imp_prelude::*, IntoDimension, IxDynImpl, ShapeError}; - -use super::arraytraits::ARRAY_FORMAT_VERSION; -use crate::iterators::Iter; - -/// Verifies that the version of the deserialized array matches the current -/// `ARRAY_FORMAT_VERSION`. -pub fn verify_version(v: u8) -> Result<(), DecodeError> { - if v != ARRAY_FORMAT_VERSION { - let err_msg = format!("unknown array version: {}", v); - Err(DecodeError::OtherString(err_msg)) - } else { - Ok(()) - } -} - -/// **Requires crate feature `"bincode"`** -impl Encode for Dim -where - I: Encode, -{ - fn encode(&self, encoder: &mut E) -> Result<(), EncodeError> { - Encode::encode(&self.ix(), encoder) - } -} - -/// **Requires crate feature `"bincode"`** -impl Decode for Dim -where - I: Decode, -{ - fn decode>(decoder: &mut D) -> Result { - Ok(Dim::new(Decode::decode(decoder)?)) - } -} - -/// **Requires crate feature `"bincode"`** -impl<'de, Context, I> BorrowDecode<'de, Context> for Dim -where - I: BorrowDecode<'de, Context>, -{ - fn borrow_decode>(decoder: &mut D) -> Result { - Ok(Dim::new(BorrowDecode::borrow_decode(decoder)?)) - } -} - -/// **Requires crate feature `"bincode"`** -impl Encode for IxDyn { - fn encode(&self, encoder: &mut E) -> Result<(), EncodeError> { - let ix: &IxDynImpl = self.ix(); - Encode::encode(&ix.len(), encoder)?; - for ix in ix.into_iter() { - Encode::encode(ix, encoder)?; - } - Ok(()) - } -} - -/// **Requires crate feature `"bincode"`** -impl Decode for IxDynImpl { - fn decode>(decoder: &mut D) -> Result { - let len: usize = Decode::decode(decoder)?; - let vals = (0..len) - .map(|_| Decode::decode(decoder)) - .collect::, DecodeError>>()?; - Ok(IxDynImpl::from(vals)) - } -} - -/// **Requires crate feature `"bincode"`** -impl<'de, Context> bincode::BorrowDecode<'de, Context> for IxDynImpl { - fn borrow_decode>(decoder: &mut D) -> Result { - let len: usize = BorrowDecode::borrow_decode(decoder)?; - let vals = (0..len) - .map(|_| BorrowDecode::borrow_decode(decoder)) - .collect::, DecodeError>>()?; - Ok(IxDynImpl::from(vals)) - } -} - -/// **Requires crate feature `"serde"`** -impl Encode for ArrayBase -where - A: Encode, - D: Dimension + Encode, - S: Data, -{ - fn encode(&self, encoder: &mut E) -> Result<(), EncodeError> { - Encode::encode(&ARRAY_FORMAT_VERSION, encoder)?; - Encode::encode(&self.raw_dim(), encoder)?; - let iter = self.iter(); - Encode::encode(&iter.len(), encoder)?; - for elt in iter.clone() { - Encode::encode(elt, encoder)?; - } - Ok(()) - } -} - -/// **Requires crate feature `"bincode"`** -impl Decode for ArrayBase -where - A: Decode, - D: Dimension + Decode, - S: DataOwned, -{ - fn decode>(decoder: &mut De) -> Result { - let data_version: u8 = Decode::decode(decoder)?; - (data_version == ARRAY_FORMAT_VERSION) - .then_some(()) - .ok_or(DecodeError::Other("ARRAY_FORMAT_VERSION not match!"))?; - let dim: D = Decode::decode(decoder)?; - let data_len: usize = Decode::decode(decoder)?; - let data = (0..data_len) - .map(|_| Decode::decode(decoder)) - .collect::, DecodeError>>()?; - let expected_size = dim.size(); - ArrayBase::from_shape_vec(dim, data).map_err(|_err: ShapeError| DecodeError::ArrayLengthMismatch { - required: expected_size, - found: data_len, - }) - } -} diff --git a/src/lib.rs b/src/lib.rs index e1be1973c..3efb378ce 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -175,8 +175,6 @@ mod itertools; mod argument_traits; #[cfg(feature = "serde")] mod array_serde; -#[cfg(feature = "bincode")] -mod array_bincode; mod arrayformat; mod arraytraits; pub use crate::argument_traits::AssignElem; @@ -226,12 +224,24 @@ pub use crate::zip::{FoldWhile, IntoNdProducer, NdProducer, Zip}; pub use crate::layout::Layout; /// Implementation's prelude. Common types used everywhere. -mod imp_prelude { +mod imp_prelude +{ pub use crate::dimension::DimensionExt; pub use crate::prelude::*; pub use crate::ArcArray; pub use crate::{ - CowRepr, Data, DataMut, DataOwned, DataShared, Ix, Ixs, RawData, RawDataMut, RawViewRepr, RemoveAxis, ViewRepr, + CowRepr, + Data, + DataMut, + DataOwned, + DataShared, + Ix, + Ixs, + RawData, + RawDataMut, + RawViewRepr, + RemoveAxis, + ViewRepr, }; } @@ -1282,8 +1292,7 @@ pub type Ixs = isize; // // [`.offset()`]: https://doc.rust-lang.org/stable/std/primitive.pointer.html#method.offset-1 pub struct ArrayBase::Elem> -where - S: RawData, +where S: RawData { /// Data buffer / ownership information. (If owned, contains the data /// buffer; if borrowed, contains the lifetime and mutability.) @@ -1392,7 +1401,8 @@ where // alter the offset of the pointer. This is allowed, as it does not // cause a pointer deref. #[derive(Debug)] -pub struct LayoutRef { +pub struct LayoutRef +{ /// A non-null pointer into the buffer held by `data`; may point anywhere /// in its range. If `S: Data`, this pointer must be aligned. ptr: std::ptr::NonNull, @@ -1633,8 +1643,10 @@ pub use data_repr::OwnedRepr; #[derive(Debug)] pub struct OwnedArcRepr(Arc>); -impl Clone for OwnedArcRepr { - fn clone(&self) -> Self { +impl Clone for OwnedArcRepr +{ + fn clone(&self) -> Self + { OwnedArcRepr(self.0.clone()) } } @@ -1645,13 +1657,16 @@ impl Clone for OwnedArcRepr { /// [`RawArrayView`] / [`RawArrayViewMut`] for the array type!* #[derive(Copy, Clone)] // This is just a marker type, to carry the mutability and element type. -pub struct RawViewRepr { +pub struct RawViewRepr +{ ptr: PhantomData, } -impl RawViewRepr { +impl RawViewRepr +{ #[inline(always)] - const fn new() -> Self { + const fn new() -> Self + { RawViewRepr { ptr: PhantomData } } } @@ -1662,13 +1677,16 @@ impl RawViewRepr { /// [`ArrayView`] / [`ArrayViewMut`] for the array type!* #[derive(Copy, Clone)] // This is just a marker type, to carry the lifetime parameter. -pub struct ViewRepr { +pub struct ViewRepr +{ life: PhantomData, } -impl ViewRepr { +impl ViewRepr +{ #[inline(always)] - const fn new() -> Self { + const fn new() -> Self + { ViewRepr { life: PhantomData } } } @@ -1677,16 +1695,19 @@ impl ViewRepr { /// /// *Don't use this type directly—use the type alias /// [`CowArray`] for the array type!* -pub enum CowRepr<'a, A> { +pub enum CowRepr<'a, A> +{ /// Borrowed data. View(ViewRepr<&'a A>), /// Owned data. Owned(OwnedRepr), } -impl CowRepr<'_, A> { +impl CowRepr<'_, A> +{ /// Returns `true` iff the data is the `View` variant. - pub fn is_view(&self) -> bool { + pub fn is_view(&self) -> bool + { match self { CowRepr::View(_) => true, CowRepr::Owned(_) => false, @@ -1694,7 +1715,8 @@ impl CowRepr<'_, A> { } /// Returns `true` iff the data is the `Owned` variant. - pub fn is_owned(&self) -> bool { + pub fn is_owned(&self) -> bool + { match self { CowRepr::View(_) => false, CowRepr::Owned(_) => true, @@ -1716,11 +1738,11 @@ mod impl_owned_array; mod impl_special_element_types; /// Private Methods -impl ArrayRef { +impl ArrayRef +{ #[inline] fn broadcast_unwrap(&self, dim: E) -> ArrayView<'_, A, E> - where - E: Dimension, + where E: Dimension { #[cold] #[inline(never)] @@ -1742,8 +1764,7 @@ impl ArrayRef { // (Checked in debug assertions). #[inline] fn broadcast_assume(&self, dim: E) -> ArrayView<'_, A, E> - where - E: Dimension, + where E: Dimension { let dim = dim.into_dimension(); debug_assert_eq!(self.shape(), dim.slice()); @@ -1760,7 +1781,8 @@ where D: Dimension, { /// Remove array axis `axis` and return the result. - fn try_remove_axis(self, axis: Axis) -> ArrayBase { + fn try_remove_axis(self, axis: Axis) -> ArrayBase + { let d = self.layout.dim.try_remove_axis(axis); let s = self.layout.strides.try_remove_axis(axis); // safe because new dimension, strides allow access to a subset of old data @@ -1800,7 +1822,8 @@ mod impl_cow; mod impl_arc_array; /// Returns `true` if the pointer is aligned. -pub(crate) fn is_aligned(ptr: *const T) -> bool { +pub(crate) fn is_aligned(ptr: *const T) -> bool +{ (ptr as usize) % ::std::mem::align_of::() == 0 } From cb691a158c58ab9ecc473ccf993027f79d85c4c6 Mon Sep 17 00:00:00 2001 From: MiyakoMeow <110924386+MiyakoMeow@users.noreply.github.com> Date: Sat, 21 Jun 2025 06:15:08 +0800 Subject: [PATCH 4/7] fix: tests --- crates/bincode-store-tests/tests/serialize.rs | 78 --------------- .../tests/store_and_extract.rs | 98 +++++++++++++++++++ src/array_bincode.rs | 12 --- 3 files changed, 98 insertions(+), 90 deletions(-) delete mode 100644 crates/bincode-store-tests/tests/serialize.rs create mode 100644 crates/bincode-store-tests/tests/store_and_extract.rs diff --git a/crates/bincode-store-tests/tests/serialize.rs b/crates/bincode-store-tests/tests/serialize.rs deleted file mode 100644 index b2f11103e..000000000 --- a/crates/bincode-store-tests/tests/serialize.rs +++ /dev/null @@ -1,78 +0,0 @@ -extern crate ndarray; - -use ndarray::{arr0, arr1, arr2, s, ArcArray, ArcArray2, ArrayD, IxDyn}; - -#[test] -fn store_many_dim_excrate() { - { - let a = arr0::(2.72); - let store_bytes = bincode::encode_to_vec(a, bincode::config::standard()).unwrap(); - println!("Bincode encode {:?} => {:?}", a, store_bytes); - let res = bincode::decode_from_slice(&store_bytes, bincode::config::standard()); - println!("{:?}", res); - assert_eq!(a, res.unwrap()); - } - - { - let a = arr1::(&[2.72, 1., 2.]); - let store_bytes = bincode::encode_to_vec(a, bincode::config::standard()).unwrap(); - println!("Bincode encode {:?} => {:?}", a, store_bytes); - let res = bincode::decode_from_slice(&store_bytes, bincode::config::standard()); - println!("{:?}", res); - assert_eq!(a, res.unwrap()); - } - - { - let a = arr2(&[[3., 1., 2.2], [3.1, 4., 7.]]); - let store_bytes = bincode::encode_to_vec(a, bincode::config::standard()).unwrap(); - println!("Bincode encode {:?} => {:?}", a, store_bytes); - let res = bincode::decode_from_slice(&store_bytes, bincode::config::standard()); - println!("{:?}", res); - assert_eq!(a, res.unwrap()); - } - - { - // Test a sliced array. - let mut a = ArcArray::from_iter(0..32) - .into_shape_with_order((2, 2, 2, 4)) - .unwrap(); - a.slice_collapse(s![..;-1, .., .., ..2]); - let store_bytes = bincode::encode_to_vec(a, bincode::config::standard()).unwrap(); - println!("Bincode encode {:?} => {:?}", a, store_bytes); - let res = bincode::decode_from_slice(&store_bytes, bincode::config::standard()); - println!("{:?}", res); - assert_eq!(a, res.unwrap()); - } -} - -#[test] -fn serial_ixdyn_serde() { - { - let a = arr0::(2.72).into_dyn(); - let store_bytes = bincode::encode_to_vec(a, bincode::config::standard()).unwrap(); - println!("Bincode encode {:?} => {:?}", a, store_bytes); - let res = bincode::decode_from_slice(&store_bytes, bincode::config::standard()); - println!("{:?}", res); - assert_eq!(a, res.unwrap()); - } - - { - let a = arr1::(&[2.72, 1., 2.]).into_dyn(); - let store_bytes = bincode::encode_to_vec(a, bincode::config::standard()).unwrap(); - println!("Bincode encode {:?} => {:?}", a, store_bytes); - let res = bincode::decode_from_slice(&store_bytes, bincode::config::standard()); - println!("{:?}", res); - assert_eq!(a, res.unwrap()); - } - - { - let a = arr2(&[[3., 1., 2.2], [3.1, 4., 7.]]) - .into_shape_with_order(IxDyn(&[3, 1, 1, 1, 2, 1])) - .unwrap(); - let store_bytes = bincode::encode_to_vec(a, bincode::config::standard()).unwrap(); - println!("Bincode encode {:?} => {:?}", a, store_bytes); - let res = bincode::decode_from_slice(&store_bytes, bincode::config::standard()); - println!("{:?}", res); - assert_eq!(a, res.unwrap()); - } -} diff --git a/crates/bincode-store-tests/tests/store_and_extract.rs b/crates/bincode-store-tests/tests/store_and_extract.rs new file mode 100644 index 000000000..325318b2c --- /dev/null +++ b/crates/bincode-store-tests/tests/store_and_extract.rs @@ -0,0 +1,98 @@ +use ndarray::{arr0, arr1, arr2, s, ArcArray, ArrayBase, Dim, IxDyn, IxDynImpl, OwnedArcRepr, OwnedRepr}; +// No test: ArcArray2, ArrayD + +#[test] +fn store_many_dim_excrate() { + { + let a = arr0::(2.72); + let store_bytes = bincode::encode_to_vec(&a, bincode::config::standard()).unwrap(); + println!("Bincode encode {:?} => {:?}", &a, store_bytes); + let res = bincode::decode_from_slice::, Dim<[usize; 0]>>, _>( + &store_bytes, + bincode::config::standard(), + ); + println!("{:?}", res); + assert_eq!(a, res.unwrap().0); + } + + { + let a = arr1::(&[2.72, 1., 2.]); + let store_bytes = bincode::encode_to_vec(&a, bincode::config::standard()).unwrap(); + println!("Bincode encode {:?} => {:?}", &a, store_bytes); + let res = bincode::decode_from_slice::, Dim<[usize; 1]>>, _>( + &store_bytes, + bincode::config::standard(), + ); + println!("{:?}", res); + assert_eq!(a, res.unwrap().0); + } + + { + let a = arr2(&[[3., 1., 2.2], [3.1, 4., 7.]]); + let store_bytes = bincode::encode_to_vec(&a, bincode::config::standard()).unwrap(); + println!("Bincode encode {:?} => {:?}", &a, store_bytes); + let res = bincode::decode_from_slice::, Dim<[usize; 2]>>, _>( + &store_bytes, + bincode::config::standard(), + ); + println!("{:?}", res); + assert_eq!(a, res.unwrap().0); + } + + { + // Test a sliced array. + let mut a = ArcArray::from_iter(0..32) + .into_shape_with_order((2, 2, 2, 4)) + .unwrap(); + a.slice_collapse(s![..;-1, .., .., ..2]); + let store_bytes = bincode::encode_to_vec(&a, bincode::config::standard()).unwrap(); + println!("Bincode encode {:?} => {:?}", &a, store_bytes); + let res = bincode::decode_from_slice::, Dim<[usize; 4]>>, _>( + &store_bytes, + bincode::config::standard(), + ); + println!("{:?}", res); + assert_eq!(a, res.unwrap().0); + } +} + +#[test] +fn serial_ixdyn_serde() { + { + let a = arr0::(2.72).into_dyn(); + let store_bytes = bincode::encode_to_vec(&a, bincode::config::standard()).unwrap(); + println!("Bincode encode {:?} => {:?}", &a, store_bytes); + let res = bincode::decode_from_slice::, Dim>, _>( + &store_bytes, + bincode::config::standard(), + ); + println!("{:?}", res); + assert_eq!(a, res.unwrap().0); + } + + { + let a = arr1::(&[2.72, 1., 2.]).into_dyn(); + let store_bytes = bincode::encode_to_vec(&a, bincode::config::standard()).unwrap(); + println!("Bincode encode {:?} => {:?}", &a, store_bytes); + let res = bincode::decode_from_slice::, Dim>, _>( + &store_bytes, + bincode::config::standard(), + ); + println!("{:?}", res); + assert_eq!(a, res.unwrap().0); + } + + { + let a = arr2(&[[3., 1., 2.2], [3.1, 4., 7.]]) + .into_shape_with_order(IxDyn(&[3, 1, 1, 1, 2, 1])) + .unwrap(); + let store_bytes = bincode::encode_to_vec(&a, bincode::config::standard()).unwrap(); + println!("Bincode encode {:?} => {:?}", &a, store_bytes); + let res = bincode::decode_from_slice::, Dim>, _>( + &store_bytes, + bincode::config::standard(), + ); + println!("{:?}", res); + assert_eq!(a, res.unwrap().0); + } +} diff --git a/src/array_bincode.rs b/src/array_bincode.rs index 257d72f19..62c654777 100644 --- a/src/array_bincode.rs +++ b/src/array_bincode.rs @@ -12,7 +12,6 @@ use bincode::{ BorrowDecode, Decode, Encode, }; -use alloc::format; #[cfg(not(feature = "std"))] use alloc::vec::Vec; @@ -20,17 +19,6 @@ use crate::{imp_prelude::*, IxDynImpl, ShapeError}; use super::arraytraits::ARRAY_FORMAT_VERSION; -/// Verifies that the version of the deserialized array matches the current -/// `ARRAY_FORMAT_VERSION`. -pub fn verify_version(v: u8) -> Result<(), DecodeError> { - if v != ARRAY_FORMAT_VERSION { - let err_msg = format!("unknown array version: {}", v); - Err(DecodeError::OtherString(err_msg)) - } else { - Ok(()) - } -} - /// **Requires crate feature `"bincode"`** impl Encode for Dim where From c4d8bc8a83caa71c22db023ee250d14e1ee59b80 Mon Sep 17 00:00:00 2001 From: MiyakoMeow <110924386+MiyakoMeow@users.noreply.github.com> Date: Sat, 21 Jun 2025 06:16:54 +0800 Subject: [PATCH 5/7] add(doc): readme --- README.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.rst b/README.rst index 49558b1c1..2a33ed807 100644 --- a/README.rst +++ b/README.rst @@ -78,6 +78,10 @@ your `Cargo.toml`. - Enables serialization support for serde 1.x +- ``bincode`` + + - Enables bincode store support for bincode 2.x + - ``rayon`` - Enables parallel iterators, parallelized methods and ``par_azip!``. From a09b891abb9953c044f25020ff0c3b1c638f4cc7 Mon Sep 17 00:00:00 2001 From: MiyakoMeow <110924386+MiyakoMeow@users.noreply.github.com> Date: Mon, 23 Jun 2025 12:49:02 +0800 Subject: [PATCH 6/7] fix: add missing feature define --- Cargo.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 434ecb295..3bddf2943 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,6 +62,8 @@ blas = ["dep:cblas-sys", "dep:libc"] serde = ["dep:serde"] +bincode = ["dep:bincode"] + std = ["num-traits/std", "matrixmultiply/std"] rayon = ["dep:rayon", "std"] From e85593643c00f8e655160d9de611676867d6b3b1 Mon Sep 17 00:00:00 2001 From: MiyakoMeow <110924386+MiyakoMeow@users.noreply.github.com> Date: Mon, 23 Jun 2025 13:05:16 +0800 Subject: [PATCH 7/7] feat: use iter when possible --- src/array_bincode.rs | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/src/array_bincode.rs b/src/array_bincode.rs index 62c654777..bf6976657 100644 --- a/src/array_bincode.rs +++ b/src/array_bincode.rs @@ -35,7 +35,7 @@ where I: Decode, { fn decode>(decoder: &mut D) -> Result { - Ok(Dim::new(Decode::decode(decoder)?)) + Decode::decode(decoder).map(Dim::new) } } @@ -45,7 +45,7 @@ where I: BorrowDecode<'de, Context>, { fn borrow_decode>(decoder: &mut D) -> Result { - Ok(Dim::new(BorrowDecode::borrow_decode(decoder)?)) + BorrowDecode::borrow_decode(decoder).map(Dim::new) } } @@ -54,10 +54,8 @@ impl Encode for IxDyn { fn encode(&self, encoder: &mut E) -> Result<(), EncodeError> { let ix: &IxDynImpl = self.ix(); Encode::encode(&ix.len(), encoder)?; - for ix in ix.into_iter() { - Encode::encode(ix, encoder)?; - } - Ok(()) + ix.into_iter() + .try_for_each(|ix| Encode::encode(ix, encoder)) } } @@ -65,10 +63,10 @@ impl Encode for IxDyn { impl Decode for IxDynImpl { fn decode>(decoder: &mut D) -> Result { let len: usize = Decode::decode(decoder)?; - let vals = (0..len) + (0..len) .map(|_| Decode::decode(decoder)) - .collect::, DecodeError>>()?; - Ok(IxDynImpl::from(vals)) + .collect::, DecodeError>>() + .map(IxDynImpl::from) } } @@ -76,10 +74,10 @@ impl Decode for IxDynImpl { impl<'de, Context> bincode::BorrowDecode<'de, Context> for IxDynImpl { fn borrow_decode>(decoder: &mut D) -> Result { let len: usize = BorrowDecode::borrow_decode(decoder)?; - let vals = (0..len) + (0..len) .map(|_| BorrowDecode::borrow_decode(decoder)) - .collect::, DecodeError>>()?; - Ok(IxDynImpl::from(vals)) + .collect::, DecodeError>>() + .map(IxDynImpl::from) } } @@ -95,10 +93,8 @@ where Encode::encode(&self.raw_dim(), encoder)?; let iter = self.iter(); Encode::encode(&iter.len(), encoder)?; - for elt in iter.clone() { - Encode::encode(elt, encoder)?; - } - Ok(()) + iter.into_iter() + .try_for_each(|elt| Encode::encode(elt, encoder)) } } @@ -116,7 +112,7 @@ where .ok_or(DecodeError::Other("ARRAY_FORMAT_VERSION not match!"))?; let dim: D = Decode::decode(decoder)?; let data_len: usize = Decode::decode(decoder)?; - let data = (0..data_len) + let data: Vec<_> = (0..data_len) .map(|_| Decode::decode(decoder)) .collect::, DecodeError>>()?; let expected_size = dim.size();