Skip to content

Commit

Permalink
Merge branch 'main' into ielashi/add_v2
Browse files Browse the repository at this point in the history
  • Loading branch information
ielashi authored Aug 17, 2023
2 parents 92177a9 + d56fba1 commit 6f0091b
Show file tree
Hide file tree
Showing 14 changed files with 610 additions and 559 deletions.
8 changes: 4 additions & 4 deletions benchmark-canisters/src/btreemap.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use crate::{count_instructions, Random};
use ic_cdk_macros::query;
use ic_stable_structures::{storable::Blob, BTreeMap, BoundedStorable, DefaultMemoryImpl};
use ic_stable_structures::{storable::Blob, BTreeMap, DefaultMemoryImpl, Storable};
use tiny_rng::{Rand, Rng};

#[query]
Expand Down Expand Up @@ -216,7 +216,7 @@ fn insert_blob_helper<const K: usize, const V: usize>() -> u64 {
}

// Profiles inserting a large number of random blobs into a btreemap.
fn insert_helper<K: Clone + Ord + BoundedStorable + Random, V: BoundedStorable + Random>() -> u64 {
fn insert_helper<K: Clone + Ord + Storable + Random, V: Storable + Random>() -> u64 {
let mut btree: BTreeMap<K, V, _> = BTreeMap::new(DefaultMemoryImpl::default());
let num_keys = 10_000;
let mut rng = Rng::from_seed(0);
Expand All @@ -241,7 +241,7 @@ fn get_blob_helper<const K: usize, const V: usize>() -> u64 {
get_helper::<Blob<K>, Blob<V>>()
}

fn get_helper<K: Clone + Ord + BoundedStorable + Random, V: BoundedStorable + Random>() -> u64 {
fn get_helper<K: Clone + Ord + Storable + Random, V: Storable + Random>() -> u64 {
let mut btree: BTreeMap<K, V, _> = BTreeMap::new(DefaultMemoryImpl::default());
let num_keys = 10_000;
let mut rng = Rng::from_seed(0);
Expand Down Expand Up @@ -271,7 +271,7 @@ fn remove_blob_helper<const K: usize, const V: usize>() -> u64 {
remove_helper::<Blob<K>, Blob<V>>()
}

fn remove_helper<K: Clone + Ord + BoundedStorable + Random, V: BoundedStorable + Random>() -> u64 {
fn remove_helper<K: Clone + Ord + Storable + Random, V: Storable + Random>() -> u64 {
let mut btree: BTreeMap<K, V, _> = BTreeMap::new(DefaultMemoryImpl::default());
let num_keys = 10_000;
let mut rng = Rng::from_seed(0);
Expand Down
12 changes: 10 additions & 2 deletions benchmark-canisters/src/main.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use ic_stable_structures::{storable::Blob, BoundedStorable};
use ic_stable_structures::storable::{Blob, Bound, Storable};
use tiny_rng::{Rand, Rng};

mod btreemap;
Expand All @@ -12,13 +12,21 @@ pub(crate) fn count_instructions<R>(f: impl FnOnce() -> R) -> u64 {
ic_cdk::api::performance_counter(0) - start
}

const fn max_size<A: Storable>() -> u32 {
if let Bound::Bounded { max_size, .. } = A::BOUND {
max_size
} else {
panic!("Cannot get max size of unbounded type.");
}
}

trait Random {
fn random(rng: &mut Rng) -> Self;
}

impl<const K: usize> Random for Blob<K> {
fn random(rng: &mut Rng) -> Self {
let size = rng.rand_u32() % Blob::<K>::MAX_SIZE;
let size = rng.rand_u32() % max_size::<Blob<K>>();
Blob::try_from(
rng.iter(Rand::rand_u8)
.take(size as usize)
Expand Down
6 changes: 3 additions & 3 deletions benchmark-canisters/src/vec.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use crate::{count_instructions, Random};
use ic_cdk_macros::query;
use ic_stable_structures::storable::Blob;
use ic_stable_structures::{BoundedStorable, DefaultMemoryImpl, StableVec};
use ic_stable_structures::{DefaultMemoryImpl, StableVec, Storable};
use tiny_rng::{Rand, Rng};

#[query]
Expand Down Expand Up @@ -78,7 +78,7 @@ fn vec_insert_blob<const N: usize>() -> u64 {
vec_insert::<Blob<N>>()
}

fn vec_insert<T: BoundedStorable + Random>() -> u64 {
fn vec_insert<T: Storable + Random>() -> u64 {
let num_items = 10_000;
let svec: StableVec<T, _> = StableVec::new(DefaultMemoryImpl::default()).unwrap();

Expand All @@ -100,7 +100,7 @@ fn vec_get_blob<const N: usize>() -> u64 {
vec_get::<Blob<N>>()
}

fn vec_get<T: BoundedStorable + Random>() -> u64 {
fn vec_get<T: Storable + Random>() -> u64 {
let num_items = 10_000;
let svec: StableVec<T, _> = StableVec::new(DefaultMemoryImpl::default()).unwrap();

Expand Down
14 changes: 8 additions & 6 deletions examples/src/custom_types_example/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
use candid::{CandidType, Decode, Deserialize, Encode};
use ic_stable_structures::memory_manager::{MemoryId, MemoryManager, VirtualMemory};
use ic_stable_structures::{BoundedStorable, DefaultMemoryImpl, StableBTreeMap, Storable};
use ic_stable_structures::{
storable::Bound, DefaultMemoryImpl, StableBTreeMap, Storable,
};
use std::{borrow::Cow, cell::RefCell};

type Memory = VirtualMemory<DefaultMemoryImpl>;
Expand All @@ -14,7 +16,7 @@ struct UserProfile {
}

// For a type to be used in a `StableBTreeMap`, it needs to implement the `Storable`
// and `BoundedStorable` traits, which specify how the type can be serialized/deserialized.
// trait, which specifies how the type can be serialized/deserialized.
//
// In this example, we're using candid to serialize/deserialize the struct, but you
// can use anything as long as you're maintaining backward-compatibility. The
Expand All @@ -31,11 +33,11 @@ impl Storable for UserProfile {
fn from_bytes(bytes: std::borrow::Cow<[u8]>) -> Self {
Decode!(bytes.as_ref(), Self).unwrap()
}
}

impl BoundedStorable for UserProfile {
const MAX_SIZE: u32 = MAX_VALUE_SIZE;
const IS_FIXED_SIZE: bool = false;
const BOUND: Bound = Bound::Bounded {
max_size: MAX_VALUE_SIZE,
is_fixed_size: false,
};
}

thread_local! {
Expand Down
18 changes: 9 additions & 9 deletions examples/src/vecs_and_strings/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
use ic_stable_structures::memory_manager::{MemoryId, MemoryManager, VirtualMemory};
use ic_stable_structures::{BoundedStorable, DefaultMemoryImpl, StableBTreeMap, Storable};
use ic_stable_structures::{storable::Bound, DefaultMemoryImpl, StableBTreeMap, Storable};
use std::cell::RefCell;

type Memory = VirtualMemory<DefaultMemoryImpl>;
Expand All @@ -25,11 +25,11 @@ impl Storable for UserName {
fn from_bytes(bytes: std::borrow::Cow<[u8]>) -> Self {
Self(String::from_bytes(bytes))
}
}

impl BoundedStorable for UserName {
const MAX_SIZE: u32 = MAX_USER_NAME_SIZE;
const IS_FIXED_SIZE: bool = false;
const BOUND: Bound = Bound::Bounded {
max_size: MAX_USER_NAME_SIZE,
is_fixed_size: false,
};
}

struct UserData(Vec<u8>);
Expand All @@ -43,11 +43,11 @@ impl Storable for UserData {
fn from_bytes(bytes: std::borrow::Cow<[u8]>) -> Self {
Self(<Vec<u8>>::from_bytes(bytes))
}
}

impl BoundedStorable for UserData {
const MAX_SIZE: u32 = MAX_USER_DATA_SIZE;
const IS_FIXED_SIZE: bool = false;
const BOUND: Bound = Bound::Bounded {
max_size: MAX_USER_DATA_SIZE,
is_fixed_size: false,
};
}

thread_local! {
Expand Down
69 changes: 36 additions & 33 deletions src/base_vec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,15 +28,12 @@
//! ```
//!
//! The `SLOT_SIZE` constant depends on the item type. If the item
//! type sets the `BoundedStorable::IS_FIXED_SIZE` flag, the
//! `SLOT_SIZE` is equal to `BoundedStorable::MAX_SIZE`. Otherwise,
//! the `SLOT_SIZE` is `BoundedStorable::MAX_SIZE` plus the number of
//! bytes required to represent integers up to
//! `BoundedStorable::MAX_SIZE`.
use crate::storable::bytes_to_store_size;
//! type is fixed in size, the `SLOT_SIZE` is equal to the max size.
//! Otherwise, the `SLOT_SIZE` is the max size plus the number of
//! bytes required to represent integers up to that max size.
use crate::storable::{bounds, bytes_to_store_size};
use crate::{
read_u32, read_u64, safe_write, write_u32, write_u64, Address, BoundedStorable, GrowFailed,
Memory,
read_u32, read_u64, safe_write, write_u32, write_u64, Address, GrowFailed, Memory, Storable,
};
use std::borrow::{Borrow, Cow};
use std::fmt;
Expand Down Expand Up @@ -66,8 +63,8 @@ pub enum InitError {
/// memory layout.
IncompatibleVersion(u8),
/// The vector type is not compatible with the current vector
/// layout: MAX_SIZE and/or IS_FIXED_SIZE differ from the original
/// initialization parameters.
/// layout: the type's bounds differ from the original initialization
/// parameters.
IncompatibleElementType,
/// Failed to allocate memory for the vector.
OutOfMemory,
Expand All @@ -85,32 +82,34 @@ impl fmt::Display for InitError {
"unsupported layout version {version}; supported version numbers are 1..={LAYOUT_VERSION}"
),
Self::IncompatibleElementType =>
write!(fmt, "either MAX_SIZE or IS_FIXED_SIZE of the element type do not match the persisted vector attributes"),
write!(fmt, "the bounds (either max_size or is_fixed_size) of the element type do not match the persisted vector attributes"),
Self::OutOfMemory => write!(fmt, "failed to allocate memory for vector metadata"),
}
}
}

impl std::error::Error for InitError {}

pub struct BaseVec<T: BoundedStorable, M: Memory> {
pub struct BaseVec<T: Storable, M: Memory> {
memory: M,
_marker: PhantomData<T>,
}

impl<T: BoundedStorable, M: Memory> BaseVec<T, M> {
impl<T: Storable, M: Memory> BaseVec<T, M> {
/// Creates a new empty vector in the specified memory,
/// overwriting any data structures the memory might have
/// contained previously.
///
/// Complexity: O(1)
pub fn new(memory: M, magic: [u8; 3]) -> Result<Self, GrowFailed> {
let t_bounds = bounds::<T>();

let header = HeaderV1 {
magic,
version: LAYOUT_VERSION,
len: 0,
max_size: T::MAX_SIZE,
is_fixed_size: T::IS_FIXED_SIZE,
max_size: t_bounds.max_size,
is_fixed_size: t_bounds.is_fixed_size,
};
Self::write_header(&header, &memory)?;
Ok(Self {
Expand Down Expand Up @@ -139,7 +138,8 @@ impl<T: BoundedStorable, M: Memory> BaseVec<T, M> {
if header.version != LAYOUT_VERSION {
return Err(InitError::IncompatibleVersion(header.version));
}
if header.max_size != T::MAX_SIZE || header.is_fixed_size != T::IS_FIXED_SIZE {
let t_bounds = bounds::<T>();
if header.max_size != t_bounds.max_size || header.is_fixed_size != t_bounds.is_fixed_size {
return Err(InitError::IncompatibleElementType);
}

Expand Down Expand Up @@ -170,7 +170,7 @@ impl<T: BoundedStorable, M: Memory> BaseVec<T, M> {

/// Sets the item at the specified index to the specified value.
///
/// Complexity: O(T::MAX_SIZE)
/// Complexity: O(max_size(T))
///
/// PRECONDITION: index < self.len()
pub fn set(&self, index: u64, item: &T) {
Expand All @@ -186,7 +186,7 @@ impl<T: BoundedStorable, M: Memory> BaseVec<T, M> {

/// Returns the item at the specified index.
///
/// Complexity: O(T::MAX_SIZE)
/// Complexity: O(max_size(T))
pub fn get(&self, index: u64) -> Option<T> {
if index < self.len() {
Some(self.read_entry(index))
Expand All @@ -197,7 +197,7 @@ impl<T: BoundedStorable, M: Memory> BaseVec<T, M> {

/// Adds a new item at the end of the vector.
///
/// Complexity: O(T::MAX_SIZE)
/// Complexity: O(max_size(T))
pub fn push(&self, item: &T) -> Result<(), GrowFailed> {
let index = self.len();
let offset = DATA_OFFSET + slot_size::<T>() as u64 * index;
Expand All @@ -212,7 +212,7 @@ impl<T: BoundedStorable, M: Memory> BaseVec<T, M> {

/// Removes the item at the end of the vector.
///
/// Complexity: O(T::MAX_SIZE)
/// Complexity: O(max_size(T))
pub fn pop(&self) -> Option<T> {
let len = self.len();
if len == 0 {
Expand Down Expand Up @@ -253,14 +253,15 @@ impl<T: BoundedStorable, M: Memory> BaseVec<T, M> {

/// Writes the size of the item at the specified offset.
fn write_entry_size(&self, offset: u64, size: u32) -> Result<u64, GrowFailed> {
debug_assert!(size <= T::MAX_SIZE);
let t_bounds = bounds::<T>();
debug_assert!(size <= t_bounds.max_size);

if T::IS_FIXED_SIZE {
if t_bounds.is_fixed_size {
Ok(offset)
} else if T::MAX_SIZE <= u8::MAX as u32 {
} else if t_bounds.max_size <= u8::MAX as u32 {
safe_write(&self.memory, offset, &[size as u8; 1])?;
Ok(offset + 1)
} else if T::MAX_SIZE <= u16::MAX as u32 {
} else if t_bounds.max_size <= u16::MAX as u32 {
safe_write(&self.memory, offset, &(size as u16).to_le_bytes())?;
Ok(offset + 2)
} else {
Expand All @@ -271,13 +272,14 @@ impl<T: BoundedStorable, M: Memory> BaseVec<T, M> {

/// Reads the size of the entry at the specified offset.
fn read_entry_size(&self, offset: u64) -> (u64, usize) {
if T::IS_FIXED_SIZE {
(offset, T::MAX_SIZE as usize)
} else if T::MAX_SIZE <= u8::MAX as u32 {
let t_bounds = bounds::<T>();
if t_bounds.is_fixed_size {
(offset, t_bounds.max_size as usize)
} else if t_bounds.max_size <= u8::MAX as u32 {
let mut size = [0u8; 1];
self.memory.read(offset, &mut size);
(offset + 1, size[0] as usize)
} else if T::MAX_SIZE <= u16::MAX as u32 {
} else if t_bounds.max_size <= u16::MAX as u32 {
let mut size = [0u8; 2];
self.memory.read(offset, &mut size);
(offset + 2, u16::from_le_bytes(size) as usize)
Expand Down Expand Up @@ -326,19 +328,20 @@ impl<T: BoundedStorable, M: Memory> BaseVec<T, M> {
}
}

impl<T: BoundedStorable + fmt::Debug, M: Memory> fmt::Debug for BaseVec<T, M> {
impl<T: Storable + fmt::Debug, M: Memory> fmt::Debug for BaseVec<T, M> {
fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
self.to_vec().fmt(fmt)
}
}

fn slot_size<T: BoundedStorable>() -> u32 {
T::MAX_SIZE + bytes_to_store_size::<T>()
fn slot_size<T: Storable>() -> u32 {
let t_bounds = bounds::<T>();
t_bounds.max_size + bytes_to_store_size(&t_bounds)
}

pub struct Iter<'a, T, M>
where
T: BoundedStorable,
T: Storable,
M: Memory,
{
vec: &'a BaseVec<T, M>,
Expand All @@ -348,7 +351,7 @@ where

impl<T, M> Iterator for Iter<'_, T, M>
where
T: BoundedStorable,
T: Storable,
M: Memory,
{
type Item = T;
Expand Down
Loading

0 comments on commit 6f0091b

Please sign in to comment.