deer: testing framework (#1701)

packages/libs/deer/Cargo.toml

@@ -25,4 +25,4 @@ std = ['serde/std', 'error-stack/std']
 arbitrary-precision = []
 
 [workspace]
-members = ['.', 'macros', 'json']
+members = ['.', 'macros', 'json', 'desert']

packages/libs/deer/desert/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "deer-desert"
+version = "0.0.0"
+edition = "2021"
+# NOTE: THIS PACKAGE IS NEVER INTENDED TO BE PUBLISHED
+publish = false
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+deer = { path = ".." }
+error-stack = { version = "0.2.4", default_features = false }
+serde_json = { version = "1.0.91", default_features = false, features = ['alloc'] }
+bitvec = { version = "1", default_features = false, features = ['alloc', 'atomic'] }

packages/libs/deer/desert/README.md

@@ -0,0 +1,6 @@
+# deer-desert
+
+desert is the the internal only deserialization testing framework used throughout the integration tests and should never
+be published.
+
+`desert` = `deser` (`deserialization`) + `t` (`test`)

packages/libs/deer/desert/src/array.rs

@@ -0,0 +1,149 @@
+use deer::{
+    error::{
+        ArrayAccessError, ArrayLengthError, BoundedContractViolationError, ExpectedLength,
+        ReceivedLength, Variant,
+    },
+    Deserialize, Deserializer as _,
+};
+use error_stack::{Report, Result, ResultExt};
+
+use crate::{
+    deserializer::{Deserializer, DeserializerNone},
+    token::Token,
+};
+
+pub struct ArrayAccess<'a, 'b, 'de: 'a> {
+    deserializer: &'a mut Deserializer<'b, 'de>,
+
+    length: Option<usize>,
+    remaining: Option<usize>,
+    consumed: usize,
+}
+
+impl<'a, 'b, 'de> ArrayAccess<'a, 'b, 'de> {
+    pub fn new(deserializer: &'a mut Deserializer<'b, 'de>, length: Option<usize>) -> Self {
+        Self {
+            deserializer,
+            consumed: 0,
+            length,
+            remaining: None,
+        }
+    }
+
+    fn scan_end(&self) -> Option<usize> {
+        let mut objects: usize = 0;
+        let mut arrays: usize = 0;
+
+        let mut n = 0;
+
+        loop {
+            let token = self.deserializer.peek_n(n)?;
+
+            match token {
+                Token::Array { .. } => arrays += 1,
+                Token::ArrayEnd if arrays == 0 && objects == 0 => {
+                    // we're at the outer layer, meaning we can know where we end
+                    return Some(n);
+                }
+                Token::ArrayEnd => arrays = arrays.saturating_sub(1),
+                Token::Object { .. } => objects += 1,
+                Token::ObjectEnd => objects = objects.saturating_sub(1),
+                _ => {}
+            }
+
+            n += 1;
+        }
+    }
+}
+
+impl<'de> deer::ArrayAccess<'de> for ArrayAccess<'_, '_, 'de> {
+    fn set_bounded(&mut self, length: usize) -> Result<(), ArrayAccessError> {
+        if self.consumed > 0 {
+            return Err(
+                Report::new(BoundedContractViolationError::SetDirty.into_error())
+                    .change_context(ArrayAccessError),
+            );
+        }
+
+        if self.remaining.is_some() {
+            return Err(Report::new(
+                BoundedContractViolationError::SetCalledMultipleTimes.into_error(),
+            )
+            .change_context(ArrayAccessError));
+        }
+
+        self.remaining = Some(length);
+
+        Ok(())
+    }
+
+    fn next<T>(&mut self) -> Option<Result<T, ArrayAccessError>>
+    where
+        T: Deserialize<'de>,
+    {
+        self.consumed += 1;
+
+        if self.deserializer.peek() == Token::ArrayEnd {
+            // we have reached the ending, if `self.remaining` is set we use the `DeserializerNone`
+            // to deserialize any values that require `None`
+            if let Some(remaining) = &mut self.remaining {
+                if *remaining == 0 {
+                    return None;
+                }
+
+                *remaining = remaining.saturating_sub(1);
+
+                let value = T::deserialize(DeserializerNone {
+                    context: self.deserializer.context(),
+                });
+
+                Some(value.change_context(ArrayAccessError))
+            } else {
+                None
+            }
+        } else {
+            let value = T::deserialize(&mut *self.deserializer);
+            Some(value.change_context(ArrayAccessError))
+        }
+    }
+
+    fn size_hint(&self) -> Option<usize> {
+        self.length
+    }
+
+    fn end(self) -> Result<(), ArrayAccessError> {
+        let mut result = Ok(());
+
+        // ensure that we consume the last token, if it is the wrong token error out
+        if self.deserializer.peek() != Token::ArrayEnd {
+            let mut error = Report::new(ArrayLengthError.into_error())
+                .attach(ExpectedLength::new(self.consumed));
+
+            if let Some(length) = self.size_hint() {
+                error = error.attach(ReceivedLength::new(length));
+            }
+
+            result = Err(error);
+        }
+
+        // bump until the very end, which ensures that deserialize calls after this might succeed!
+        let bump = self
+            .scan_end()
+            .unwrap_or_else(|| self.deserializer.tape().remaining());
+        self.deserializer.tape_mut().bump_n(bump);
+
+        if let Some(remaining) = self.remaining {
+            if remaining > 0 {
+                let error =
+                    Report::new(BoundedContractViolationError::EndRemainingItems.into_error());
+
+                match &mut result {
+                    Err(result) => result.extend_one(error),
+                    result => *result = Err(error),
+                }
+            }
+        }
+
+        result.change_context(ArrayAccessError)
+    }
+}

packages/libs/deer/desert/src/assert.rs

@@ -0,0 +1,50 @@
+use core::fmt::Debug;
+
+use deer::{error::ReportExt, Context, Deserialize};
+use serde_json::to_value;
+
+use crate::{deserializer::Deserializer, token::Token};
+
+pub fn assert_tokens_with_context<'de, T>(expected: &T, tokens: &'de [Token], context: &Context)
+where
+    T: Deserialize<'de> + PartialEq + Debug,
+{
+    let mut de = Deserializer::new(tokens, context);
+    let received = T::deserialize(&mut de).expect("should deserialize");
+
+    if de.remaining() > 0 {
+        panic!("{} remaining tokens", de.remaining());
+    }
+
+    assert_eq!(received, *expected);
+}
+
+pub fn assert_tokens<'de, T>(value: &T, tokens: &'de [Token])
+where
+    T: Deserialize<'de> + PartialEq + Debug,
+{
+    assert_tokens_with_context(value, tokens, &Context::new());
+}
+
+pub fn assert_tokens_with_context_error<'de, T>(
+    error: &serde_json::Value,
+    tokens: &'de [Token],
+    context: &Context,
+) where
+    T: Deserialize<'de> + Debug,
+{
+    let mut de = Deserializer::new(tokens, context);
+    let received = T::deserialize(&mut de).expect_err("value of type T should fail serialization");
+
+    let received = received.export();
+    let received = to_value(received).expect("error should serialize");
+
+    assert_eq!(received, *error)
+}
+
+pub fn assert_tokens_error<'de, T>(error: &serde_json::Value, tokens: &'de [Token])
+where
+    T: Deserialize<'de> + Debug,
+{
+    assert_tokens_with_context_error::<T>(error, tokens, &Context::new());
+}

packages/libs/deer/desert/src/deserializer.rs

@@ -0,0 +1,145 @@
+use alloc::borrow::ToOwned;
+use core::ops::Range;
+
+use deer::{error::DeserializerError, Context, Visitor};
+use error_stack::{Result, ResultExt};
+
+use crate::{array::ArrayAccess, object::ObjectAccess, tape::Tape, token::Token};
+
+macro_rules! forward {
+    ($($method:ident),*) => {
+        $(
+        fn $method<V>(self, visitor: V) -> Result<V::Value, DeserializerError>
+        where
+            V: Visitor<'de>,
+        {
+            self.deserialize_any(visitor)
+        }
+        )*
+    };
+}
+
+#[derive(Debug)]
+pub struct Deserializer<'a, 'de> {
+    context: &'a Context,
+    tape: Tape<'a, 'de>,
+}
+
+impl<'a, 'de> Deserializer<'a, 'de> {
+    pub(crate) fn erase(&mut self, range: Range<usize>) {
+        self.tape.set_trivia(range);
+    }
+}
+
+impl<'a, 'de> deer::Deserializer<'de> for &mut Deserializer<'a, 'de> {
+    forward!(
+        deserialize_null,
+        deserialize_bool,
+        deserialize_number,
+        deserialize_char,
+        deserialize_string,
+        deserialize_str,
+        deserialize_bytes,
+        deserialize_bytes_buffer,
+        deserialize_array,
+        deserialize_object
+    );
+
+    fn context(&self) -> &Context {
+        self.context
+    }
+
+    fn deserialize_any<V>(self, visitor: V) -> Result<V::Value, DeserializerError>
+    where
+        V: Visitor<'de>,
+    {
+        let token = self.next();
+
+        match token {
+            Token::Bool(value) => visitor.visit_bool(value),
+            Token::Number(value) => visitor.visit_number(value.clone()),
+            Token::Char(value) => visitor.visit_char(value),
+            Token::Str(value) => visitor.visit_str(value),
+            Token::BorrowedStr(value) => visitor.visit_borrowed_str(value),
+            Token::String(value) => visitor.visit_string(value.to_owned()),
+            Token::Bytes(value) => visitor.visit_bytes(value),
+            Token::BorrowedBytes(value) => visitor.visit_borrowed_bytes(value),
+            Token::BytesBuf(value) => visitor.visit_bytes_buffer(value.to_vec()),
+            Token::Array { length } => visitor.visit_array(ArrayAccess::new(self, length)),
+            Token::Object { length } => visitor.visit_object(ObjectAccess::new(self, length)),
+            _ => {
+                panic!("Deserializer did not expect {token}");
+            }
+        }
+        .change_context(DeserializerError)
+    }
+}
+
+impl<'a, 'de> Deserializer<'a, 'de> {
+    pub(crate) fn new_bare(tape: Tape<'a, 'de>, context: &'a Context) -> Self {
+        Self { tape, context }
+    }
+
+    pub fn new(tokens: &'de [Token], context: &'a Context) -> Self {
+        Self::new_bare(tokens.into(), context)
+    }
+
+    pub(crate) fn peek(&self) -> Token {
+        self.tape.peek().expect("should have token to deserialize")
+    }
+
+    pub(crate) fn peek_n(&self, n: usize) -> Option<Token> {
+        self.tape.peek_n(n)
+    }
+
+    pub(crate) fn next(&mut self) -> Token {
+        self.tape.next().expect("should have token to deserialize")
+    }
+
+    pub(crate) fn tape(&self) -> &Tape<'a, 'de> {
+        &self.tape
+    }
+
+    pub(crate) fn tape_mut(&mut self) -> &mut Tape<'a, 'de> {
+        &mut self.tape
+    }
+
+    pub fn remaining(&self) -> usize {
+        self.tape.remaining()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.tape.is_empty()
+    }
+}
+
+#[derive(Debug)]
+pub(crate) struct DeserializerNone<'a> {
+    pub(crate) context: &'a Context,
+}
+
+impl<'de> deer::Deserializer<'de> for DeserializerNone<'_> {
+    forward!(
+        deserialize_null,
+        deserialize_bool,
+        deserialize_number,
+        deserialize_char,
+        deserialize_string,
+        deserialize_str,
+        deserialize_bytes,
+        deserialize_bytes_buffer,
+        deserialize_array,
+        deserialize_object
+    );
+
+    fn context(&self) -> &Context {
+        self.context
+    }
+
+    fn deserialize_any<V>(self, visitor: V) -> Result<V::Value, DeserializerError>
+    where
+        V: Visitor<'de>,
+    {
+        visitor.visit_none().change_context(DeserializerError)
+    }
+}

packages/libs/deer/desert/src/lib.rs

@@ -0,0 +1,16 @@
+#![no_std]
+
+extern crate alloc;
+
+pub(crate) mod array;
+mod assert;
+mod deserializer;
+pub(crate) mod object;
+pub(crate) mod tape;
+mod token;
+
+pub use assert::{
+    assert_tokens, assert_tokens_error, assert_tokens_with_context,
+    assert_tokens_with_context_error,
+};
+pub use token::Token;