Merge pull request #3132 from stacks-network/feat/to-from-buff

Implement consensus de/serialization in Clarity

clarity/src/vm/analysis/arithmetic_checker/mod.rs

@@ -181,7 +181,7 @@ impl<'a> ArithmeticOnlyChecker<'a> {
             | FetchEntry | SetEntry | DeleteEntry | InsertEntry | SetVar | MintAsset
             | MintToken | TransferAsset | TransferToken | ContractCall | StxTransfer
             | StxTransferMemo | StxBurn | AtBlock | GetStxBalance | GetTokenSupply | BurnToken
-            | BurnAsset | StxGetAccount => {
+            | FromConsensusBuff | ToConsensusBuff | BurnAsset | StxGetAccount => {
                 return Err(Error::FunctionNotPermitted(function));
             }
             Append | Concat | AsMaxLen | ContractOf | PrincipalOf | ListCons | Print

clarity/src/vm/analysis/errors.rs

@@ -68,6 +68,7 @@ pub enum CheckErrors {
     ExpectedOptionalOrResponseValue(Value),
     CouldNotDetermineResponseOkType,
     CouldNotDetermineResponseErrType,
+    CouldNotDetermineSerializationType,
     UncheckedIntermediaryResponses,
 
     CouldNotDetermineMatchTypes,
@@ -425,6 +426,7 @@ impl DiagnosableError for CheckErrors {
             },
             CheckErrors::UncheckedIntermediaryResponses => format!("intermediary responses in consecutive statements must be checked"),
             CheckErrors::CostComputationFailed(s) => format!("contract cost computation failed: {}", s),
+            CheckErrors::CouldNotDetermineSerializationType => format!("could not determine the input type for the serialization function"),
         }
     }
 

clarity/src/vm/analysis/read_only_checker/mod.rs

@@ -282,13 +282,19 @@ impl<'a, 'b> ReadOnlyChecker<'a, 'b> {
             | UnwrapErrRet | IsOkay | IsNone | Asserts | Unwrap | UnwrapErr | Match | IsErr
             | IsSome | TryRet | ToUInt | ToInt | BuffToIntLe | BuffToUIntLe | BuffToIntBe
             | BuffToUIntBe | IntToAscii | IntToUtf8 | StringToInt | StringToUInt | IsStandard
-            | PrincipalDestruct | PrincipalConstruct | Append | Concat | AsMaxLen | ContractOf
-            | PrincipalOf | ListCons | GetBlockInfo | GetBurnBlockInfo | TupleGet | TupleMerge
-            | Len | Print | AsContract | Begin | FetchVar | GetStxBalance | StxGetAccount
-            | GetTokenBalance | GetAssetOwner | GetTokenSupply | ElementAt | IndexOf | Slice => {
+            | ToConsensusBuff | PrincipalDestruct | PrincipalConstruct | Append | Concat
+            | AsMaxLen | ContractOf | PrincipalOf | ListCons | GetBlockInfo | GetBurnBlockInfo
+            | TupleGet | TupleMerge | Len | Print | AsContract | Begin | FetchVar
+            | GetStxBalance | StxGetAccount | GetTokenBalance | GetAssetOwner | GetTokenSupply
+            | ElementAt | IndexOf | Slice => {
                 // Check all arguments.
                 self.check_each_expression_is_read_only(args)
             }
+            FromConsensusBuff => {
+                // Check only the second+ arguments: the first argument is a type parameter
+                check_argument_count(2, args)?;
+                self.check_each_expression_is_read_only(&args[1..])
+            }
             AtBlock => {
                 check_argument_count(2, args)?;
 

clarity/src/vm/analysis/type_checker/natives/conversions.rs

@@ -0,0 +1,39 @@
+use crate::vm::analysis::read_only_checker::check_argument_count;
+use crate::vm::analysis::type_checker::contexts::TypingContext;
+use crate::vm::analysis::type_checker::{TypeChecker, TypeResult};
+use crate::vm::analysis::CheckError;
+use crate::vm::types::{BufferLength, SequenceSubtype, TypeSignature};
+use crate::vm::SymbolicExpression;
+
+/// to-consensus-buff admits exactly one argument:
+///   * the Clarity value to serialize
+/// it returns an `(optional (buff x))` where `x` is the maximum possible
+/// consensus buffer length based on the inferred type of the supplied value.
+pub fn check_special_to_consensus_buff(
+    checker: &mut TypeChecker,
+    args: &[SymbolicExpression],
+    context: &TypingContext,
+) -> TypeResult {
+    check_argument_count(1, args)?;
+    let input_type = checker.type_check(&args[0], context)?;
+    let buffer_max_len = BufferLength::try_from(input_type.max_serialized_size()?)?;
+    TypeSignature::new_option(TypeSignature::SequenceType(SequenceSubtype::BufferType(
+        buffer_max_len,
+    )))
+    .map_err(CheckError::from)
+}
+
+/// from-consensus-buff admits exactly two arguments:
+///   * a type signature indicating the expected return type `t1`
+///   * a buffer (of up to max length)
+/// it returns an `(optional t1)`
+pub fn check_special_from_consensus_buff(
+    checker: &mut TypeChecker,
+    args: &[SymbolicExpression],
+    context: &TypingContext,
+) -> TypeResult {
+    check_argument_count(2, args)?;
+    let result_type = TypeSignature::parse_type_repr(&args[0], checker)?;
+    checker.type_check_expects(&args[1], context, &TypeSignature::max_buffer())?;
+    TypeSignature::new_option(result_type).map_err(CheckError::from)
+}

clarity/src/vm/analysis/type_checker/natives/mod.rs

@@ -39,6 +39,7 @@ use crate::vm::costs::{
 };
 
 mod assets;
+mod conversions;
 mod maps;
 mod options;
 mod sequences;
@@ -882,6 +883,12 @@ impl TypedNativeFunction {
             IsNone => Special(SpecialNativeFunction(&options::check_special_is_optional)),
             IsSome => Special(SpecialNativeFunction(&options::check_special_is_optional)),
             AtBlock => Special(SpecialNativeFunction(&check_special_at_block)),
+            ToConsensusBuff => Special(SpecialNativeFunction(
+                &conversions::check_special_to_consensus_buff,
+            )),
+            FromConsensusBuff => Special(SpecialNativeFunction(
+                &conversions::check_special_from_consensus_buff,
+            )),
         }
     }
 }

clarity/src/vm/analysis/type_checker/tests/mod.rs

@@ -27,6 +27,7 @@ use crate::vm::analysis::AnalysisDatabase;
 use crate::vm::ast::errors::ParseErrors;
 use crate::vm::ast::{build_ast, parse};
 use crate::vm::contexts::OwnedEnvironment;
+use crate::vm::execute_v2;
 use crate::vm::representations::SymbolicExpression;
 use crate::vm::types::{
     BufferLength, FixedFunction, FunctionType, PrincipalData, QualifiedContractIdentifier,
@@ -74,6 +75,145 @@ fn ascii_type(size: u32) -> TypeSignature {
     TypeSignature::SequenceType(StringType(ASCII(size.try_into().unwrap()))).into()
 }
 
+#[test]
+fn test_from_consensus_buff() {
+    let good = [
+        ("(from-consensus-buff int 0x00)", "(optional int)"),
+        (
+            "(from-consensus-buff { a: uint, b: principal } 0x00)",
+            "(optional (tuple (a uint) (b principal)))",
+        ),
+    ];
+
+    let bad = [
+        (
+            "(from-consensus-buff)",
+            CheckErrors::IncorrectArgumentCount(2, 0),
+        ),
+        (
+            "(from-consensus-buff 0x00 0x00 0x00)",
+            CheckErrors::IncorrectArgumentCount(2, 3),
+        ),
+        (
+            "(from-consensus-buff 0x00 0x00)",
+            CheckErrors::InvalidTypeDescription,
+        ),
+        (
+            "(from-consensus-buff int u6)",
+            CheckErrors::TypeError(TypeSignature::max_buffer(), TypeSignature::UIntType),
+        ),
+        (
+            "(from-consensus-buff (buff 1048576) 0x00)",
+            CheckErrors::ValueTooLarge,
+        ),
+    ];
+
+    for (good_test, expected) in good.iter() {
+        let type_result = type_check_helper(good_test).unwrap();
+        assert_eq!(expected, &type_result.to_string());
+
+        assert!(
+            type_result.admits(&execute_v2(good_test).unwrap().unwrap()),
+            "The analyzed type must admit the evaluated type"
+        );
+    }
+
+    for (bad_test, expected) in bad.iter() {
+        assert_eq!(expected, &type_check_helper(&bad_test).unwrap_err().err);
+    }
+}
+
+#[test]
+fn test_to_consensus_buff() {
+    let good = [
+        (
+            "(to-consensus-buff (if true (some u1) (some u2)))",
+            "(optional (buff 18))",
+        ),
+        (
+            "(to-consensus-buff (if true (ok u1) (ok u2)))",
+            "(optional (buff 18))",
+        ),
+        (
+            "(to-consensus-buff (if true (ok 1) (err u2)))",
+            "(optional (buff 18))",
+        ),
+        (
+            "(to-consensus-buff (if true (ok 1) (err true)))",
+            "(optional (buff 18))",
+        ),
+        (
+            "(to-consensus-buff (if true (ok false) (err true)))",
+            "(optional (buff 2))",
+        ),
+        (
+            "(to-consensus-buff (if true (err u1) (err u2)))",
+            "(optional (buff 18))",
+        ),
+        ("(to-consensus-buff none)", "(optional (buff 1))"),
+        ("(to-consensus-buff 0x00)", "(optional (buff 6))"),
+        ("(to-consensus-buff \"a\")", "(optional (buff 6))"),
+        ("(to-consensus-buff u\"ab\")", "(optional (buff 13))"),
+        ("(to-consensus-buff 'STB44HYPYAT2BB2QE513NSP81HTMYWBJP02HPGK6)", "(optional (buff 151))"),
+        ("(to-consensus-buff 'STB44HYPYAT2BB2QE513NSP81HTMYWBJP02HPGK6.abcdeabcdeabcdeabcdeabcdeabcdeabcdeabcde)", "(optional (buff 151))"),
+        ("(to-consensus-buff true)", "(optional (buff 1))"),
+        ("(to-consensus-buff -1)", "(optional (buff 17))"),
+        ("(to-consensus-buff u1)", "(optional (buff 17))"),
+        ("(to-consensus-buff (list 1 2 3 4))", "(optional (buff 73))"),
+        (
+            "(to-consensus-buff { apple: u1, orange: 2, blue: true })",
+            "(optional (buff 58))",
+        ),
+        (
+            "(define-private (my-func (x (buff 1048566)))
+           (to-consensus-buff x))
+          (my-func 0x001122334455)
+         ",
+            "(optional (buff 1048571))",
+        ),
+    ];
+
+    let bad = [
+        (
+            "(to-consensus-buff)",
+            CheckErrors::IncorrectArgumentCount(1, 0),
+        ),
+        (
+            "(to-consensus-buff 0x00 0x00)",
+            CheckErrors::IncorrectArgumentCount(1, 2),
+        ),
+        (
+            "(define-private (my-func (x (buff 1048576)))
+           (to-consensus-buff x))",
+            CheckErrors::ValueTooLarge,
+        ),
+        (
+            "(define-private (my-func (x (buff 1048570)))
+           (to-consensus-buff x))",
+            CheckErrors::ValueTooLarge,
+        ),
+        (
+            "(define-private (my-func (x (buff 1048567)))
+           (to-consensus-buff x))",
+            CheckErrors::ValueTooLarge,
+        ),
+    ];
+
+    for (good_test, expected) in good.iter() {
+        let type_result = type_check_helper(good_test).unwrap();
+        assert_eq!(expected, &type_result.to_string());
+
+        assert!(
+            type_result.admits(&execute_v2(good_test).unwrap().unwrap()),
+            "The analyzed type must admit the evaluated type"
+        );
+    }
+
+    for (bad_test, expected) in bad.iter() {
+        assert_eq!(expected, &type_check_helper(&bad_test).unwrap_err().err);
+    }
+}
+
 #[test]
 fn test_get_block_info() {
     let good = [

clarity/src/vm/docs/mod.rs

@@ -1978,6 +1978,55 @@ one of the following error codes:
 "
 };
 
+const TO_CONSENSUS_BUFF: SpecialAPI = SpecialAPI {
+    input_type: "any",
+    output_type: "(optional buff)",
+    signature: "(to-consensus-buff value)",
+    description: "`to-consensus-buff` is a special function that will serialize any
+Clarity value into a buffer, using the SIP-005 serialization of the
+Clarity value. Not all values can be serialized: some value's
+consensus serialization is too large to fit in a Clarity buffer (this
+is because of the type prefix in the consensus serialization).
+
+If the value cannot fit as serialized into the maximum buffer size,
+this returns `none`, otherwise, it will be
+`(some consensus-serialized-buffer)`. During type checking, the
+analyzed type of the result of this method will be the maximum possible
+consensus buffer length based on the inferred type of the supplied value.
+",
+    example: r#"
+(to-consensus-buff 1) ;; Returns (some 0x0000000000000000000000000000000001)
+(to-consensus-buff u1) ;; Returns (some 0x0100000000000000000000000000000001)
+(to-consensus-buff true) ;; Returns (some 0x03)
+(to-consensus-buff false) ;; Returns (some 0x04)
+(to-consensus-buff none) ;; Returns (some 0x09)
+(to-consensus-buff 'SZ2J6ZY48GV1EZ5V2V5RB9MP66SW86PYKKQ9H6DPR) ;; Returns (some 0x051fa46ff88886c2ef9762d970b4d2c63678835bd39d)
+(to-consensus-buff { abc: 3, def: 4 }) ;; Returns (some 0x0c00000002036162630000000000000000000000000000000003036465660000000000000000000000000000000004)
+"#,
+};
+
+const FROM_CONSENSUS_BUFF: SpecialAPI = SpecialAPI {
+    input_type: "type-signature(t), buff",
+    output_type: "(optional t)",
+    signature: "(from-consensus-buff type-signature buffer)",
+    description: "`from-consensus-buff` is a special function that will deserialize a
+buffer into a Clarity value, using the SIP-005 serialization of the
+Clarity value. The type that `from-consensus-buff` tries to deserialize
+into is provided by the first parameter to the function. If it fails
+to deserialize the type, the method returns `none`.
+",
+    example: r#"
+(from-consensus-buff int 0x0000000000000000000000000000000001) ;; Returns (some 1)
+(from-consensus-buff uint 0x0000000000000000000000000000000001) ;; Returns none
+(from-consensus-buff uint 0x0100000000000000000000000000000001) ;; Returns (some u1)
+(from-consensus-buff bool 0x0000000000000000000000000000000001) ;; Returns none
+(from-consensus-buff bool 0x03) ;; Returns (some true)
+(from-consensus-buff bool 0x04) ;; Returns (some false)
+(from-consensus-buff principal 0x051fa46ff88886c2ef9762d970b4d2c63678835bd39d) ;; Returns (some SZ2J6ZY48GV1EZ5V2V5RB9MP66SW86PYKKQ9H6DPR)
+(from-consensus-buff { abc: int, def: int } 0x0c00000002036162630000000000000000000000000000000003036465660000000000000000000000000000000004) ;; Returns (some (tuple (abc 3) (def 4)))
+"#,
+};
+
 fn make_api_reference(function: &NativeFunctions) -> FunctionAPI {
     use crate::vm::functions::NativeFunctions::*;
     let name = function.get_name();
@@ -2079,6 +2128,8 @@ fn make_api_reference(function: &NativeFunctions) -> FunctionAPI {
         StxTransfer => make_for_special(&STX_TRANSFER, name),
         StxTransferMemo => make_for_special(&STX_TRANSFER_MEMO, name),
         StxBurn => make_for_simple_native(&STX_BURN, &StxBurn, name),
+        ToConsensusBuff => make_for_special(&TO_CONSENSUS_BUFF, name),
+        FromConsensusBuff => make_for_special(&FROM_CONSENSUS_BUFF, name),
     }
 }
 

clarity/src/vm/functions/conversions.rs

@@ -14,6 +14,8 @@
 // You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+use stacks_common::codec::StacksMessageCodec;
+
 use crate::vm::costs::cost_functions::ClarityCostFunction;
 use crate::vm::costs::runtime_cost;
 use crate::vm::errors::{check_argument_count, CheckErrors, InterpreterResult as Result};
@@ -209,3 +211,58 @@ pub fn native_int_to_utf8(value: Value) -> Result<Value> {
     // Given a string representing an integer, convert this to Clarity UTF8 value.
     native_int_to_string_generic(value, Value::string_utf8_from_bytes)
 }
+
+/// Returns `value` consensus serialized into a `(optional buff)` object.
+/// If the value cannot fit as serialized into the maximum buffer size,
+/// this returns `none`, otherwise, it will be `(some consensus-serialized-buffer)`
+pub fn to_consensus_buff(value: Value) -> Result<Value> {
+    let clar_buff_serialized = match Value::buff_from(value.serialize_to_vec()) {
+        Ok(x) => x,
+        Err(_) => return Ok(Value::none()),
+    };
+
+    match Value::some(clar_buff_serialized) {
+        Ok(x) => Ok(x),
+        Err(_) => Ok(Value::none()),
+    }
+}
+
+/// Deserialize a Clarity value from a consensus serialized buffer.
+/// If the supplied buffer either fails to deserialize or deserializes
+/// to an unexpected type, returns `none`. Otherwise, it will be `(some value)`
+pub fn from_consensus_buff(
+    args: &[SymbolicExpression],
+    env: &mut Environment,
+    context: &LocalContext,
+) -> Result<Value> {
+    check_argument_count(2, args)?;
+
+    let type_arg = TypeSignature::parse_type_repr(&args[0], env)?;
+    let value = eval(&args[1], env, context)?;
+
+    // get the buffer bytes from the supplied value. if not passed a buffer,
+    // this is a type error
+    let input_bytes = if let Value::Sequence(SequenceData::Buffer(buff_data)) = value {
+        Ok(buff_data.data)
+    } else {
+        Err(CheckErrors::TypeValueError(
+            TypeSignature::max_buffer(),
+            value,
+        ))
+    }?;
+
+    runtime_cost(ClarityCostFunction::Unimplemented, env, input_bytes.len())?;
+
+    // Perform the deserialization and check that it deserialized to the expected
+    // type. A type mismatch at this point is an error that should be surfaced in
+    // Clarity (as a none return).
+    let result = match Value::try_deserialize_bytes_exact(&input_bytes, &type_arg) {
+        Ok(value) => value,
+        Err(_) => return Ok(Value::none()),
+    };
+    if !type_arg.admits(&result) {
+        return Ok(Value::none());
+    }
+
+    Value::some(result)
+}