Added marshalling for preprocessed public data, LPC commitment state,…

… polys_evaluator.
NilFoundation · Sep 8, 2024 · eef407b · eef407b
1 parent 4a10666
commit eef407b
Show file tree

Hide file tree

Showing 73 changed files with 3,199 additions and 1,100 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -53,7 +53,7 @@ else()
     target_architecture(CMAKE_TARGET_ARCHITECTURE)
 endif()
 
-find_package(Boost REQUIRED COMPONENTS container random filesystem log log_setup program_options thread system unit_test_framework timer)
+find_package(Boost REQUIRED COMPONENTS container random filesystem log log_setup program_options thread system unit_test_framework timer log)
 
 add_subdirectories("${CMAKE_CURRENT_LIST_DIR}/libs/")
 add_subdirectories("${CMAKE_CURRENT_LIST_DIR}/libs/marshalling")

diff --git a/libs/algebra/crypto3.algebra.podspec.json b/libs/algebra/crypto3.algebra.podspec.json
@@ -18,7 +18,7 @@
     "branch": "master"
   },
   "xcconfig": {
-    "CLANG_CXX_LANGUAGE_STANDARD": "c++14",
+    "CLANG_CXX_LANGUAGE_STANDARD": "c++17",
     "CLANG_CXX_LIBRARY": "libc++",
     "HEADER_SEARCH_PATHS": "\"${PODS_ROOT}/include/\""
   },

diff --git a/libs/block/crypto3.block.podspec.json b/libs/block/crypto3.block.podspec.json
@@ -17,7 +17,7 @@
     "branch": "master"
   },
   "xcconfig": {
-    "CLANG_CXX_LANGUAGE_STANDARD": "c++14",
+    "CLANG_CXX_LANGUAGE_STANDARD": "c++17",
     "CLANG_CXX_LIBRARY": "libc++",
     "HEADER_SEARCH_PATHS": "\"${PODS_ROOT}/include/\""
   },

diff --git a/libs/block/test/CMakeLists.txt b/libs/block/test/CMakeLists.txt
@@ -21,7 +21,7 @@ macro(define_block_cipher_test name)
                                ${Boost_INCLUDE_DIRS})
 
     if(NOT CMAKE_CXX_STANDARD)
-        set_target_properties(${test_name} PROPERTIES CXX_STANDARD 14)
+        set_target_properties(${test_name} PROPERTIES CXX_STANDARD 17)
     endif()
 
     string(CONCAT TEST_DATA ${CMAKE_CURRENT_SOURCE_DIR} "/data/" "${name}" ".json")

diff --git a/libs/blueprint/include/nil/blueprint/basic_non_native_policy.hpp b/libs/blueprint/include/nil/blueprint/basic_non_native_policy.hpp
@@ -29,6 +29,8 @@
 #include <nil/crypto3/algebra/curves/pallas.hpp>
 #include <nil/crypto3/algebra/curves/vesta.hpp>
 #include <nil/crypto3/algebra/curves/ed25519.hpp>
+#include <nil/crypto3/algebra/fields/bls12/base_field.hpp>
+#include <nil/crypto3/algebra/fields/bls12/scalar_field.hpp>
 
 #include <boost/multiprecision/cpp_int.hpp>
 #include <nil/crypto3/zk/snark/arithmetization/plonk/constraint_system.hpp>
@@ -251,8 +253,6 @@ namespace nil {
                 typedef std::array<var, ratio> non_native_var_type;
             };
 
-
-
             /*
              * Native element type.
              */

diff --git a/libs/codec/crypto3.codec.podspec.json b/libs/codec/crypto3.codec.podspec.json
@@ -17,7 +17,7 @@
     "branch": "master"
   },
   "xcconfig": {
-    "CLANG_CXX_LANGUAGE_STANDARD": "c++14",
+    "CLANG_CXX_LANGUAGE_STANDARD": "c++17",
     "CLANG_CXX_LIBRARY": "libc++",
     "HEADER_SEARCH_PATHS": "\"${PODS_ROOT}/include/\""
   },

diff --git a/libs/codec/test/CMakeLists.txt b/libs/codec/test/CMakeLists.txt
@@ -24,7 +24,7 @@ macro(define_codec_test name)
 
     if(NOT CMAKE_CXX_STANDARD)
         set_target_properties(codec_${name}_test PROPERTIES
-                              CXX_STANDARD 14
+                              CXX_STANDARD 17
                               CXX_STANDARD_REQUIRED TRUE)
     endif()
 

diff --git a/libs/containers/include/nil/crypto3/container/merkle/tree.hpp b/libs/containers/include/nil/crypto3/container/merkle/tree.hpp
@@ -175,6 +175,7 @@ namespace nil {
 
                     typedef typename node_type::value_type value_type;
                     constexpr static const std::size_t value_bits = node_type::value_bits;
+                    constexpr static const std::size_t arity = Arity;
 
                     typedef std::vector<value_type> container_type;
 
@@ -253,15 +254,15 @@ namespace nil {
                     }
 
                     bool operator==(const merkle_tree_impl &rhs) const {
-                        return _hashes == rhs.val;
+                        return _hashes == rhs._hashes;
                     }
 
                     bool operator!=(const merkle_tree_impl &rhs) const {
                         return !(rhs == *this);
                     }
 
                     allocator_type get_allocator() const BOOST_NOEXCEPT {
-                        return this->val.__alloc();
+                        return this->_hashes.__alloc();
                     }
 
                     iterator begin() BOOST_NOEXCEPT {

diff --git a/libs/hash/crypto3.hash.podspec.json b/libs/hash/crypto3.hash.podspec.json
@@ -17,7 +17,7 @@
     "branch": "master"
   },
   "xcconfig": {
-    "CLANG_CXX_LANGUAGE_STANDARD": "c++14",
+    "CLANG_CXX_LANGUAGE_STANDARD": "c++17",
     "CLANG_CXX_LIBRARY": "libc++",
     "HEADER_SEARCH_PATHS": "\"${PODS_ROOT}/include/\""
   },

diff --git a/libs/kdf/crypto3.kdf.podspec.json b/libs/kdf/crypto3.kdf.podspec.json
@@ -17,7 +17,7 @@
     "branch": "master"
   },
   "xcconfig": {
-    "CLANG_CXX_LANGUAGE_STANDARD": "c++14",
+    "CLANG_CXX_LANGUAGE_STANDARD": "c++17",
     "CLANG_CXX_LIBRARY": "libc++",
     "HEADER_SEARCH_PATHS": "\"${PODS_ROOT}/include/\""
   },

diff --git a/libs/mac/crypto3.mac.podspec.json b/libs/mac/crypto3.mac.podspec.json
@@ -17,7 +17,7 @@
     "branch": "master"
   },
   "xcconfig": {
-    "CLANG_CXX_LANGUAGE_STANDARD": "c++14",
+    "CLANG_CXX_LANGUAGE_STANDARD": "c++17",
     "CLANG_CXX_LIBRARY": "libc++",
     "HEADER_SEARCH_PATHS": "\"${PODS_ROOT}/include/\""
   },

diff --git a/libs/marshalling/algebra/include/nil/crypto3/marshalling/algebra/types/field_element.hpp b/libs/marshalling/algebra/include/nil/crypto3/marshalling/algebra/types/field_element.hpp
@@ -686,12 +686,16 @@ namespace nil {
                 //     return field;
                 // }
 
-                template<typename FieldValueType, typename Endianness>
-                nil::marshalling::types::array_list<
-                    nil::marshalling::field_type<Endianness>,
-                    field_element<nil::marshalling::field_type<Endianness>, FieldValueType>,
+
+                template<typename FieldValueType, typename TTypeBase>
+                using field_element_vector = nil::marshalling::types::array_list<
+                    TTypeBase,
+                    field_element<TTypeBase, FieldValueType>,
                     nil::marshalling::option::sequence_size_field_prefix<
-                        nil::marshalling::types::integral<nil::marshalling::field_type<Endianness>, std::size_t>>>
+                        nil::marshalling::types::integral<TTypeBase, std::size_t>>>;
+
+                template<typename FieldValueType, typename Endianness>
+                field_element_vector<FieldValueType, nil::marshalling::field_type<Endianness>>
                     fill_field_element_vector(const std::vector<FieldValueType> &field_elem_vector) {
 
                     using TTypeBase = nil::marshalling::field_type<Endianness>;
@@ -711,12 +715,7 @@ namespace nil {
 
                 template<typename FieldValueType, typename Endianness>
                 std::vector<FieldValueType> make_field_element_vector(
-                    const nil::marshalling::types::array_list<
-                        nil::marshalling::field_type<Endianness>,
-                        field_element<nil::marshalling::field_type<Endianness>, FieldValueType>,
-                        nil::marshalling::option::sequence_size_field_prefix<
-                            nil::marshalling::types::integral<nil::marshalling::field_type<Endianness>, std::size_t>>>
-                        &field_elem_vector) {
+                    const field_element_vector<FieldValueType, nil::marshalling::field_type<Endianness>>& field_elem_vector) {
 
                     std::vector<FieldValueType> result;
                     result.reserve(field_elem_vector.value().size());

diff --git a/libs/marshalling/core/CMakeLists.txt b/libs/marshalling/core/CMakeLists.txt
@@ -19,6 +19,10 @@ if(CRYPTO3_MARSHALLING_THROWS)
     add_definitions(-DCRYPTO3_MARSHALLING_THROWS)
 endif()
 
+if(NOT CMAKE_CXX_STANDARD)
+    set_target_properties(${test_name} PROPERTIES CXX_STANDARD 17)
+endif()
+
 cm_setup_version(VERSION 0.1.0 PREFIX ${CMAKE_WORKSPACE_NAME}_${CURRENT_PROJECT_NAME})
 
 add_library(${CMAKE_WORKSPACE_NAME}_${CURRENT_PROJECT_NAME} INTERFACE)

diff --git a/libs/marshalling/core/include/nil/marshalling/options.hpp b/libs/marshalling/core/include/nil/marshalling/options.hpp
@@ -43,6 +43,12 @@
 
 namespace nil {
     namespace marshalling {
+        namespace types {
+            // We cannot include integral.hpp, it includes this file. So just declare the class.
+            template<typename TFieldBase, typename T, typename... TOptions>
+            class integral;
+        }
+
         namespace option {
             namespace detail {
 
@@ -286,6 +292,10 @@ namespace nil {
             template<typename TField>
             struct sequence_size_field_prefix { };
 
+            template <typename TTypeBase>
+                using size_t_sequence_size_field_prefix = sequence_size_field_prefix<
+                    nil::marshalling::types::integral<TTypeBase, std::size_t>>;
+
             /// @brief Option that modifies the default behaviour of collection fields to
             ///     prepend the serialized data with number of @b bytes information.
             /// @details Similar to @ref sequence_size_field_prefix, but instead of

diff --git a/libs/marshalling/core/include/nil/marshalling/types/adapter/sequence_size_field_prefix.hpp b/libs/marshalling/core/include/nil/marshalling/types/adapter/sequence_size_field_prefix.hpp
@@ -32,6 +32,7 @@
 namespace nil {
     namespace marshalling {
         namespace types {
+
             namespace adapter {
 
                 template<typename TSizeField, typename TBase>
@@ -129,7 +130,6 @@ namespace nil {
                         base_impl_type::write_no_status(iter);
                     }
                 };
-
             }    // namespace adapter
         }        // namespace types
     }            // namespace marshalling

diff --git a/libs/marshalling/core/include/nil/marshalling/types/array_list.hpp b/libs/marshalling/core/include/nil/marshalling/types/array_list.hpp
@@ -26,13 +26,16 @@
 #ifndef MARSHALLING_ARRAY_LIST_HPP
 #define MARSHALLING_ARRAY_LIST_HPP
 
+#include <functional>
 #include <vector>
+#include <map>
 
 #include <nil/marshalling/status_type.hpp>
 #include <nil/marshalling/types/array_list/behaviour.hpp>
 #include <nil/marshalling/types/detail/options_parser.hpp>
 
 #include <nil/marshalling/types/tag.hpp>
+#include <nil/detail/type_traits.hpp>
 
 namespace nil {
     namespace marshalling {
@@ -404,6 +407,79 @@ namespace nil {
                 return field;
             }
 
+            // We use this type of array_list waay too often, so this is a shortcut, not to copy-paste it all the time.
+            template<typename TFieldBase, typename TElement>
+            using standard_array_list = array_list<
+                TFieldBase,
+                TElement,
+                nil::marshalling::option::size_t_sequence_size_field_prefix<TFieldBase>>;
+
+            // Very often we just need an array list of std::size_t, so here's another shortcut.
+            template<typename TFieldBase>
+            using standard_size_t_array_list = array_list<
+                TFieldBase,
+                nil::marshalling::types::integral<TFieldBase, std::size_t>,
+                nil::marshalling::option::size_t_sequence_size_field_prefix<TFieldBase>>;
+
+            // Helper functions to convert to/from an arraylist.
+            template<typename TFieldBase, typename TMarshalledElement, typename Range>
+            typename std::enable_if<
+                nil::detail::is_range<Range>::value, 
+                standard_array_list<TFieldBase, TMarshalledElement>>::type 
+            fill_standard_array_list(
+                    const Range& input_range,
+                    std::function<TMarshalledElement(const typename Range::value_type&)> element_marshalling) {
+                standard_array_list<TFieldBase, TMarshalledElement> result;
+                for (const auto& v: input_range) {
+                    result.value().push_back(element_marshalling(v));
+                }
+                return result;
+            }
+
+            template<typename TFieldBase, typename TElement, typename TMarshalledElement> 
+            std::vector<TElement> make_standard_array_list(
+                    const standard_array_list<TFieldBase, TMarshalledElement>& filled_array,
+                    std::function<TElement(const TMarshalledElement&)> element_de_marshalling) {
+                std::vector<TElement> result;
+                for (const auto& v: filled_array.value()) {
+                    result.push_back(element_de_marshalling(v));
+                }
+                return result;
+            }
+
+            // Helper functions to marshall an std::map.
+            // We keep TKey, TValue at the end, because they can be decuded from the map type, but the other 3
+            // arguments must be provided explicitly.
+            template<typename TFieldBase, typename TMarshalledKey, typename TMarshalledValue, typename TKey, typename TValue>
+            std::pair<standard_array_list<TFieldBase, TMarshalledKey>, standard_array_list<TFieldBase, TMarshalledValue>>
+            fill_std_map(
+                    const std::map<TKey, TValue>& input_map,
+                    std::function<TMarshalledKey(const TKey&)> key_marshalling,
+                    std::function<TMarshalledValue(const TValue&)> value_marshalling) {
+                standard_array_list<TFieldBase, TMarshalledKey> result_keys;
+                standard_array_list<TFieldBase, TMarshalledValue> result_values;
+                for (const auto& [k, v]: input_map) {
+                    result_keys.value().push_back(key_marshalling(k));
+                    result_values.value().push_back(value_marshalling(v));
+                }
+                return {result_keys, result_values};
+            }
+
+            template<typename TFieldBase, typename TKey, typename TValue, typename TMarshalledKey, typename TMarshalledValue>
+            std::map<TKey, TValue> make_std_map(
+                    const standard_array_list<TFieldBase, TMarshalledKey>& filled_keys,
+                    const standard_array_list<TFieldBase, TMarshalledValue>& filled_values,
+                    std::function<TKey(const TMarshalledKey&)> key_de_marshalling,
+                    std::function<TValue(const TMarshalledValue&)> value_de_marshalling) {
+                assert(filled_keys.value().size() == filled_values.value().size());
+
+                std::map<TKey, TValue> result;
+                for (std::size_t i = 0; i < filled_keys.value().size(); ++i) {
+                    result[key_de_marshalling(filled_keys.value()[i])] = value_de_marshalling(filled_values.value()[i]);
+                }
+                return result;
+            }
+
         }    // namespace types
     }        // namespace marshalling
 }    // namespace nil

diff --git a/libs/marshalling/core/include/nil/marshalling/types/integral.hpp b/libs/marshalling/core/include/nil/marshalling/types/integral.hpp
@@ -439,6 +439,19 @@ namespace nil {
                 return field;
             }
 
+            // This is a helper, frequently used to convert size_t. It just shortens our code.
+            template<typename TFieldBase>
+            integral<TFieldBase, std::size_t> fill_size_t(const std::size_t& v) {
+                integral<TFieldBase, std::size_t> result;
+                result.value() = v;
+                return result;
+            }
+
+            template<typename TFieldBase>
+            std::size_t make_size_t(const integral<TFieldBase, std::size_t>& v) {
+                return v.value();
+            }
+
         }    // namespace types
     }        // namespace marshalling
 }    // namespace nil