Fix handling memory of max size on 32-bit systems: check for size_t overflow

lib/fizzy/execute.cpp

@@ -151,13 +151,15 @@ inline uint32_t grow_memory(
 
     const auto new_pages = static_cast<uint32_t>(new_pages_u64);
 
+    const uint64_t new_bytes = memory_pages_to_bytes(new_pages);
+    if (!can_narrow<size_t>(new_bytes))
+        return static_cast<uint32_t>(-1);
+
     try
     {
-        // new_pages <= memory_pages_limit <= MaxMemoryPagesLimit guarantees memory_pages_to_bytes
-        // will not overflow uint32_t.
-        assert(memory_pages_to_bytes(new_pages) <= std::numeric_limits<uint32_t>::max());
-        static_assert(sizeof(size_t) >= sizeof(uint32_t));
-        memory.resize(static_cast<size_t>(memory_pages_to_bytes(new_pages)));
+        // can_narrow guarantees that new_bytes won't overflow size_t
+        assert(new_bytes <= std::numeric_limits<size_t>::max());
+        memory.resize(static_cast<size_t>(new_bytes));
         return static_cast<uint32_t>(cur_pages);
     }
     catch (...)

lib/fizzy/instantiate.cpp

@@ -204,9 +204,16 @@ std::tuple<bytes_ptr, Limits> allocate_memory(const std::vector<Memory>& module_
                                     " bytes"};
         }
 
+        const uint64_t memory_min_bytes = memory_pages_to_bytes(memory_min);
+        if (!can_narrow<size_t>(memory_min_bytes))
+        {
+            throw instantiate_error{"cannot allocate more than " +
+                                    std::to_string(std::numeric_limits<size_t>::max()) + " bytes"};
+        }
         // NOTE: fill it with zeroes
-        bytes_ptr memory{
-            new bytes(static_cast<size_t>(memory_pages_to_bytes(memory_min)), 0), bytes_delete};
+        // can_narrow guarantees that memory_min_bytes won't overflow size_t
+        assert(memory_min_bytes <= std::numeric_limits<size_t>::max());
+        bytes_ptr memory{new bytes(static_cast<size_t>(memory_min_bytes), 0), bytes_delete};
         return {std::move(memory), module_memories[0].limits};
     }
     else if (imported_memories.size() == 1)

lib/fizzy/limits.hpp

@@ -5,6 +5,8 @@
 #pragma once
 
 #include <cstdint>
+#include <limits>
+#include <type_traits>
 
 namespace fizzy
 {
@@ -17,6 +19,21 @@ inline constexpr uint64_t memory_pages_to_bytes(uint32_t pages) noexcept
     return uint64_t{pages} * PageSize;
 }
 
+/// Check if an integral value can be casted to a narrower type without an overflow.
+template <typename TypeTo, typename TypeFrom,
+    typename =
+        typename std::enable_if_t<std::is_integral_v<TypeFrom> && std::is_integral_v<TypeTo> &&
+                                  sizeof(TypeFrom) >= sizeof(TypeTo)>>
+inline constexpr bool can_narrow([[maybe_unused]] TypeFrom value) noexcept
+{
+    if constexpr (sizeof(TypeFrom) > sizeof(TypeTo))
+    {
+        return value <= std::numeric_limits<TypeTo>::max();
+    }
+    else
+        return true;
+}
+
 /// The maximum memory page limit as defined by the specification.
 /// It is only possible to address 4 GB (32-bit) of memory.
 constexpr uint32_t MaxMemoryPagesLimit = (4 * 1024 * 1024 * 1024ULL) / PageSize;

test/unittests/execute_test.cpp

@@ -698,7 +698,15 @@ TEST(execute, memory_grow_custom_hard_limit)
         const auto instance_huge_hard_limit = instantiate(*module, {}, {}, {}, {}, 65536);
         // For huge hard limit we test only failure cases, because allocating 4GB of memory would
         // be too slow for unit tests.
-        // EXPECT_THAT(execute(*instance_huge_hard_limit, 0, {65535}), Result(1));
+        if constexpr (sizeof(size_t) < sizeof(uint64_t))
+        {
+            // On 32-bit 4 GB can't be allocated
+            EXPECT_THAT(execute(*instance_huge_hard_limit, 0, {65535}), Result(-1));
+        }
+        else
+        {
+            // EXPECT_THAT(execute(*instance_huge_hard_limit, 0, {65535}), Result(1));
+        }
         EXPECT_THAT(execute(*instance_huge_hard_limit, 0, {65536}), Result(-1));
         EXPECT_THAT(execute(*instance_huge_hard_limit, 0, {0xffffffff}), Result(-1));
     }

test/unittests/instantiate_test.cpp

@@ -535,6 +535,19 @@ TEST(instantiate, memory_single_custom_hard_limit)
         instantiate_error, "hard memory limit cannot exceed 4294967296 bytes");
 }
 
+TEST(instantiate, memory_32bit_cant_allocate_max_memory)
+{
+    if constexpr (sizeof(size_t) < sizeof(uint64_t))
+    {
+        /* wat2wasm
+          (memory 65536)
+        */
+        const auto bin_max_memory = from_hex("0061736d0100000005050100808004");
+        EXPECT_THROW_MESSAGE(instantiate(parse(bin_max_memory), {}, {}, {}, {}, 65536),
+            instantiate_error, "cannot allocate more than 4294967295 bytes");
+    }
+}
+
 TEST(instantiate, imported_memory_custom_hard_limit)
 {
     /* wat2wasm
@@ -588,6 +601,26 @@ TEST(instantiate, memory_pages_to_bytes)
     EXPECT_EQ(memory_pages_to_bytes(MaxMemoryPagesLimit), 4 * 1024 * 1024 * 1024ULL);
 }
 
+TEST(instantiate, can_narrow)
+{
+    EXPECT_TRUE(can_narrow<size_t>(std::numeric_limits<size_t>::max()));
+    EXPECT_TRUE(can_narrow<size_t>(uint64_t{std::numeric_limits<size_t>::max()}));
+
+    EXPECT_TRUE(can_narrow<size_t>(uint64_t{0}));
+    EXPECT_TRUE(can_narrow<size_t>(uint64_t{1}));
+    EXPECT_TRUE(can_narrow<size_t>(uint64_t{std::numeric_limits<size_t>::max()}));
+
+    if constexpr (sizeof(size_t) < sizeof(uint64_t))
+    {
+        EXPECT_FALSE(can_narrow<size_t>(uint64_t{std::numeric_limits<size_t>::max()} + 1));
+        EXPECT_FALSE(can_narrow<size_t>(std::numeric_limits<uint64_t>::max()));
+    }
+    else
+    {
+        EXPECT_TRUE(can_narrow<size_t>(std::numeric_limits<uint64_t>::max()));
+    }
+}
+
 TEST(instantiate, element_section)
 {
     /* wat2wasm