Use address space reservation to minimize allocations.

CHANGELOG.md

@@ -5,6 +5,7 @@
 * Add support for building with Xcode 14 using the CMake project ([PR #5577](https://github.com/realm/realm-core/pull/5577)).
 * Expose MongoDB client interface in the C API. ([PR #5638](https://github.com/realm/realm-core/pull/5638)).
 * Add support in the C API for constructing a new `realm_app_t` object via `realm_app_create`. ([PR #5570](https://github.com/realm/realm-core/issues/5570))
+* Reduce use of memory mappings and virtual address space ([PR #5645](https://github.com/realm/realm-core/pull/5645)). Also fixes some errors (see below)
 
 ### Fixed
 * <How do the end-user experience this issue? what was the impact?> ([#????](https://github.com/realm/realm-core/issues/????), since v?.?.?)
@@ -14,6 +15,7 @@
 * Fix compilation failures on watchOS platforms which do not support thread-local storage. ([#7694](https://github.com/realm/realm-swift/issues/7694), [#7695](https://github.com/realm/realm-swift/issues/7695) since v11.7.0)
 * Fix a data race when committing a transaction while multiple threads are waiting for the write lock on platforms using emulated interprocess condition variables (most platforms other than non-Android Linux).
 * Fix a data race when writing audit events which could occur if the sync client thread was busy with other work when the event Realm was opened.
+* Fix some cases of running out of virtual address space (seen/reported as mmap failures) ([PR #5645](https://github.com/realm/realm-core/pull/5645))
 * Audit event scopes containing only write events and no read events would occasionally throw a `BadVersion` exception when a write transaction was committed (since v11.17.0).
 
 ### Breaking changes

src/realm/alloc_slab.cpp

@@ -1122,8 +1122,14 @@ void SlabAlloc::update_reader_view(size_t file_size)
         if (old_baseline < old_slab_base) {
             // old_slab_base should be 0 if we had no mappings previously
             REALM_ASSERT(old_num_mappings > 0);
-            replace_last_mapping = true;
-            --old_num_mappings;
+            // try to extend the old mapping in-place instead of replacing it.
+            MapEntry& cur_entry = m_mappings.back();
+            const size_t section_start_offset = get_section_base(old_num_mappings - 1);
+            const size_t section_size = std::min<size_t>(1 << section_shift, file_size - section_start_offset);
+            if (!cur_entry.primary_mapping.try_extend_to(section_size)) {
+                replace_last_mapping = true;
+                --old_num_mappings;
+            }
         }
 
         // Create new mappings covering from the end of the last complete
@@ -1134,8 +1140,25 @@ void SlabAlloc::update_reader_view(size_t file_size)
         for (size_t k = old_num_mappings; k < num_mappings; ++k) {
             const size_t section_start_offset = get_section_base(k);
             const size_t section_size = std::min<size_t>(1 << section_shift, file_size - section_start_offset);
-            new_mappings.push_back(
-                {util::File::Map<char>(m_file, section_start_offset, File::access_ReadOnly, section_size)});
+            if (section_size == (1 << section_shift)) {
+                new_mappings.push_back(
+                    {util::File::Map<char>(m_file, section_start_offset, File::access_ReadOnly, section_size)});
+            }
+            else {
+                new_mappings.push_back({util::File::Map<char>()});
+                auto& mapping = new_mappings.back().primary_mapping;
+                bool reserved =
+                    mapping.try_reserve(m_file, File::access_ReadOnly, 1 << section_shift, section_start_offset);
+                if (reserved) {
+                    // if reservation is supported, first attempt at extending must succeed
+                    if (!mapping.try_extend_to(section_size))
+                        throw std::bad_alloc();
+                }
+                else {
+                    new_mappings.back().primary_mapping.map(m_file, File::access_ReadOnly, section_size, 0,
+                                                            section_start_offset);
+                }
+            }
         }
     }
 
@@ -1169,7 +1192,7 @@ void SlabAlloc::update_reader_view(size_t file_size)
 
     // Build the fast path mapping
 
-    // The fast path mapping is an array which will is used from multiple threads
+    // The fast path mapping is an array which is used from multiple threads
     // without locking - see translate().
 
     // Addition of a new mapping may require a completely new fast mapping table.

src/realm/util/encrypted_file_mapping.cpp

@@ -560,7 +560,6 @@ void EncryptedFileMapping::mark_for_refresh(size_t ref_start, size_t ref_end)
     }
 }
 
-
 void EncryptedFileMapping::write_and_update_all(size_t local_page_ndx, size_t begin_offset,
                                                 size_t end_offset) noexcept
 {
@@ -860,13 +859,24 @@ void EncryptedFileMapping::read_barrier(const void* addr, size_t size, Header_to
     }
 }
 
+void EncryptedFileMapping::extend_to(size_t offset, size_t new_size)
+{
+    REALM_ASSERT(new_size % (1ULL << m_page_shift) == 0);
+    size_t num_pages = new_size >> m_page_shift;
+    m_page_state.resize(num_pages, PageState::Clean);
+    m_chunk_dont_scan.resize((num_pages + page_to_chunk_factor - 1) >> page_to_chunk_shift, false);
+    m_file.cryptor.set_file_size((off_t)(offset + new_size));
+}
 
 void EncryptedFileMapping::set(void* new_addr, size_t new_size, size_t new_file_offset)
 {
     REALM_ASSERT(new_file_offset % (1ULL << m_page_shift) == 0);
     REALM_ASSERT(new_size % (1ULL << m_page_shift) == 0);
-    REALM_ASSERT(new_size > 0);
 
+    // This seems dangerous - correct operation in a setting with multiple (partial)
+    // mappings of the same file would rely on ordering of individual mapping requests.
+    // Currently we only ever extend the file - but when we implement continuous defrag,
+    // this design should be revisited.
     m_file.cryptor.set_file_size(off_t(new_size + new_file_offset));
 
     flush();

src/realm/util/encrypted_file_mapping.hpp

@@ -72,6 +72,10 @@ class EncryptedFileMapping {
     // Flushes any remaining dirty pages from the old mapping
     void set(void* new_addr, size_t new_size, size_t new_file_offset);
 
+    // Extend the size of this mapping. Memory holding decrypted pages must
+    // have been allocated earlier
+    void extend_to(size_t offset, size_t new_size);
+
     size_t collect_decryption_count()
     {
         return m_num_decrypted;
@@ -104,6 +108,7 @@ class EncryptedFileMapping {
     size_t m_num_decrypted; // 1 for every page decrypted
 
     enum PageState {
+        Clean = 0,
         Touched = 1,         // a ref->ptr translation has taken place
         UpToDate = 2,        // the page is fully up to date
         RefetchRequired = 4, // the page is valid for old translations, but requires re-decryption for new

src/realm/util/file.cpp

@@ -1691,30 +1691,96 @@ void File::MapBase::map(const File& f, AccessMode a, size_t size, int map_flags,
 #else
     m_addr = f.map(a, size, map_flags, offset);
 #endif
-    m_size = size;
+    m_size = m_reservation_size = size;
     m_fd = f.m_fd;
     m_offset = offset;
+    m_a = a;
 }
 
 
 void File::MapBase::unmap() noexcept
 {
     if (!m_addr)
         return;
-    REALM_ASSERT(m_size);
-    File::unmap(m_addr, m_size);
-    m_addr = nullptr;
-    m_size = 0;
+    REALM_ASSERT(m_reservation_size);
 #if REALM_ENABLE_ENCRYPTION
-    m_encrypted_mapping = nullptr;
+    if (m_encrypted_mapping) {
+        m_encrypted_mapping = nullptr;
+        util::remove_encrypted_mapping(m_addr, m_size);
+    }
 #endif
+    ::munmap(m_addr, m_reservation_size);
+    m_addr = nullptr;
+    m_size = 0;
+    m_reservation_size = 0;
 }
 
 void File::MapBase::remap(const File& f, AccessMode a, size_t size, int map_flags)
 {
     REALM_ASSERT(m_addr);
     m_addr = f.remap(m_addr, m_size, a, size, map_flags);
-    m_size = size;
+    m_size = m_reservation_size = size;
+}
+
+bool File::MapBase::try_reserve(const File& file, AccessMode a, size_t size, size_t offset)
+{
+#ifdef _WIN32
+    // unsupported for now
+    return false;
+#else
+    void* addr = ::mmap(0, size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    if (addr == MAP_FAILED)
+        return false;
+    m_addr = addr;
+    REALM_ASSERT(m_size == 0);
+    m_a = a;
+    m_reservation_size = size;
+    m_fd = file.get_descriptor();
+    m_offset = offset;
+#if REALM_ENABLE_ENCRYPTION
+    if (file.m_encryption_key) {
+        m_encrypted_mapping = util::reserve_mapping(addr, m_fd, offset, a, file.m_encryption_key.get());
+    }
+#endif
+#endif
+    return true;
+}
+
+bool File::MapBase::try_extend_to(size_t size) noexcept
+{
+    if (size > m_reservation_size) {
+        return false;
+    }
+    // return false;
+#ifndef _WIN32
+    char* extension_start_addr = (char*)m_addr + m_size;
+    size_t extension_size = size - m_size;
+    size_t extension_start_offset = m_offset + m_size;
+#if REALM_ENABLE_ENCRYPTION
+    if (m_encrypted_mapping) {
+        void* got_addr = ::mmap(extension_start_addr, extension_size, PROT_READ | PROT_WRITE,
+                                MAP_ANON | MAP_PRIVATE | MAP_FIXED, -1, 0);
+        if (got_addr == MAP_FAILED)
+            return false;
+        REALM_ASSERT(got_addr == extension_start_addr);
+        util::extend_encrypted_mapping(m_encrypted_mapping, m_addr, m_offset, m_size, size);
+        m_size = size;
+        return true;
+    }
+#endif
+    try {
+        void* got_addr =
+            util::mmap_fixed(m_fd, extension_start_addr, extension_size, m_a, extension_start_offset, nullptr);
+        if (got_addr == extension_start_addr) {
+            m_size = size;
+            return true;
+        }
+    }
+    catch (...) {
+        return false;
+    }
+#endif
+    return false;
 }
 
 void File::MapBase::sync()

src/realm/util/file.hpp

@@ -646,8 +646,10 @@ class File {
     struct MapBase {
         void* m_addr = nullptr;
         mutable size_t m_size = 0;
+        size_t m_reservation_size = 0;
         size_t m_offset = 0;
         FileDesc m_fd;
+        AccessMode m_a;
 
         MapBase() noexcept;
         ~MapBase() noexcept;
@@ -659,12 +661,18 @@ class File {
 
         // Use
         void map(const File&, AccessMode, size_t size, int map_flags, size_t offset = 0);
+        // reserve address space for later mapping operations.
+        // returns false if reservation can't be done.
+        bool try_reserve(const File&, AccessMode, size_t size, size_t offset = 0);
         void remap(const File&, AccessMode, size_t size, int map_flags);
         void unmap() noexcept;
         // fully update any process shared representation (e.g. buffer cache).
         // other processes will be able to see changes, but a full platform crash
         // may loose data
         void flush();
+        // try to extend the mapping in-place. Virtual address space must have
+        // been set aside earlier by a call to reserve()
+        bool try_extend_to(size_t size) noexcept;
         // fully synchronize any underlying storage. After completion, a full platform
         // crash will *not* have lost data.
         void sync();
@@ -764,11 +772,13 @@ class File::Map : private MapBase {
             unmap();
         m_addr = other.get_addr();
         m_size = other.m_size;
+        m_a = other.m_a;
+        m_reservation_size = other.m_reservation_size;
         m_offset = other.m_offset;
         m_fd = other.m_fd;
         other.m_offset = 0;
         other.m_addr = nullptr;
-        other.m_size = 0;
+        other.m_size = other.m_reservation_size = 0;
 #if REALM_ENABLE_ENCRYPTION
         m_encrypted_mapping = other.m_encrypted_mapping;
         other.m_encrypted_mapping = nullptr;
@@ -793,6 +803,8 @@ class File::Map : private MapBase {
     /// currently attached to a memory mapped file.
     void unmap() noexcept;
 
+    bool try_reserve(const File&, AccessMode a = access_ReadOnly, size_t size = sizeof(T), size_t offset = 0);
+
     /// See File::remap().
     ///
     /// Calling this function on a Map instance that is not currently
@@ -801,6 +813,9 @@ class File::Map : private MapBase {
     /// returned by get_addr().
     T* remap(const File&, AccessMode = access_ReadOnly, size_t size = sizeof(T), int map_flags = 0);
 
+    /// Try to extend the existing mapping to a given size
+    bool try_extend_to(size_t size) noexcept;
+
     /// See File::sync_map().
     ///
     /// Calling this function on an instance that is not currently
@@ -1187,6 +1202,12 @@ inline T* File::Map<T>::map(const File& f, AccessMode a, size_t size, int map_fl
     return static_cast<T*>(m_addr);
 }
 
+template <class T>
+inline bool File::Map<T>::try_reserve(const File& f, AccessMode a, size_t size, size_t offset)
+{
+    return MapBase::try_reserve(f, a, size, offset);
+}
+
 template <class T>
 inline void File::Map<T>::unmap() noexcept
 {
@@ -1204,6 +1225,12 @@ inline T* File::Map<T>::remap(const File& f, AccessMode a, size_t size, int map_
     return static_cast<T*>(m_addr);
 }
 
+template <class T>
+inline bool File::Map<T>::try_extend_to(size_t size) noexcept
+{
+    return MapBase::try_extend_to(sizeof(T) * size);
+}
+
 template <class T>
 inline void File::Map<T>::sync()
 {