Test to minic how Realm Java compact an encrypted and populated file

release_notes.md

@@ -2,7 +2,8 @@
 
 ### Bugfixes:
 
-* Lorem ipsum.
+* Fixed a bug in SharedGroup::Compact() which could leave the database in an
+  inconsistent state.
 
 ### API breaking changes:
 

src/realm/alloc_slab.cpp

@@ -503,7 +503,6 @@ ref_type SlabAlloc::attach_file(const std::string& path, Config& cfg)
     try {
         File::Map<char> map(m_file, File::access_ReadOnly, size); // Throws
 
-        m_file_on_streaming_form = false; // May be updated by validate_buffer()
         if (!cfg.skip_validate) {
             // Verify the data structures
             validate_buffer(map.get_addr(), initial_size_of_file, path, top_ref, cfg.is_shared); // Throws
@@ -526,9 +525,11 @@ ref_type SlabAlloc::attach_file(const std::string& path, Config& cfg)
         }
 
         {
-            const Header* header = reinterpret_cast<const Header*>(map.get_addr());
-            int select_field = ((header->m_flags & SlabAlloc::flags_SelectBit) != 0 ? 1 : 0);
-            m_file_format = header->m_file_format[select_field];
+            const Header& header = reinterpret_cast<const Header&>(*map.get_addr());
+            int select_field = ((header.m_flags & SlabAlloc::flags_SelectBit) != 0 ? 1 : 0);
+            m_file_format = header.m_file_format[select_field];
+            uint_fast64_t ref = uint_fast64_t(header.m_top_ref[select_field]);
+            m_file_on_streaming_form = (select_field == 0 && ref == 0xFFFFFFFFFFFFFFFFULL);
         }
 
         m_data        = map.release();
@@ -553,7 +554,7 @@ ref_type SlabAlloc::attach_file(const std::string& path, Config& cfg)
         Header* header = reinterpret_cast<Header*>(m_data);
         static_cast<void>(header);
 
-        // Don't compare file format version fields as they are allowed to differ. 
+        // Don't compare file format version fields as they are allowed to differ.
         // Also don't compare reserved fields (todo, is it correct to ignore?)
         REALM_ASSERT_3(header->m_flags, == , streaming_header.m_flags);
         REALM_ASSERT_3(header->m_mnemonic[0], == , streaming_header.m_mnemonic[0]);
@@ -588,16 +589,17 @@ ref_type SlabAlloc::attach_buffer(char* data, size_t size)
     REALM_ASSERT(!is_attached());
 
     // Verify the data structures
-    m_file_on_streaming_form = false; // May be updated by validate_buffer()
     std::string path; // No path
     ref_type top_ref;
     bool is_shared = false;
     validate_buffer(data, size, path, top_ref, is_shared); // Throws
 
     {
-        const Header* header = reinterpret_cast<const Header*>(data);
-        int select_field = ((header->m_flags & SlabAlloc::flags_SelectBit) != 0 ? 1 : 0);
-        m_file_format = header->m_file_format[select_field];
+        const Header& header = reinterpret_cast<const Header&>(*data);
+        int select_field = ((header.m_flags & SlabAlloc::flags_SelectBit) != 0 ? 1 : 0);
+        m_file_format = header.m_file_format[select_field];
+        uint_fast64_t ref = uint_fast64_t(header.m_top_ref[select_field]);
+        m_file_on_streaming_form = (select_field == 0 && ref == 0xFFFFFFFFFFFFFFFFULL);
     }
 
     m_data        = data;
@@ -629,30 +631,28 @@ void SlabAlloc::attach_empty()
     m_initial_mapping_size = m_baseline;
 }
 
+
 void SlabAlloc::validate_buffer(const char* data, size_t size, const std::string& path,
                                 ref_type& top_ref, bool is_shared)
 {
     // Verify that size is sane and 8-byte aligned
     if (REALM_UNLIKELY(size < sizeof (Header) || size % 8 != 0))
         throw InvalidDatabase("Realm file has bad size", path);
 
-    // File header is 24 bytes, composed of three 64-bit
-    // blocks. The two first being top_refs (only one valid
-    // at a time) and the last being the info block.
-    const char* file_header = data;
+    const Header& header = *reinterpret_cast<const Header*>(data);
 
     // First four bytes of info block is file format id
-    if (REALM_UNLIKELY(!(file_header[16] == 'T' &&
-                         file_header[17] == '-' &&
-                         file_header[18] == 'D' &&
-                         file_header[19] == 'B')))
+    if (REALM_UNLIKELY(!(char(header.m_mnemonic[0]) == 'T' &&
+                         char(header.m_mnemonic[1]) == '-' &&
+                         char(header.m_mnemonic[2]) == 'D' &&
+                         char(header.m_mnemonic[3]) == 'B')))
         throw InvalidDatabase("Not a Realm file", path);
 
     // Last bit in info block indicates which top_ref block is valid
-    int valid_part = file_header[16 + 7] & 0x1;
+    int select_field = ((header.m_flags & SlabAlloc::flags_SelectBit) != 0 ? 1 : 0);
 
     // Byte 4 and 5 (depending on valid_part) in the info block is version
-    int file_format = static_cast<unsigned char>(file_header[16 + 4 + valid_part]);
+    int file_format = int(header.m_file_format[select_field]);
     bool bad_file_format = (file_format != library_file_format);
 
     // As a special case, allow upgrading from version 2 to 3, but only when
@@ -664,16 +664,14 @@ void SlabAlloc::validate_buffer(const char* data, size_t size, const std::string
         throw InvalidDatabase("Unsupported Realm file format version", path);
 
     // Top_ref should always point within buffer
-    const uint64_t* top_refs = reinterpret_cast<const uint64_t*>(data);
-    uint_fast64_t ref = top_refs[valid_part];
-    if (valid_part == 0 && ref == 0xFFFFFFFFFFFFFFFFULL) {
+    uint_fast64_t ref = uint_fast64_t(header.m_top_ref[select_field]);
+    if (select_field == 0 && ref == 0xFFFFFFFFFFFFFFFFULL) {
         if (REALM_UNLIKELY(size < sizeof (Header) + sizeof (StreamingFooter)))
             throw InvalidDatabase("Realm file in streaming form has bad size", path);
-        const StreamingFooter* footer = reinterpret_cast<const StreamingFooter*>(data+size) - 1;
-        ref = footer->m_top_ref;
-        if (REALM_UNLIKELY(footer->m_magic_cookie != footer_magic_cookie))
+        const StreamingFooter& footer = *(reinterpret_cast<const StreamingFooter*>(data+size) - 1);
+        ref = footer.m_top_ref;
+        if (REALM_UNLIKELY(footer.m_magic_cookie != footer_magic_cookie))
             throw InvalidDatabase("Bad Realm file header (#1)", path);
-        m_file_on_streaming_form = true;
     }
     if (REALM_UNLIKELY(ref % 8 != 0))
         throw InvalidDatabase("Bad Realm file header (#2)", path);

src/realm/alloc_slab.hpp

@@ -397,8 +397,6 @@ class SlabAlloc: public Allocator {
     /// Throws InvalidDatabase if the file is not a Realm file, if the file is
     /// corrupted, or if the specified encryption key is incorrect. This
     /// function will not detect all forms of corruption, though.
-    ///
-    /// Initializes `m_file_on_streaming_form`.
     void validate_buffer(const char* data, std::size_t len, const std::string& path,
                          ref_type& top_ref, bool is_shared);
 
@@ -438,7 +436,7 @@ class SlabAlloc: public Allocator {
     /// Find a possible allocation of 'request_size' that will fit into a section
     /// which is inside the range from 'start_pos' to 'start_pos'+'free_chunk_size'
     /// If found return the position, if not return 0.
-    std::size_t find_section_in_range(std::size_t start_pos, std::size_t free_chunk_size, 
+    std::size_t find_section_in_range(std::size_t start_pos, std::size_t free_chunk_size,
                                       std::size_t request_size) const noexcept;
 
     friend class Group;

test/test_lang_bind_helper.cpp

@@ -8968,4 +8968,48 @@ TEST(LangBindHelper_RollbackToInitialState2)
     sg_w.rollback();
 }
 
+TEST(LangBindHelper_Compact)
+{
+    SHARED_GROUP_TEST_PATH(path);
+    size_t N = 100;
+
+    {
+        std::unique_ptr<ClientHistory> hist_w(make_client_history(path, crypt_key()));
+        SharedGroup sg_w(*hist_w, SharedGroup::durability_Full, crypt_key());
+        WriteTransaction w(sg_w);
+        TableRef table = w.get_or_add_table("test");
+        table->add_column(type_Int, "int");
+        for (size_t i = 0; i < N; ++i) {
+            table->add_empty_row();
+            table->set_int(0, i, i);
+        }
+        w.commit();
+        sg_w.close();
+    }
+    {
+        std::unique_ptr<ClientHistory> hist(make_client_history(path, crypt_key()));
+        SharedGroup sg(*hist, SharedGroup::durability_Full, crypt_key());
+        ReadTransaction r(sg);
+        ConstTableRef table = r.get_table("test");
+        CHECK_EQUAL(N, table->size());
+        sg.close();
+    }
+
+    {
+        std::unique_ptr<ClientHistory> hist(make_client_history(path, crypt_key()));
+        SharedGroup sg(*hist, SharedGroup::durability_Full, crypt_key());
+        CHECK_EQUAL(true, sg.compact());
+        sg.close();
+    }
+
+    {
+        std::unique_ptr<ClientHistory> hist(make_client_history(path, crypt_key()));
+        SharedGroup sg(*hist, SharedGroup::durability_Full, crypt_key());
+        ReadTransaction r(sg);
+        ConstTableRef table = r.get_table("test");
+        CHECK_EQUAL(N, table->size());
+        sg.close();
+    }
+}
+
 #endif

test/test_shared.cpp

@@ -277,6 +277,12 @@ TEST(Shared_CompactingOnTheFly)
         CHECK_EQUAL(true, sg2.compact());
         ReadTransaction rt2(sg2);
         rt2.get_group().verify();
+        sg2.close();
+    }
+    {
+        SharedGroup sg2(path, true, SharedGroup::durability_Full, crypt_key());
+        ReadTransaction rt2(sg2);
+        rt2.get_group().verify();
     }
 }