Pass sequential test

src/extmap.cc

@@ -1,3 +1,4 @@
+#include <cassert>
 #include <cstdint>
 #include <folly/logging/xlog.h>
 #include <memory>
@@ -8,20 +9,14 @@
 
 Task<vec<std::pair<usize, S3Ext>>> ExtMap::lookup(usize offset, usize len)
 {
+    assert(offset + len <= total_len);
     auto l = co_await mtx.co_scoped_lock_shared();
     vec<std::pair<usize, S3Ext>> res;
 
     // upper_bound returns the first element that is > offset, so the element
     // at which we start is always the one before that
     auto it = map.upper_bound(offset);
 
-    // the whole address space should be mapped, so if we don't find anything
-    // that means we're looking at a corrupted map
-    if (it == map.end()) {
-        XLOGF(ERR, "Corrupt map: no entry found for offset {}", offset);
-        co_return res;
-    }
-
     // Go back one so we start at the right place
     it--;
 
@@ -100,7 +95,7 @@ void ExtMap::unmap_locked(usize offset, usize len)
     // BUG: not sure how
     while (it != map.end() && it->first < end) {
         auto &[base, ext] = *it;
-        if (base + ext.len < end)
+        if (base + ext.len <= end)
             it = map.erase(it);
         else
             break;
@@ -131,6 +126,9 @@ Task<void> ExtMap::update(usize base, usize len, S3Ext ext)
     auto l = co_await mtx.co_scoped_lock();
     unmap_locked(base, len);
     map[base] = ext;
+
+    if (VERIFY_MAP_INTEGRITY_ON_UPDATE)
+        verify_integrity();
 }
 
 Task<void> ExtMap::unmap(usize base, usize len)
@@ -140,6 +138,19 @@ Task<void> ExtMap::unmap(usize base, usize len)
     map[base] = {0, 0, len};
 }
 
+void ExtMap::verify_integrity()
+{
+    usize last_base = 0, last_len = 0, total = 0;
+    for (auto &[base, ext] : map) {
+        assert(base >= last_base);
+        assert(base == last_base + last_len);
+        last_base = base;
+        last_len = ext.len;
+        total += ext.len;
+    }
+    assert(total == total_len);
+}
+
 Task<vec<byte>> ExtMap::serialise()
 {
     vec<byte> buf;
@@ -165,9 +176,7 @@ uptr<ExtMap> ExtMap::deserialise(vec<byte> buf)
 
 uptr<ExtMap> ExtMap::create_empty(usize len)
 {
-    auto m = std::unique_ptr<ExtMap>(new ExtMap());
-    m->map.insert({0, S3Ext{0, 0, len}});
-    return m;
+    return std::unique_ptr<ExtMap>(new ExtMap(len));
 }
 
 Task<std::string> ExtMap::to_string()

src/extmap.h

@@ -6,6 +6,10 @@
 #include "representation.h"
 #include "utils.h"
 
+#ifndef VERIFY_MAP_INTEGRITY_ON_UPDATE
+#define VERIFY_MAP_INTEGRITY_ON_UPDATE true
+#endif
+
 /**
 Maps a range of addresses to an extent on the backend.
 
@@ -35,12 +39,18 @@ class ExtMap
     extents with seqnum = 0 are ignored and can be of any value
      */
     std::map<usize, S3Ext> map;
+    usize total_len = 0;
 
     // Internal use, assumes the lock is already held
     void unmap_locked(usize base, usize len);
+    void verify_integrity();
 
-    ExtMap() {}
-    ExtMap(std::map<usize, S3Ext> map) : map(std::move(map)) {}
+    ExtMap(usize len) : total_len(len) { map.insert({0, S3Ext{0, 0, len}}); }
+    ExtMap(std::map<usize, S3Ext> map_) : map(std::move(map_))
+    {
+        for (auto &[base, ext] : map)
+            total_len += ext.len;
+    }
 
   public:
     ~ExtMap() {}

src/image.cc

@@ -56,6 +56,11 @@ class LogObj
 
 ResTask<void> LsvdImage::read(off_t offset, smartiov iovs)
 {
+    assert(offset >= 0);
+    assert(iovs.bytes() > 0 && iovs.bytes() % block_size == 0);
+    if (offset + iovs.bytes() > superblock.image_size)
+        co_return outcome::failure(std::errc::invalid_argument);
+
     // Since backend objects are immutable, we just need a read lock for as long
     // as we're getting the extents. Once we have the extents, the map can be
     // moified and we don't care.

src/journal.cc

@@ -8,7 +8,7 @@ ResTask<uptr<Journal>> Journal::open(fstr path)
 
 ResTask<void> Journal::record_write(off_t offset, iovec iov, S3Ext ext)
 {
-    todo();
+    // todo();
     co_return outcome::success();
 }
 

src/main.cc

@@ -8,13 +8,11 @@
 #include "representation.h"
 #include "utils.h"
 
-FOLLY_INIT_LOGGING_CONFIG(
-    ".=DBG,folly=INFO,cachelib=WARNING; default:async=true");
+FOLLY_INIT_LOGGING_CONFIG(".=DBG,folly=INFO");
 
 ResTask<std::string> main_task()
 {
     sptr<ObjStore> pool = ObjStore::connect_to_pool("pone").value();
-    (co_await LsvdImage::create(pool, "testimg", 1 * 1024 * 1024)).value();
     auto img = (co_await LsvdImage::mount(pool, "testimg", "")).value();
 
     vec<byte> bufw(4096);

src/read_cache.cc

@@ -59,7 +59,7 @@ class SharedCache
     }
 
     // returns true iff item was found
-    auto read(strv key, usize adjust, smartiov &dest) -> Task<bool>
+    auto read(strv key, usize adjust, smartiov dest) -> Task<bool>
     {
         auto h = co_await cache->find(key).toSemiFuture();
 
@@ -112,7 +112,7 @@ class ImageObjCache : public ReadCache
     }
 
     auto read_chunk(seqnum_t seq, usize offset, usize adjust,
-                    smartiov &dest) -> ResTask<void>
+                    smartiov dest) -> ResTask<void>
     {
         auto k = get_key(seq, offset);
         auto in_cache = co_await cache->read(k, adjust, dest);
@@ -130,7 +130,7 @@ class ImageObjCache : public ReadCache
         co_return outcome::success();
     }
 
-    ResTask<void> read(S3Ext ext, smartiov &dest) override
+    ResTask<void> read(S3Ext ext, smartiov dest) override
     {
         /**
         |-----------------------entire object---------------------------------|
@@ -150,6 +150,7 @@ class ImageObjCache : public ReadCache
             auto iov = dest.slice(bytes_read, bytes_read + to_read);
             tasks.push_back(
                 read_chunk(ext.seqnum, chunk_off, adjust, iov).semi());
+            bytes_read += to_read;
         }
 
         auto all = co_await folly::collectAll(tasks);

src/read_cache.h

@@ -9,7 +9,7 @@ class ReadCache
   public:
     virtual ~ReadCache() {}
 
-    virtual ResTask<void> read(S3Ext ext, smartiov &dest) = 0;
+    virtual ResTask<void> read(S3Ext ext, smartiov dest) = 0;
     virtual ResTask<void> insert_obj(seqnum_t seqnum, buffer iov) = 0;
 
     static void init_cache(usize mem_bytes, usize nvm_bytes, fstr nvm_path);

test/test_sequential_rw.cc

@@ -1,14 +1,17 @@
+#include "folly/String.h"
+#include "folly/init/Init.h"
 #include <array>
 #include <cassert>
+#include <folly/logging/Init.h>
 #include <iostream>
 #include <rados/librados.h>
 
 #include "backend.h"
-#include "folly/String.h"
-#include "folly/init/Init.h"
 #include "image.h"
 #include "utils.h"
 
+FOLLY_INIT_LOGGING_CONFIG(".=INFO,folly=INFO");
+
 const size_t LSVD_BLOCK_SIZE = 4096;
 using comp_buf = std::array<uint8_t, LSVD_BLOCK_SIZE>;
 
@@ -61,10 +64,10 @@ bool verify_buf(uint64_t block_num, const comp_buf &buf,
     fill_buf(block_num, expected_buf);
 
     if (buf != expected_buf) {
-        XLOGF(ERR, "Error reading block {:08x}.\nExpected:\n{}\nActual:\n{}",
-              block_num,
-              folly::hexDump(expected_buf.data(), expected_buf.size()),
-              folly::hexDump(buf.data(), buf.size()));
+        fmt::println("Error reading block {:08x}.\nExpected:\n{}\nActual:\n{}",
+                     block_num,
+                     folly::hexDump(expected_buf.data(), expected_buf.size()),
+                     folly::hexDump(buf.data(), buf.size()));
         return false;
     }
 
@@ -76,15 +79,14 @@ auto run_test(sptr<ObjStore> s3) -> Task<void>
     XLOGF(INFO, "Starting sequential write then readback test");
 
     XLOGF(INFO, "Removing old image if one exists");
-    auto res = co_await LsvdImage::remove(s3, "random-test-img");
-    res.value();
+    std::ignore = co_await LsvdImage::remove(s3, "random-test-img");
 
     // size_t img_size = 1 * 1024 * 1024 * 1024;
     size_t img_size = 100 * 1024 * 1024;
 
     // create the image for our own use
     XLOGF(INFO, "Creating image {} of size {}", "random-test-img", img_size);
-    res = co_await LsvdImage::create(s3, "random-test-img", img_size);
+    auto res = co_await LsvdImage::create(s3, "random-test-img", img_size);
     res.value();
 
     // open the image
@@ -140,6 +142,10 @@ int main(int argc, char *argv[])
     std::string pool_name = "pone";
     auto s3 = ObjStore::connect_to_pool(pool_name).value();
 
-    run_test(s3).scheduleOn(folly::getGlobalCPUExecutor()).start().wait();
+    auto ret =
+        run_test(s3).scheduleOn(folly::getGlobalCPUExecutor()).start().wait();
+    auto res = ret.result();
+    if (res.hasException())
+        XLOGF(FATAL, "Error:\n{}", res.exception().what());
     return 0;
 }