Recover ledger last idx from read-only ledger directory on recovery (#…

…1953)

src/host/ledger.h

@@ -625,6 +625,35 @@ namespace asynchost
           max_chunk_threshold_size));
       }
 
+      // Recover last idx from read-only ledger directories
+      for (const auto& read_dir : read_ledger_dirs)
+      {
+        LOG_DEBUG_FMT("Recovering read-only ledger directory \"{}\"", read_dir);
+        if (!fs::is_directory(read_dir))
+        {
+          throw std::logic_error(
+            fmt::format("\"{}\" is not a directory", read_dir));
+        }
+
+        for (auto const& f : fs::directory_iterator(read_dir))
+        {
+          auto last_idx_ = get_last_idx_from_file_name(f.path().filename());
+          if (!last_idx_.has_value())
+          {
+            LOG_DEBUG_FMT(
+              "Read-only ledger file \"{}\" is ignored as not committed",
+              f.path().filename());
+            continue;
+          }
+
+          if (last_idx_.value() > last_idx)
+          {
+            last_idx = last_idx_.value();
+            committed_idx = last_idx;
+          }
+        }
+      }
+
       if (fs::is_directory(ledger_dir))
       {
         // If the ledger directory exists, recover ledger files from it
@@ -649,7 +678,17 @@ namespace asynchost
           return a->get_last_idx() < b->get_last_idx();
         });
 
-        last_idx = get_latest_file()->get_last_idx();
+        auto main_ledger_dir_last_idx = get_latest_file()->get_last_idx();
+        if (main_ledger_dir_last_idx < last_idx)
+        {
+          throw std::logic_error(fmt::format(
+            "Ledger directory last idx ({}) is less than read-only "
+            "ledger directories last idx ({})",
+            main_ledger_dir_last_idx,
+            last_idx));
+        }
+
+        last_idx = main_ledger_dir_last_idx;
 
         for (auto f = files.begin(); f != files.end();)
         {
@@ -686,6 +725,11 @@ namespace asynchost
         }
         require_new_file = true;
       }
+
+      LOG_INFO_FMT(
+        "Recovered ledger entries up to {}, committed to {}",
+        last_idx,
+        committed_idx);
     }
 
     Ledger(const Ledger& that) = delete;
@@ -895,4 +939,4 @@ namespace asynchost
         });
     }
   };
-}
+}

src/host/test/ledger.cpp

@@ -1,9 +1,9 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the Apache 2.0 License.
 #define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN
-#include "../ledger.h"
+#include "host/ledger.h"
 
-#include "../ds/serialized.h"
+#include "ds/serialized.h"
 
 #include <doctest/doctest.h>
 #include <string>
@@ -818,4 +818,46 @@ TEST_CASE("Multiple ledger paths")
     // cannot be read
     REQUIRE_FALSE(ledger.read_entry(last_idx).has_value());
   }
+}
+
+TEST_CASE("Recover from read-only ledger directory only")
+{
+  static constexpr auto ledger_dir_2 = "ledger_dir_2";
+
+  fs::remove_all(ledger_dir);
+  fs::remove_all(ledger_dir_2);
+
+  size_t max_read_cache_size = 2;
+  size_t chunk_threshold = 30;
+  size_t chunk_count = 5;
+
+  size_t last_idx = 0;
+
+  INFO("Write many entries on first ledger");
+  {
+    Ledger ledger(ledger_dir, wf, chunk_threshold);
+    TestEntrySubmitter entry_submitter(ledger);
+
+    // Writing some committed chunks
+    initialise_ledger(entry_submitter, chunk_threshold, chunk_count);
+    last_idx = entry_submitter.get_last_idx();
+    ledger.commit(last_idx);
+  }
+
+  INFO("Recover from read-only ledger entry only");
+  {
+    Ledger ledger(
+      ledger_dir_2, wf, chunk_threshold, max_read_cache_size, {ledger_dir});
+
+    read_entries_range_from_ledger(ledger, 1, last_idx);
+
+    TestEntrySubmitter entry_submitter(ledger, last_idx);
+
+    for (size_t i = 0; i < chunk_count; i++)
+    {
+      entry_submitter.write(true);
+    }
+
+    read_entries_range_from_ledger(ledger, 1, entry_submitter.get_last_idx());
+  }
 }