schema

silkworm/db/blocks/schema_config.cpp

@@ -20,12 +20,25 @@
 
 namespace silkworm::db::blocks {
 
+snapshots::Schema::RepositoryDef make_blocks_repository_schema() {
+    snapshots::Schema::RepositoryDef schema;
+    schema
+        .segment(kHeaderSegmentName)
+        .rec_split_index(kIdxHeaderHashName)
+        .segment(kBodySegmentName)
+        .rec_split_index(kIdxBodyNumberName)
+        .segment(kTxnSegmentName)
+        .rec_split_index(kIdxTxnHashName)
+        .rec_split_index(kIdxTxnHash2BlockName);
+    return schema;
+}
+
 snapshots::SnapshotRepository make_blocks_repository(std::filesystem::path dir_path, bool open) {
     return snapshots::SnapshotRepository{
         std::move(dir_path),
         open,
         std::make_unique<snapshots::StepToBlockNumConverter>(),
-        std::make_unique<db::SnapshotBundleFactoryImpl>(),
+        std::make_unique<db::SnapshotBundleFactoryImpl>(make_blocks_repository_schema()),
     };
 }
 

silkworm/db/blocks/schema_config.hpp

@@ -17,12 +17,31 @@
 #pragma once
 
 #include "../datastore/common/entity_name.hpp"
+#include "../datastore/snapshots/schema.hpp"
 #include "../datastore/snapshots/snapshot_repository.hpp"
 
 namespace silkworm::db::blocks {
 
 inline constexpr datastore::EntityName kBlocksRepositoryName{"Blocks"};
 
-snapshots::SnapshotRepository make_blocks_repository(std::filesystem::path dir_path, bool open = true);
+snapshots::Schema::RepositoryDef make_blocks_repository_schema();
+
+snapshots::SnapshotRepository make_blocks_repository(
+    std::filesystem::path dir_path,
+    bool open = true);
+
+inline constexpr datastore::EntityName kHeaderSegmentName{"headers"};
+//! Index header_hash -> block_num -> headers_segment_offset
+inline constexpr datastore::EntityName kIdxHeaderHashName{"headers"};
+
+inline constexpr datastore::EntityName kBodySegmentName{"bodies"};
+//! Index block_num -> bodies_segment_offset
+inline constexpr datastore::EntityName kIdxBodyNumberName{"bodies"};
+
+inline constexpr datastore::EntityName kTxnSegmentName{"transactions"};
+//! Index transaction_hash -> txn_id -> transactions_segment_offset
+inline constexpr datastore::EntityName kIdxTxnHashName{"transactions"};
+//! Index transaction_hash -> block_num
+inline constexpr datastore::EntityName kIdxTxnHash2BlockName{"transactions_to_block"};
 
 }  // namespace silkworm::db::blocks

silkworm/db/data_store.cpp

@@ -0,0 +1,35 @@
+/*
+   Copyright 2024 The Silkworm Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+#include "data_store.hpp"
+
+namespace silkworm::db {
+
+datastore::Schema DataStore::make_schema(
+    bool enabled_state_repository) {
+    snapshots::Schema snapshots;
+    snapshots.repository(blocks::kBlocksRepositoryName) = blocks::make_blocks_repository_schema();
+
+    if (enabled_state_repository) {
+        snapshots.repository(state::kStateRepositoryName) = state::make_state_repository_schema();
+    }
+
+    return {
+        std::move(snapshots),
+    };
+}
+
+}  // namespace silkworm::db

silkworm/db/data_store.hpp

@@ -37,6 +37,7 @@ class DataStore {
         snapshots::SnapshotRepository blocks_repository,
         std::optional<snapshots::SnapshotRepository> state_repository = std::nullopt)
         : store_{
+              make_schema(state_repository.has_value()),
               std::move(chaindata_env),
               make_repositories_map(std::move(blocks_repository), std::move(state_repository)),
           } {}
@@ -61,6 +62,8 @@ class DataStore {
     db::RWAccess chaindata_rw() const { return store_.chaindata_rw(); }
 
   private:
+    static datastore::Schema make_schema(bool enabled_state_repository);
+
     static std::map<datastore::EntityName, std::unique_ptr<snapshots::SnapshotRepository>> make_repositories_map(
         snapshots::SnapshotRepository blocks_repository,
         std::optional<snapshots::SnapshotRepository> state_repository) {

silkworm/db/datastore/data_store.hpp

@@ -21,16 +21,19 @@
 
 #include "common/entity_name.hpp"
 #include "mdbx/mdbx.hpp"
+#include "schema.hpp"
 #include "snapshots/snapshot_repository.hpp"
 
 namespace silkworm::datastore {
 
 class DataStore {
   public:
     DataStore(
+        Schema schema,
         mdbx::env_managed chaindata_env,
         std::map<EntityName, std::unique_ptr<snapshots::SnapshotRepository>> repositories)
-        : chaindata_env_{std::move(chaindata_env)},
+        : schema_{std::move(schema)},
+          chaindata_env_{std::move(chaindata_env)},
           repositories_{std::move(repositories)} {}
 
     void close() {
@@ -39,11 +42,14 @@ class DataStore {
             entry.second->close();
     }
 
+    const Schema& schema() const { return schema_; }
     db::ROAccess chaindata() const { return db::ROAccess{chaindata_env_}; }
     db::RWAccess chaindata_rw() const { return db::RWAccess{chaindata_env_}; }
+
     snapshots::SnapshotRepository& repository(const EntityName& name) const { return *repositories_.at(name); }
 
   private:
+    Schema schema_;
     mdbx::env_managed chaindata_env_;
     std::map<EntityName, std::unique_ptr<snapshots::SnapshotRepository>> repositories_;
 };

silkworm/db/datastore/schema.hpp

@@ -0,0 +1,27 @@
+/*
+   Copyright 2024 The Silkworm Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+#pragma once
+
+#include "snapshots/schema.hpp"
+
+namespace silkworm::datastore {
+
+struct Schema {
+    snapshots::Schema snapshots;
+};
+
+}  // namespace silkworm::datastore

silkworm/db/datastore/snapshots/schema.cpp

@@ -0,0 +1,76 @@
+/*
+   Copyright 2024 The Silkworm Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+#include "schema.hpp"
+
+#include <magic_enum.hpp>
+
+namespace silkworm::snapshots {
+
+std::map<datastore::EntityName, SnapshotPath> Schema::RepositoryDef::make_segment_paths(
+    const std::filesystem::path& dir_path,
+    StepRange range) const {
+    std::map<datastore::EntityName, SnapshotPath> results;
+    for (auto& entry : segment_defs_) {
+        auto type = *magic_enum::enum_cast<SnapshotType>(entry.first.name);
+        results.emplace(entry.first, SnapshotPath::make(dir_path, kSnapshotV1, range, type));
+    }
+    return results;
+}
+
+std::map<datastore::EntityName, SegmentFileReader> Schema::RepositoryDef::make_segments(
+    const std::filesystem::path& dir_path,
+    StepRange range) const {
+    std::map<datastore::EntityName, SegmentFileReader> results;
+    for (auto& entry : make_segment_paths(dir_path, range)) {
+        results.emplace(entry.first, SegmentFileReader{entry.second});
+    }
+    return results;
+}
+
+std::map<datastore::EntityName, SnapshotPath> Schema::RepositoryDef::make_rec_split_index_paths(
+    const std::filesystem::path& dir_path,
+    StepRange range) const {
+    std::map<datastore::EntityName, SnapshotPath> results;
+    for (auto& entry : rec_split_index_defs_) {
+        auto type = *magic_enum::enum_cast<SnapshotType>(entry.first.name);
+        results.emplace(entry.first, SnapshotPath::make(dir_path, kSnapshotV1, range, type, kIdxExtension));
+    }
+    return results;
+}
+
+std::map<datastore::EntityName, Index> Schema::RepositoryDef::make_rec_split_indexes(
+    const std::filesystem::path& dir_path,
+    StepRange range) const {
+    std::map<datastore::EntityName, Index> results;
+    for (auto& entry : make_rec_split_index_paths(dir_path, range)) {
+        results.emplace(entry.first, Index{entry.second});
+    }
+    return results;
+}
+
+std::vector<SnapshotPath> Schema::RepositoryDef::make_all_paths(
+    const std::filesystem::path& dir_path,
+    StepRange range) const {
+    std::vector<SnapshotPath> results;
+    for (auto& entry : make_segment_paths(dir_path, range))
+        results.push_back(std::move(entry.second));
+    for (auto& entry : make_rec_split_index_paths(dir_path, range))
+        results.push_back(std::move(entry.second));
+    return results;
+}
+
+}  // namespace silkworm::snapshots

silkworm/db/datastore/snapshots/schema.hpp

@@ -0,0 +1,63 @@
+/*
+   Copyright 2024 The Silkworm Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+#pragma once
+
+#include <map>
+#include <variant>
+#include <vector>
+
+#include "../common/entity_name.hpp"
+#include "common/snapshot_path.hpp"
+#include "rec_split_index/index.hpp"
+#include "segment/segment_reader.hpp"
+
+namespace silkworm::snapshots {
+
+class Schema {
+  public:
+    class RepositoryDef {
+      public:
+        RepositoryDef& segment(datastore::EntityName name) {
+            segment_defs_.try_emplace(name);
+            return *this;
+        }
+
+        RepositoryDef& rec_split_index(datastore::EntityName name) {
+            rec_split_index_defs_.try_emplace(name);
+            return *this;
+        }
+
+        std::map<datastore::EntityName, SnapshotPath> make_segment_paths(const std::filesystem::path& dir_path, StepRange range) const;
+        std::map<datastore::EntityName, SegmentFileReader> make_segments(const std::filesystem::path& dir_path, StepRange range) const;
+        std::map<datastore::EntityName, SnapshotPath> make_rec_split_index_paths(const std::filesystem::path& dir_path, StepRange range) const;
+        std::map<datastore::EntityName, Index> make_rec_split_indexes(const std::filesystem::path& dir_path, StepRange range) const;
+        std::vector<SnapshotPath> make_all_paths(const std::filesystem::path& dir_path, StepRange range) const;
+
+      private:
+        std::map<datastore::EntityName, std::monostate> segment_defs_;
+        std::map<datastore::EntityName, std::monostate> rec_split_index_defs_;
+    };
+
+    RepositoryDef& repository(datastore::EntityName name) {
+        return repository_defs_[name];
+    }
+
+  private:
+    std::map<datastore::EntityName, RepositoryDef> repository_defs_;
+};
+
+}  // namespace silkworm::snapshots

silkworm/db/datastore/snapshots/snapshot_bundle.cpp

@@ -18,6 +18,8 @@
 
 #include <silkworm/infra/common/ensure.hpp>
 
+#include "common/util/iterator/map_values_view.hpp"
+
 namespace silkworm::snapshots {
 
 SnapshotBundle::~SnapshotBundle() {
@@ -72,23 +74,15 @@ std::vector<SnapshotPath> SnapshotBundle::segment_paths() {
 }
 
 std::vector<SnapshotPath> SnapshotBundlePaths::segment_paths() const {
-    return {
-        header_segment_path,
-        body_segment_path,
-        txn_segment_path,
-    };
+    MapValuesView view{schema_.make_segment_paths(dir_path_, step_range_)};
+    return std::vector<SnapshotPath>{view.begin(), view.end()};
 }
 
 std::vector<std::filesystem::path> SnapshotBundlePaths::files() const {
-    return std::vector<std::filesystem::path>{
-        header_segment_path.path(),
-        idx_header_hash_path.path(),
-        body_segment_path.path(),
-        idx_body_number_path.path(),
-        txn_segment_path.path(),
-        idx_txn_hash_path.path(),
-        idx_txn_hash_2_block_path.path(),
-    };
+    std::vector<std::filesystem::path> results;
+    for (auto& path : schema_.make_all_paths(dir_path_, step_range_))
+        results.push_back(path.path());
+    return results;
 }
 
 }  // namespace silkworm::snapshots

silkworm/db/datastore/snapshots/snapshot_bundle.hpp

@@ -25,6 +25,7 @@
 
 #include "common/snapshot_path.hpp"
 #include "rec_split_index/index.hpp"
+#include "schema.hpp"
 #include "segment/segment_reader.hpp"
 #include "segment_and_index.hpp"
 
@@ -49,21 +50,10 @@ struct SnapshotBundleData {
     static constexpr size_t kIndexesCount = 4;
 };
 
-struct SnapshotBundlePathsData {
-    SnapshotPath header_segment_path;
-    SnapshotPath idx_header_hash_path;
-
-    SnapshotPath body_segment_path;
-    SnapshotPath idx_body_number_path;
-
-    SnapshotPath txn_segment_path;
-    SnapshotPath idx_txn_hash_path;
-    SnapshotPath idx_txn_hash_2_block_path;
-};
-
-struct SnapshotBundlePaths : public SnapshotBundlePathsData {
-    SnapshotBundlePaths(StepRange step_range, SnapshotBundlePathsData bundle)
-        : SnapshotBundlePathsData{std::move(bundle)},
+struct SnapshotBundlePaths {
+    SnapshotBundlePaths(Schema::RepositoryDef schema, std::filesystem::path dir_path, StepRange step_range)
+        : schema_{std::move(schema)},
+          dir_path_{std::move(dir_path)},
           step_range_{step_range} {}
 
     StepRange step_range() const { return step_range_; }
@@ -72,6 +62,8 @@ struct SnapshotBundlePaths : public SnapshotBundlePathsData {
     std::vector<SnapshotPath> segment_paths() const;
 
   private:
+    Schema::RepositoryDef schema_;
+    std::filesystem::path dir_path_;
     StepRange step_range_;
 };