Merge pull request #169 from Syndica/dnut/repair3

feat(repair): link shred collector with snapshot - root slot, leader schedule, verify shreds
Syndica · Jul 1, 2024 · 6ce1445 · 6ce1445
2 parents 220f002 + a65ebf7
commit 6ce1445
Show file tree

Hide file tree

Showing 23 changed files with 1,842 additions and 342 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,14 +1,8 @@
 .zig-cache/
+zig-cache/
 zig-out/
 data/
 
-# unpacking test snapshots
-test_data/version
-test_data/accounts/
-test_data/snapshots/
-test_data/tmp/
-test_data/bench_snapshot/
-test_data/accountsdb_fuzz/
 index_storage/
 
 /gossip-dumps

diff --git a/src/accountsdb/genesis_config.zig b/src/accountsdb/genesis_config.zig
@@ -108,10 +108,10 @@ pub const EpochSchedule = extern struct {
     first_normal_slot: Slot,
 
     pub fn getEpoch(self: *const EpochSchedule, slot: Slot) Epoch {
-        return self.getEpochAndSlotIndex(slot).epoch;
+        return self.getEpochAndSlotIndex(slot)[0];
     }
 
-    pub fn getEpochAndSlotIndex(self: *const EpochSchedule, slot: Slot) struct { epoch: Epoch, slot_index: Slot } {
+    pub fn getEpochAndSlotIndex(self: *const EpochSchedule, slot: Slot) struct { Epoch, Slot } {
         if (slot < self.first_normal_slot) {
             var epoch = slot +| MINIMUM_SLOTS_PER_EPOCH +| 1;
             epoch = @ctz(std.math.ceilPowerOfTwo(u64, epoch) catch {
@@ -123,23 +123,26 @@ pub const EpochSchedule = extern struct {
 
             const slot_index = slot -| (epoch_len -| MINIMUM_SLOTS_PER_EPOCH);
 
-            return .{
-                .epoch = epoch,
-                .slot_index = slot_index,
-            };
+            return .{ epoch, slot_index };
         } else {
             const normal_slot_index = slot -| self.first_normal_slot;
             const normal_epoch_index = std.math.divTrunc(u64, normal_slot_index, self.slots_per_epoch) catch 0;
 
             const epoch = self.first_normal_epoch +| normal_epoch_index;
             const slot_index = std.math.rem(u64, normal_slot_index, self.slots_per_epoch) catch 0;
 
-            return .{
-                .epoch = epoch,
-                .slot_index = slot_index,
-            };
+            return .{ epoch, slot_index };
         }
     }
+
+    /// get the length of the given epoch (in slots)
+    pub fn getSlotsInEpoch(self: *const EpochSchedule, epoch: Epoch) Slot {
+        comptime std.debug.assert(std.math.isPowerOfTwo(MINIMUM_SLOTS_PER_EPOCH));
+        return if (epoch < self.first_normal_epoch)
+            @as(Slot, 1) <<| epoch +| @ctz(MINIMUM_SLOTS_PER_EPOCH)
+        else
+            self.slots_per_epoch;
+    }
 };
 
 pub const ClusterType = enum(u8) {

diff --git a/src/accountsdb/lib.zig b/src/accountsdb/lib.zig
@@ -12,6 +12,7 @@ pub const AccountsDB = db.AccountsDB;
 pub const AllSnapshotFields = snapshots.AllSnapshotFields;
 pub const Bank = bank.Bank;
 pub const GenesisConfig = genesis_config.GenesisConfig;
+pub const SnapshotFields = snapshots.SnapshotFields;
 pub const SnapshotFieldsAndPaths = snapshots.SnapshotFieldsAndPaths;
 pub const SnapshotFiles = snapshots.SnapshotFiles;
 pub const StatusCache = snapshots.StatusCache;

diff --git a/src/accountsdb/snapshots.zig b/src/accountsdb/snapshots.zig
@@ -55,16 +55,69 @@ pub const Stakes = struct {
 };
 
 pub const VoteAccounts = struct {
-    vote_accounts: HashMap(Pubkey, struct { u64, Account }),
+    vote_accounts: HashMap(Pubkey, struct { u64, VoteAccount }),
 
     staked_nodes: ?HashMap(
         Pubkey, // VoteAccount.vote_state.node_pubkey.
         u64, // Total stake across all vote-accounts.
     ) = null,
 
     pub const @"!bincode-config:staked_nodes" = bincode.FieldConfig(?HashMap(Pubkey, u64)){ .skip = true };
+
+    const Self = @This();
+
+    pub fn stakedNodes(self: *Self, allocator: std.mem.Allocator) !*const HashMap(Pubkey, u64) {
+        if (self.staked_nodes) |*staked_nodes| {
+            return staked_nodes;
+        }
+        const vote_accounts = self.vote_accounts;
+        var staked_nodes = HashMap(Pubkey, u64).init(allocator);
+        var iter = vote_accounts.iterator();
+        while (iter.next()) |vote_entry| {
+            const vote_state = try vote_entry.value_ptr[1].voteState();
+            const node_entry = try staked_nodes.getOrPut(vote_state.node_pubkey);
+            if (!node_entry.found_existing) {
+                node_entry.value_ptr.* = 0;
+            }
+            node_entry.value_ptr.* += vote_entry.value_ptr[0];
+        }
+        self.staked_nodes = staked_nodes;
+        return &self.staked_nodes.?;
+    }
 };
 
+pub const VoteAccount = struct {
+    account: Account,
+    vote_state: ?anyerror!VoteState = null,
+
+    pub const @"!bincode-config:vote_state" = bincode.FieldConfig(?anyerror!VoteState){ .skip = true };
+
+    pub fn voteState(self: *@This()) !VoteState {
+        if (self.vote_state) |vs| {
+            return vs;
+        }
+        self.vote_state = bincode.readFromSlice(undefined, VoteState, self.account.data, .{});
+        return self.vote_state.?;
+    }
+};
+
+pub const VoteState = struct {
+    /// The variant of the rust enum
+    tag: u32, // TODO: consider varint bincode serialization (in rust this is enum)
+    /// the node that votes in this account
+    node_pubkey: Pubkey,
+};
+
+test "deserialize VoteState.node_pubkey" {
+    const bytes = .{
+        2,  0,   0,   0, 60,  155, 13,  144, 187, 252, 153, 72,  190, 35,  87,  94,  7,  178,
+        90, 174, 158, 6, 199, 179, 134, 194, 112, 248, 166, 232, 144, 253, 128, 249, 67, 118,
+    } ++ .{0} ** 1586 ++ .{ 31, 0, 0, 0, 0, 0, 0, 0, 1 } ++ .{0} ** 24;
+    const vote_state = try bincode.readFromSlice(undefined, VoteState, &bytes, .{});
+    const expected_pubkey = try Pubkey.fromString("55abJrqFnjm7ZRB1noVdh7BzBe3bBSMFT3pt16mw6Vad");
+    try std.testing.expect(expected_pubkey.equals(&vote_state.node_pubkey));
+}
+
 pub const Delegation = struct {
     /// to whom the stake is delegated
     voter_pubkey: Pubkey,