feat(repair): link shred collector with snapshot - root slot, leader schedule, verify shreds

.gitignore

@@ -1,14 +1,8 @@
 .zig-cache/
+zig-cache/
 zig-out/
 data/
 
-# unpacking test snapshots
-test_data/version
-test_data/accounts/
-test_data/snapshots/
-test_data/tmp/
-test_data/bench_snapshot/
-test_data/accountsdb_fuzz/
 index_storage/
 
 /gossip-dumps

src/accountsdb/genesis_config.zig

@@ -108,10 +108,10 @@ pub const EpochSchedule = extern struct {
  first_normal_slot: Slot,
 
  pub fn getEpoch(self: *const EpochSchedule, slot: Slot) Epoch {
- return self.getEpochAndSlotIndex(slot).epoch;
+ return self.getEpochAndSlotIndex(slot)[0];
  }
 
- pub fn getEpochAndSlotIndex(self: *const EpochSchedule, slot: Slot) struct { epoch: Epoch, slot_index: Slot } {
+ pub fn getEpochAndSlotIndex(self: *const EpochSchedule, slot: Slot) struct { Epoch, Slot } {
  if (slot < self.first_normal_slot) {
  var epoch = slot +| MINIMUM_SLOTS_PER_EPOCH +| 1;
  epoch = @ctz(std.math.ceilPowerOfTwo(u64, epoch) catch {
@@ -123,23 +123,26 @@ pub const EpochSchedule = extern struct {
 
  const slot_index = slot -| (epoch_len -| MINIMUM_SLOTS_PER_EPOCH);
 
- return .{
- .epoch = epoch,
- .slot_index = slot_index,
- };
+ return .{ epoch, slot_index };
  } else {
  const normal_slot_index = slot -| self.first_normal_slot;
  const normal_epoch_index = std.math.divTrunc(u64, normal_slot_index, self.slots_per_epoch) catch 0;
 
  const epoch = self.first_normal_epoch +| normal_epoch_index;
  const slot_index = std.math.rem(u64, normal_slot_index, self.slots_per_epoch) catch 0;
 
- return .{
- .epoch = epoch,
- .slot_index = slot_index,
- };
+ return .{ epoch, slot_index };
  }
  }
+
+ /// get the length of the given epoch (in slots)
+ pub fn getSlotsInEpoch(self: *const EpochSchedule, epoch: Epoch) Slot {
+ comptime std.debug.assert(std.math.isPowerOfTwo(MINIMUM_SLOTS_PER_EPOCH));
+ return if (epoch < self.first_normal_epoch)
+ @as(Slot, 1) <<| epoch +| @ctz(MINIMUM_SLOTS_PER_EPOCH)
+ else
+ self.slots_per_epoch;
+ }
 };
 
 pub const ClusterType = enum(u8) {

src/accountsdb/lib.zig

@@ -12,6 +12,7 @@ pub const AccountsDB = db.AccountsDB;
 pub const AllSnapshotFields = snapshots.AllSnapshotFields;
 pub const Bank = bank.Bank;
 pub const GenesisConfig = genesis_config.GenesisConfig;
+pub const SnapshotFields = snapshots.SnapshotFields;
 pub const SnapshotFieldsAndPaths = snapshots.SnapshotFieldsAndPaths;
 pub const SnapshotFiles = snapshots.SnapshotFiles;
 pub const StatusCache = snapshots.StatusCache;

src/accountsdb/snapshots.zig

@@ -55,16 +55,69 @@ pub const Stakes = struct {
 };
 
 pub const VoteAccounts = struct {
- vote_accounts: HashMap(Pubkey, struct { u64, Account }),
+ vote_accounts: HashMap(Pubkey, struct { u64, VoteAccount }),
 
  staked_nodes: ?HashMap(
  Pubkey, // VoteAccount.vote_state.node_pubkey.
  u64, // Total stake across all vote-accounts.
  ) = null,
 
  pub const @"!bincode-config:staked_nodes" = bincode.FieldConfig(?HashMap(Pubkey, u64)){ .skip = true };
+
+ const Self = @This();
+
+ pub fn stakedNodes(self: *Self, allocator: std.mem.Allocator) !*const HashMap(Pubkey, u64) {
+ if (self.staked_nodes) |*staked_nodes| {
+ return staked_nodes;
+ }
+ const vote_accounts = self.vote_accounts;
+ var staked_nodes = HashMap(Pubkey, u64).init(allocator);
+ var iter = vote_accounts.iterator();
+ while (iter.next()) |vote_entry| {
+ const vote_state = try vote_entry.value_ptr[1].voteState();
+ const node_entry = try staked_nodes.getOrPut(vote_state.node_pubkey);
+ if (!node_entry.found_existing) {
+ node_entry.value_ptr.* = 0;
+ }
+ node_entry.value_ptr.* += vote_entry.value_ptr[0];
+ }
+ self.staked_nodes = staked_nodes;
+ return &self.staked_nodes.?;
+ }
 };
 
+pub const VoteAccount = struct {
+ account: Account,
+ vote_state: ?anyerror!VoteState = null,
+
+ pub const @"!bincode-config:vote_state" = bincode.FieldConfig(?anyerror!VoteState){ .skip = true };
+
+ pub fn voteState(self: *@This()) !VoteState {
+ if (self.vote_state) |vs| {
+ return vs;
+ }
+ self.vote_state = bincode.readFromSlice(undefined, VoteState, self.account.data, .{});
+ return self.vote_state.?;
+ }
+};
+
+pub const VoteState = struct {
+ /// The variant of the rust enum
+ tag: u32, // TODO: consider varint bincode serialization (in rust this is enum)
+ /// the node that votes in this account
+ node_pubkey: Pubkey,
+};
+
+test "deserialize VoteState.node_pubkey" {
+ const bytes = .{
+ 2, 0, 0, 0, 60, 155, 13, 144, 187, 252, 153, 72, 190, 35, 87, 94, 7, 178,
+ 90, 174, 158, 6, 199, 179, 134, 194, 112, 248, 166, 232, 144, 253, 128, 249, 67, 118,
+ } ++ .{0} ** 1586 ++ .{ 31, 0, 0, 0, 0, 0, 0, 0, 1 } ++ .{0} ** 24;
+ const vote_state = try bincode.readFromSlice(undefined, VoteState, &bytes, .{});
+ const expected_pubkey = try Pubkey.fromString("55abJrqFnjm7ZRB1noVdh7BzBe3bBSMFT3pt16mw6Vad");
+ try std.testing.expect(expected_pubkey.equals(&vote_state.node_pubkey));
+}
+
 pub const Delegation = struct {
  /// to whom the stake is delegated
  voter_pubkey: Pubkey,