feat(accountsdb): Generate snapshots, fix UAF, improve bincode

build.zig.zon

@@ -24,8 +24,8 @@
             .hash = "1220b8a918dfcee4fc8326ec337776e2ffd3029511c35f6b96d10aa7be98ca2faf99",
         },
         .zstd = .{
-            .url = "https://github.com/Syndica/zstd.zig/archive/a052e839a3dfc44feb00c2eb425815baa3c76e0d.tar.gz",
-            .hash = "122001d56e43ef94e31243739ae83d7508abf0b8102795aff1ac91446e7ff450d875",
+            .url = "https://github.com/Syndica/zstd.zig/archive/20a21798a253ea1f0388ee633f4110e1deb2ddef.tar.gz",
+            .hash = "1220e27e4fece8ff0cabb2f7439891919e8ed294dc936c2a54f0702a2f0ec96f4b6c",
         },
         .curl = .{
             .url = "https://github.com/jiacai2050/zig-curl/archive/8a3f45798a80a5de4c11c6fa44dab8785c421d27.tar.gz",

src/accountsdb/accounts_file.zig

@@ -14,14 +14,16 @@ const AccountFileInfo = @import("snapshots.zig").AccountFileInfo;
 /// Simple strictly-typed alias for an integer, used to represent a file ID.
 ///
 /// Analogous to [AccountsFileId](https://github.com/anza-xyz/agave/blob/4c921ca276bbd5997f809dec1dd3937fb06463cc/accounts-db/src/accounts_db.rs#L824)
-pub const FileId = enum(u32) {
+pub const FileId = enum(Int) {
     _,
 
+    pub const Int = u32;
+
     pub inline fn fromInt(int: u32) FileId {
         return @enumFromInt(int);
     }
 
-    pub inline fn toInt(file_id: FileId) u32 {
+    pub inline fn toInt(file_id: FileId) Int {
         return @intFromEnum(file_id);
     }
 
@@ -222,7 +224,7 @@ pub const AccountInFile = struct {
 pub const AccountFile = struct {
     // file contents
     memory: []align(std.mem.page_size) u8,
-    id: usize,
+    id: FileId,
     slot: Slot,
     // number of bytes used
     length: usize,
@@ -395,7 +397,7 @@ test "core.accounts_file: verify accounts file" {
     const path = "test_data/test_account_file";
     const file = try std.fs.cwd().openFile(path, .{ .mode = .read_write });
     const file_info = AccountFileInfo{
-        .id = 0,
+        .id = FileId.fromInt(0),
         .length = 162224,
     };
     var accounts_file = try AccountFile.init(file, file_info, 10);

src/accountsdb/index.zig

@@ -67,8 +67,6 @@ pub const AccountRef = struct {
     }
 };
 
-const ReferenceMemory = std.AutoHashMap(Slot, ArrayList(AccountRef));
-
 /// stores the mapping from Pubkey to the account location (AccountRef)
 ///
 /// Analogous to [AccountsIndex](https://github.com/anza-xyz/agave/blob/a6b2283142192c5360ad0f53bec1eb4a9fb36154/accounts-db/src/accounts_index.rs#L644)
@@ -79,6 +77,7 @@ pub const AccountIndex = struct {
     bins: []RwMux(RefMap),
     calculator: PubkeyBinCalculator,
 
+    pub const ReferenceMemory = std.AutoHashMap(Slot, ArrayList(AccountRef));
     pub const RefMap = SwissMap(Pubkey, AccountReferenceHead, pubkey_hash, pubkey_eql);
 
     const Self = @This();
@@ -387,7 +386,7 @@ pub const AccountIndex = struct {
                 .slot = accounts_file.slot,
                 .location = .{
                     .File = .{
-                        .file_id = FileId.fromInt(@intCast(accounts_file.id)),
+                        .file_id = accounts_file.id,
                         .offset = offset,
                     },
                 },

src/accountsdb/readme.md

@@ -188,7 +188,7 @@ loading from a snapshot begins in `accounts_db.loadFromSnapshot` is a very
 expensive operation. 
 
 the steps include:
-- reads and load all the account files 
+- reads and load all the account files based on the snapshot manifest's file map 
 - validates + indexes every account in each file (in parallel) 
 - combines the results across the threads (also in parallel) 
 
@@ -233,6 +233,21 @@ after validating accounts-db data, we also validate a few key structs:
 - `Bank` : contains `bank_fields` which is in the snapshot metadata (not used right now)
 - `StatusCache / SlotHistory Sysvar` : additional validation performed in `status_cache.validate`
 
+## generating a snapshot
+
+*note:* at the time of writing, this functionality is in its infancy.
+
+The core logic for generating a snapshot lives in `accounts_db.db.writeSnapshotTarWithFields`; the principle entrypoint is `AccountsDB.writeSnapshotTar`.
+The procedure consists of writing the version file, the status cache (`snapshots/status_cache`) file, the snapshot manifest (`snapshots/{SLOT}/{SLOT}`),
+and the account files (`accounts/{SLOT}.{FILE_ID}`). This is all written to a stream in the TAR archive format.
+
+The snapshot manifest file content is comprised of the bincoded (bincode-encoded) data structure `SnapshotFields`, which is an aggregate of:
+* implicit state: data derived from the current state of AccountsDB, like the file map for all the account which exist at that snapshot, or which have
+  changed relative to a full snapshot in an incremental one
+* configuration state: data that is used to communicate details about the snapshot, like the full slot to which an incremental snapshot is relative.
+
+For full snapshots, we write all account files present in AccountsDB which are rooted - as in, less than or equal to the latest rooted slot.
+
 ## read/write benchmarks 
 `BenchArgs` contains all the configuration of a benchmark (comments describe each parameter) 
 - found at the bottom of `db.zig`
@@ -257,4 +272,4 @@ swissmapBenchmark(500k accounts)          1    7715875    7715875          0
         WRITE: 17.163ms (1.44x faster than std)
         READ: 50.975ms (0.70x faster than std)
 swissmapBenchmark(1m accounts)            1   17163500   17163500          0   17163500
-```
+```