Bincode simplifications

src/accountsdb/snapshots.zig

@@ -317,9 +317,10 @@ pub const BankFields = struct {
         epoch_reward_status: ?EpochRewardStatus = null,
 
         // TODO: do a thorough review on this, this seems to work by chance with the test data, but I don't trust it yet
-        pub const @"!bincode-config:snapshot_persistence" = bincode.FieldConfig(?BankIncrementalSnapshotPersistence){ .default_on_eof = true };
-        pub const @"!bincode-config:epoch_accounts_hash" = bincode.FieldConfig(?Hash){ .default_on_eof = true };
-        pub const @"!bincode-config:epoch_reward_status" = bincode.FieldConfig(?EpochRewardStatus){ .default_on_eof = true, .skip_write_fn = bincode.skipIfNull };
+
+        pub const @"!bincode-config:snapshot_persistence" = bincode.optional.defaultToNullOnEof(BankIncrementalSnapshotPersistence, .{ .encode_optional = true });
+        pub const @"!bincode-config:epoch_accounts_hash" = bincode.optional.defaultToNullOnEof(Hash, .{ .encode_optional = true });
+        pub const @"!bincode-config:epoch_reward_status" = bincode.optional.defaultToNullOnEof(EpochRewardStatus, .{ .encode_optional = false });
     };
 };
 
@@ -404,11 +405,11 @@ pub const AccountsDbFields = struct {
 pub const SnapshotFields = struct {
     bank_fields: BankFields,
     accounts_db_fields: AccountsDbFields,
-    lamports_per_signature: u64 = 0,
+    lamports_per_signature: u64,
     /// incremental snapshot fields (to accompany added to bank_fields)
     bank_fields_inc: BankFields.Incremental = .{},
 
-    pub const @"!bincode-config:lamports_per_signature" = bincode.FieldConfig(u64){ .default_on_eof = true };
+    pub const @"!bincode-config:lamports_per_signature" = bincode.int.defaultOnEof(u64, 0);
 
     pub fn readFromFilePath(allocator: std.mem.Allocator, path: []const u8) !SnapshotFields {
         const file = std.fs.cwd().openFile(path, .{}) catch |err| {

src/bincode/arraylist.zig

@@ -1,46 +1,21 @@
 const std = @import("std");
 const sig = @import("../lib.zig");
-
 const bincode = sig.bincode;
 
-pub fn ArrayListConfig(comptime Child: type) bincode.FieldConfig(std.ArrayList(Child)) {
-    const S = struct {
-        pub fn serialize(writer: anytype, data: anytype, params: bincode.Params) !void {
-            const list: std.ArrayList(Child) = data;
-            try bincode.write(writer, @as(u64, list.items.len), params);
-            for (list.items) |item| {
-                try bincode.write(writer, item, params);
-            }
-            return;
-        }
-
-        pub fn deserialize(allocator: std.mem.Allocator, reader: anytype, params: bincode.Params) !std.ArrayList(Child) {
-            const len = try bincode.read(allocator, u64, reader, params);
-            var list = try std.ArrayList(Child).initCapacity(allocator, @as(usize, len));
-            for (0..len) |_| {
-                const item = try bincode.read(allocator, Child, reader, params);
-                try list.append(item);
-            }
-            return list;
-        }
-
-        pub fn free(allocator: std.mem.Allocator, data: anytype) void {
-            _ = allocator;
-            data.deinit();
-        }
-    };
-
-    return bincode.FieldConfig(std.ArrayList(Child)){
-        .serializer = S.serialize,
-        .deserializer = S.deserialize,
-        .free = S.free,
-    };
-}
-
 pub fn defaultArrayListUnmanagedOnEOFConfig(comptime T: type) bincode.FieldConfig(std.ArrayListUnmanaged(T)) {
     const S = struct {
-        fn defaultEOF(_: std.mem.Allocator) std.ArrayListUnmanaged(T) {
-            return .{};
+        fn deserialize(allocator: std.mem.Allocator, reader: anytype, params: bincode.Params) anyerror!std.ArrayListUnmanaged(T) {
+            const len = if (bincode.readIntAsLength(usize, reader, params)) |maybe_len|
+                (maybe_len orelse return error.ArrayListTooBig)
+            else |err| {
+                if (err == error.EndOfStream) return .{};
+                return err;
+            };
+
+            const slice = try allocator.alloc(T, len);
+            errdefer allocator.free(slice);
+
+            return std.ArrayListUnmanaged(T).fromOwnedSlice(slice);
         }
 
         fn free(allocator: std.mem.Allocator, data: anytype) void {
@@ -49,15 +24,15 @@ pub fn defaultArrayListUnmanagedOnEOFConfig(comptime T: type) bincode.FieldConfi
     };
 
     return .{
-        .default_on_eof = true,
+        .deserializer = S.deserialize,
         .free = S.free,
-        .default_fn = S.defaultEOF,
     };
 }
 pub fn defaultArrayListOnEOFConfig(comptime T: type) bincode.FieldConfig(std.ArrayList(T)) {
     const S = struct {
-        fn defaultEOF(allocator: std.mem.Allocator) std.ArrayList(T) {
-            return std.ArrayList(T).init(allocator);
+        fn deserialize(allocator: std.mem.Allocator, reader: anytype, params: bincode.Params) anyerror!std.ArrayList(T) {
+            var unmanaged = try defaultArrayListUnmanagedOnEOFConfig(T).deserializer.?(allocator, reader, params);
+            return unmanaged.toManaged(allocator);
         }
 
         fn free(_: std.mem.Allocator, data: anytype) void {
@@ -66,8 +41,7 @@ pub fn defaultArrayListOnEOFConfig(comptime T: type) bincode.FieldConfig(std.Arr
     };
 
     return .{
-        .default_on_eof = true,
+        .deserializer = S.deserialize,
         .free = S.free,
-        .default_fn = S.defaultEOF,
     };
 }

src/bincode/bincode.zig

@@ -2,7 +2,8 @@ pub const arraylist = @import("arraylist.zig");
 pub const shortvec = @import("shortvec.zig");
 pub const varint = @import("varint.zig");
 pub const optional = @import("optional.zig");
-pub const list = @import("slice.zig");
+pub const list = @import("list.zig");
+pub const int = @import("int.zig");
 
 const std = @import("std");
 const sig = @import("../lib.zig");
@@ -314,19 +315,7 @@ fn readFieldWithConfig(
         return try readHashMap(allocator, reader, params, field.type, field_config.hashmap);
     }
 
-    return bincode.read(allocator, field.type, reader, params) catch |err| blk: {
-        if (field_config.default_on_eof and err == error.EndOfStream) {
-            if (field.default_value) |default_value| {
-                break :blk @as(*const field.type, @ptrCast(@alignCast(default_value))).*;
-            } else if (field_config.default_fn) |default_fcn| {
-                break :blk default_fcn(allocator);
-            } else {
-                return error.MissingFieldDefaultValue;
-            }
-        } else {
-            return err;
-        }
-    };
+    return try bincode.read(allocator, field.type, reader, params);
 }
 
 pub fn readIntAsLength(comptime T: type, reader: anytype, params: bincode.Params) !?T {
@@ -540,7 +529,6 @@ fn writeFieldWithConfig(
 
     if (maybe_field_config) |field_config| {
         if (field_config.skip) return;
-        if (field_config.skip_write_fn(data)) return;
         if (field_config.serializer) |ser_fcn| {
             try ser_fcn(writer, data, params);
             return;
@@ -675,17 +663,6 @@ pub fn FieldConfig(comptime T: type) type {
         serializer: ?SerializeFunction = null,
         free: ?FreeFunction = null,
         skip: bool = false,
-        default_on_eof: bool = false,
-        default_fn: ?fn (alloc: std.mem.Allocator) T = null,
-        /// A predicate which returns true for a given field value if it should be skipped
-        /// during serialization, and false otherwise.
-        ///
-        /// This predicate is needed in tandem with `default_on_eof` in order to help
-        /// ensure values which were never read (and thus set to a default) aren't written
-        /// back. It is defaulted to `neverSkip`, which simply always returns false.
-        /// The programmer overriding this field is encouraged to use a predicate which
-        /// coordinates with `default_fn`.
-        skip_write_fn: fn (value: anytype) bool = neverSkip,
         hashmap: if (hashMapInfo(T)) |hm_info| bincode.HashMapConfig(hm_info) else void = if (hashMapInfo(T) != null) .{} else {},
     };
 }
@@ -812,7 +789,7 @@ test "bincode: default on eof" {
         value: u8 = 0,
         accounts: std.ArrayList(u64),
         pub const @"!bincode-config:accounts" = defaultArrayListOnEOFConfig(u64);
-        pub const @"!bincode-config:value" = .{ .default_on_eof = true };
+        pub const @"!bincode-config:value" = int.defaultOnEof(u8, 0);
     };
 
     var buf: [1]u8 = .{1};

src/bincode/int.zig

@@ -0,0 +1,32 @@
+const std = @import("std");
+const sig = @import("../lib.zig");
+const bincode = sig.bincode;
+
+pub fn defaultOnEof(comptime T: type, comptime eof_value: T) bincode.FieldConfig(T) {
+    const gen = struct {
+        fn deserializer(
+            _: std.mem.Allocator,
+            reader: anytype,
+            params: bincode.Params,
+        ) anyerror!T {
+            var fba = comptime std.heap.FixedBufferAllocator.init(&.{});
+            return bincode.read(fba.allocator(), T, reader, params) catch |err| switch (err) {
+                error.EndOfStream => eof_value,
+                else => |e| e,
+            };
+        }
+
+        fn serializer(
+            writer: anytype,
+            data: anytype,
+            params: bincode.Params,
+        ) anyerror!void {
+            if (data == eof_value) return;
+            try bincode.write(writer, data, params);
+        }
+    };
+    return .{
+        .deserializer = gen.deserializer,
+        .serializer = gen.serializer,
+    };
+}

src/bincode/slice.zig → src/bincode/list.zig

@@ -38,9 +38,6 @@ pub fn valueEncodedAsSlice(
         .serializer = gen.serializeImpl,
         .free = config.free,
         .skip = config.skip,
-        .default_on_eof = config.default_on_eof,
-        .default_fn = config.default_fn,
-        .skip_write_fn = config.skip_write_fn,
         .hashmap = config.hashmap,
     };
 }

src/bincode/optional.zig

@@ -3,17 +3,25 @@ const sig = @import("../lib.zig");
 const bincode = sig.bincode;
 const hashMapInfo = sig.utils.types.hashMapInfo;
 
-pub fn defaultToNullOnEof(comptime T: type, fields: struct {
-    free: ?fn (allocator: std.mem.Allocator, data: anytype) void = null,
-    hashmap: if (hashMapInfo(T)) |hm_info| bincode.HashMapConfig(hm_info) else void = if (hashMapInfo(T) != null) .{} else {},
-}) bincode.FieldConfig(?T) {
+pub fn defaultToNullOnEof(
+    comptime T: type,
+    comptime options: struct {
+        /// When this is false, the field will be encoded and decoded as a non-optional value, only reading as null on eof, and not written when it is null.
+        /// When this is true, the field will be encoded and decoded as an optional value, defaulting to null on eof while reading.
+        encode_optional: bool = false,
+
+        free: ?fn (allocator: std.mem.Allocator, data: anytype) void = null,
+        hashmap: if (hashMapInfo(T)) |hm_info| bincode.HashMapConfig(hm_info) else void = if (hashMapInfo(T) != null) .{} else {},
+    },
+) bincode.FieldConfig(?T) {
     const gen = struct {
         fn deserializer(
             allocator: std.mem.Allocator,
             reader: anytype,
             params: bincode.Params,
         ) anyerror!?T {
-            return bincode.read(allocator, T, reader, params) catch |err| switch (err) {
+            const EncodedType = if (options.encode_optional) ?T else T;
+            return bincode.read(allocator, EncodedType, reader, params) catch |err| switch (err) {
                 error.EndOfStream => null,
                 else => |e| e,
             };
@@ -24,26 +32,19 @@ pub fn defaultToNullOnEof(comptime T: type, fields: struct {
             maybe_data: anytype,
             params: bincode.Params,
         ) anyerror!void {
-            const data = maybe_data orelse return;
-            try bincode.write(writer, data, params);
-        }
-
-        fn default(_: std.mem.Allocator) ?T {
-            return null;
-        }
-
-        fn skipWrite(maybe_value: anytype) bool {
-            return maybe_value == null;
+            if (options.encode_optional) {
+                return try bincode.write(writer, maybe_data, params);
+            } else {
+                const data = maybe_data orelse return;
+                return try bincode.write(writer, data, params);
+            }
         }
     };
     return .{
         .deserializer = gen.deserializer,
         .serializer = gen.serializer,
-        .free = fields.free,
+        .free = options.free,
         .skip = false,
-        .default_on_eof = true,
-        .default_fn = gen.default,
-        .skip_write_fn = gen.skipWrite,
-        .hashmap = fields.hashmap,
+        .hashmap = options.hashmap,
     };
 }

src/bloom/bloom.zig

@@ -10,7 +10,6 @@ const ArrayList = std.ArrayList;
 const DynamicArrayBitSet = sig.bloom.bit_set.DynamicArrayBitSet;
 const BitVec = sig.bloom.bit_vec.BitVec;
 const BitVecConfig = sig.bloom.bit_vec.BitVecConfig;
-const ArrayListConfig = bincode.arraylist.ArrayListConfig;
 const FnvHasher = sig.crypto.FnvHasher;
 
 /// A bloom filter whose bitset is made up of u64 blocks
@@ -19,7 +18,6 @@ pub const Bloom = struct {
     bits: DynamicArrayBitSet(u64),
     num_bits_set: u64,
 
-    pub const @"!bincode-config:keys" = ArrayListConfig(u64);
     pub const @"!bincode-config:bits" = BitVecConfig(u64);
 
     const Self = @This();