Merge pull request #2 from 0xNineteen/gossip-protocol-implementation

sync: misc gossip protocol updates

build.zig

@@ -133,4 +133,4 @@ pub fn build(b: *std.Build) void {
     // This will evaluate the `run` step rather than the default, which is "install".
     const run_step = b.step("run", "Run the app");
     run_step.dependOn(&run_cmd.step);
-}
+}

build.zig.zon

@@ -11,8 +11,8 @@
             .hash = "122090e7cb4459c2224399a45c56f47462733b919aa547c96b8c14ee705bfa22976e",
         },
         .@"bincode-zig" = .{
-            .url = "https://github.com/ultd/bincode-zig/archive/0e38a40b5e962d3dd7ce3c4ec8ccfc43fcbc26c6.tar.gz",
-            .hash = "1220d47d04a72072a4bb1ab3ebfc34b3e3873d77d58486cde1f5bc82d22c5c776ae6",
+            .url = "https://github.com/0xNineteen/bincode-zig/archive/6b8cbfe412ce87a8f21062c3c53e67e56c5f9b94.tar.gz",
+            .hash = "122024b878fc84c240e5e3551cf86dd9c8ac5ff547deddf34e978f8c45b591cc7543",
         },
         .@"zig-cli" = .{
             .url = "https://github.com/sam701/zig-cli/archive/ea92821a797bdefb5df41ce602f8c7f387bcfb93.tar.gz",

src/bloom/bitvec.zig

@@ -1,6 +1,5 @@
 const std = @import("std");
 const DynamicBitSet = std.bit_set.DynamicBitSet;
-const Option = @import("../option.zig").Option;
 const bincode = @import("bincode-zig");
 const testing = std.testing;
 
@@ -16,33 +15,32 @@ var gpa = gpa_allocator.allocator();
 /// serializes/deserializes differently thus why we need this
 /// intermediary type.
 pub const BitVec = struct {
-    bits: Option([]u64),
+    bits: ?[]u64,
     len: u64,
 
     const Self = @This();
 
     pub fn initFromBitSet(bitset: DynamicBitSet) Self {
         if (bitset.capacity() > 0) {
-            return Self{
-                .bits = Option([]u64).Some(bitset.unmanaged.masks[0..(bitset.unmanaged.bit_length / 64)]),
+            return Self {
+                .bits = bitset.unmanaged.masks[0..(bitset.unmanaged.bit_length / 64)],
                 .len = @as(u64, bitset.unmanaged.bit_length),
             };
         }
         return Self{
-            .bits = Option([]u64).None(),
+            .bits = null,
             .len = @as(u64, bitset.unmanaged.bit_length),
         };
     }
 
     pub fn toBitSet(self: *const Self, allocator: std.mem.Allocator) !DynamicBitSet {
         var bitset = try DynamicBitSet.initEmpty(allocator, self.len);
-        switch (self.bits) {
-            .some => |bits| {
-                for (0..(self.len / 64)) |i| {
-                    bitset.unmanaged.masks[i] = bits[i];
-                }
-            },
-            else => {},
+        errdefer bitset.deinit();
+
+        if (self.bits) |bits| { 
+            for (0..(self.len / 64)) |i| {
+                bitset.unmanaged.masks[i] = bits[i];
+            }
         }
         return bitset;
     }
@@ -75,6 +73,6 @@ test "bitvec serializes/deserializes and matches Rust's BitVec" {
     var deserialied = try bincode.readFromSlice(testing.allocator, BitVec, out, bincode.Params.standard);
     defer bincode.readFree(testing.allocator, deserialied);
 
-    try testing.expect(std.mem.eql(u64, original.bits.some[0..], deserialied.bits.some[0..]));
-    try testing.expect(std.mem.eql(u8, rust_bit_vec_serialized[0..], out[0..]));
+    try testing.expect(std.mem.eql(u64, original.bits.?[0..], deserialied.bits.?[0..]));
+    try testing.expectEqualSlices(u8, rust_bit_vec_serialized[0..], out[0..]);
 }

src/bloom/bloom.zig

@@ -21,51 +21,46 @@ pub const Bloom = struct {
 
     const Self = @This();
 
-    pub fn init(allocator: std.mem.Allocator) Self {
+    pub fn init(allocator: std.mem.Allocator, num_bits: u64) Self {
+        // need to be power of 2 for serialization to match rust
+        if (num_bits != 0) { 
+            std.debug.assert((num_bits & (num_bits - 1) == 0));
+        }
         return Self{
             .keys = ArrayList(u64).init(allocator),
-            .bits = DynamicBitSet.initEmpty(allocator, 0) catch unreachable,
+            .bits = DynamicBitSet.initEmpty(allocator, num_bits) catch unreachable,
             .num_bits_set = 0,
         };
     }
 
-    pub fn deinit(self: Self) void {
+    pub fn deinit(self: *Self) void {
         self.bits.deinit();
         self.keys.deinit();
     }
 
-    pub fn add(self: *Self, key: u64) void {
-        for (self.keys.items) |k| {
-            var i = self.pos(key, k);
+    pub fn add_key(self: *Self, key: u64) !void { 
+        try self.keys.append(key);
+    }
+
+    pub fn add(self: *Self, key: []const u8) void {
+        for (self.keys.items) |hash_index| {
+            var i = self.pos(key, hash_index);
             if (!self.bits.isSet(i)) {
                 self.num_bits_set +|= 1;
                 self.bits.set(i);
             }
         }
     }
 
-    pub fn pos(self: *Self, key: u64, k: u64) u64 {
-        return self.hash_at_index(key, k) % @bitSizeOf(u64);
+    pub fn pos(self: *Self, bytes: []const u8, hash_index: u64) u64 {
+        return hash_at_index(bytes, hash_index) % @as(u64, self.bits.capacity());
     }
 
-    pub fn hash_at_index(key: u64, hash_index: u64) u64 {
-        var bytes = std.mem.asBytes(&key);
+    pub fn hash_at_index(bytes: []const u8, hash_index: u64) u64 {
         var hasher = FnvHasher.initWithOffset(hash_index);
         hasher.update(bytes);
         return hasher.final();
     }
-
-    // pub fn pos()
-
-    // pub fn add(self: *Self) void {
-    //      for (self.keys) |key| {
-    //         let pos = self.pos(key, *k);
-    //         if !self.bits.get(pos) {
-    //             self.num_bits_set = self.num_bits_set.saturating_add(1);
-    //             self.bits.set(pos, true);
-    //         }
-    //     }
-    // }
 };
 
 fn bincode_serialize_bit_vec(writer: anytype, data: anytype, params: bincode.Params) !void {
@@ -77,21 +72,81 @@ fn bincode_serialize_bit_vec(writer: anytype, data: anytype, params: bincode.Par
 
 fn bincode_deserialize_bit_vec(allocator: std.mem.Allocator, comptime T: type, reader: anytype, params: bincode.Params) !T {
     var bitvec = try bincode.read(allocator, BitVec, reader, params);
-    return try bitvec.toBitSet(allocator);
+    defer bincode.readFree(allocator, bitvec);
+
+    var dynamic_bitset = try bitvec.toBitSet(allocator);
+    return dynamic_bitset;
 }
 
 // TODO: Finish test
-test "bloom serializes/deserializes correctly" {
-    var keys = ArrayList(u64).init(testing.allocator);
-    _ = keys;
-    var bloom = Bloom.init(testing.allocator);
-    var buf: [10000]u8 = undefined;
+test "bloom: serializes/deserializes correctly" {
+    var bloom = Bloom.init(testing.allocator, 0);
 
+    var buf: [10000]u8 = undefined;
     var out = try bincode.writeToSlice(buf[0..], bloom, bincode.Params.standard);
-
     std.log.debug("out: {any}", .{out});
 
     var deserialized = try bincode.readFromSlice(testing.allocator, Bloom, out, bincode.Params.standard);
+    try testing.expect(bloom.num_bits_set == deserialized.num_bits_set);
+}
+
+test "bloom: serializes/deserializes correctly with set bits" {
+    var bloom = Bloom.init(testing.allocator, 128);
+    try bloom.add_key(10);
+    // required for memory leaks
+    defer bloom.deinit();
+
+    var buf: [10000]u8 = undefined;
+    var out = try bincode.writeToSlice(buf[0..], bloom, bincode.Params.standard);
+    std.log.debug("out: {any}", .{out});
+
+    var deserialized: Bloom = try bincode.readFromSlice(testing.allocator, Bloom, out, bincode.Params.standard);
+    defer deserialized.deinit();
 
     try testing.expect(bloom.num_bits_set == deserialized.num_bits_set);
 }
+
+test "bloom: rust: serialized bytes equal rust (no keys)" {
+    // note: need to init with len 2^i
+    var bloom = Bloom.init(testing.allocator, 128);
+    defer bloom.deinit();
+    try bloom.add_key(1);
+
+    const v: [1]u8 = .{ 1 };
+    bloom.add(&v);
+
+    var buf: [10000]u8 = undefined;
+    var bytes = try bincode.writeToSlice(buf[0..], bloom, bincode.Params.standard);
+
+    const rust_bytes = .{1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0}; 
+
+    try testing.expectEqualSlices(u8, &rust_bytes, bytes[0..bytes.len]);
+}
+
+test "bloom: rust: serialized bytes equal rust (multiple keys)" {
+    var bloom = Bloom.init(testing.allocator, 128);
+    defer bloom.deinit();
+
+    try bloom.add_key(1);
+    try bloom.add_key(2);
+    try bloom.add_key(3);
+
+    var buf: [10000]u8 = undefined;
+
+    const v: [2]u8 = .{ 1, 2 };
+    bloom.add(&v);
+
+    const x: [2]u8 = .{ 3, 4 };
+    bloom.add(&x);
+
+    var bytes = try bincode.writeToSlice(buf[0..], bloom, bincode.Params.standard);
+
+    // let mut bloom = Bloom::new(128, vec![1, 2, 3]);
+    // bloom.add(&[1, 2]);
+    // bloom.add(&[3, 4]);
+    // println!("{:?}", bincode::serialize(&bloom).unwrap());
+
+    const rust_bytes = .{3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 66, 16, 32, 0, 128, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0}; 
+
+    try testing.expectEqualSlices(u8, &rust_bytes, bytes[0..bytes.len]);
+}

src/cmd/cmd.zig

@@ -42,13 +42,15 @@ var app = &cli.App{
     },
 };
 
+// prints (and creates if DNE) pubkey in ~/.sig/identity.key
 fn identity(_: []const []const u8) !void {
     const id = try gossipCmd.getOrInitIdentity(gpa);
     var pk: [50]u8 = undefined;
     var size = try base58Encoder.encode(&id.public_key.toBytes(), &pk);
     try std.io.getStdErr().writer().print("Identity: {s}\n", .{pk[0..size]});
 }
 
+// gossip entrypoint
 fn gossip(_: []const []const u8) !void {
     var gossip_port: u16 = @intCast(gossip_port_option.value.int.?);
     var entrypoints = std.ArrayList(LegacyContactInfo).init(gpa);

src/crypto/fnv.zig

@@ -1,3 +1,5 @@
+const std = @import("std");
+
 /// ***FnvHasher*** is a FNV-1 (64-bit) hasher implementation.
 /// See: https://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function
 pub const FnvHasher = struct {
@@ -9,25 +11,25 @@ pub const FnvHasher = struct {
 
     pub fn init() Self {
         return Self{
-            .value = DEFAULT_OFFSET,
+            .hashh = DEFAULT_OFFSET,
         };
     }
 
     pub fn initWithOffset(offset: u64) Self {
         return Self{
-            .value = offset,
+            .hashh = offset,
         };
     }
 
     pub fn update(self: *Self, input: []const u8) void {
         for (input) |byte| {
-            self.value ^= byte;
-            self.value *%= prime;
+            self.hashh ^= byte;
+            self.hashh *%= prime;
         }
     }
 
     pub fn final(self: *Self) u64 {
-        return self.value;
+        return self.hashh;
     }
 
     pub fn hash(input: []const u8) u64 {
@@ -42,3 +44,21 @@ pub const FnvHasher = struct {
         return c.final();
     }
 };
+
+test "fnv hasher with default is correct" {
+    const exp: u64 = 12638152016183539244; 
+
+    var hasher = FnvHasher.init();
+    hasher.update(&.{ 1 });
+    const result = hasher.final();
+    try std.testing.expectEqual(exp, result);
+}
+
+test "fnv hasher with offset is correct" {
+    const exp: u64 = 11233064889143142093; 
+
+    var hasher = FnvHasher.initWithOffset(19);
+    hasher.update(&.{ 1, 2, 3 });
+    const result = hasher.final();
+    try std.testing.expectEqual(exp, result);
+}

src/gossip/cluster_info.zig

@@ -102,12 +102,4 @@ pub const ClusterInfo = struct {
         }
     }
 
-    pub fn processPackets(self: *Self, allocator: std.mem.Allocator, receiver: *Channel(Packet)) !void {
-        _ = self;
-        while (receiver.receive()) |p| {
-            var protocol_message = try bincode.readFromSlice(allocator, Protocol, p.data[0..p.size], bincode.Params.standard);
-
-            logger.debug("got a protocol message: {any}", .{protocol_message});
-        }
-    }
 };

src/gossip/cmd.zig

@@ -12,11 +12,15 @@ const LegacyContactInfo = @import("crds.zig").LegacyContactInfo;
 var gpa = std.heap.GeneralPurposeAllocator(.{}){};
 var gpa_allocator = gpa.allocator();
 
-const IDENTITY_KEYPAIR_PATH = "/.sig/identity.key";
+const logger = std.log.scoped(.cmd);
+
+const IDENTITY_KEYPAIR_DIR = "/.sig";
+const IDENTITY_KEYPAIR_PATH = "/identity.key";
 
 pub fn getOrInitIdentity(allocator: std.mem.Allocator) !Keypair {
     const home_dir = std.os.getenv("HOME") orelse return error.UnableDetectHomeDir;
-    var path = try std.mem.concat(allocator, u8, &[_][]const u8{ home_dir, IDENTITY_KEYPAIR_PATH });
+    var path = try std.mem.concat(allocator, u8, &[_][]const u8{ home_dir, IDENTITY_KEYPAIR_DIR, IDENTITY_KEYPAIR_PATH });
+
     if (std.fs.openFileAbsolute(path, .{})) |file| {
         try file.seekTo(0);
 
@@ -29,11 +33,16 @@ pub fn getOrInitIdentity(allocator: std.mem.Allocator) !Keypair {
     } else |err| {
         switch (err) {
             error.FileNotFound => {
+                // create ~/.sig dir
+                var dir = try std.mem.concat(allocator, u8, &[_][]const u8{ home_dir, IDENTITY_KEYPAIR_DIR });
+                std.fs.makeDirAbsolute(dir) catch { logger.debug("sig directory already exists...", .{}); };
+
+                // create new keypair 
                 const file = try std.fs.createFileAbsolute(path, .{ .truncate = true });
                 defer file.close();
 
                 const kp = try Keypair.create(null);
-                try file.writeAll(kp.secret_key.toBytes()[0..]);
+                try file.writeAll(&kp.secret_key.toBytes());
 
                 return kp;
             },