diff --git a/agreement/vote.go b/agreement/vote.go index b9261a4853..599023ada0 100644 --- a/agreement/vote.go +++ b/agreement/vote.go @@ -146,6 +146,12 @@ func (uv unauthenticatedVote) verify(l LedgerReader) (vote, error) { return vote{R: rv, Cred: cred, Sig: uv.Sig}, nil } +var ( + // testMakeVoteCheck is a function that can be set to check every + // unauthenticatedVote before it is returned by makeVote. It is only set by tests. + testMakeVoteCheck func(*unauthenticatedVote) error +) + // makeVote creates a new unauthenticated vote from its constituent components. // // makeVote returns an error if it fails. @@ -178,7 +184,15 @@ func makeVote(rv rawVote, voting crypto.OneTimeSigner, selection *crypto.VRFSecr } cred := committee.MakeCredential(&selection.SK, m.Selector) - return unauthenticatedVote{R: rv, Cred: cred, Sig: sig}, nil + ret := unauthenticatedVote{R: rv, Cred: cred, Sig: sig} + + // for use when running in tests + if testMakeVoteCheck != nil { + if testErr := testMakeVoteCheck(&ret); testErr != nil { + return unauthenticatedVote{}, fmt.Errorf("makeVote: testMakeVoteCheck failed: %w", testErr) + } + } + return ret, nil } // ToBeHashed implements the Hashable interface. diff --git a/agreement/vote_test.go b/agreement/vote_test.go index 9f5b9c9aed..9c5f74c8c7 100644 --- a/agreement/vote_test.go +++ b/agreement/vote_test.go @@ -17,6 +17,10 @@ package agreement import ( + "bytes" + "encoding/base64" + "encoding/hex" + "fmt" "os" "testing" @@ -27,10 +31,38 @@ import ( "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/committee" "github.com/algorand/go-algorand/logging" + "github.com/algorand/go-algorand/network/vpack" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/test/partitiontest" ) +func init() { + testMakeVoteCheck = testVPackMakeVote +} + +func testVPackMakeVote(v *unauthenticatedVote) error { + vbuf := protocol.Encode(v) + enc := vpack.NewStatelessEncoder() + dec := vpack.NewStatelessDecoder() + encBuf, err := enc.CompressVote(nil, vbuf) + if err != nil { + return fmt.Errorf("makeVote: failed to parse vote msgpack: %v", err) + } + decBuf, err := dec.DecompressVote(nil, encBuf) + if err != nil { + return fmt.Errorf("makeVote: failed to decompress vote msgpack: %v", err) + } + if !bytes.Equal(vbuf, decBuf) { + fmt.Printf("vote: %+v\n", v) + fmt.Printf("oldbuf: %s\n", hex.EncodeToString(vbuf)) + fmt.Printf("decbuf: %s\n", hex.EncodeToString(decBuf)) + fmt.Printf("base64 oldbuf: %s\n", base64.StdEncoding.EncodeToString(vbuf)) + fmt.Printf("base64 decbuf: %s\n", base64.StdEncoding.EncodeToString(decBuf)) + return fmt.Errorf("makeVote: decompressed vote msgpack does not match original") + } + return nil +} + // error is set if this address is not selected func makeVoteTesting(addr basics.Address, vrfSecs *crypto.VRFSecrets, otSecs crypto.OneTimeSigner, ledger Ledger, round basics.Round, period period, step step, digest crypto.Digest) (vote, error) { var proposal proposalValue diff --git a/config/localTemplate.go b/config/localTemplate.go index f8b673d195..db32ce742c 100644 --- a/config/localTemplate.go +++ b/config/localTemplate.go @@ -44,7 +44,7 @@ type Local struct { // Version tracks the current version of the defaults so we can migrate old -> new // This is specifically important whenever we decide to change the default value // for an existing parameter. This field tag must be updated any time we add a new version. - Version uint32 `version[0]:"0" version[1]:"1" version[2]:"2" version[3]:"3" version[4]:"4" version[5]:"5" version[6]:"6" version[7]:"7" version[8]:"8" version[9]:"9" version[10]:"10" version[11]:"11" version[12]:"12" version[13]:"13" version[14]:"14" version[15]:"15" version[16]:"16" version[17]:"17" version[18]:"18" version[19]:"19" version[20]:"20" version[21]:"21" version[22]:"22" version[23]:"23" version[24]:"24" version[25]:"25" version[26]:"26" version[27]:"27" version[28]:"28" version[29]:"29" version[30]:"30" version[31]:"31" version[32]:"32" version[33]:"33" version[34]:"34" version[35]:"35"` + Version uint32 `version[0]:"0" version[1]:"1" version[2]:"2" version[3]:"3" version[4]:"4" version[5]:"5" version[6]:"6" version[7]:"7" version[8]:"8" version[9]:"9" version[10]:"10" version[11]:"11" version[12]:"12" version[13]:"13" version[14]:"14" version[15]:"15" version[16]:"16" version[17]:"17" version[18]:"18" version[19]:"19" version[20]:"20" version[21]:"21" version[22]:"22" version[23]:"23" version[24]:"24" version[25]:"25" version[26]:"26" version[27]:"27" version[28]:"28" version[29]:"29" version[30]:"30" version[31]:"31" version[32]:"32" version[33]:"33" version[34]:"34" version[35]:"35" version[36]:"36"` // Archival nodes retain a full copy of the block history. Non-Archival nodes will delete old blocks and only retain what's need to properly validate blockchain messages (the precise number of recent blocks depends on the consensus parameters. Currently the last 1321 blocks are required). This means that non-Archival nodes require significantly less storage than Archival nodes. If setting this to true for the first time, the existing ledger may need to be deleted to get the historical values stored as the setting only affects current blocks forward. To do this, shutdown the node and delete all .sqlite files within the data/testnet-version directory, except the crash.sqlite file. Restart the node and wait for the node to sync. Archival bool `version[0]:"false"` @@ -643,6 +643,9 @@ type Local struct { // GoMemLimit provides the Go runtime with a soft memory limit. The default behavior is no limit, // unless the GOMEMLIMIT environment variable is set. GoMemLimit uint64 `version[34]:"0"` + + // EnableVoteCompression controls whether vote compression is enabled for websocket networks + EnableVoteCompression bool `version[36]:"true"` } // DNSBootstrapArray returns an array of one or more DNS Bootstrap identifiers diff --git a/config/local_defaults.go b/config/local_defaults.go index fcc6e3d3d8..70df924d87 100644 --- a/config/local_defaults.go +++ b/config/local_defaults.go @@ -20,7 +20,7 @@ package config var defaultLocal = Local{ - Version: 35, + Version: 36, AccountUpdatesStatsInterval: 5000000000, AccountsRebuildSynchronousMode: 1, AgreementIncomingBundlesQueueLength: 15, @@ -89,6 +89,7 @@ var defaultLocal = Local{ EnableTxnEvalTracer: false, EnableUsageLog: false, EnableVerbosedTransactionSyncLogging: false, + EnableVoteCompression: true, EndpointAddress: "127.0.0.1:0", FallbackDNSResolverAddress: "", ForceFetchTransactions: false, diff --git a/go.mod b/go.mod index 24d12e640a..4a8c36580b 100644 --- a/go.mod +++ b/go.mod @@ -58,7 +58,7 @@ require ( golang.org/x/sys v0.30.0 golang.org/x/text v0.22.0 gopkg.in/sohlich/elogrus.v3 v3.0.0-20180410122755-1fa29e2f2009 - pgregory.net/rapid v0.6.2 + pgregory.net/rapid v1.2.0 ) require ( diff --git a/go.sum b/go.sum index f5ad16f484..e5d42b917d 100644 --- a/go.sum +++ b/go.sum @@ -970,8 +970,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= lukechampine.com/blake3 v1.3.0 h1:sJ3XhFINmHSrYCgl958hscfIa3bw8x4DqMP3u1YvoYE= lukechampine.com/blake3 v1.3.0/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= -pgregory.net/rapid v0.6.2 h1:ErW5sL+UKtfBfUTsWHDCoeB+eZKLKMxrSd1VJY6W4bw= -pgregory.net/rapid v0.6.2/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= +pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= +pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= diff --git a/installer/config.json.example b/installer/config.json.example index 1e7937d455..fec7d7bf6d 100644 --- a/installer/config.json.example +++ b/installer/config.json.example @@ -1,5 +1,5 @@ { - "Version": 35, + "Version": 36, "AccountUpdatesStatsInterval": 5000000000, "AccountsRebuildSynchronousMode": 1, "AgreementIncomingBundlesQueueLength": 15, @@ -68,6 +68,7 @@ "EnableTxnEvalTracer": false, "EnableUsageLog": false, "EnableVerbosedTransactionSyncLogging": false, + "EnableVoteCompression": true, "EndpointAddress": "127.0.0.1:0", "FallbackDNSResolverAddress": "", "ForceFetchTransactions": false, diff --git a/network/msgCompressor.go b/network/msgCompressor.go index a91dbf69a8..7eec31286d 100644 --- a/network/msgCompressor.go +++ b/network/msgCompressor.go @@ -24,6 +24,7 @@ import ( "github.com/DataDog/zstd" "github.com/algorand/go-algorand/logging" + "github.com/algorand/go-algorand/network/vpack" "github.com/algorand/go-algorand/protocol" ) @@ -52,19 +53,39 @@ func zstdCompressMsg(tbytes []byte, d []byte) ([]byte, string) { return mbytesComp, "" } +func vpackCompressVote(tbytes []byte, d []byte) ([]byte, string) { + var enc vpack.StatelessEncoder + bound := vpack.MaxCompressedVoteSize + // Pre-allocate buffer for tag bytes and compressed data + mbytesComp := make([]byte, len(tbytes)+bound) + copy(mbytesComp, tbytes) + comp, err := enc.CompressVote(mbytesComp[len(tbytes):], d) + if err != nil { + // fallback and reuse non-compressed original data + logMsg := fmt.Sprintf("failed to compress vote into buffer of len %d: %v", len(d), err) + copied := copy(mbytesComp[len(tbytes):], d) + return mbytesComp[:len(tbytes)+copied], logMsg + } + + result := mbytesComp[:len(tbytes)+len(comp)] + return result, "" +} + // MaxDecompressedMessageSize defines a maximum decompressed data size // to prevent zip bombs. This depends on MaxTxnBytesPerBlock consensus parameter // and should be larger. const MaxDecompressedMessageSize = 20 * 1024 * 1024 // some large enough value -// wsPeerMsgDataConverter performs optional incoming messages conversion. -// At the moment it only supports zstd decompression for payload proposal -type wsPeerMsgDataConverter struct { +// wsPeerMsgDataDecoder performs optional incoming messages conversion. +// At the moment it only supports zstd decompression for payload proposal, +// and vpack decompression for votes. +type wsPeerMsgDataDecoder struct { log logging.Logger origin string // actual converter(s) ppdec zstdProposalDecompressor + avdec vpackVoteDecompressor } type zstdProposalDecompressor struct{} @@ -73,6 +94,15 @@ func (dec zstdProposalDecompressor) accept(data []byte) bool { return len(data) > 4 && bytes.Equal(data[:4], zstdCompressionMagic[:]) } +type vpackVoteDecompressor struct { + enabled bool + dec *vpack.StatelessDecoder +} + +func (dec vpackVoteDecompressor) convert(data []byte) ([]byte, error) { + return dec.dec.DecompressVote(nil, data) +} + func (dec zstdProposalDecompressor) convert(data []byte) ([]byte, error) { r := zstd.NewReader(bytes.NewReader(data)) defer r.Close() @@ -96,7 +126,7 @@ func (dec zstdProposalDecompressor) convert(data []byte) ([]byte, error) { } } -func (c *wsPeerMsgDataConverter) convert(tag protocol.Tag, data []byte) ([]byte, error) { +func (c *wsPeerMsgDataDecoder) convert(tag protocol.Tag, data []byte) ([]byte, error) { if tag == protocol.ProposalPayloadTag { // sender might support compressed payload but fail to compress for whatever reason, // in this case it sends non-compressed payload - the receiver decompress only if it is compressed. @@ -108,16 +138,33 @@ func (c *wsPeerMsgDataConverter) convert(tag protocol.Tag, data []byte) ([]byte, return res, nil } c.log.Warnf("peer %s supported zstd but sent non-compressed data", c.origin) + } else if tag == protocol.AgreementVoteTag { + if c.avdec.enabled { + res, err := c.avdec.convert(data) + if err != nil { + c.log.Warnf("peer %s vote decompress error: %v", c.origin, err) + // fall back to original data + return data, nil + } + return res, nil + } } return data, nil } -func makeWsPeerMsgDataConverter(wp *wsPeer) *wsPeerMsgDataConverter { - c := wsPeerMsgDataConverter{ +func makeWsPeerMsgDataDecoder(wp *wsPeer) *wsPeerMsgDataDecoder { + c := wsPeerMsgDataDecoder{ log: wp.log, origin: wp.originAddress, } c.ppdec = zstdProposalDecompressor{} + // have both ends advertised support for compression? + if wp.enableVoteCompression && wp.vpackVoteCompressionSupported() { + c.avdec = vpackVoteDecompressor{ + enabled: true, + dec: vpack.NewStatelessDecoder(), + } + } return &c } diff --git a/network/msgCompressor_test.go b/network/msgCompressor_test.go index 88273e37cf..d64b8fb54f 100644 --- a/network/msgCompressor_test.go +++ b/network/msgCompressor_test.go @@ -76,7 +76,7 @@ func (cl *converterTestLogger) Warnf(s string, args ...interface{}) { func TestWsPeerMsgDataConverterConvert(t *testing.T) { partitiontest.PartitionTest(t) - c := wsPeerMsgDataConverter{} + c := wsPeerMsgDataDecoder{} c.ppdec = zstdProposalDecompressor{} tag := protocol.AgreementVoteTag data := []byte("data") diff --git a/network/vpack/README.md b/network/vpack/README.md new file mode 100644 index 0000000000..1bb6dcd2c5 --- /dev/null +++ b/network/vpack/README.md @@ -0,0 +1,72 @@ +# Stateless *vpack* wire format + +This document specifies the byte‑level (on‑wire) layout produced by `StatelessEncoder.CompressVote` and accepted by `StatelessDecoder.DecompressVote`. +The goal is to minimize vote size while retaining a 1‑to‑1, loss‑free mapping to the canonical msgpack representation of `agreement.UnauthenticatedVote`. +The canonical msgpack representation we rely on is provided by agreement/msgp_gen.go, generated by our [custom msgpack code generator](https://github.com/algorand/msgp) +which ensures fields are generated in lexicographic order, omit empty key-value pairs, and use specific formats for certain types as defined in +[our specification](https://github.com/algorandfoundation/specs/blob/c0331123148971e4705f25b9c937cb23e5ee28d1/dev/crypto.md#L22-L40). + +--- + +## 1. High‑level structure + +``` ++---------+-----------------+---------------------+--------------------------+ +| Header | VrfProof ("pf") | rawVote ("r") | OneTimeSignature ("sig") | +| 2 bytes | 80 bytes | variable length | 256 bytes | ++---------+-----------------+---------------------+--------------------------+ +``` + +All fields appear exactly once, and in the fixed order above. The presence of optional sub‑fields inside `rawVote` are indicated by a 1‑byte bitmask in the header. +No field names appear, only values. + +--- + +## 2. Header (2 bytes) + +| Offset | Description | +| ------ | -------------------------------------------------------------- | +| `0` | Presence flags for optional values (LSB first, see table). | +| `1` | Reserved, currently zero. | + +### 2.1 Bit‑mask layout (byte 0) + +| Bit | Flag | Field enabled | Encoded size | +| --- | ----------- | -------------------------------- | ------------ | +| 0 | `bitPer` | `r.per` (varuint) | 1 - 9 bytes | +| 1 | `bitDig` | `r.prop.dig` (digest) | 32 bytes | +| 2 | `bitEncDig` | `r.prop.encdig` (digest) | 32 bytes | +| 3 | `bitOper` | `r.prop.oper` (varuint) | 1 - 9 bytes | +| 4 | `bitOprop` | `r.prop.oprop` (address) | 32 bytes | +| 5 | `bitStep` | `r.step` (varuint) | 1 - 9 bytes | + +Binary fields are represented by their 32-, 64-, and 80-byte values without markers. +Integers use msgpack's variable-length unsigned integer encoding: +- `fixint` (≤ 127), 1 byte in length (values 0x00-0x7f) +- `uint8` 2 bytes in length (marker byte 0xcc + 1-byte value) +- `uint16` 3 bytes in length (marker byte 0xcd + 2-byte value) +- `uint32` 5 bytes in length (marker byte 0xce + 4-byte value) +- `uint64` 9 bytes in length (marker byte 0xcf + 8-byte value) + +--- + +## 3. Field serialization order + +After the 2-byte header, the encoder emits values in the following order: + +| Field | Type | Encoded size | Presence flag | +| -------------- | ------------------------------ | ------------ | ------------- | +| `pf` | VRF credential | 80 bytes | Required | +| `r.per` | Period | varuint | `bitPer` | +| `r.prop.dig` | Proposal digest | 32 bytes | `bitDig` | +| `r.prop.encdig`| Digest of encoded proposal | 32 bytes | `bitEncDig` | +| `r.prop.oper` | Proposal's original period | varuint | `bitOper` | +| `r.prop.oprop` | Proposal's original proposer | 32 bytes | `bitOprop` | +| `r.rnd` | Round number | varuint | Required | +| `r.snd` | Voter's (sender) address | 32 bytes | Required | +| `r.step` | Step | varuint | `bitStep` | +| `sig.p` | Ed25519 public key | 32 bytes | Required | +| `sig.p1s` | Signature over offset ID | 64 bytes | Required | +| `sig.p2` | Second-tier Ed25519 public key | 32 bytes | Required | +| `sig.p2s` | Signature over batch ID | 64 bytes | Required | +| `sig.s` | Signature over vote using `p` | 64 bytes | Required | diff --git a/network/vpack/msgp.go b/network/vpack/msgp.go new file mode 100644 index 0000000000..0346eae866 --- /dev/null +++ b/network/vpack/msgp.go @@ -0,0 +1,197 @@ +// Copyright (C) 2019-2025 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package vpack + +import ( + "fmt" +) + +// Minimal msgpack constants used here +const ( + msgpFixMapMask = 0x80 + msgpFixMapMax = 0x8f + msgpFixStrMask = 0xa0 + msgpFixStrMax = 0xbf + msgpBin8 = 0xc4 + msgpBin8Len32 = "\xc4\x20" // bin8 marker with 32 items + msgpBin8Len64 = "\xc4\x40" // bin8 marker with 64 items + msgpBin8Len80 = "\xc4\x50" // bin8 marker with 80 items + msgpUint8 = 0xcc + msgpUint16 = 0xcd + msgpUint32 = 0xce + msgpUint64 = 0xcf + + msgpFixstrCred = "\xa4cred" + msgpFixstrDig = "\xa3dig" + msgpFixstrEncdig = "\xa6encdig" + msgpFixstrOper = "\xa4oper" + msgpFixstrOprop = "\xa5oprop" + msgpFixstrP = "\xa1p" + msgpFixstrP1s = "\xa3p1s" + msgpFixstrP2 = "\xa2p2" + msgpFixstrP2s = "\xa3p2s" + msgpFixstrPer = "\xa3per" + msgpFixstrPf = "\xa2pf" + msgpFixstrProp = "\xa4prop" + msgpFixstrPs = "\xa2ps" + msgpFixstrR = "\xa1r" + msgpFixstrRnd = "\xa3rnd" + msgpFixstrS = "\xa1s" + msgpFixstrSig = "\xa3sig" + msgpFixstrSnd = "\xa3snd" + msgpFixstrStep = "\xa4step" +) + +func isMsgpFixint(b byte) bool { + return b>>7 == 0 +} + +// msgpVoteParser provides a zero-allocation msgpVoteParser for vote messages. +type msgpVoteParser struct { + data []byte + pos int +} + +func newMsgpVoteParser(data []byte) *msgpVoteParser { + return &msgpVoteParser{data: data} +} + +// Error if we need more bytes than available +func (p *msgpVoteParser) ensureBytes(n int) error { + if p.pos+n > len(p.data) { + return fmt.Errorf("unexpected EOF: need %d bytes, have %d", n, len(p.data)-p.pos) + } + return nil +} + +// Read a single byte +func (p *msgpVoteParser) readByte() (byte, error) { + if err := p.ensureBytes(1); err != nil { + return 0, err + } + b := p.data[p.pos] + p.pos++ + return b, nil +} + +// Read a fixmap header and return the count +func (p *msgpVoteParser) readFixMap() (uint8, error) { + b, err := p.readByte() + if err != nil { + return 0, err + } + + if b < msgpFixMapMask || b > msgpFixMapMax { + return 0, fmt.Errorf("expected fixmap, got 0x%02x", b) + } + + return b & 0x0f, nil +} + +// Zero-allocation string reading that returns a slice of the original data +func (p *msgpVoteParser) readString() ([]byte, error) { + b, err := p.readByte() + if err != nil { + return nil, err + } + if b < msgpFixStrMask || b > msgpFixStrMax { + return nil, fmt.Errorf("readString: expected fixstr, got 0x%02x", b) + } + length := int(b & 0x1f) + if err := p.ensureBytes(length); err != nil { + return nil, err + } + s := p.data[p.pos : p.pos+length] + p.pos += length + return s, nil +} + +func (p *msgpVoteParser) readBin80() ([80]byte, error) { + const sz = 80 + var data [sz]byte + if err := p.ensureBytes(sz + 2); err != nil { + return data, err + } + if p.data[p.pos] != msgpBin8 || p.data[p.pos+1] != sz { + return data, fmt.Errorf("expected bin8 length %d, got %d", sz, int(p.data[p.pos+1])) + } + copy(data[:], p.data[p.pos+2:p.pos+sz+2]) + p.pos += sz + 2 + return data, nil +} + +func (p *msgpVoteParser) readBin32() ([32]byte, error) { + const sz = 32 + var data [sz]byte + if err := p.ensureBytes(sz + 2); err != nil { + return data, err + } + if p.data[p.pos] != msgpBin8 || p.data[p.pos+1] != sz { + return data, fmt.Errorf("expected bin8 length %d, got %d", sz, int(p.data[p.pos+1])) + } + copy(data[:], p.data[p.pos+2:p.pos+sz+2]) + p.pos += sz + 2 + return data, nil +} + +func (p *msgpVoteParser) readBin64() ([64]byte, error) { + const sz = 64 + var data [sz]byte + if err := p.ensureBytes(sz + 2); err != nil { + return data, err + } + if p.data[p.pos] != msgpBin8 || p.data[p.pos+1] != sz { + return data, fmt.Errorf("expected bin8 length %d, got %d", sz, int(p.data[p.pos+1])) + } + copy(data[:], p.data[p.pos+2:p.pos+sz+2]) + p.pos += sz + 2 + return data, nil +} + +// readUintBytes reads a variable-length msgpack unsigned integer from the reader. +// It will return a zero-length/nil slice iff err != nil. +func (p *msgpVoteParser) readUintBytes() ([]byte, error) { + startPos := p.pos + // read marker byte + b, err := p.readByte() + if err != nil { + return nil, err + } + // fixint is a single byte containing marker and value + if isMsgpFixint(b) { + return p.data[startPos : startPos+1], nil + } + // otherwise, we expect a tag byte followed by the value + var dataSize int + switch b { + case msgpUint8: + dataSize = 1 + case msgpUint16: + dataSize = 2 + case msgpUint32: + dataSize = 4 + case msgpUint64: + dataSize = 8 + default: + return nil, fmt.Errorf("expected uint tag, got 0x%02x", b) + } + if err := p.ensureBytes(dataSize); err != nil { + return nil, err + } + p.pos += dataSize + return p.data[startPos : startPos+dataSize+1], nil +} diff --git a/network/vpack/parse.go b/network/vpack/parse.go new file mode 100644 index 0000000000..17598ee76f --- /dev/null +++ b/network/vpack/parse.go @@ -0,0 +1,255 @@ +// Copyright (C) 2019-2025 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package vpack + +import ( + "fmt" +) + +type voteValueType uint8 + +const ( + credPfVoteValue voteValueType = iota + rPerVoteValue + rPropDigVoteValue + rPropEncdigVoteValue + rPropOperVoteValue + rPropOpropVoteValue + rRndVoteValue + rSndVoteValue + rStepVoteValue + sigP1sVoteValue + sigP2VoteValue + sigP2sVoteValue + sigPVoteValue + sigPsVoteValue + sigSVoteValue +) + +func parseMsgpVote(msgpData []byte, c *StatelessEncoder) error { + p := newMsgpVoteParser(msgpData) + + // Parse unauthenticatedVote + cnt, err := p.readFixMap() + if err != nil { + return fmt.Errorf("reading map for unauthenticatedVote: %w", err) + } + // Assert unauthenticatedVote struct has 3 fields + if cnt != 3 { + return fmt.Errorf("expected fixed map size 3 for unauthenticatedVote, got %d", cnt) + } + // Required nested map in unauthenticatedVote: cred + if s, rErr := p.readString(); rErr != nil || string(s) != "cred" { + return fmt.Errorf("expected string cred, got %s: %w", s, rErr) + } + + // Parse UnauthenticatedCredential + cnt, err = p.readFixMap() + if err != nil { + return fmt.Errorf("reading map for UnauthenticatedCredential: %w", err) + } + // Assert UnauthenticatedCredential struct has 1 fields + if cnt != 1 { + return fmt.Errorf("expected fixed map size 1 for UnauthenticatedCredential, got %d", cnt) + } + // Required field name for UnauthenticatedCredential: pf + if s, rErr := p.readString(); rErr != nil || string(s) != "pf" { + return fmt.Errorf("expected string pf, got %s: %w", s, rErr) + } + val, err := p.readBin80() + if err != nil { + return fmt.Errorf("reading pf: %w", err) + } + c.writeBin80(credPfVoteValue, val) + + // Required nested map in unauthenticatedVote: r + if s, rErr := p.readString(); rErr != nil || string(s) != "r" { + return fmt.Errorf("expected string r, got %s: %w", s, rErr) + } + + // Parse rawVote fixmap + cnt, err = p.readFixMap() + if err != nil { + return fmt.Errorf("reading map for rawVote: %w", err) + } + if cnt < 1 || cnt > 5 { + return fmt.Errorf("expected fixmap size for rawVote 1 <= cnt <= 5, got %d", cnt) + } + for range cnt { + voteKey, err1 := p.readString() + if err1 != nil { + return fmt.Errorf("reading key for rawVote: %w", err1) + } + switch string(voteKey) { + case "per": + val, err1 := p.readUintBytes() + if err1 != nil { + return fmt.Errorf("reading per: %w", err1) + } + c.writeVaruint(rPerVoteValue, val) + case "prop": + // Parse proposalValue fixmap + propCnt, err1 := p.readFixMap() + if err1 != nil { + return fmt.Errorf("reading map for proposalValue: %w", err1) + } + if propCnt < 1 || propCnt > 4 { + return fmt.Errorf("expected fixmap size for proposalValue 1 <= cnt <= 4, got %d", propCnt) + } + for range propCnt { + propKey, err2 := p.readString() + if err2 != nil { + return fmt.Errorf("reading key for proposalValue: %w", err2) + } + switch string(propKey) { + case "dig": + val, err2 := p.readBin32() + if err2 != nil { + return fmt.Errorf("reading dig: %w", err2) + } + c.writeBin32(rPropDigVoteValue, val) + + case "encdig": + val, err2 := p.readBin32() + if err2 != nil { + return fmt.Errorf("reading encdig: %w", err2) + } + c.writeBin32(rPropEncdigVoteValue, val) + + case "oper": + val, err2 := p.readUintBytes() + if err2 != nil { + return fmt.Errorf("reading oper: %w", err2) + } + c.writeVaruint(rPropOperVoteValue, val) + case "oprop": + val, err2 := p.readBin32() + if err2 != nil { + return fmt.Errorf("reading oprop: %w", err2) + } + c.writeBin32(rPropOpropVoteValue, val) + + default: + return fmt.Errorf("unexpected field in proposalValue: %q", propKey) + } + } + case "rnd": + val, err1 := p.readUintBytes() + if err1 != nil { + return fmt.Errorf("reading rnd: %w", err1) + } + c.writeVaruint(rRndVoteValue, val) + case "snd": + val, err1 := p.readBin32() + if err1 != nil { + return fmt.Errorf("reading snd: %w", err1) + } + c.writeBin32(rSndVoteValue, val) + + case "step": + val, err1 := p.readUintBytes() + if err1 != nil { + return fmt.Errorf("reading step: %w", err1) + } + c.writeVaruint(rStepVoteValue, val) + default: + return fmt.Errorf("unexpected field in rawVote: %q", voteKey) + } + } + + // Required nested map in unauthenticatedVote: sig + if s, rErr := p.readString(); rErr != nil || string(s) != "sig" { + return fmt.Errorf("expected string sig, got %s: %w", s, rErr) + } + + // Parse OneTimeSignature fixmap + cnt, err = p.readFixMap() + if err != nil { + return fmt.Errorf("reading map for OneTimeSignature: %w", err) + } + // Assert OneTimeSignature struct has 6 fields + if cnt != 6 { + return fmt.Errorf("expected fixed map size 6 for OneTimeSignature, got %d", cnt) + } + // Required field for OneTimeSignature: p + if s, rErr := p.readString(); rErr != nil || string(s) != "p" { + return fmt.Errorf("expected string p, got %s: %w", s, rErr) + } + val32, err := p.readBin32() + if err != nil { + return fmt.Errorf("reading p: %w", err) + } + c.writeBin32(sigPVoteValue, val32) + + // Required field for OneTimeSignature: p1s + if s, rErr := p.readString(); rErr != nil || string(s) != "p1s" { + return fmt.Errorf("expected string p1s, got %s: %w", s, rErr) + } + val64, err := p.readBin64() + if err != nil { + return fmt.Errorf("reading p1s: %w", err) + } + c.writeBin64(sigP1sVoteValue, val64) + + // Required field for OneTimeSignature: p2 + if s, rErr := p.readString(); rErr != nil || string(s) != "p2" { + return fmt.Errorf("expected string p2, got %s: %w", s, rErr) + } + val32, err = p.readBin32() + if err != nil { + return fmt.Errorf("reading p2: %w", err) + } + c.writeBin32(sigP2VoteValue, val32) + + // Required field for OneTimeSignature: p2s + if s, rErr := p.readString(); rErr != nil || string(s) != "p2s" { + return fmt.Errorf("expected string p2s, got %s: %w", s, rErr) + } + val64, err = p.readBin64() + if err != nil { + return fmt.Errorf("reading p2s: %w", err) + } + c.writeBin64(sigP2sVoteValue, val64) + + // Required field for OneTimeSignature: ps + if s, rErr := p.readString(); rErr != nil || string(s) != "ps" { + return fmt.Errorf("expected string ps, got %s: %w", s, rErr) + } + val64, err = p.readBin64() + if err != nil { + return fmt.Errorf("reading ps: %w", err) + } + if val64 != [64]byte{} { + return fmt.Errorf("expected empty array for ps, got %v", val64) + } + + // Required field for OneTimeSignature: s + if s, rErr := p.readString(); rErr != nil || string(s) != "s" { + return fmt.Errorf("expected string s, got %s: %w", s, rErr) + } + val64, err = p.readBin64() + if err != nil { + return fmt.Errorf("reading s: %w", err) + } + c.writeBin64(sigSVoteValue, val64) + + // Check for trailing bytes + if p.pos < len(p.data) { + return fmt.Errorf("unexpected trailing data: %d bytes remain unprocessed", len(p.data)-p.pos) + } + return nil +} diff --git a/network/vpack/parse_test.go b/network/vpack/parse_test.go new file mode 100644 index 0000000000..0ad6230c8a --- /dev/null +++ b/network/vpack/parse_test.go @@ -0,0 +1,189 @@ +// Copyright (C) 2019-2025 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package vpack + +import ( + "testing" + + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/test/partitiontest" + "github.com/stretchr/testify/assert" +) + +// a string that is greater than the max 5-bit fixmap size +const gtFixMapString = "12345678901234567890123456789012" + +var parseVoteTestCases = []struct { + obj any + errContains string +}{ + // vote + {map[string]string{"a": "1", "b": "2"}, + "expected fixed map size 3 for unauthenticatedVote, got 2"}, + {map[string]any{"a": 1, "b": 2, "c": 3}, + "expected string cred, got a"}, + {[]int{1, 2, 3}, + "reading map for unauthenticatedVote"}, + {map[string]string{"a": "1", "b": "2", "c": "3", "d": "4", "e": "5", "f": "6", "g": "7"}, + "expected fixed map size 3 for unauthenticatedVote, got 7"}, + {map[string]string{gtFixMapString: "1", "b": "2", "c": "3"}, + "readString: expected fixstr, got 0xd9"}, + + // cred + {map[string]string{"cred": "1", "d": "2", "e": "3"}, + "reading map for UnauthenticatedCredential"}, + {map[string]any{"cred": map[string]int{"pf": 1, "q": 2}, "d": "2", "e": "3"}, + "expected fixed map size 1 for UnauthenticatedCredential, got 2"}, + {map[string]any{"cred": map[string]int{gtFixMapString: 1}, "d": "2", "e": "3"}, + "readString: expected fixstr, got 0xd9"}, + {map[string]any{"cred": map[string]string{"invalid": "1"}, "r": "2", "sig": "3"}, + "expected string pf, got invalid"}, + {map[string]any{"cred": map[string]any{"pf": []byte{1, 2, 3}}, "r": "2", "sig": "3"}, + "reading pf"}, + {map[string]any{"cred": map[string]any{"pf": [100]byte{1, 2, 3}}, "r": "2", "sig": "3"}, + "reading pf: expected bin8 length 80, got 100"}, + + // rawVote + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{1}}, "r": []int{1, 2, 3}, "sig": "3"}, + "reading map for rawVote"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{1}}, "r": map[string]string{}, "sig": "3"}, + "expected fixmap size for rawVote 1 <= cnt <= 5, got 0"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{1}}, "r": map[string]string{"a": "1", "b": "2", "c": "3", "d": "4", "e": "5", "f": "6"}, "sig": "3"}, + "expected fixmap size for rawVote 1 <= cnt <= 5, got 6"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{1}}, "r": map[string]string{gtFixMapString: "1"}, "sig": "3"}, + "reading key for rawVote"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{1}}, "r": map[string]string{"invalid": "1"}, "sig": "3"}, + "unexpected field in rawVote"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{1}}, "r": map[string]any{"per": "not-a-number"}, "sig": "3"}, + "reading per"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{1}}, "r": map[string]any{"rnd": "not-a-number"}, "sig": "3"}, + "reading rnd"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{1}}, "r": map[string]any{"step": "not-a-number"}, "sig": "3"}, + "reading step"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{1}}, "r": map[string]any{"prop": "not-a-map"}, "sig": "3"}, + "reading map for proposalValue"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{1}}, "r": map[string]any{"snd": []int{1, 2, 3}}, "sig": "3"}, + "reading snd: unexpected EOF"}, + + // proposalValue + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{1}}, "r": map[string]any{"prop": map[string]string{"invalid": "1"}}, "sig": "3"}, + "unexpected field in proposalValue"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{1}}, "r": map[string]any{"prop": map[string]string{gtFixMapString: "1"}}, "sig": "3"}, + "reading key for proposalValue"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{1}}, "r": map[string]any{"prop": map[string]any{"dig": []int{1, 2, 3}}}, "sig": "3"}, + "reading dig"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{1}}, "r": map[string]any{"prop": map[string]any{"encdig": []int{1, 2, 3}}}, "sig": "3"}, + "reading encdig"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{1}}, "r": map[string]any{"prop": map[string]any{"oper": "not-a-number"}}, "sig": "3"}, + "reading oper"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{1}}, "r": map[string]any{"prop": map[string]any{"oprop": []int{1, 2, 3}}}, "sig": "3"}, + "reading oprop"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{1}}, "r": map[string]any{"prop": map[string]any{"a": 1, "b": 2, "c": 3, "d": 4, "e": 5}}, "sig": "3"}, + "expected fixmap size for proposalValue 1 <= cnt <= 4, got 5"}, + + // sig + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{1}}, "r": map[string]any{"rnd": 1}, "sig": []int{1, 2, 3}}, + "reading map for OneTimeSignature"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{1}}, "r": map[string]any{"rnd": 1}, "sig": map[string]any{}}, + "expected fixed map size 6 for OneTimeSignature, got 0"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{1}}, "r": map[string]any{"rnd": 1}, "sig": map[string]any{"p": []int{1}}}, + "expected fixed map size 6 for OneTimeSignature, got 1"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{1}}, "r": map[string]any{"rnd": 1}, "sig": map[string]any{ + gtFixMapString: "1", "a": 1, "b": 2, "c": 3, "d": 4, "e": 5}}, + "readString: expected fixstr, got 0xd9"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{1}}, "r": map[string]any{"rnd": 1}, "sig": map[string]any{ + "a": 1, "b": 2, "c": 3, "d": 4, "e": 5, "f": 6}}, + "expected string p, got a"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{1}}, "r": map[string]any{"rnd": 1}, "sig": map[string]any{ + "p": []int{1}, "p1s": [64]byte{}, "p2": [32]byte{}, "p2s": [64]byte{}, "ps": [64]byte{}, "s": [64]byte{}}}, + "reading p: expected bin8 length 32"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{1}}, "r": map[string]any{"rnd": 1}, "sig": map[string]any{ + "p": [32]byte{}, "p1s": []int{1}, "p2": [32]byte{}, "p2s": [64]byte{}, "ps": [64]byte{}, "s": [64]byte{}}}, + "reading p1s: expected bin8 length 64"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{1}}, "r": map[string]any{"rnd": 1}, "sig": map[string]any{ + "p": [32]byte{}, "p1s": [64]byte{}, "p2": []int{1}, "p2s": [64]byte{}, "ps": [64]byte{}, "s": [64]byte{}}}, + "reading p2: expected bin8 length 32"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{1}}, "r": map[string]any{"rnd": 1}, "sig": map[string]any{ + "p": [32]byte{}, "p1s": [64]byte{}, "p2": [32]byte{}, "p2s": []int{1}, "ps": [64]byte{}, "s": [64]byte{}}}, + "reading p2s: expected bin8 length 64"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{1}}, "r": map[string]any{"rnd": 1}, "sig": map[string]any{ + "p": [32]byte{}, "p1s": [64]byte{}, "p2": [32]byte{}, "p2s": [64]byte{}, "ps": []int{1}, "s": [64]byte{}}}, + "reading ps: expected bin8 length 64"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{1}}, "r": map[string]any{"rnd": 1}, "sig": map[string]any{ + "p": [32]byte{}, "p1s": [64]byte{}, "p2": [32]byte{}, "p2s": [64]byte{}, "ps": [64]byte{}, "s": []int{1}}}, + "reading s: unexpected EOF"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{}}, "ra": 1, "sig": map[string]any{}}, + "expected string r"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{}}, "r": map[string]any{"rnd": uint64(1)}, "snd": 1}, + "expected string sig, got snd"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{}}, "r": map[string]any{"rnd": uint64(1)}, "sig": map[string]any{ + "p": [32]byte{}, "p1x": [64]byte{}, "p2": [32]byte{}, "p2s": [64]byte{}, "ps": [64]byte{}, "s": [64]byte{}}}, + "expected string p1s, got p1x"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{}}, "r": map[string]any{"rnd": uint64(1)}, "sig": map[string]any{ + "p": [32]byte{}, "p1s": [64]byte{}, "p2": [32]byte{}, "p2x": [64]byte{}, "ps": [64]byte{}, "s": [64]byte{}}}, + "expected string p2s, got p2x"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{}}, "r": map[string]any{"rnd": uint64(1)}, "sig": map[string]any{ + "p": [32]byte{}, "p1s": [64]byte{}, "p1x": [64]byte{}, "p2": [32]byte{}, "ps": [64]byte{}, "s": [64]byte{}}}, + "expected string p2, got p1x"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{}}, "r": map[string]any{"rnd": uint64(1)}, "sig": map[string]any{ + "p": [32]byte{}, "p1s": [64]byte{}, "p2": [32]byte{}, "p2s": [64]byte{}, "pt": [64]byte{}, "s": [64]byte{}}}, + "expected string ps, got pt"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{}}, "r": map[string]any{"rnd": uint64(1)}, "sig": map[string]any{ + "p": [32]byte{}, "p1s": [64]byte{}, "p2": [32]byte{}, "p2s": [64]byte{}, "ps": [64]byte{1}, "s": [64]byte{}}}, + "expected empty array for ps"}, + {map[string]any{"cred": map[string]any{"pf": crypto.VrfProof{}}, "r": map[string]any{"rnd": uint64(1)}, "sig": map[string]any{ + "p": [32]byte{}, "p1s": [64]byte{}, "p2": [32]byte{}, "p2s": [64]byte{}, "ps": [64]byte{}, "sa": [64]byte{}}}, + "expected string s, got sa"}, +} + +// TestParseVoteErrors tests error cases of the parseMsgpVote function +func TestParseVoteErrors(t *testing.T) { + partitiontest.PartitionTest(t) + + for _, tc := range parseVoteTestCases { + t.Run(tc.errContains, func(t *testing.T) { + buf := protocol.EncodeReflect(tc.obj) + se := NewStatelessEncoder() + _, err := se.CompressVote(nil, buf) + assert.ErrorContains(t, err, tc.errContains) + }) + } +} + +func TestParseVoteTrailingDataErr(t *testing.T) { + partitiontest.PartitionTest(t) + + // Build minimal valid vote + obj := map[string]any{ + "cred": map[string]any{"pf": crypto.VrfProof{}}, + "r": map[string]any{"rnd": uint64(1)}, + "sig": map[string]any{ + "p": [32]byte{}, + "p1s": [64]byte{}, + "p2": [32]byte{}, + "p2s": [64]byte{}, + "ps": [64]byte{}, + "s": [64]byte{}, + }, + } + buf := protocol.EncodeReflect(obj) + buf = append(buf, 0xFF) + se := NewStatelessEncoder() + _, err := se.CompressVote(nil, buf) + assert.ErrorContains(t, err, "unexpected trailing data") +} diff --git a/network/vpack/rapid_test.go b/network/vpack/rapid_test.go new file mode 100644 index 0000000000..28f1942a0a --- /dev/null +++ b/network/vpack/rapid_test.go @@ -0,0 +1,264 @@ +// Copyright (C) 2019-2025 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package vpack + +import ( + "bytes" + "math" + "reflect" + "testing" + "unsafe" + + "github.com/algorand/go-algorand/agreement" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/test/partitiontest" + "github.com/stretchr/testify/require" + "pgregory.net/rapid" +) + +// TestCheckStatelessEncoder tests the StatelessEncoder/Decoder using randomly generated votes +// generated by rapid. +func TestCheckStatelessEncoder(t *testing.T) { + partitiontest.PartitionTest(t) + rapid.Check(t, checkStatelessEncoder) +} + +// FuzzCheckStatelessEncoder is the same as TestCheckStatelessEncoder, but is a fuzz test. +func FuzzCheckStatelessEncoder(f *testing.F) { + f.Fuzz(rapid.MakeFuzz(checkStatelessEncoder)) +} + +func checkStatelessEncoder(t *rapid.T) { + // Generate a random vote + v0 := generateRandomVote().Draw(t, "vote") + + // Convert to msgpack + msgpBuf := protocol.EncodeMsgp(v0) + require.LessOrEqual(t, len(msgpBuf), MaxMsgpackVoteSize) + + // Try to compress with StatelessEncoder + encBM := NewStatelessEncoder() + encBufBM, err := encBM.CompressVote(nil, msgpBuf) + require.NoError(t, err) + require.LessOrEqual(t, len(encBufBM), MaxCompressedVoteSize) + + // Verify the bitmask is at the beginning + require.GreaterOrEqual(t, len(encBufBM), 2, "Compressed data should have at least 2 bytes for header") + // Decompress with StatelessDecoder + decBM := NewStatelessDecoder() + decBufBM, err := decBM.DecompressVote(nil, encBufBM) + require.NoError(t, err) + + // Ensure the decompressed data matches the original msgpack data + require.Equal(t, msgpBuf, decBufBM) + + // Decode the decompressed data and verify it matches the original vote + var v1 agreement.UnauthenticatedVote + err = protocol.Decode(decBufBM, &v1) + require.NoError(t, err) + require.Equal(t, *v0, v1) +} + +// generateRandomVote creates a random vote generator using rapid +func generateRandomVote() *rapid.Generator[*agreement.UnauthenticatedVote] { + return rapid.Custom(func(t *rapid.T) *agreement.UnauthenticatedVote { + v := &agreement.UnauthenticatedVote{} + + filterZeroBytes := func(b []byte) bool { + for _, v := range b { + if v != 0 { + return true + } + } + return false + } + + // Generate random sender address (32 bytes) + addressBytes := rapid.SliceOfN(rapid.Byte(), 32, 32).Filter(filterZeroBytes).Draw(t, "sender") + copy(v.R.Sender[:], addressBytes) + + // Create an equal distribution generator for different integer ranges + // This will test different MessagePack varuint encodings (uint8, uint16, uint32, uint64) + integerRangeGen := rapid.OneOf( + rapid.Uint64Range(0, 255), // uint8 range + rapid.Uint64Range(256, 65535), // uint16 range + rapid.Uint64Range(65536, 4294967295), // uint32 range + rapid.Uint64Range(4294967296, math.MaxUint64), // uint64 range + ) + + // Generate non-zero round using the range generator + roundNum := integerRangeGen.Filter(func(n uint64) bool { + return n > 0 // Ensure non-zero round + }).Draw(t, "round") + v.R.Round = basics.Round(roundNum) + + // Use reflection to set the unexported period field with the range generator + rPeriodField := reflect.ValueOf(&v.R).Elem().FieldByName("Period") + rPeriodField = reflect.NewAt(rPeriodField.Type(), unsafe.Pointer(rPeriodField.UnsafeAddr())).Elem() + rPeriodField.SetUint(integerRangeGen.Draw(t, "period")) + + // Create a biased generator for steps to emphasize early steps (0, 1, 2, 3) + stepGen := rapid.OneOf( + rapid.Just(uint64(0)), // Explicitly test step 0 + rapid.Just(uint64(1)), // Explicitly test step 1 + rapid.Just(uint64(2)), // Explicitly test step 2 + rapid.Just(uint64(3)), // Explicitly test step 3 + integerRangeGen, // Test other steps with less probability + ) + + // Use reflection to set the unexported step field + rStepField := reflect.ValueOf(&v.R).Elem().FieldByName("Step") + rStepField = reflect.NewAt(rStepField.Type(), unsafe.Pointer(rStepField.UnsafeAddr())).Elem() + rStepField.SetUint(stepGen.Draw(t, "step")) + + // Decide whether to include a proposal or leave it empty + // If empty, the default zero values will be used + includeProposal := rapid.Bool().Draw(t, "includeProposal") + if includeProposal { + // Use reflection to set the OriginalPeriod field in the proposal + propVal := reflect.ValueOf(&v.R.Proposal).Elem() + origPeriodField := propVal.FieldByName("OriginalPeriod") + origPeriodField = reflect.NewAt(origPeriodField.Type(), unsafe.Pointer(origPeriodField.UnsafeAddr())).Elem() + origPeriodField.SetUint(integerRangeGen.Draw(t, "originalPeriod")) + // Generate random OpropField, BlockDigest, and EncodingDigest bytes (32 bytes each) + // But sometimes make them empty to test edge cases + makeBytesFn := func(name string) []byte { + generator := rapid.OneOf( + rapid.Just([]byte{}), // Empty case + rapid.SliceOfN(rapid.Byte(), 32, 32), // Full case + ) + return generator.Draw(t, name) + } + opropBytes := makeBytesFn("oprop") + digestBytes := makeBytesFn("digest") + encDigestBytes := makeBytesFn("encDigest") + + copy(v.R.Proposal.OriginalProposer[:], opropBytes) + copy(v.R.Proposal.BlockDigest[:], digestBytes) + copy(v.R.Proposal.EncodingDigest[:], encDigestBytes) + + } + + // Generate random proof bytes (80 bytes) + pfBytes := rapid.SliceOfN(rapid.Byte(), 80, 80).Filter(filterZeroBytes).Draw(t, "proof") + copy(v.Cred.Proof[:], pfBytes) + + // Generate signature fields (variable sizes) + sigBytes := rapid.SliceOfN(rapid.Byte(), 64, 64).Filter(filterZeroBytes).Draw(t, "sig") + pkBytes := rapid.SliceOfN(rapid.Byte(), 32, 32).Filter(filterZeroBytes).Draw(t, "pk") + p2Bytes := rapid.SliceOfN(rapid.Byte(), 32, 32).Filter(filterZeroBytes).Draw(t, "pk2") + p1sBytes := rapid.SliceOfN(rapid.Byte(), 64, 64).Filter(filterZeroBytes).Draw(t, "pk1sig") + p2sBytes := rapid.SliceOfN(rapid.Byte(), 64, 64).Filter(filterZeroBytes).Draw(t, "pk2sig") + copy(v.Sig.Sig[:], sigBytes) + copy(v.Sig.PK[:], pkBytes) + copy(v.Sig.PK2[:], p2Bytes) + copy(v.Sig.PK1Sig[:], p1sBytes) + copy(v.Sig.PK2Sig[:], p2sBytes) + + // PKSigOld is deprecated and always zero when encoded with StatelessEncoder + v.Sig.PKSigOld = [64]byte{} + + return v + }) +} + +// FuzzStatelessEncoder is a fuzz test that generates random votes, encodes them as +// msgpack, and uses them to seed the Go fuzzer. If they are valid, they will also be +// decompressed. +// +// Since the fuzzer is generating random data that StatelessEncoder +// expects to be valid msgpack-encoded votes, this test is only ensures that +// StatelessEncoder and StatelessDecoder don't crash on malformed data. +func FuzzStatelessEncoder(f *testing.F) { + // Seed with valid compressed votes from random vote generator + voteGen := generateRandomVote() + var msgpBuf []byte + for i := range 100 { + vote := voteGen.Example(i) + msgpBuf := protocol.EncodeMsgp(vote) + f.Add(msgpBuf) // Add seed corpus for the fuzzer + } + // Provide truncated versions of the last valid compressed vote + for i := 1; i < len(msgpBuf); i++ { + f.Add(msgpBuf[:i]) + } + // Add parseVote test cases + for _, tc := range parseVoteTestCases { + f.Add(protocol.EncodeReflect(tc.obj)) + } + + // Use a separate function that properly utilizes the fuzzer input + f.Fuzz(func(t *testing.T, msgpBuf []byte) { + // Try to compress the input + enc := NewStatelessEncoder() + compressed, err := enc.CompressVote(nil, msgpBuf) + if err != nil { + // Not valid msgpack data for a vote, skip + return + } + + // Then decompress it + dec := NewStatelessDecoder() + decompressed, err := dec.DecompressVote(nil, compressed) + if err != nil { + t.Fatalf("Failed to decompress valid compressed data: %v", err) + } + + // Verify the decompressed data matches the original + if !bytes.Equal(msgpBuf, decompressed) { + t.Fatalf("Decompressed data does not match original") + } + }) +} + +// FuzzStatelessDecoder is a fuzz test specifically targeting the StatelessDecoder +// with potentially malformed input. +func FuzzStatelessDecoder(f *testing.F) { + // Add valid compressed votes from random vote generator + voteGen := generateRandomVote() + var msgpBuf []byte + for i := range 100 { + vote := voteGen.Example(i) // Use deterministic seeds + msgpBuf = protocol.EncodeMsgp(vote) + require.LessOrEqual(f, len(msgpBuf), MaxMsgpackVoteSize) + enc := NewStatelessEncoder() + compressedVote, err := enc.CompressVote(nil, msgpBuf) + if err != nil { + continue + } + require.LessOrEqual(f, len(compressedVote), MaxCompressedVoteSize) + f.Add(compressedVote) + } + // Provide truncated versions of the last valid compressed vote + for i := 1; i < len(msgpBuf); i++ { + f.Add(msgpBuf[:i]) + } + + f.Add([]byte{}) + f.Add([]byte{0x01}) + f.Add([]byte{0x01, 0x02}) + f.Add([]byte{0xFF, 0xFF}) + f.Add([]byte{0x00, 0x00}) + f.Add([]byte{0x00, 0x00, 0xFF}) + + f.Fuzz(func(t *testing.T, data []byte) { + dec := NewStatelessDecoder() + decbuf, _ := dec.DecompressVote(nil, data) // Ensure it doesn't crash + require.LessOrEqual(t, len(decbuf), MaxMsgpackVoteSize) + }) +} diff --git a/network/vpack/vpack.go b/network/vpack/vpack.go new file mode 100644 index 0000000000..05e2c5861d --- /dev/null +++ b/network/vpack/vpack.go @@ -0,0 +1,407 @@ +// Copyright (C) 2019-2025 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package vpack + +import ( + "fmt" + "math/bits" +) + +// A vote is made up of 14 values, some of which are optional. +// The required values are: cred.pf, r.rnd, r.snd, sig.p, sig.p1s, sig.p2, +// sig.p2s, sig.s (sig.ps is always zero). +// The remaining 6 optional values are either present or omitted, and their +// presence is indicated in a 1-byte bitmask in the header. +const ( + bitPer uint8 = 1 << iota // r.per + bitDig // r.prop.dig + bitEncDig // r.prop.encdig + bitOper // r.prop.oper + bitOprop // r.prop.oprop + bitStep // r.step + + propFieldsMask uint8 = bitDig | bitEncDig | bitOper | bitOprop + totalRequiredFields = 8 +) + +const ( + headerSize = 2 // 1 byte for mask, 1 byte for future use + + maxMsgpVaruintSize = 9 // max size of a varuint is 8 bytes + 1 byte for the marker + msgpBin8Len32Size = len(msgpBin8Len32) + 32 + msgpBin8Len64Size = len(msgpBin8Len64) + 64 + msgpBin8Len80Size = len(msgpBin8Len80) + 80 + msgpFixMapMarkerSize = 1 + + // MaxMsgpackVoteSize is the maximum size of a vote, including msgpack control characters + // and all required and optional data fields. + MaxMsgpackVoteSize = msgpFixMapMarkerSize + // top-level fixmap + len(msgpFixstrCred) + msgpFixMapMarkerSize + // cred: fixmap + len(msgpFixstrPf) + msgpBin8Len80Size + // cred.pf: bin8(80) + len(msgpFixstrR) + msgpFixMapMarkerSize + // r: fixmap + len(msgpFixstrPer) + maxMsgpVaruintSize + // r.per: varuint + len(msgpFixstrProp) + msgpFixMapMarkerSize + // r.prop: fixmap + len(msgpFixstrDig) + msgpBin8Len32Size + // r.prop.dig: bin8(32) + len(msgpFixstrEncdig) + msgpBin8Len32Size + // r.prop.encdig: bin8(32) + len(msgpFixstrOper) + maxMsgpVaruintSize + // r.prop.oper: varuint + len(msgpFixstrOprop) + msgpBin8Len32Size + // r.prop.oprop: bin8(32) + len(msgpFixstrRnd) + maxMsgpVaruintSize + // r.rnd: varuint + len(msgpFixstrSnd) + msgpBin8Len32Size + // r.snd: bin8(32) + len(msgpFixstrStep) + maxMsgpVaruintSize + // r.step: varuint + len(msgpFixstrSig) + msgpFixMapMarkerSize + // sig: fixmap + len(msgpFixstrP) + msgpBin8Len32Size + // sig.p: bin8(32) + len(msgpFixstrP1s) + msgpBin8Len64Size + // sig.p1s: bin8(64) + len(msgpFixstrP2) + msgpBin8Len32Size + // sig.p2: bin8(32) + len(msgpFixstrP2s) + msgpBin8Len64Size + // sig.p2s: bin8(64) + len(msgpFixstrPs) + msgpBin8Len64Size + // sig.ps: bin8(64) + len(msgpFixstrS) + msgpBin8Len64Size // sig.s: bin8(64) + + // MaxCompressedVoteSize is the maximum size of a compressed vote using StatelessEncoder, + // including all required and optional fields. + MaxCompressedVoteSize = headerSize + + 80 + // cred.pf + maxMsgpVaruintSize*4 + // r.rnd, r.per, r.step, r.prop.oper + 32*6 + // r.prop.dig, r.prop.encdig, r.prop.oprop, r.snd, sig.p, sig.p2 + 64*3 // sig.p1s, sig.p2s, sig.s (sig.ps is omitted) +) + +// StatelessEncoder compresses a msgpack-encoded vote by stripping all msgpack +// formatting and field names, replacing them with a bitmask indicating which +// fields are present. It is not thread-safe. +type StatelessEncoder struct { + cur []byte + pos int + mask uint8 + + requiredFields uint8 +} + +// NewStatelessEncoder returns a new StatelessEncoder. +func NewStatelessEncoder() *StatelessEncoder { + return &StatelessEncoder{} +} + +// ErrBufferTooSmall is returned when the destination buffer is too small +var ErrBufferTooSmall = fmt.Errorf("destination buffer too small") + +// CompressVote compresses a vote in src and writes it to dst. +// If the provided buffer dst is nil or too small, a new slice is allocated. +// The returned slice may be the same as dst. +// To re-use dst, run like: dst = enc.CompressVote(dst[:0], src) +func (e *StatelessEncoder) CompressVote(dst, src []byte) ([]byte, error) { + bound := MaxCompressedVoteSize + // Reuse dst if it's big enough, otherwise allocate a new buffer + if cap(dst) >= bound { + dst = dst[0:bound] // Reuse dst buffer with its full capacity + } else { + dst = make([]byte, bound) + } + + // Reset our position to the beginning + e.cur = dst + e.mask = 0 + e.requiredFields = 0 + // put empty header at beginning, to fill in later + e.pos = headerSize + err := parseMsgpVote(src, e) + if err != nil { + return nil, err + } + + // Check if we overflowed the buffer + if e.pos > len(dst) { + return nil, ErrBufferTooSmall + } + + if e.requiredFields != totalRequiredFields { + return nil, fmt.Errorf("missing required fields") + } + // fill in header's first byte with mask + e.cur[0] = e.mask + + // Return only the portion that was used + return dst[:e.pos], nil + +} + +// writeBytes writes multiple bytes to the current buffer +// This is optimized to avoid per-byte bounds checking when possible +func (e *StatelessEncoder) writeBytes(bytes []byte) { + // If we have enough room in the buffer, use direct copy + if e.pos+len(bytes) <= len(e.cur) { + copy(e.cur[e.pos:], bytes) + } + // Always increment pos, so CompressVote will return ErrBufferTooSmall + e.pos += len(bytes) +} + +func (e *StatelessEncoder) updateMask(field voteValueType) { + switch field { + case rPerVoteValue: + e.mask |= bitPer + case rPropDigVoteValue: + e.mask |= bitDig + case rPropEncdigVoteValue: + e.mask |= bitEncDig + case rPropOperVoteValue: + e.mask |= bitOper + case rPropOpropVoteValue: + e.mask |= bitOprop + case rStepVoteValue: + e.mask |= bitStep + default: + // all other fields are required + e.requiredFields++ + } +} + +func (e *StatelessEncoder) writeVaruint(field voteValueType, b []byte) { + e.updateMask(field) + e.writeBytes(b) +} + +func (e *StatelessEncoder) writeBin32(field voteValueType, b [32]byte) { + e.updateMask(field) + e.writeBytes(b[:]) +} + +func (e *StatelessEncoder) writeBin64(field voteValueType, b [64]byte) { + e.updateMask(field) + e.writeBytes(b[:]) +} + +func (e *StatelessEncoder) writeBin80(field voteValueType, b [80]byte) { + e.updateMask(field) + e.writeBytes(b[:]) +} + +// StatelessDecoder decompresses votes that were compressed by StatelessEncoder. +type StatelessDecoder struct { + dst, src []byte + pos int +} + +// NewStatelessDecoder returns a new StatelessDecoder. +func NewStatelessDecoder() *StatelessDecoder { + return &StatelessDecoder{} +} + +func (d *StatelessDecoder) rawVoteMapSize(mask uint8) (cnt uint8) { + // Count how many of per, step are set (rnd & snd must be present) + cnt = 2 + uint8(bits.OnesCount8(mask&(bitPer|bitStep))) + // Add 1 if any prop bits are set + if mask&propFieldsMask != 0 { + cnt++ + } + return +} + +func (d *StatelessDecoder) proposalValueMapSize(mask uint8) uint8 { + // Count how many of dig, encdig, oper, oprop are set + return uint8(bits.OnesCount8(mask & (bitDig | bitEncDig | bitOper | bitOprop))) +} + +// DecompressVote decodes a compressed vote in src and appends it to dst. +// To re-use dst, run like: dst = dec.DecompressVote(dst[:0], src) +func (d *StatelessDecoder) DecompressVote(dst, src []byte) ([]byte, error) { + if len(src) < 2 { + return nil, fmt.Errorf("header missing") + } + mask := uint8(src[0]) + d.pos = 2 + d.src = src + d.dst = dst + if d.dst == nil { // allocate a new buffer if dst is nil + d.dst = make([]byte, 0, MaxMsgpackVoteSize) + } + + // top-level UnauthenticatedVote: fixmap(3) { cred, rawVote, sig } + d.dst = append(d.dst, msgpFixMapMask|3) + + // cred: fixmap(1) { pf: bin8(80) } + d.dst = append(d.dst, msgpFixstrCred...) + d.dst = append(d.dst, msgpFixMapMask|1) + + // cred.pf is always present + if err := d.bin80(msgpFixstrPf); err != nil { + return nil, err + } + + // rawVote: fixmap { per, prop, rnd, snd, step } + d.dst = append(d.dst, msgpFixstrR...) + d.dst = append(d.dst, msgpFixMapMask|d.rawVoteMapSize(mask)) + + // rawVote.per + if (mask & bitPer) != 0 { + if err := d.varuint(msgpFixstrPer); err != nil { + return nil, err + } + } + + // rawVote.prop could be zero (bottom vote is empty value) + if (mask & propFieldsMask) != 0 { + // proposalValue: fixmap { dig, encdig, oper, oprop } + d.dst = append(d.dst, msgpFixstrProp...) + d.dst = append(d.dst, msgpFixMapMask|d.proposalValueMapSize(mask)) + // prop.dig + if (mask & bitDig) != 0 { + if err := d.bin32(msgpFixstrDig); err != nil { + return nil, err + } + } + // prop.encdig + if (mask & bitEncDig) != 0 { + if err := d.bin32(msgpFixstrEncdig); err != nil { + return nil, err + } + } + // prop.oper + if (mask & bitOper) != 0 { + if err := d.varuint(msgpFixstrOper); err != nil { + return nil, err + } + } + // prop.oprop + if (mask & bitOprop) != 0 { + if err := d.bin32(msgpFixstrOprop); err != nil { + return nil, err + } + } + } + + // rawVote.rnd is always present + if err := d.varuint(msgpFixstrRnd); err != nil { + return nil, err + } + + // rawVote.snd is always present + if err := d.bin32(msgpFixstrSnd); err != nil { + return nil, err + } + + // rawVote.step + if (mask & bitStep) != 0 { + if err := d.varuint(msgpFixstrStep); err != nil { + return nil, err + } + } + + // crypto.OneTimeSignature does not use omitempty; all fields are required + // and always present. + + // sig: fixmap(6) { p, p1s, p2, p2s, ps, s } + d.dst = append(d.dst, msgpFixstrSig...) + d.dst = append(d.dst, msgpFixMapMask|6) + // sig.p + if err := d.bin32(msgpFixstrP); err != nil { + return nil, err + } + // sig.p1s + if err := d.bin64(msgpFixstrP1s); err != nil { + return nil, err + } + // sig.p2 + if err := d.bin32(msgpFixstrP2); err != nil { + return nil, err + } + // sig.p2s + if err := d.bin64(msgpFixstrP2s); err != nil { + return nil, err + } + // sig.ps is always zero + d.dst = append(d.dst, msgpFixstrPs...) + d.dst = append(d.dst, msgpBin8Len64...) + d.dst = append(d.dst, make([]byte, 64)...) + // sig.s + if err := d.bin64(msgpFixstrS); err != nil { + return nil, err + } + + if d.pos < len(d.src) { + return nil, fmt.Errorf("unexpected trailing data: %d bytes remain", len(d.src)-d.pos) + } + + return d.dst, nil +} + +func (d *StatelessDecoder) bin64(fieldStr string) error { + if d.pos+64 > len(d.src) { + return fmt.Errorf("not enough data to read value for field %s", fieldStr) + } + d.dst = append(d.dst, fieldStr...) + d.dst = append(d.dst, msgpBin8Len64...) + d.dst = append(d.dst, d.src[d.pos:d.pos+64]...) + d.pos += 64 + return nil +} + +func (d *StatelessDecoder) bin32(fieldStr string) error { + if d.pos+32 > len(d.src) { + return fmt.Errorf("not enough data to read value for field %s", fieldStr) + } + d.dst = append(d.dst, fieldStr...) + d.dst = append(d.dst, msgpBin8Len32...) + d.dst = append(d.dst, d.src[d.pos:d.pos+32]...) + d.pos += 32 + return nil +} + +func (d *StatelessDecoder) bin80(fieldStr string) error { + if d.pos+80 > len(d.src) { + return fmt.Errorf("not enough data to read value for field %s, d.pos=%d, len(src)=%d", fieldStr, d.pos, len(d.src)) + } + d.dst = append(d.dst, fieldStr...) + d.dst = append(d.dst, msgpBin8Len80...) + d.dst = append(d.dst, d.src[d.pos:d.pos+80]...) + d.pos += 80 + return nil +} + +func (d *StatelessDecoder) varuint(fieldName string) error { + if d.pos+1 > len(d.src) { + return fmt.Errorf("not enough data to read varuint marker for field %s", fieldName) + } + marker := d.src[d.pos] // read msgpack varuint marker + moreBytes := 0 + switch marker { + case msgpUint8: + moreBytes = 1 + case msgpUint16: + moreBytes = 2 + case msgpUint32: + moreBytes = 4 + case msgpUint64: + moreBytes = 8 + default: // fixint uses a single byte for marker+value + if !isMsgpFixint(marker) { + return fmt.Errorf("not a fixint for field %s, got %d", fieldName, marker) + } + moreBytes = 0 + } + + if d.pos+1+moreBytes > len(d.src) { + return fmt.Errorf("not enough data for varuint (need %d bytes) for field %s", moreBytes, fieldName) + } + d.dst = append(d.dst, fieldName...) + d.dst = append(d.dst, marker) + if moreBytes > 0 { + d.dst = append(d.dst, d.src[d.pos+1:d.pos+moreBytes+1]...) + } + d.pos += moreBytes + 1 // account for marker byte + value bytes + + return nil +} diff --git a/network/vpack/vpack_test.go b/network/vpack/vpack_test.go new file mode 100644 index 0000000000..f0b11a51a2 --- /dev/null +++ b/network/vpack/vpack_test.go @@ -0,0 +1,386 @@ +// Copyright (C) 2019-2025 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package vpack + +import ( + "encoding/json" + "fmt" + "reflect" + "slices" + "testing" + "unsafe" + + "github.com/algorand/go-algorand/agreement" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/test/partitiontest" + "github.com/stretchr/testify/require" +) + +// checkVoteValid analyzes a vote to determine if it would cause compression errors and what kind. +// Returns (true, expectedError) if an error is expected during compression, (false, "") otherwise. +func checkVoteValid(vote *agreement.UnauthenticatedVote) (ok bool, expectedError string) { + if vote.R.MsgIsZero() || vote.Cred.MsgIsZero() || vote.Sig.MsgIsZero() { + return false, "expected fixed map size 3 for unauthenticatedVote" + } + if vote.Cred.Proof.MsgIsZero() { + return false, "expected fixed map size 1 for UnauthenticatedCredential" + } + if !vote.Sig.PKSigOld.MsgIsZero() { + return false, "expected empty array for ps" + } + if vote.R.Round == 0 { + return false, "missing required fields" + } + if vote.R.Sender.IsZero() { + return false, "missing required fields" + } + + return true, "" +} + +// based on RunEncodingTest from protocol/codec_tester.go +func TestEncodingTest(t *testing.T) { + partitiontest.PartitionTest(t) + + var errorCount int + const iters = 20000 + for range iters { + v0obj, err := protocol.RandomizeObject(&agreement.UnauthenticatedVote{}, + protocol.RandomizeObjectWithZeroesEveryN(10), + protocol.RandomizeObjectWithAllUintSizes(), + ) + require.NoError(t, err) + + v0 := v0obj.(*agreement.UnauthenticatedVote) + if *v0 == (agreement.UnauthenticatedVote{}) { + continue // don't try to encode or compress empty votes (a single byte, 0x80) + } + // zero out ps, always empty + v0.Sig.PKSigOld = [64]byte{} + + var expectError string + if ok, errorMsg := checkVoteValid(v0); !ok { + expectError = errorMsg + } + + msgpBuf := protocol.EncodeMsgp(v0) + require.LessOrEqual(t, len(msgpBuf), MaxMsgpackVoteSize) + enc := NewStatelessEncoder() + encBuf, err := enc.CompressVote(nil, msgpBuf) + require.LessOrEqual(t, len(encBuf), MaxCompressedVoteSize) + if expectError != "" { + // skip expected errors + require.ErrorContains(t, err, expectError, "expected error: %s", expectError) + require.Nil(t, encBuf) + errorCount++ + continue + } + require.NoError(t, err) + + // decompress and compare to original + dec := NewStatelessDecoder() + decMsgpBuf, err := dec.DecompressVote(nil, encBuf) + jsonBuf, _ := json.MarshalIndent(v0, "", " ") + require.NoError(t, err, "got vote %s", jsonBuf) + require.Equal(t, msgpBuf, decMsgpBuf) // msgp encoding matches + var v1 agreement.UnauthenticatedVote + err = protocol.Decode(decMsgpBuf, &v1) + require.NoError(t, err) + require.Equal(t, *v0, v1) // vote objects match + } + t.Logf("TestEncodingTest: %d expected errors out of %d iterations", errorCount, iters) +} + +func TestStatelessDecoderErrors(t *testing.T) { + partitiontest.PartitionTest(t) + + z := func(n int) []byte { return make([]byte, n) } + h := func(m uint8) []byte { return []byte{m, 0x00} } + + fix1 := []byte{0x01} // msgpack fixint 1 (rnd = 1) + pf := z(80) // cred.pf (always present) + z32, z64 := z(32), z(64) + + type tc struct { + name, want string + buf func() []byte + } + + cases := []tc{ + // ---------- cred ---------- + {"pf-bin80", fmt.Sprintf("field %s", msgpFixstrPf), + func() []byte { return slices.Concat(h(0), z(79)) }}, + + // ---------- r.per ---------- + {"per-varuint-marker", fmt.Sprintf("field %s", msgpFixstrPer), + func() []byte { return slices.Concat(h(bitPer), pf) }}, + + // ---------- r.prop.* ---------- + {"dig-bin32", fmt.Sprintf("field %s", msgpFixstrDig), + func() []byte { return slices.Concat(h(bitDig), pf, z(10)) }}, + {"encdig-bin32", fmt.Sprintf("field %s", msgpFixstrEncdig), + func() []byte { return slices.Concat(h(bitDig|bitEncDig), pf, z32, z(10)) }}, + {"oper-varuint-marker", fmt.Sprintf("field %s", msgpFixstrOper), + func() []byte { return slices.Concat(h(bitOper), pf) }}, + {"oprop-bin32", fmt.Sprintf("field %s", msgpFixstrOprop), + func() []byte { return slices.Concat(h(bitOprop), pf, z(5)) }}, + + // ---------- r.rnd ---------- + {"rnd-varuint-marker", fmt.Sprintf("field %s", msgpFixstrRnd), + func() []byte { return slices.Concat(h(0), pf) }}, + {"rnd-varuint-trunc", fmt.Sprintf("not enough data for varuint (need 4 bytes) for field %s", msgpFixstrRnd), + func() []byte { return slices.Concat(h(0), pf, []byte{msgpUint32, 0x00}) }}, + + // ---------- r.snd / r.step ---------- + {"snd-bin32", fmt.Sprintf("field %s", msgpFixstrSnd), + func() []byte { return slices.Concat(h(0), pf, fix1) }}, + {"step-varuint-marker", fmt.Sprintf("field %s", msgpFixstrStep), + func() []byte { return slices.Concat(h(bitStep), pf, fix1, z32) }}, + + // ---------- sig.* ---------- + {"p-bin32", fmt.Sprintf("field %s", msgpFixstrP), + func() []byte { return slices.Concat(h(0), pf, fix1, z32) }}, + {"p1s-bin64", fmt.Sprintf("field %s", msgpFixstrP1s), + func() []byte { return slices.Concat(h(0), pf, fix1, z32, z32, z(12)) }}, + {"p2-bin32", fmt.Sprintf("field %s", msgpFixstrP2), + func() []byte { return slices.Concat(h(0), pf, fix1, z32, z32, z64) }}, + {"p2s-bin64", fmt.Sprintf("field %s", msgpFixstrP2s), + func() []byte { return slices.Concat(h(0), pf, fix1, z32, z32, z64, z32, z(3)) }}, + {"s-bin64", fmt.Sprintf("field %s", msgpFixstrS), + func() []byte { return slices.Concat(h(0), pf, fix1, z32, z32, z64, z32, z64) }}, + + // ---------- trailing data ---------- + {"trailing-bytes", "unexpected trailing data", + func() []byte { + return slices.Concat( + h(0), pf, fix1, // header, pf, rnd + z32, // snd + z32, z64, z32, z64, // p, p1s, p2, p2s + z64, // s + []byte{0xFF}, // extra byte + ) + }}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dec := NewStatelessDecoder() + _, err := dec.DecompressVote(nil, tc.buf()) + require.ErrorContains(t, err, tc.want) + }) + } +} + +// FuzzMsgpVote is a fuzz test for parseVote, CompressVote and DecompressVote. +// It generates random msgp-encoded votes, then compresses & decompresses them. +func FuzzMsgpVote(f *testing.F) { + addVote := func(obj any) []byte { + var buf []byte + if v, ok := obj.(*agreement.UnauthenticatedVote); ok { + buf = protocol.Encode(v) + } else { + buf = protocol.EncodeReflect(obj) + } + f.Add(buf) + f.Add(append([]byte{0x80}, buf...)) // add a prefix + f.Add(append([]byte{0x00}, buf...)) // add a prefix + f.Add(append(buf, 0x80)) // add a suffix + f.Add(append(buf, 0x00)) // add a suffix + return buf + } + // error cases (weird msgp bufs) + for _, tc := range parseVoteTestCases { + addVote(tc.obj) + } + for range 100 { // random valid votes + v, err := protocol.RandomizeObject(&agreement.UnauthenticatedVote{}, + protocol.RandomizeObjectWithZeroesEveryN(10), + protocol.RandomizeObjectWithAllUintSizes()) + require.NoError(f, err) + msgpbuf := addVote(v) + require.LessOrEqual(f, len(msgpbuf), MaxMsgpackVoteSize) + for i := range len(msgpbuf) { + f.Add(msgpbuf[:i]) + } + } + + f.Fuzz(func(t *testing.T, buf []byte) { + enc := NewStatelessEncoder() + encBuf, err := enc.CompressVote(nil, buf) + require.LessOrEqual(t, len(encBuf), MaxCompressedVoteSize) + if err != nil { + // invalid msgpbuf, skip + return + } + dec := NewStatelessDecoder() + decBuf, err := dec.DecompressVote(nil, encBuf) + require.LessOrEqual(t, len(decBuf), MaxMsgpackVoteSize) + require.NoError(t, err) + require.Equal(t, buf, decBuf) + }) +} + +func FuzzVoteFields(f *testing.F) { + f.Fuzz(func(t *testing.T, snd []byte, rnd, per, step uint64, + oper uint64, oprop, dig, encdig []byte, + pf []byte, s, p, ps, p2, p1s, p2s []byte) { + var v0 agreement.UnauthenticatedVote + copy(v0.R.Sender[:], snd) + v0.R.Round = basics.Round(rnd) + // Use reflection to set the unexported period field + rPeriodField := reflect.ValueOf(&v0.R).Elem().FieldByName("Period") + rPeriodField = reflect.NewAt(rPeriodField.Type(), unsafe.Pointer(rPeriodField.UnsafeAddr())).Elem() + rPeriodField.SetUint(per) + require.EqualValues(t, per, v0.R.Period) + // Use reflection to set the unexported step field + rStepField := reflect.ValueOf(&v0.R).Elem().FieldByName("Step") + rStepField = reflect.NewAt(rStepField.Type(), unsafe.Pointer(rStepField.UnsafeAddr())).Elem() + rStepField.SetUint(step) + require.EqualValues(t, step, v0.R.Step) + // Use reflection to set the OriginalPeriod field in the proposal + propVal := reflect.ValueOf(&v0.R.Proposal).Elem() + origPeriodField := propVal.FieldByName("OriginalPeriod") + origPeriodField = reflect.NewAt(origPeriodField.Type(), unsafe.Pointer(origPeriodField.UnsafeAddr())).Elem() + origPeriodField.SetUint(oper) + require.EqualValues(t, oper, v0.R.Proposal.OriginalPeriod) + + copy(v0.R.Proposal.OriginalProposer[:], oprop) + copy(v0.R.Proposal.BlockDigest[:], dig) + copy(v0.R.Proposal.EncodingDigest[:], encdig) + copy(v0.Cred.Proof[:], pf) + copy(v0.Sig.Sig[:], s) + copy(v0.Sig.PK[:], p) + copy(v0.Sig.PKSigOld[:], ps) + copy(v0.Sig.PK2[:], p2) + copy(v0.Sig.PK1Sig[:], p1s) + copy(v0.Sig.PK2Sig[:], p2s) + + var expectError string + if ok, errorMsg := checkVoteValid(&v0); !ok { + expectError = errorMsg + } + + msgpBuf := protocol.Encode(&v0) + require.LessOrEqual(t, len(msgpBuf), MaxMsgpackVoteSize) + enc := NewStatelessEncoder() + encBuf, err := enc.CompressVote(nil, msgpBuf) + require.LessOrEqual(t, len(encBuf), MaxCompressedVoteSize) + if expectError != "" { + // skip expected errors + require.ErrorContains(t, err, expectError) + require.Nil(t, encBuf) + return + } + require.NoError(t, err) + dec := NewStatelessDecoder() + decBuf, err := dec.DecompressVote(nil, encBuf) + require.NoError(t, err) + require.Equal(t, msgpBuf, decBuf) + var v1 agreement.UnauthenticatedVote + err = protocol.Decode(decBuf, &v1) + require.NoError(t, err) + require.Equal(t, v0, v1) + }) +} + +// TestEncoderReuse specifically tests the reuse of a StatelessEncoder instance across +// multiple compression operations. This test would have caught the bug where +// the encoder's position wasn't being reset between calls. +func TestEncoderReuse(t *testing.T) { + partitiontest.PartitionTest(t) + + // Create several random votes + const numVotes = 10 + msgpBufs := make([][]byte, 0, numVotes) + voteGen := generateRandomVote() + + // Generate random votes and encode them + for i := 0; i < numVotes; i++ { + msgpBufs = append(msgpBufs, protocol.EncodeMsgp(voteGen.Example(i))) + require.LessOrEqual(t, len(msgpBufs[i]), MaxMsgpackVoteSize) + } + + // Test reusing the same encoder multiple times + enc := NewStatelessEncoder() + var compressedBufs [][]byte + + // First case: Create a new buffer for each compression + for i, msgpBuf := range msgpBufs { + compressedBuf, err := enc.CompressVote(nil, msgpBuf) + require.NoError(t, err, "Vote %d failed to compress with new buffer", i) + require.LessOrEqual(t, len(compressedBuf), MaxCompressedVoteSize) + compressedBufs = append(compressedBufs, compressedBuf) + } + + // Verify all compressed buffers can be decompressed correctly + dec := NewStatelessDecoder() + for i, compressedBuf := range compressedBufs { + decompressedBuf, err := dec.DecompressVote(nil, compressedBuf) + require.LessOrEqual(t, len(decompressedBuf), MaxMsgpackVoteSize) + require.NoError(t, err, "Vote %d failed to decompress", i) + require.Equal(t, msgpBufs[i], decompressedBuf, "Vote %d decompressed incorrectly", i) + } + + // Second case: Reuse a single pre-allocated buffer + compressedBufs = compressedBufs[:0] // Clear + reusedBuffer := make([]byte, 0, 4096) + + for i, msgpBuf := range msgpBufs { + // Save the compressed result and create a new copy + // to avoid the buffer being modified by subsequent operations + compressed, err := enc.CompressVote(reusedBuffer[:0], msgpBuf) + require.NoError(t, err, "Vote %d failed to compress with reused buffer", i) + require.LessOrEqual(t, len(compressed), MaxCompressedVoteSize) + compressedCopy := make([]byte, len(compressed)) + copy(compressedCopy, compressed) + compressedBufs = append(compressedBufs, compressedCopy) + } + + // Verify all compressed buffers with reused buffer can be decompressed correctly + for i, compressedBuf := range compressedBufs { + decompressedBuf, err := dec.DecompressVote(nil, compressedBuf) + require.NoError(t, err, "Vote %d failed to decompress (reused buffer)", i) + require.LessOrEqual(t, len(decompressedBuf), MaxMsgpackVoteSize) + require.Equal(t, msgpBufs[i], decompressedBuf, "Vote %d decompressed incorrectly (reused buffer)", i) + } + + // Third case: Test with varying buffer sizes to ensure we handle capacity changes correctly + compressedBufs = compressedBufs[:0] // Clear + varyingBuffer := make([]byte, 0, 10) // Start with a small buffer + + for i, msgpBuf := range msgpBufs { + // This will cause the buffer to be reallocated sometimes + compressed, err := enc.CompressVote(varyingBuffer[:0], msgpBuf) + require.NoError(t, err, "Vote %d failed to compress with varying buffer", i) + require.LessOrEqual(t, len(compressed), MaxCompressedVoteSize) + compressedCopy := make([]byte, len(compressed)) + copy(compressedCopy, compressed) + compressedBufs = append(compressedBufs, compressedCopy) + + // Update the buffer for next iteration - it might have grown + varyingBuffer = compressed + } + + // Verify all compressed buffers with varying buffer can be decompressed correctly + for i, compressedBuf := range compressedBufs { + decompressedBuf, err := dec.DecompressVote(nil, compressedBuf) + require.NoError(t, err, "Vote %d failed to decompress (varying buffer)", i) + require.LessOrEqual(t, len(decompressedBuf), MaxMsgpackVoteSize) + require.Equal(t, msgpBufs[i], decompressedBuf, "Vote %d decompressed incorrectly (varying buffer)", i) + } +} diff --git a/network/wsNetwork.go b/network/wsNetwork.go index a3544c50ab..3dc8b0849a 100644 --- a/network/wsNetwork.go +++ b/network/wsNetwork.go @@ -302,6 +302,8 @@ type msgBroadcaster struct { broadcastQueueBulk chan broadcastRequest // slowWritingPeerMonitorInterval defines the interval between two consecutive tests for slow peer writing slowWritingPeerMonitorInterval time.Duration + // enableVoteCompression controls whether vote compression is enabled + enableVoteCompression bool } // msgHandler contains the logic for handling incoming messages and managing a readBuffer. It provides @@ -582,6 +584,7 @@ func (wn *WebsocketNetwork) setup() { config: wn.config, broadcastQueueHighPrio: make(chan broadcastRequest, wn.outgoingMessagesBufferSize), broadcastQueueBulk: make(chan broadcastRequest, 100), + enableVoteCompression: wn.config.EnableVoteCompression, } if wn.broadcaster.slowWritingPeerMonitorInterval == 0 { wn.broadcaster.slowWritingPeerMonitorInterval = slowWritingPeerMonitorInterval @@ -1001,7 +1004,11 @@ func (wn *WebsocketNetwork) ServeHTTP(response http.ResponseWriter, request *htt responseHeader.Set(ProtocolVersionHeader, matchingVersion) responseHeader.Set(GenesisHeader, wn.GenesisID) // set the features we support - responseHeader.Set(PeerFeaturesHeader, PeerFeatureProposalCompression) + features := []string{PeerFeatureProposalCompression} + if wn.config.EnableVoteCompression { + features = append(features, PeerFeatureVoteVpackCompression) + } + responseHeader.Set(PeerFeaturesHeader, strings.Join(features, ",")) var challenge string if wn.prioScheme != nil { challenge = wn.prioScheme.NewPrioChallenge() @@ -1035,18 +1042,19 @@ func (wn *WebsocketNetwork) ServeHTTP(response http.ResponseWriter, request *htt client, _ := wn.GetHTTPClient(trackedRequest.remoteAddress()) peer := &wsPeer{ - wsPeerCore: makePeerCore(wn.ctx, wn, wn.log, wn.handler.readBuffer, trackedRequest.remoteAddress(), client, trackedRequest.remoteHost), - conn: wsPeerWebsocketConnImpl{conn}, - outgoing: false, - InstanceName: trackedRequest.otherInstanceName, - incomingMsgFilter: wn.incomingMsgFilter, - prioChallenge: challenge, - createTime: trackedRequest.created, - version: matchingVersion, - identity: peerID, - identityChallenge: peerIDChallenge, - identityVerified: atomic.Uint32{}, - features: decodePeerFeatures(matchingVersion, request.Header.Get(PeerFeaturesHeader)), + wsPeerCore: makePeerCore(wn.ctx, wn, wn.log, wn.handler.readBuffer, trackedRequest.remoteAddress(), client, trackedRequest.remoteHost), + conn: wsPeerWebsocketConnImpl{conn}, + outgoing: false, + InstanceName: trackedRequest.otherInstanceName, + incomingMsgFilter: wn.incomingMsgFilter, + prioChallenge: challenge, + createTime: trackedRequest.created, + version: matchingVersion, + identity: peerID, + identityChallenge: peerIDChallenge, + identityVerified: atomic.Uint32{}, + features: decodePeerFeatures(matchingVersion, request.Header.Get(PeerFeaturesHeader)), + enableVoteCompression: wn.config.EnableVoteCompression, } peer.TelemetryGUID = trackedRequest.otherTelemetryGUID peer.init(wn.config, wn.outgoingMessagesBufferSize) @@ -1331,17 +1339,18 @@ func (wn *WebsocketNetwork) getPeersChangeCounter() int32 { // preparePeerData prepares batches of data for sending. // It performs zstd compression for proposal massages if they this is a prio request and has proposal. -func (wn *msgBroadcaster) preparePeerData(request broadcastRequest, prio bool) ([]byte, crypto.Digest) { +func (wn *msgBroadcaster) preparePeerData(request broadcastRequest, prio bool) ([]byte, []byte, crypto.Digest) { tbytes := []byte(request.tag) mbytes := make([]byte, len(tbytes)+len(request.data)) copy(mbytes, tbytes) copy(mbytes[len(tbytes):], request.data) + var compressedData []byte var digest crypto.Digest if request.tag != protocol.MsgDigestSkipTag && len(request.data) >= messageFilterSize { digest = crypto.Hash(mbytes) } - + // Compress proposals -- all proposals are compressed as of wsnet 2.2 if prio && request.tag == protocol.ProposalPayloadTag { compressed, logMsg := zstdCompressMsg(tbytes, request.data) if len(logMsg) > 0 { @@ -1349,7 +1358,15 @@ func (wn *msgBroadcaster) preparePeerData(request broadcastRequest, prio bool) ( } mbytes = compressed } - return mbytes, digest + // Optionally compress votes: only supporting peers will receive it. + if prio && request.tag == protocol.AgreementVoteTag && wn.enableVoteCompression { + var logMsg string + compressedData, logMsg = vpackCompressVote(tbytes, request.data) + if len(logMsg) > 0 { + wn.log.Warn(logMsg) + } + } + return mbytes, compressedData, digest } // prio is set if the broadcast is a high-priority broadcast. @@ -1366,7 +1383,7 @@ func (wn *msgBroadcaster) innerBroadcast(request broadcastRequest, prio bool, pe } start := time.Now() - data, digest := wn.preparePeerData(request, prio) + data, dataWithCompression, digest := wn.preparePeerData(request, prio) // first send to all the easy outbound peers who don't block, get them started. sentMessageCount := 0 @@ -1377,7 +1394,13 @@ func (wn *msgBroadcaster) innerBroadcast(request broadcastRequest, prio bool, pe if Peer(peer) == request.except { continue } - ok := peer.writeNonBlock(request.ctx, data, prio, digest, request.enqueueTime) + dataToSend := data + // check whether to send a compressed vote. dataWithCompression will be empty if this node + // has not enabled vote compression. + if peer.vpackVoteCompressionSupported() && len(dataWithCompression) > 0 { + dataToSend = dataWithCompression + } + ok := peer.writeNonBlock(request.ctx, dataToSend, prio, digest, request.enqueueTime) if ok { sentMessageCount++ continue @@ -1883,6 +1906,10 @@ const PeerFeaturesHeader = "X-Algorand-Peer-Features" // supports proposal payload compression with zstd const PeerFeatureProposalCompression = "ppzstd" +// PeerFeatureVoteVpackCompression is a value for PeerFeaturesHeader indicating peer +// supports agreement vote message compression with vpack +const PeerFeatureVoteVpackCompression = "avvpack" + var websocketsScheme = map[string]string{"http": "ws", "https": "wss"} var errBadAddr = errors.New("bad address") @@ -2013,7 +2040,11 @@ func (wn *WebsocketNetwork) tryConnect(netAddr, gossipAddr string) { // for backward compatibility, include the ProtocolVersion header as well. requestHeader.Set(ProtocolVersionHeader, wn.protocolVersion) // set the features header (comma-separated list) - requestHeader.Set(PeerFeaturesHeader, PeerFeatureProposalCompression) + features := []string{PeerFeatureProposalCompression} + if wn.config.EnableVoteCompression { + features = append(features, PeerFeatureVoteVpackCompression) + } + requestHeader.Set(PeerFeaturesHeader, strings.Join(features, ",")) SetUserAgentHeader(requestHeader) myInstanceName := wn.log.GetInstanceName() requestHeader.Set(InstanceNameHeader, myInstanceName) @@ -2118,6 +2149,7 @@ func (wn *WebsocketNetwork) tryConnect(netAddr, gossipAddr string) { version: matchingVersion, identity: peerID, features: decodePeerFeatures(matchingVersion, response.Header.Get(PeerFeaturesHeader)), + enableVoteCompression: wn.config.EnableVoteCompression, } peer.TelemetryGUID, peer.InstanceName, _ = getCommonHeaders(response.Header) diff --git a/network/wsNetwork_test.go b/network/wsNetwork_test.go index 65cedb245a..4950f62bba 100644 --- a/network/wsNetwork_test.go +++ b/network/wsNetwork_test.go @@ -490,6 +490,89 @@ func TestWebsocketProposalPayloadCompression(t *testing.T) { } } +// Set up two nodes, send vote to test vote compression feature +func TestWebsocketVoteCompression(t *testing.T) { + partitiontest.PartitionTest(t) + + type testDef struct { + netAEnableCompression, netBEnableCompression bool + } + + var tests []testDef = []testDef{ + {true, true}, // both nodes with compression enabled + {true, false}, // node A with compression, node B without + {false, true}, // node A without compression, node B with compression + {false, false}, // both nodes with compression disabled + } + + for _, test := range tests { + t.Run(fmt.Sprintf("A_compression_%v+B_compression_%v", test.netAEnableCompression, test.netBEnableCompression), func(t *testing.T) { + cfgA := defaultConfig + cfgA.GossipFanout = 1 + cfgA.EnableVoteCompression = test.netAEnableCompression + netA := makeTestWebsocketNodeWithConfig(t, cfgA) + netA.Start() + defer netStop(t, netA, "A") + + cfgB := defaultConfig + cfgB.GossipFanout = 1 + cfgB.EnableVoteCompression = test.netBEnableCompression + netB := makeTestWebsocketNodeWithConfig(t, cfgB) + + addrA, postListen := netA.Address() + require.True(t, postListen) + t.Log(addrA) + netB.phonebook.ReplacePeerList([]string{addrA}, "default", phonebook.RelayRole) + netB.Start() + defer netStop(t, netB, "B") + + // ps is empty, so this is a valid vote + vote1 := map[string]any{ + "cred": map[string]any{"pf": crypto.VrfProof{1}}, + "r": map[string]any{"rnd": uint64(2), "snd": [32]byte{3}}, + "sig": map[string]any{ + "p": [32]byte{4}, "p1s": [64]byte{5}, "p2": [32]byte{6}, + "p2s": [64]byte{7}, "ps": [64]byte{}, "s": [64]byte{9}, + }, + } + // ps is not empty: vpack compression will fail, but it will still be sent through + vote2 := map[string]any{ + "cred": map[string]any{"pf": crypto.VrfProof{10}}, + "r": map[string]any{"rnd": uint64(11), "snd": [32]byte{12}}, + "sig": map[string]any{ + "p": [32]byte{13}, "p1s": [64]byte{14}, "p2": [32]byte{15}, + "p2s": [64]byte{16}, "ps": [64]byte{17}, "s": [64]byte{18}, + }, + } + // Send a totally invalid message to ensure that it goes through. Even though vpack compression + // and decompression will fail, the message should still go through (as an intended fallback). + vote3 := []byte("hello") + messages := [][]byte{protocol.EncodeReflect(vote1), protocol.EncodeReflect(vote2), vote3} + matcher := newMessageMatcher(t, messages) + counterDone := matcher.done + netB.RegisterHandlers([]TaggedMessageHandler{{Tag: protocol.AgreementVoteTag, MessageHandler: matcher}}) + + readyTimeout := time.NewTimer(2 * time.Second) + waitReady(t, netA, readyTimeout.C) + t.Log("a ready") + waitReady(t, netB, readyTimeout.C) + t.Log("b ready") + + for _, msg := range messages { + netA.Broadcast(context.Background(), protocol.AgreementVoteTag, msg, true, nil) + } + + select { + case <-counterDone: + case <-time.After(2 * time.Second): + t.Errorf("timeout, count=%d, wanted %d", len(matcher.received), len(messages)) + } + + require.True(t, matcher.Match()) + }) + } +} + // Repeat basic, but test a unicast func TestWebsocketNetworkUnicast(t *testing.T) { partitiontest.PartitionTest(t) @@ -3686,37 +3769,60 @@ func BenchmarkVariableTransactionMessageBlockSizes(t *testing.B) { func TestPreparePeerData(t *testing.T) { partitiontest.PartitionTest(t) - // no compression + vote := map[string]any{ + "cred": map[string]any{"pf": crypto.VrfProof{}}, + "r": map[string]any{"rnd": uint64(1), "snd": [32]byte{}}, + "sig": map[string]any{ + "p": [32]byte{}, "p1s": [64]byte{}, "p2": [32]byte{}, + "p2s": [64]byte{}, "ps": [64]byte{}, "s": [64]byte{}, + }, + } reqs := []broadcastRequest{ - {tag: protocol.AgreementVoteTag, data: []byte("test")}, + {tag: protocol.AgreementVoteTag, data: protocol.EncodeReflect(vote)}, {tag: protocol.ProposalPayloadTag, data: []byte("data")}, + {tag: protocol.TxnTag, data: []byte("txn")}, + {tag: protocol.StateProofSigTag, data: []byte("stateproof")}, } wn := WebsocketNetwork{} + wn.broadcaster.log = logging.TestingLog(t) + // Enable vote compression for the test + wn.broadcaster.enableVoteCompression = true data := make([][]byte, len(reqs)) + compressedData := make([][]byte, len(reqs)) digests := make([]crypto.Digest, len(reqs)) + + // Test without compression (prio = false) for i, req := range reqs { - data[i], digests[i] = wn.broadcaster.preparePeerData(req, false) + data[i], compressedData[i], digests[i] = wn.broadcaster.preparePeerData(req, false) require.NotEmpty(t, data[i]) require.Empty(t, digests[i]) // small messages have no digest } for i := range data { require.Equal(t, append([]byte(reqs[i].tag), reqs[i].data...), data[i]) + require.Empty(t, compressedData[i]) // No compression when prio = false } + // Test with compression (prio = true) for i, req := range reqs { - data[i], digests[i] = wn.broadcaster.preparePeerData(req, true) + data[i], compressedData[i], digests[i] = wn.broadcaster.preparePeerData(req, true) require.NotEmpty(t, data[i]) require.Empty(t, digests[i]) // small messages have no digest } for i := range data { - if reqs[i].tag != protocol.ProposalPayloadTag { + if reqs[i].tag == protocol.AgreementVoteTag { + // For votes with prio=true, the main data remains uncompressed, but compressedData is filled require.Equal(t, append([]byte(reqs[i].tag), reqs[i].data...), data[i]) - require.Equal(t, data[i], data[i]) - } else { + require.NotEmpty(t, compressedData[i], "Vote messages should have compressed data when prio=true") + } else if reqs[i].tag == protocol.ProposalPayloadTag { + // For proposals with prio=true, the main data is compressed with zstd require.Equal(t, append([]byte(reqs[i].tag), zstdCompressionMagic[:]...), data[i][:len(reqs[i].tag)+len(zstdCompressionMagic)]) + require.Empty(t, compressedData[i], "Proposal messages should not have separate compressed data") + } else { + require.Equal(t, append([]byte(reqs[i].tag), reqs[i].data...), data[i]) + require.Empty(t, compressedData[i]) } } } diff --git a/network/wsPeer.go b/network/wsPeer.go index ef851905c2..c102ce48f8 100644 --- a/network/wsPeer.go +++ b/network/wsPeer.go @@ -238,6 +238,9 @@ type wsPeer struct { // peer features derived from the peer version features peerFeatureFlag + // enableCompression specifies whether this node can compress or decompress votes (and whether it has advertised this) + enableVoteCompression bool + // responseChannels used by the client to wait on the response of the request responseChannels map[uint64]chan *Response @@ -522,7 +525,7 @@ func (wp *wsPeer) readLoop() { }() wp.conn.SetReadLimit(MaxMessageLength) slurper := MakeLimitedReaderSlurper(averageMessageLength, MaxMessageLength) - dataConverter := makeWsPeerMsgDataConverter(wp) + dataConverter := makeWsPeerMsgDataDecoder(wp) for { msg := IncomingMessage{} @@ -1086,11 +1089,16 @@ func (wp *wsPeer) OnClose(f func()) { wp.closers = append(wp.closers, f) } +func (wp *wsPeer) vpackVoteCompressionSupported() bool { + return wp.features&pfCompressedVoteVpack != 0 +} + //msgp:ignore peerFeatureFlag type peerFeatureFlag int const ( pfCompressedProposal peerFeatureFlag = 1 << iota + pfCompressedVoteVpack ) // versionPeerFeatures defines protocol version when peer features were introduced @@ -1135,6 +1143,9 @@ func decodePeerFeatures(version string, announcedFeatures string) peerFeatureFla if part == PeerFeatureProposalCompression { features |= pfCompressedProposal } + if part == PeerFeatureVoteVpackCompression { + features |= pfCompressedVoteVpack + } } return features } diff --git a/network/wsPeer_test.go b/network/wsPeer_test.go index 707dc210ea..02fa324a08 100644 --- a/network/wsPeer_test.go +++ b/network/wsPeer_test.go @@ -166,6 +166,8 @@ func TestVersionToFeature(t *testing.T) { {"2.2", PeerFeatureProposalCompression, pfCompressedProposal}, {"2.2", strings.Join([]string{PeerFeatureProposalCompression, "test"}, ","), pfCompressedProposal}, {"2.2", strings.Join([]string{PeerFeatureProposalCompression, "test"}, ", "), pfCompressedProposal}, + {"2.2", strings.Join([]string{PeerFeatureProposalCompression, PeerFeatureVoteVpackCompression}, ","), pfCompressedVoteVpack | pfCompressedProposal}, + {"2.2", PeerFeatureVoteVpackCompression, pfCompressedVoteVpack}, {"2.3", PeerFeatureProposalCompression, pfCompressedProposal}, } for i, test := range tests { diff --git a/protocol/codec_tester.go b/protocol/codec_tester.go index a81113e745..7e6cb02919 100644 --- a/protocol/codec_tester.go +++ b/protocol/codec_tester.go @@ -45,27 +45,59 @@ type msgpMarshalUnmarshal interface { var rawMsgpType = reflect.TypeOf(msgp.Raw{}) var errSkipRawMsgpTesting = fmt.Errorf("skipping msgp.Raw serializing, since it won't be the same across go-codec and msgp") +func oneOf(n int) bool { + return (rand.Int() % n) == 0 +} + +type randomizeObjectCfg struct { + // ZeroesEveryN will increase the chance of zero values being generated. + ZeroesEveryN int + // AllUintSizes will be equally likely to generate 8-bit, 16-bit, 32-bit, or 64-bit uints. + AllUintSizes bool +} + +// RandomizeObjectOption is an option for RandomizeObject +type RandomizeObjectOption func(*randomizeObjectCfg) + +// RandomizeObjectWithZeroesEveryN sets the chance of zero values being generated (one in n) +func RandomizeObjectWithZeroesEveryN(n int) RandomizeObjectOption { + return func(cfg *randomizeObjectCfg) { cfg.ZeroesEveryN = n } +} + +// RandomizeObjectWithAllUintSizes will be equally likely to generate 8-bit, 16-bit, 32-bit, or 64-bit uints. +func RandomizeObjectWithAllUintSizes() RandomizeObjectOption { + return func(cfg *randomizeObjectCfg) { cfg.AllUintSizes = true } +} + // RandomizeObject returns a random object of the same type as template -func RandomizeObject(template interface{}) (interface{}, error) { +func RandomizeObject(template interface{}, opts ...RandomizeObjectOption) (interface{}, error) { + cfg := randomizeObjectCfg{} + for _, opt := range opts { + opt(&cfg) + } tt := reflect.TypeOf(template) if tt.Kind() != reflect.Ptr { return nil, fmt.Errorf("RandomizeObject: must be ptr") } v := reflect.New(tt.Elem()) changes := int(^uint(0) >> 1) - err := randomizeValue(v.Elem(), tt.String(), "", &changes, make(map[reflect.Type]bool)) + err := randomizeValue(v.Elem(), 0, tt.String(), "", &changes, cfg, make(map[reflect.Type]bool)) return v.Interface(), err } // RandomizeObjectField returns a random object of the same type as template where a single field was modified. -func RandomizeObjectField(template interface{}) (interface{}, error) { +func RandomizeObjectField(template interface{}, opts ...RandomizeObjectOption) (interface{}, error) { + cfg := randomizeObjectCfg{} + for _, opt := range opts { + opt(&cfg) + } tt := reflect.TypeOf(template) if tt.Kind() != reflect.Ptr { return nil, fmt.Errorf("RandomizeObject: must be ptr") } v := reflect.New(tt.Elem()) changes := 1 - err := randomizeValue(v.Elem(), tt.String(), "", &changes, make(map[reflect.Type]bool)) + err := randomizeValue(v.Elem(), 0, tt.String(), "", &changes, cfg, make(map[reflect.Type]bool)) return v.Interface(), err } @@ -207,14 +239,14 @@ func checkBoundsLimitingTag(val reflect.Value, datapath string, structTag string return } -func randomizeValue(v reflect.Value, datapath string, tag string, remainingChanges *int, seenTypes map[reflect.Type]bool) error { +func randomizeValue(v reflect.Value, depth int, datapath string, tag string, remainingChanges *int, cfg randomizeObjectCfg, seenTypes map[reflect.Type]bool) error { if *remainingChanges == 0 { return nil } - /*if oneOf(5) { + if depth != 0 && cfg.ZeroesEveryN > 0 && oneOf(cfg.ZeroesEveryN) { // Leave zero value return nil - }*/ + } /* Consider cutting off recursive structures by stopping at some datapath depth. @@ -231,7 +263,22 @@ func randomizeValue(v reflect.Value, datapath string, tag string, remainingChang // generate value that will avoid protocol.ErrInvalidObject from HashType.Validate() v.SetUint(rand.Uint64() % 3) // 3 is crypto.MaxHashType } else { - v.SetUint(rand.Uint64()) + var num uint64 + if cfg.AllUintSizes { + switch rand.Intn(4) { + case 0: // fewer than 8 bits + num = uint64(rand.Intn(1 << 8)) // 0 to 255 + case 1: // fewer than 16 bits + num = uint64(rand.Intn(1 << 16)) // 0 to 65535 + case 2: // fewer than 32 bits + num = uint64(rand.Uint32()) // 0 to 2^32-1 + case 3: // fewer than 64 bits + num = rand.Uint64() // 0 to 2^64-1 + } + } else { + num = rand.Uint64() + } + v.SetUint(num) } *remainingChanges-- case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: @@ -260,7 +307,7 @@ func randomizeValue(v reflect.Value, datapath string, tag string, remainingChang *remainingChanges-- case reflect.Ptr: v.Set(reflect.New(v.Type().Elem())) - err := randomizeValue(reflect.Indirect(v), datapath, tag, remainingChanges, seenTypes) + err := randomizeValue(reflect.Indirect(v), depth+1, datapath, tag, remainingChanges, cfg, seenTypes) if err != nil { return err } @@ -288,7 +335,7 @@ func randomizeValue(v reflect.Value, datapath string, tag string, remainingChang if rawMsgpType == f.Type { return errSkipRawMsgpTesting } - err := randomizeValue(v.Field(fieldIdx), datapath+"/"+f.Name, string(tag), remainingChanges, seenTypes) + err := randomizeValue(v.Field(fieldIdx), depth+1, datapath+"/"+f.Name, string(tag), remainingChanges, cfg, seenTypes) if err != nil { return err } @@ -300,7 +347,7 @@ func randomizeValue(v reflect.Value, datapath string, tag string, remainingChang case reflect.Array: indicesOrder := rand.Perm(v.Len()) for i := 0; i < v.Len(); i++ { - err := randomizeValue(v.Index(indicesOrder[i]), fmt.Sprintf("%s/%d", datapath, indicesOrder[i]), "", remainingChanges, seenTypes) + err := randomizeValue(v.Index(indicesOrder[i]), depth+1, fmt.Sprintf("%s/%d", datapath, indicesOrder[i]), "", remainingChanges, cfg, seenTypes) if err != nil { return err } @@ -321,7 +368,7 @@ func randomizeValue(v reflect.Value, datapath string, tag string, remainingChang s := reflect.MakeSlice(v.Type(), l, l) indicesOrder := rand.Perm(l) for i := 0; i < l; i++ { - err := randomizeValue(s.Index(indicesOrder[i]), fmt.Sprintf("%s/%d", datapath, indicesOrder[i]), "", remainingChanges, seenTypes) + err := randomizeValue(s.Index(indicesOrder[i]), depth+1, fmt.Sprintf("%s/%d", datapath, indicesOrder[i]), "", remainingChanges, cfg, seenTypes) if err != nil { return err } @@ -345,13 +392,13 @@ func randomizeValue(v reflect.Value, datapath string, tag string, remainingChang indicesOrder := rand.Perm(l) for i := 0; i < l; i++ { mk := reflect.New(mt.Key()) - err := randomizeValue(mk.Elem(), fmt.Sprintf("%s/%d", datapath, indicesOrder[i]), "", remainingChanges, seenTypes) + err := randomizeValue(mk.Elem(), depth+1, fmt.Sprintf("%s/%d", datapath, indicesOrder[i]), "", remainingChanges, cfg, seenTypes) if err != nil { return err } mv := reflect.New(mt.Elem()) - err = randomizeValue(mv.Elem(), fmt.Sprintf("%s/%d", datapath, indicesOrder[i]), "", remainingChanges, seenTypes) + err = randomizeValue(mv.Elem(), depth+1, fmt.Sprintf("%s/%d", datapath, indicesOrder[i]), "", remainingChanges, cfg, seenTypes) if err != nil { return err } diff --git a/test/testdata/configs/config-v36.json b/test/testdata/configs/config-v36.json new file mode 100644 index 0000000000..fec7d7bf6d --- /dev/null +++ b/test/testdata/configs/config-v36.json @@ -0,0 +1,147 @@ +{ + "Version": 36, + "AccountUpdatesStatsInterval": 5000000000, + "AccountsRebuildSynchronousMode": 1, + "AgreementIncomingBundlesQueueLength": 15, + "AgreementIncomingProposalsQueueLength": 50, + "AgreementIncomingVotesQueueLength": 20000, + "AnnounceParticipationKey": true, + "Archival": false, + "BaseLoggerDebugLevel": 4, + "BlockDBDir": "", + "BlockServiceCustomFallbackEndpoints": "", + "BlockServiceMemCap": 500000000, + "BroadcastConnectionsLimit": -1, + "CadaverDirectory": "", + "CadaverSizeTarget": 0, + "CatchpointDir": "", + "CatchpointFileHistoryLength": 365, + "CatchpointInterval": 10000, + "CatchpointTracking": 0, + "CatchupBlockDownloadRetryAttempts": 1000, + "CatchupBlockValidateMode": 0, + "CatchupFailurePeerRefreshRate": 10, + "CatchupGossipBlockFetchTimeoutSec": 4, + "CatchupHTTPBlockFetchTimeoutSec": 4, + "CatchupLedgerDownloadRetryAttempts": 50, + "CatchupParallelBlocks": 16, + "ColdDataDir": "", + "ConnectionsRateLimitingCount": 60, + "ConnectionsRateLimitingWindowSeconds": 1, + "CrashDBDir": "", + "DNSBootstrapID": ".algorand.network?backup=.algorand.net&dedup=.algorand-.(network|net)", + "DNSSecurityFlags": 9, + "DeadlockDetection": 0, + "DeadlockDetectionThreshold": 30, + "DisableAPIAuth": false, + "DisableLedgerLRUCache": false, + "DisableLocalhostConnectionRateLimit": true, + "DisableNetworking": false, + "DisableOutgoingConnectionThrottling": false, + "EnableAccountUpdatesStats": false, + "EnableAgreementReporting": false, + "EnableAgreementTimeMetrics": false, + "EnableAssembleStats": false, + "EnableBlockService": false, + "EnableDHTProviders": false, + "EnableDeveloperAPI": false, + "EnableExperimentalAPI": false, + "EnableFollowMode": false, + "EnableGossipBlockService": true, + "EnableGossipService": true, + "EnableIncomingMessageFilter": false, + "EnableLedgerService": false, + "EnableMetricReporting": false, + "EnableNetDevMetrics": false, + "EnableOutgoingNetworkMessageFiltering": true, + "EnableP2P": false, + "EnableP2PHybridMode": false, + "EnablePingHandler": true, + "EnablePrivateNetworkAccessHeader": false, + "EnableProcessBlockStats": false, + "EnableProfiler": false, + "EnableRequestLogger": false, + "EnableRuntimeMetrics": false, + "EnableTopAccountsReporting": false, + "EnableTxBacklogAppRateLimiting": true, + "EnableTxBacklogRateLimiting": true, + "EnableTxnEvalTracer": false, + "EnableUsageLog": false, + "EnableVerbosedTransactionSyncLogging": false, + "EnableVoteCompression": true, + "EndpointAddress": "127.0.0.1:0", + "FallbackDNSResolverAddress": "", + "ForceFetchTransactions": false, + "ForceRelayMessages": false, + "GoMemLimit": 0, + "GossipFanout": 4, + "HeartbeatUpdateInterval": 600, + "HotDataDir": "", + "IncomingConnectionsLimit": 2400, + "IncomingMessageFilterBucketCount": 5, + "IncomingMessageFilterBucketSize": 512, + "LedgerSynchronousMode": 2, + "LogArchiveDir": "", + "LogArchiveMaxAge": "", + "LogArchiveName": "node.archive.log", + "LogFileDir": "", + "LogSizeLimit": 1073741824, + "MaxAPIBoxPerApplication": 100000, + "MaxAPIResourcesPerAccount": 100000, + "MaxAcctLookback": 4, + "MaxBlockHistoryLookback": 0, + "MaxCatchpointDownloadDuration": 43200000000000, + "MaxConnectionsPerIP": 8, + "MinCatchpointFileDownloadBytesPerSecond": 20480, + "NetAddress": "", + "NetworkMessageTraceServer": "", + "NetworkProtocolVersion": "", + "NodeExporterListenAddress": ":9100", + "NodeExporterPath": "./node_exporter", + "OptimizeAccountsDatabaseOnStartup": false, + "OutgoingMessageFilterBucketCount": 3, + "OutgoingMessageFilterBucketSize": 128, + "P2PHybridIncomingConnectionsLimit": 1200, + "P2PHybridNetAddress": "", + "P2PPersistPeerID": false, + "P2PPrivateKeyLocation": "", + "ParticipationKeysRefreshInterval": 60000000000, + "PeerConnectionsUpdateInterval": 3600, + "PeerPingPeriodSeconds": 0, + "PriorityPeers": {}, + "ProposalAssemblyTime": 500000000, + "PublicAddress": "", + "ReconnectTime": 60000000000, + "ReservedFDs": 256, + "RestConnectionsHardLimit": 2048, + "RestConnectionsSoftLimit": 1024, + "RestReadTimeoutSeconds": 15, + "RestWriteTimeoutSeconds": 120, + "RunHosted": false, + "StateproofDir": "", + "StorageEngine": "sqlite", + "SuggestedFeeBlockHistory": 3, + "SuggestedFeeSlidingWindowSize": 50, + "TLSCertFile": "", + "TLSKeyFile": "", + "TelemetryToLog": true, + "TrackerDBDir": "", + "TransactionSyncDataExchangeRate": 0, + "TransactionSyncSignificantMessageThreshold": 0, + "TxBacklogAppRateLimitingCountERLDrops": false, + "TxBacklogAppTxPerSecondRate": 100, + "TxBacklogAppTxRateLimiterMaxSize": 1048576, + "TxBacklogRateLimitingCongestionPct": 50, + "TxBacklogReservedCapacityPerPeer": 20, + "TxBacklogServiceRateWindowSeconds": 10, + "TxBacklogSize": 26000, + "TxIncomingFilterMaxSize": 500000, + "TxIncomingFilteringFlags": 1, + "TxPoolExponentialIncreaseFactor": 2, + "TxPoolSize": 75000, + "TxSyncIntervalSeconds": 60, + "TxSyncServeResponseSize": 1000000, + "TxSyncTimeoutSeconds": 30, + "UseXForwardedForAddressField": "", + "VerifiedTranscationsCacheSize": 150000 +} diff --git a/tools/block-generator/go.sum b/tools/block-generator/go.sum index bb102ccb7b..67095905ea 100644 --- a/tools/block-generator/go.sum +++ b/tools/block-generator/go.sum @@ -901,8 +901,8 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= lukechampine.com/blake3 v1.3.0 h1:sJ3XhFINmHSrYCgl958hscfIa3bw8x4DqMP3u1YvoYE= lukechampine.com/blake3 v1.3.0/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= -pgregory.net/rapid v0.6.2 h1:ErW5sL+UKtfBfUTsWHDCoeB+eZKLKMxrSd1VJY6W4bw= -pgregory.net/rapid v0.6.2/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= +pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= +pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck=