PoS Add QC fields to block header schema (#558)

* Initial commit

* Add tests

* Fix broken tests

* Better comments and edge case handling

* Revert unnecessary comments

* Address Nina's feedback

* Fix postgres tests

* Fix compile error when the relic build tag isn't defined

* Address Piotr's comments

* Address Nina's comments

lib/block_view_types.go

@@ -2,6 +2,7 @@ package lib
 
 import (
  "bytes"
+ "encoding/binary"
  "encoding/hex"
  "encoding/json"
  "fmt"
@@ -4647,6 +4648,58 @@ func DecodeByteArray(reader io.Reader) ([]byte, error) {
  }
 }
 
+func EncodeUint64Array(uint64s []uint64) []byte {
+ var data []byte
+
+ data = append(data, UintToBuf(uint64(len(uint64s)))...)
+ for _, uint64 := range uint64s {
+ data = append(data, UintToBuf(uint64)...)
+ }
+
+ return data
+}
+
+func DecodeUint64Array(reader io.Reader) ([]uint64, error) {
+ arrLen, err := ReadUvarint(reader)
+ if err != nil {
+ return nil, errors.Wrapf(err, "DecodeUint64Array: Problem reading array length")
+ }
+
+ if arrLen == 0 {
+ return nil, nil
+ }
+
+ var result []uint64
+ result, err = SafeMakeSliceWithLength[uint64](arrLen)
+ if err != nil {
+ return nil, errors.Wrapf(err, "DecodeUint64Array: Problem creating slice")
+ }
+
+ for ii := uint64(0); ii < arrLen; ii++ {
+ result[ii], err = ReadUvarint(reader)
+ if err != nil {
+ return nil, errors.Wrapf(err, "DecodeUint64Array: Problem reading uint64")
+ }
+ }
+
+ return result, nil
+}
+
+func EncodeUint64BigEndian(val uint64) []byte {
+ encodedBytes := [8]byte{}
+ binary.BigEndian.PutUint64(encodedBytes[:], val)
+ return encodedBytes[:]
+}
+
+func DecodeUint64BigEndian(rr io.Reader) (uint64, error) {
+ scratchBytes := [8]byte{}
+ _, err := io.ReadFull(rr, scratchBytes[:])
+ if err != nil {
+ return 0, err
+ }
+ return binary.BigEndian.Uint64(scratchBytes[:]), nil
+}
+
 func EncodePKIDuint64Map(pkidMap map[PKID]uint64) []byte {
  var data []byte
 

lib/constants.go

@@ -85,9 +85,11 @@ const (
  NetworkType_TESTNET NetworkType = 2
 )
 
+type MsgDeSoHeaderVersion = uint32
+
 const (
  // This is the header version that the blockchain started with.
- HeaderVersion0 = uint32(0)
+ HeaderVersion0 = MsgDeSoHeaderVersion(0)
  // This version made several changes to the previous header encoding format:
  // - The Nonce field was expanded to 64 bits
  // - Another ExtraNonce field was added to provide *another* 64 bits of entropy,
@@ -101,7 +103,24 @@ const (
  //
  // At the time of this writing, the intent is to deploy it in a backwards-compatible
  // fashion, with the eventual goal of phasing out blocks with the previous version.
- HeaderVersion1 = uint32(1)
+ HeaderVersion1 = MsgDeSoHeaderVersion(1)
+ // This version introduces the transition from Proof of Work to Proof of Stake blocks.
+ // It includes several changes to the header format:
+ // - Nonce field is deprecated
+ // - ExtraNonce field is deprecated
+ // - ValidatorsVoteQC field is added
+ // - ValidatorsTimeoutAggregateQC field is added
+ //
+ // This format change is a breaking change that is not backwards-compatible with
+ // versions 0 and 1.
+ HeaderVersion2 = MsgDeSoHeaderVersion(2)
+ // This CurrentHeaderVersion is an implicit version type that represents the latest
+ // backwards compatible Proof of Work header format. This value is now locked to
+ // HeaderVersion1 since versions 2 and onwards will be used for Proof of Stake formats.
+ //
+ // TODO: rename this constant to "LatestProofOfWorkHeaderVersion". Note, doing so will
+ // be a breaking change for 3rd party applications that import core and use this
+ // constant.
  CurrentHeaderVersion = HeaderVersion1
 )
 

lib/db_utils_test.go

@@ -61,7 +61,7 @@ func _GetTestBlockNode() *BlockNode {
 
  // Header (make a copy)
  bs.Header = NewMessage(MsgTypeHeader).(*MsgDeSoHeader)
- headerBytes, _ := expectedBlockHeader.ToBytes(false)
+ headerBytes, _ := expectedBlockHeaderVersion1.ToBytes(false)
  bs.Header.FromBytes(headerBytes)
 
  // Status

lib/network.go

@@ -1892,16 +1892,41 @@ type MsgDeSoHeader struct {
  // The height of the block this header corresponds to.
  Height uint64
 
+ // Nonce is only used for Proof of Work blocks, and will only be populated
+ // in MsgDeSoHeader versions 0 and 1. For all later versions, this field will
+ // default to a value of zero.
+ //
  // The nonce that is used by miners in order to produce valid blocks.
  //
  // Note: Before the upgrade from HeaderVersion0 to HeaderVersion1, miners would make
  // use of ExtraData in the BlockRewardMetadata to get extra nonces. However, this is
  // no longer needed since HeaderVersion1 upgraded the nonce to 64 bits from 32 bits.
  Nonce uint64
 
- // An extra nonce that can be used to provice *even more* entropy for miners, in the
+ // ExtraNonce is only used for Proof of Work blocks, and will only be populated
+ // in MsgDeSoHeader versions 0 and 1. For all later versions, this field will
+ // default to zero.
+ //
+ // An extra nonce that can be used to provide *even more* entropy for miners, in the
  // event that ASICs become powerful enough to have birthday problems in the future.
  ExtraNonce uint64
+
+ // ValidatorsVoteQC is only used for Proof of Stake blocks, and will only be
+ // populated in MsgDeSoHeader versions 2 and higher. For all earlier version, this
+ // field will be null.
+ //
+ // This corresponds to QC containing votes from 2/3 of validators for weighted by stake.
+ ValidatorsVoteQC *QuorumCertificate
+
+ // ValidatorsTimeoutAggregateQC is only used for Proof of Stake blocks, and will only be
+ // populated in MsgDeSoHeader versions 2 and higher. For all earlier version, this field
+ // will be null.
+ //
+ // In the event of a timeout, this field will contain the aggregate QC constructed from
+ // timeout messages from 2/3 of validators weighted by stake, and proves that they have
+ // time out. This value is set to null in normal cases where a regular block vote has
+ // taken place.
+ ValidatorsTimeoutAggregateQC *TimeoutAggregateQuorumCertificate
 }
 
 func HeaderSizeBytes() int {
@@ -2030,13 +2055,72 @@ func (msg *MsgDeSoHeader) EncodeHeaderVersion1(preSignature bool) ([]byte, error
  return retBytes, nil
 }
 
+func (msg *MsgDeSoHeader) EncodeHeaderVersion2(preSignature bool) ([]byte, error) {
+ retBytes := []byte{}
+
+ // Version
+ {
+ scratchBytes := [4]byte{}
+ binary.BigEndian.PutUint32(scratchBytes[:], msg.Version)
+ retBytes = append(retBytes, scratchBytes[:]...)
+ }
+
+ // PrevBlockHash
+ prevBlockHash := msg.PrevBlockHash
+ if prevBlockHash == nil {
+ prevBlockHash = &BlockHash{}
+ }
+ retBytes = append(retBytes, prevBlockHash[:]...)
+
+ // TransactionMerkleRoot
+ transactionMerkleRoot := msg.TransactionMerkleRoot
+ if transactionMerkleRoot == nil {
+ transactionMerkleRoot = &BlockHash{}
+ }
+ retBytes = append(retBytes, transactionMerkleRoot[:]...)
+
+ // TstampSecs: this field can be encoded to take up the full 64 bits now
+ // that MsgDeSoHeader version 2 does not need to be backwards compatible.
+ retBytes = append(retBytes, EncodeUint64BigEndian(msg.TstampSecs)...)
+
+ // Height
+ retBytes = append(retBytes, EncodeUint64BigEndian(msg.Height)...)
+
+ // The Nonce and ExtraNonce fields are unused in version 2. We skip them
+ // during both encoding and decoding.
+
+ // ValidatorsVoteQC
+ if msg.ValidatorsVoteQC == nil {
+ return nil, fmt.Errorf("EncodeHeaderVersion2: ValidatorsVoteQC must be non-nil")
+ }
+ encodedValidatorsVoteQC, err := msg.ValidatorsVoteQC.ToBytes()
+ if err != nil {
+ return nil, errors.Wrapf(err, "EncodeHeaderVersion2: error encoding ValidatorsVoteQC")
+ }
+ retBytes = append(retBytes, encodedValidatorsVoteQC...)
+
+ // ValidatorsTimeoutAggregateQC
+ if msg.ValidatorsTimeoutAggregateQC == nil {
+ return nil, fmt.Errorf("EncodeHeaderVersion2: ValidatorsTimeoutAggregateQC must be non-nil")
+ }
+ encodedValidatorsTimeoutAggregateQC, err := msg.ValidatorsTimeoutAggregateQC.ToBytes()
+ if err != nil {
+ return nil, errors.Wrapf(err, "EncodeHeaderVersion2: error encoding ValidatorsTimeoutAggregateQC")
+ }
+ retBytes = append(retBytes, encodedValidatorsTimeoutAggregateQC...)
+
+ return retBytes, nil
+}
+
 func (msg *MsgDeSoHeader) ToBytes(preSignature bool) ([]byte, error) {
 
  // Depending on the version, we decode the header differently.
  if msg.Version == HeaderVersion0 {
  return msg.EncodeHeaderVersion0(preSignature)
  } else if msg.Version == HeaderVersion1 {
  return msg.EncodeHeaderVersion1(preSignature)
+ } else if msg.Version == HeaderVersion2 {
+ return msg.EncodeHeaderVersion2(preSignature)
  } else {
  // If we have an unrecognized version then we default to serializing with
  // version 0. This is necessary because there are places where we use a
@@ -2151,6 +2235,53 @@ func DecodeHeaderVersion1(rr io.Reader) (*MsgDeSoHeader, error) {
  return retHeader, nil
 }
 
+func DecodeHeaderVersion2(rr io.Reader) (*MsgDeSoHeader, error) {
+ retHeader := NewMessage(MsgTypeHeader).(*MsgDeSoHeader)
+
+ // PrevBlockHash
+ _, err := io.ReadFull(rr, retHeader.PrevBlockHash[:])
+ if err != nil {
+ return nil, errors.Wrapf(err, "MsgDeSoHeader.FromBytes: Problem decoding PrevBlockHash")
+ }
+
+ // TransactionMerkleRoot
+ _, err = io.ReadFull(rr, retHeader.TransactionMerkleRoot[:])
+ if err != nil {
+ return nil, errors.Wrapf(err, "MsgDeSoHeader.FromBytes: Problem decoding TransactionMerkleRoot")
+ }
+
+ // TstampSecs
+ retHeader.TstampSecs, err = DecodeUint64BigEndian(rr)
+ if err != nil {
+ return nil, errors.Wrapf(err, "MsgDeSoHeader.FromBytes: Problem decoding TstampSecs")
+ }
+
+ // Height
+ retHeader.Height, err = DecodeUint64BigEndian(rr)
+ if err != nil {
+ return nil, errors.Wrapf(err, "MsgDeSoHeader.FromBytes: Problem decoding Height")
+ }
+
+ // The Nonce and ExtraNonce fields are unused in version 2. We skip them
+ // during both encoding and decoding.
+ retHeader.Nonce = 0
+ retHeader.ExtraNonce = 0
+
+ // ValidatorsVoteQC
+ retHeader.ValidatorsVoteQC, err = DecodeQuorumCertificate(rr)
+ if err != nil {
+ return nil, errors.Wrapf(err, "MsgDeSoHeader.FromBytes: Problem decoding ValidatorsVoteQC")
+ }
+
+ // ValidatorsTimeoutAggregateQC
+ retHeader.ValidatorsTimeoutAggregateQC = &TimeoutAggregateQuorumCertificate{}
+ if err = retHeader.ValidatorsTimeoutAggregateQC.FromBytes(rr); err != nil {
+ return nil, errors.Wrapf(err, "MsgDeSoHeader.FromBytes: Problem decoding ValidatorsTimeoutAggregateQC")
+ }
+
+ return retHeader, nil
+}
+
 func DecodeHeader(rr io.Reader) (*MsgDeSoHeader, error) {
  // Read the version to determine
  scratchBytes := [4]byte{}
@@ -2165,15 +2296,16 @@ func DecodeHeader(rr io.Reader) (*MsgDeSoHeader, error) {
  ret, err = DecodeHeaderVersion0(rr)
  } else if headerVersion == HeaderVersion1 {
  ret, err = DecodeHeaderVersion1(rr)
+ } else if headerVersion == HeaderVersion2 {
+ ret, err = DecodeHeaderVersion2(rr)
  } else {
- // If we have an unrecognized version then we default to de-serializing with
- // version 0. This is necessary because there are places where we use a
- // MsgDeSoHeader struct to store Bitcoin headers.
- ret, err = DecodeHeaderVersion0(rr)
+ // If we have an unrecognized version then we return an error. The schema
+ // differences between header versions 0, 1, 2, and beyond will be large
+ // enough that no one decoder is a safe fallback.
+ err = fmt.Errorf("DecodeHeader: Unrecognized header version: %v", headerVersion)
  }
  if err != nil {
- return nil, fmt.Errorf(
- "DecodeHeader: Unrecognized header version: %v", headerVersion)
+ return nil, errors.Wrapf(err, "DecodeHeader: Error parsing header:")
  }
  // Set the version since it's not decoded in the version-specific handlers.
  ret.Version = headerVersion