Merge Signer-indices to master

cmd/bootstrap/cmd/clusters.go

@@ -5,6 +5,8 @@ import (
 	model "github.com/onflow/flow-go/model/bootstrap"
 	"github.com/onflow/flow-go/model/cluster"
 	"github.com/onflow/flow-go/model/flow"
+	"github.com/onflow/flow-go/model/flow/assignment"
+	"github.com/onflow/flow-go/model/flow/factory"
 	"github.com/onflow/flow-go/model/flow/filter"
 )
 
@@ -23,20 +25,22 @@ func constructClusterAssignment(partnerNodes, internalNodes []model.NodeInfo, se
 	internals = internals.DeterministicShuffle(seed)
 
 	nClusters := flagCollectionClusters
-	assignments := make(flow.AssignmentList, nClusters)
+	identifierLists := make([]flow.IdentifierList, nClusters)
 
 	// first, round-robin internal nodes into each cluster
 	for i, node := range internals {
-		assignments[i%len(assignments)] = append(assignments[i%len(assignments)], node.NodeID)
+		identifierLists[i%len(identifierLists)] = append(identifierLists[i%len(identifierLists)], node.NodeID)
 	}
 
 	// next, round-robin partner nodes into each cluster
 	for i, node := range partners {
-		assignments[i%len(assignments)] = append(assignments[i%len(assignments)], node.NodeID)
+		identifierLists[i%len(identifierLists)] = append(identifierLists[i%len(identifierLists)], node.NodeID)
 	}
 
+	assignments := assignment.FromIdentifierLists(identifierLists)
+
 	collectors := append(partners, internals...)
-	clusters, err := flow.NewClusterList(assignments, collectors)
+	clusters, err := factory.NewClusterList(assignments, collectors)
 	if err != nil {
 		log.Fatal().Err(err).Msg("could not create cluster list")
 	}

cmd/bootstrap/cmd/seal.go

@@ -7,6 +7,7 @@ import (
 	"github.com/onflow/flow-go/model/dkg"
 	"github.com/onflow/flow-go/model/flow"
 	"github.com/onflow/flow-go/model/flow/order"
+	"github.com/onflow/flow-go/module/signature"
 )
 
 func constructRootResultAndSeal(
@@ -43,9 +44,24 @@ func constructRootResultAndSeal(
 		RandomSource:       flagBootstrapRandomSeed,
 	}
 
+	qcsWithSignerIDs := make([]*flow.QuorumCertificateWithSignerIDs, 0, len(clusterQCs))
+	for i, clusterQC := range clusterQCs {
+		members := assignments[i]
+		signerIDs, err := signature.DecodeSignerIndicesToIdentifiers(members, clusterQC.SignerIndices)
+		if err != nil {
+			log.Fatal().Err(err).Msgf("could not decode signer IDs from clusterQC at index %v", i)
+		}
+		qcsWithSignerIDs = append(qcsWithSignerIDs, &flow.QuorumCertificateWithSignerIDs{
+			View:      clusterQC.View,
+			BlockID:   clusterQC.BlockID,
+			SignerIDs: signerIDs,
+			SigData:   clusterQC.SigData,
+		})
+	}
+
 	epochCommit := &flow.EpochCommit{
 		Counter:            flagEpochCounter,
-		ClusterQCs:         flow.ClusterQCVoteDatasFromQCs(clusterQCs),
+		ClusterQCs:         flow.ClusterQCVoteDatasFromQCs(qcsWithSignerIDs),
 		DKGGroupKey:        dkgData.PubGroupKey,
 		DKGParticipantKeys: dkgData.PubKeyShares,
 	}

cmd/bootstrap/run/block.go

@@ -19,7 +19,7 @@ func GenerateRootBlock(chainID flow.ChainID, parentID flow.Identifier, height ui
 		PayloadHash:        payload.Hash(),
 		Timestamp:          timestamp,
 		View:               0,
-		ParentVoterIDs:     nil,
+		ParentVoterIndices: nil,
 		ParentVoterSigData: nil,
 		ProposerID:         flow.ZeroID,
 		ProposerSigData:    nil,

cmd/bootstrap/run/cluster_qc.go

@@ -15,6 +15,7 @@ import (
 	"github.com/onflow/flow-go/model/bootstrap"
 	"github.com/onflow/flow-go/model/cluster"
 	"github.com/onflow/flow-go/model/flow"
+	"github.com/onflow/flow-go/model/flow/order"
 	"github.com/onflow/flow-go/module/local"
 )
 
@@ -29,7 +30,8 @@ func GenerateClusterRootQC(signers []bootstrap.NodeInfo, allCommitteeMembers flo
 	}
 
 	// STEP 2: create VoteProcessor
-	committee, err := committees.NewStaticCommittee(allCommitteeMembers, flow.Identifier{}, nil, nil)
+	ordered := allCommitteeMembers.Sort(order.Canonical)
+	committee, err := committees.NewStaticCommittee(ordered, flow.Identifier{}, nil, nil)
 	if err != nil {
 		return nil, err
 	}

cmd/bootstrap/run/qc_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/onflow/flow-go/crypto"
 	"github.com/onflow/flow-go/model/bootstrap"
 	"github.com/onflow/flow-go/model/flow"
+	"github.com/onflow/flow-go/model/flow/order"
 	"github.com/onflow/flow-go/module/signature"
 	"github.com/onflow/flow-go/utils/unittest"
 )
@@ -32,7 +33,7 @@ func TestGenerateRootQC(t *testing.T) {
 }
 
 func createSignerData(t *testing.T, n int) *ParticipantData {
-	identities := unittest.IdentityListFixture(n)
+	identities := unittest.IdentityListFixture(n).Sort(order.Canonical)
 
 	networkingKeys := unittest.NetworkingKeys(n)
 	stakingKeys := unittest.StakingKeys(n)

cmd/util/cmd/epochs/cmd/reset_test.go

@@ -30,7 +30,7 @@ func TestReset_LocalSnapshot(t *testing.T) {
 		unittest.RunWithTempDir(t, func(bootDir string) {
 
 			// create a root snapshot
-			rootSnapshot := unittest.RootSnapshotFixture(unittest.IdentityListFixture(10))
+			rootSnapshot := unittest.RootSnapshotFixture(unittest.IdentityListFixture(10, unittest.WithAllRoles()))
 
 			// write snapshot to correct path in bootDir
 			err := writeRootSnapshot(bootDir, rootSnapshot)
@@ -62,7 +62,7 @@ func TestReset_LocalSnapshot(t *testing.T) {
 		unittest.RunWithTempDir(t, func(bootDir string) {
 
 			// create a root snapshot
-			rootSnapshot := unittest.RootSnapshotFixture(unittest.IdentityListFixture(10))
+			rootSnapshot := unittest.RootSnapshotFixture(unittest.IdentityListFixture(10, unittest.WithAllRoles()))
 
 			// write snapshot to correct path in bootDir
 			err := writeRootSnapshot(bootDir, rootSnapshot)

cmd/util/cmd/exec-data-json-export/block_exporter.go

@@ -16,12 +16,12 @@ import (
 )
 
 type blockSummary struct {
-	BlockHeight    uint64   `json:"block_height"`
-	BlockID        string   `json:"block_id"`
-	ParentBlockID  string   `json:"parent_block_id"`
-	ParentVoterIDs []string `json:"parent_voter_ids"`
-	// ParentVoterSigData []string  `json:"parent_voter_sig"`
-	ProposerID string `json:"proposer_id"`
+	BlockHeight        uint64 `json:"block_height"`
+	BlockID            string `json:"block_id"`
+	ParentBlockID      string `json:"parent_block_id"`
+	ParentVoterIndices string `json:"parent_voter_indices"`
+	ParentVoterSigData string `json:"parent_voter_sig"`
+	ProposerID         string `json:"proposer_id"`
 	// ProposerSigData    string  `json:"proposer_sig"`
 	Timestamp         time.Time `json:"timestamp"`
 	CollectionIDs     []string  `json:"collection_ids"`
@@ -87,22 +87,18 @@ func ExportBlocks(blockID flow.Identifier, dbPath string, outputPath string) (fl
 			sealsStates = append(sealsStates, hex.EncodeToString(s.FinalState[:]))
 		}
 
-		pvIDs := make([]string, 0)
-		for _, i := range header.ParentVoterIDs {
-			pvIDs = append(pvIDs, hex.EncodeToString(i[:]))
-		}
-
 		b := blockSummary{
-			BlockID:           hex.EncodeToString(activeBlockID[:]),
-			BlockHeight:       header.Height,
-			ParentBlockID:     hex.EncodeToString(header.ParentID[:]),
-			ParentVoterIDs:    pvIDs,
-			ProposerID:        hex.EncodeToString(header.ProposerID[:]),
-			Timestamp:         header.Timestamp,
-			CollectionIDs:     cols,
-			SealedBlocks:      seals,
-			SealedResults:     sealsResults,
-			SealedFinalStates: sealsStates,
+			BlockID:            hex.EncodeToString(activeBlockID[:]),
+			BlockHeight:        header.Height,
+			ParentBlockID:      hex.EncodeToString(header.ParentID[:]),
+			ParentVoterIndices: hex.EncodeToString(header.ParentVoterIndices),
+			ParentVoterSigData: hex.EncodeToString(header.ParentVoterSigData),
+			ProposerID:         hex.EncodeToString(header.ProposerID[:]),
+			Timestamp:          header.Timestamp,
+			CollectionIDs:      cols,
+			SealedBlocks:       seals,
+			SealedResults:      sealsResults,
+			SealedFinalStates:  sealsStates,
 		}
 
 		jsonData, err := json.Marshal(b)

consensus/follower_test.go

@@ -5,6 +5,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/onflow/flow-go/module/signature"
+
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
@@ -68,9 +70,9 @@ func (s *HotStuffFollowerSuite) SetupTest() {
 
 	// mock consensus committee
 	s.committee = &mockhotstuff.Committee{}
-	s.committee.On("Identities", mock.Anything, mock.Anything).Return(
-		func(blockID flow.Identifier, selector flow.IdentityFilter) flow.IdentityList {
-			return identities.Filter(selector)
+	s.committee.On("Identities", mock.Anything).Return(
+		func(blockID flow.Identifier) flow.IdentityList {
+			return identities
 		},
 		nil,
 	)
@@ -105,10 +107,13 @@ func (s *HotStuffFollowerSuite) SetupTest() {
 		Height:    21053,
 		View:      52078,
 	}
+
+	signerIndices, err := signature.EncodeSignersToIndices(identities.NodeIDs(), identities.NodeIDs()[:3])
+	require.NoError(s.T(), err)
 	s.rootQC = &flow.QuorumCertificate{
-		View:      s.rootHeader.View,
-		BlockID:   s.rootHeader.ID(),
-		SignerIDs: identities.NodeIDs()[:3],
+		View:          s.rootHeader.View,
+		BlockID:       s.rootHeader.ID(),
+		SignerIndices: signerIndices,
 	}
 
 	// we start with the latest finalized block being the root block
@@ -335,6 +340,7 @@ func (mc *MockConsensus) extendBlock(blockView uint64, parent *flow.Header) *flo
 	nextBlock := unittest.BlockHeaderWithParentFixture(parent)
 	nextBlock.View = blockView
 	nextBlock.ProposerID = mc.identities[int(blockView)%len(mc.identities)].NodeID
-	nextBlock.ParentVoterIDs = mc.identities.NodeIDs()
+	signerIndices, _ := signature.EncodeSignersToIndices(mc.identities.NodeIDs(), mc.identities.NodeIDs())
+	nextBlock.ParentVoterIndices = signerIndices
 	return &nextBlock
 }

consensus/hotstuff/blockproducer/block_producer.go

@@ -33,7 +33,7 @@ func (bp *BlockProducer) MakeBlockProposal(qc *flow.QuorumCertificate, view uint
 	// in hotstuff, we use this for view number and signature-related fields
 	setHotstuffFields := func(header *flow.Header) error {
 		header.View = view
-		header.ParentVoterIDs = qc.SignerIDs
+		header.ParentVoterIndices = qc.SignerIndices
 		header.ParentVoterSigData = qc.SigData
 		header.ProposerID = bp.committee.Self()
 

consensus/hotstuff/committee.go

@@ -6,7 +6,7 @@ import (
 )
 
 // Committee accounts for the fact that we might have multiple HotStuff instances
-// (collector committees and main consensus committee). Each hostuff instance is supposed to
+// (collector committees and main consensus committee). Each HotStuff instance is supposed to
 // have a dedicated Committee state.
 // A Committee provides subset of the protocol.State, which is restricted to exactly those
 // nodes that participate in the current HotStuff instance: the state of all legitimate HotStuff
@@ -17,14 +17,12 @@ import (
 // Given a collector block, some logic is required to find the main consensus block
 // for determining the valid collector-HotStuff participants.
 type Committee interface {
-
 	// Identities returns a IdentityList with legitimate HotStuff participants for the specified block.
-	// The list of participants is filtered by the provided selector. The returned list of HotStuff participants
+	// The returned list of HotStuff participants
 	//   * contains nodes that are allowed to sign the specified block (legitimate participants with NON-ZERO WEIGHT)
 	//   * is ordered in the canonical order
 	//   * contains no duplicates.
-	// The list of all legitimate HotStuff participants for the specified block can be obtained by using `filter.Any`
-	Identities(blockID flow.Identifier, selector flow.IdentityFilter) (flow.IdentityList, error)
+	Identities(blockID flow.Identifier) (flow.IdentityList, error)
 
 	// Identity returns the full Identity for specified HotStuff participant.
 	// The node must be a legitimate HotStuff participant with NON-ZERO WEIGHT at the specified block.

consensus/hotstuff/committees/cluster_committee.go

@@ -61,8 +61,12 @@ func NewClusterCommittee(
 	return com, nil
 }
 
-func (c *Cluster) Identities(blockID flow.Identifier, selector flow.IdentityFilter) (flow.IdentityList, error) {
-
+// Identities returns the identities of all cluster members that are authorized to
+// participate at the given block. The order of the identities is the canonical order.
+func (c *Cluster) Identities(blockID flow.Identifier) (flow.IdentityList, error) {
+	// blockID is a collection block not a block produced by consensus,
+	// to query the identities from protocol state, we need to use the reference block id from the payload
+	//
 	// first retrieve the cluster block payload
 	payload, err := c.payloads.ByBlockID(blockID)
 	if err != nil {
@@ -74,14 +78,12 @@ func (c *Cluster) Identities(blockID flow.Identifier, selector flow.IdentityFilt
 
 	// use the initial cluster members for root block
 	if isRootBlock {
-		return c.initialClusterMembers.Filter(selector), nil
+		return c.initialClusterMembers, nil
 	}
 
 	// otherwise use the snapshot given by the reference block
-	identities, err := c.state.AtBlockID(payload.ReferenceBlockID).Identities(filter.And(
-		selector,
-		c.clusterMemberFilter,
-	))
+	identities, err := c.state.AtBlockID(payload.ReferenceBlockID).Identities(c.clusterMemberFilter) // remove ejected nodes
+
 	return identities, err
 }
 

consensus/hotstuff/committees/consensus_committee.go

@@ -68,11 +68,10 @@ func NewConsensusCommittee(state protocol.State, me flow.Identifier) (*Consensus
 	return com, nil
 }
 
-func (c *Consensus) Identities(blockID flow.Identifier, selector flow.IdentityFilter) (flow.IdentityList, error) {
-	il, err := c.state.AtBlockID(blockID).Identities(filter.And(
-		filter.IsVotingConsensusCommitteeMember,
-		selector,
-	))
+// Identities returns the identities of all authorized consensus participants at the given block.
+// The order of the identities is the canonical order.
+func (c *Consensus) Identities(blockID flow.Identifier) (flow.IdentityList, error) {
+	il, err := c.state.AtBlockID(blockID).Identities(filter.IsVotingConsensusCommitteeMember)
 	return il, err
 }