From 8f28c20374cdc5f0fef246bccd79568aa8ffcab1 Mon Sep 17 00:00:00 2001 From: JimboJ <40345116+jimjbrettj@users.noreply.github.com> Date: Wed, 13 Dec 2023 11:05:02 -0700 Subject: [PATCH] client(consensus/grandpa): implement finality proof logic (#3589) Co-authored-by: Timothy Wu --- client/api/backend.go | 26 - client/consensus/grandpa/authorities.go | 75 +-- client/consensus/grandpa/authorities_test.go | 22 +- client/consensus/grandpa/aux_schema.go | 64 +-- client/consensus/grandpa/aux_schema_test.go | 35 +- client/consensus/grandpa/finality_proof.go | 246 +++++++++ .../consensus/grandpa/finality_proof_test.go | 501 ++++++++++++++++++ client/consensus/grandpa/helpers_test.go | 50 ++ client/consensus/grandpa/interfaces.go | 127 ++++- client/consensus/grandpa/justification.go | 98 +++- .../consensus/grandpa/justification_test.go | 110 ++-- client/consensus/grandpa/lib.go | 3 + .../consensus/grandpa/mocks_backend_test.go | 173 ++++++ .../grandpa/mocks_blockchainBackend_test.go | 288 ++++++++++ .../consensus/grandpa/mocks_generate_test.go | 8 + .../grandpa/mocks_headerBackend_test.go | 234 ++++++++ 16 files changed, 1813 insertions(+), 247 deletions(-) delete mode 100644 client/api/backend.go create mode 100644 client/consensus/grandpa/finality_proof.go create mode 100644 client/consensus/grandpa/finality_proof_test.go create mode 100644 client/consensus/grandpa/helpers_test.go create mode 100644 client/consensus/grandpa/mocks_backend_test.go create mode 100644 client/consensus/grandpa/mocks_blockchainBackend_test.go create mode 100644 client/consensus/grandpa/mocks_generate_test.go create mode 100644 client/consensus/grandpa/mocks_headerBackend_test.go diff --git a/client/api/backend.go b/client/api/backend.go deleted file mode 100644 index 97b1eccc07..0000000000 --- a/client/api/backend.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2023 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package api - -type Key []byte - -type KeyValue struct { - Key Key - Value []byte -} - -// AuxStore is part of the substrate backend. -// Provides access to an auxiliary database. -// -// This is a simple global database not aware of forks. Can be used for storing auxiliary -// information like total block weight/difficulty for fork resolution purposes as a common use -// case. -type AuxStore interface { - // Insert auxiliary data into key-Value store. - // - // Deletions occur after insertions. - Insert(insert []KeyValue, delete []Key) error - // Get Query auxiliary data from key-Value store. - Get(key Key) (*[]byte, error) -} diff --git a/client/consensus/grandpa/authorities.go b/client/consensus/grandpa/authorities.go index 2b5462685f..77ccf4d881 100644 --- a/client/consensus/grandpa/authorities.go +++ b/client/consensus/grandpa/authorities.go @@ -8,7 +8,6 @@ import ( "fmt" "sync" - "github.com/ChainSafe/gossamer/pkg/scale" "golang.org/x/exp/constraints" "golang.org/x/exp/slices" ) @@ -705,64 +704,33 @@ func (asc *AuthoritySetChanges[N]) append(setID uint64, blockNumber N) { }) } -type authoritySetChangeID scale.VaryingDataType +type authoritySetChangeID any -// Set will set a VaryingDataTypeValue using the underlying VaryingDataType -func (asc *authoritySetChangeID) Set(val scale.VaryingDataTypeValue) (err error) { - vdt := scale.VaryingDataType(*asc) - err = vdt.Set(val) - if err != nil { - return - } - *asc = authoritySetChangeID(vdt) - return +type authoritySetChangeIDs[N constraints.Unsigned] interface { + authoritySetChangeIDLatest | authoritySetChangeIDSet[N] | authoritySetChangeIDUnknown } -// Value will return value from underying VaryingDataType -func (asc *authoritySetChangeID) Value() (val scale.VaryingDataTypeValue, err error) { - vdt := scale.VaryingDataType(*asc) - return vdt.Value() +func newAuthoritySetID[N constraints.Unsigned, ID authoritySetChangeIDs[N]](authSetChangeID ID) authoritySetChangeID { + return authoritySetChangeID(authSetChangeID) } -func newAuthoritySetChangeID[N constraints.Unsigned]() authoritySetChangeID { - vdt := scale.MustNewVaryingDataType(latest{}, set[N]{}, unknown{}) - return authoritySetChangeID(vdt) -} - -type latest struct{} - -func (latest) Index() uint { - return 0 -} +type authoritySetChangeIDLatest struct{} -type set[N constraints.Unsigned] struct { +type authoritySetChangeIDSet[N constraints.Unsigned] struct { inner setIDNumber[N] } -func (set[N]) Index() uint { - return 1 -} - -type unknown struct{} - -func (unknown) Index() uint { - return 2 -} +type authoritySetChangeIDUnknown struct{} // Three states that can be returned: Latest, Set (tuple), Unknown -func (asc *AuthoritySetChanges[N]) getSetID(blockNumber N) (authSetChangeID authoritySetChangeID, err error) { +func (asc *AuthoritySetChanges[N]) getSetID(blockNumber N) (authoritySetChangeID, error) { if asc == nil { - return authSetChangeID, fmt.Errorf("getSetID: authSetChanges is nil") + return nil, fmt.Errorf("getSetID: authSetChanges is nil") } - authSetChangeID = newAuthoritySetChangeID[N]() authSet := *asc last := authSet[len(authSet)-1] if last.BlockNumber < blockNumber { - err = authSetChangeID.Set(latest{}) - if err != nil { - return authSetChangeID, err - } - return authSetChangeID, nil + return newAuthoritySetID[N](authoritySetChangeIDLatest{}), nil } idx, _ := slices.BinarySearchFunc( @@ -786,26 +754,15 @@ func (asc *AuthoritySetChanges[N]) getSetID(blockNumber N) (authSetChangeID auth // if this is the first index but not the first set id then we are missing data. if idx == 0 && authChange.SetID != 0 { - err = authSetChangeID.Set(unknown{}) - if err != nil { - return authSetChangeID, err - } - return authSetChangeID, nil + return newAuthoritySetID[N](authoritySetChangeIDUnknown{}), nil } - err = authSetChangeID.Set(set[N]{ + + return newAuthoritySetID[N](authoritySetChangeIDSet[N]{ authChange, - }) - if err != nil { - return authSetChangeID, err - } - return authSetChangeID, nil + }), nil } - err = authSetChangeID.Set(unknown{}) - if err != nil { - return authSetChangeID, err - } - return authSetChangeID, nil + return newAuthoritySetID[N](authoritySetChangeIDUnknown{}), nil } func (asc *AuthoritySetChanges[N]) insert(blockNumber N) { diff --git a/client/consensus/grandpa/authorities_test.go b/client/consensus/grandpa/authorities_test.go index 6b786d4e87..db90e54d61 100644 --- a/client/consensus/grandpa/authorities_test.go +++ b/client/consensus/grandpa/authorities_test.go @@ -3,7 +3,6 @@ package grandpa import ( - "fmt" "strings" "testing" @@ -1205,24 +1204,19 @@ func TestCleanUpStaleForcedChangesWhenApplyingStandardChangeAlternateCase(t *tes func assertExpectedSet(t *testing.T, authSetID authoritySetChangeID, expected setIDNumber[uint]) { t.Helper() - authSetVal, err := authSetID.Value() - require.NoError(t, err) - switch val := authSetVal.(type) { - case set[uint]: + switch val := authSetID.(type) { + case authoritySetChangeIDSet[uint]: require.Equal(t, expected, val.inner) default: - err = fmt.Errorf("invalid authSetID type") + t.FailNow() } - require.NoError(t, err) } func assertUnknown(t *testing.T, authSetID authoritySetChangeID) { t.Helper() - authSetVal, err := authSetID.Value() - require.NoError(t, err) isUnknown := false - switch authSetVal.(type) { - case unknown: + switch authSetID.(type) { + case authoritySetChangeIDUnknown: isUnknown = true } require.True(t, isUnknown) @@ -1230,11 +1224,9 @@ func assertUnknown(t *testing.T, authSetID authoritySetChangeID) { func assertLatest(t *testing.T, authSetID authoritySetChangeID) { t.Helper() - authSetVal, err := authSetID.Value() - require.NoError(t, err) isLatest := false - switch authSetVal.(type) { - case latest: + switch authSetID.(type) { + case authoritySetChangeIDLatest: isLatest = true } require.True(t, isLatest) diff --git a/client/consensus/grandpa/aux_schema.go b/client/consensus/grandpa/aux_schema.go index eedde8a3d0..e070758922 100644 --- a/client/consensus/grandpa/aux_schema.go +++ b/client/consensus/grandpa/aux_schema.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" - "github.com/ChainSafe/gossamer/client/api" grandpa "github.com/ChainSafe/gossamer/pkg/finality-grandpa" "github.com/ChainSafe/gossamer/pkg/scale" "golang.org/x/exp/constraints" @@ -22,7 +21,7 @@ var ( errValueNotFound = errors.New("value not found") ) -type writeAux func(insertions []api.KeyValue) error +type writeAux func(insertions []KeyValue) error type getGenesisAuthorities[ID AuthorityID] func() ([]Authority[ID], error) @@ -31,7 +30,7 @@ type persistentData[H comparable, N constraints.Unsigned, ID AuthorityID, Sig Au setState SharedVoterSetState[H, N, ID, Sig] } -func loadDecoded(store api.AuxStore, key []byte, destination any) error { +func loadDecoded(store AuxStore, key []byte, destination any) error { encodedValue, err := store.Get(key) if err != nil { return err @@ -50,7 +49,7 @@ func loadDecoded(store api.AuxStore, key []byte, destination any) error { } func loadPersistent[H comparable, N constraints.Unsigned, ID AuthorityID, Sig AuthoritySignature]( - store api.AuxStore, + store AuxStore, genesisHash H, genesisNumber N, genesisAuths getGenesisAuthorities[ID]) (*persistentData[H, N, ID, Sig], error) { @@ -85,13 +84,11 @@ func loadPersistent[H comparable, N constraints.Unsigned, ID AuthorityID, Sig Au } } - newSharedVoterSetState := sharedVoterSetState[H, N, ID, Sig]{ - Inner: setState, - } - return &persistentData[H, N, ID, Sig]{ authoritySet: SharedAuthoritySet[H, N, ID]{inner: *authSet}, - setState: SharedVoterSetState[H, N, ID, Sig]{Inner: newSharedVoterSetState}, //nolint + setState: SharedVoterSetState[H, N, ID, Sig]{Inner: sharedVoterSetState[H, N, ID, Sig]{ + Inner: setState, + }}, }, nil } @@ -116,9 +113,9 @@ func loadPersistent[H comparable, N constraints.Unsigned, ID AuthorityID, Sig Au return nil, err } - insert := []api.KeyValue{ - {authoritySetKey, scale.MustMarshal(*genesisSet)}, //nolint - {setStateKey, scale.MustMarshal(genesisState)}, //nolint + insert := []KeyValue{ + {authoritySetKey, scale.MustMarshal(*genesisSet)}, + {setStateKey, scale.MustMarshal(genesisState)}, } err = store.Insert(insert, nil) @@ -126,13 +123,11 @@ func loadPersistent[H comparable, N constraints.Unsigned, ID AuthorityID, Sig Au return nil, err } - newSharedVoterSetState := sharedVoterSetState[H, N, ID, Sig]{ - Inner: genesisState, - } - return &persistentData[H, N, ID, Sig]{ authoritySet: SharedAuthoritySet[H, N, ID]{inner: *genesisSet}, - setState: SharedVoterSetState[H, N, ID, Sig]{Inner: newSharedVoterSetState}, //nolint + setState: SharedVoterSetState[H, N, ID, Sig]{Inner: sharedVoterSetState[H, N, ID, Sig]{ + Inner: genesisState, + }}, }, nil } @@ -145,7 +140,6 @@ func UpdateAuthoritySet[H comparable, N constraints.Unsigned, ID AuthorityID, Si set AuthoritySet[H, N, ID], newSet *NewAuthoritySetStruct[H, N, ID], write writeAux) error { - // TODO make sure that Insert has affect of both insert and update depending on use case encodedAuthSet, err := scale.Marshal(set) if err != nil { return err @@ -169,9 +163,9 @@ func UpdateAuthoritySet[H comparable, N constraints.Unsigned, ID AuthorityID, Si return err } - insert := []api.KeyValue{ - {authoritySetKey, encodedAuthSet}, //nolint - {setStateKey, encodedVoterSet}, //nolint + insert := []KeyValue{ + {authoritySetKey, encodedAuthSet}, + {setStateKey, encodedVoterSet}, } err = write(insert) if err != nil { @@ -179,8 +173,8 @@ func UpdateAuthoritySet[H comparable, N constraints.Unsigned, ID AuthorityID, Si } } else { - insert := []api.KeyValue{ - {authoritySetKey, encodedAuthSet}, //nolint + insert := []KeyValue{ + {authoritySetKey, encodedAuthSet}, } err = write(insert) @@ -201,16 +195,16 @@ func updateBestJustification[ N constraints.Unsigned, S comparable, ID AuthorityID, - H Header[Hash, N]]( - justification Justification[Hash, N, S, ID, H], +]( + justification GrandpaJustification[Hash, N, S, ID], write writeAux) error { encodedJustificaiton, err := scale.Marshal(justification) if err != nil { return fmt.Errorf("marshalling: %w", err) } - insert := []api.KeyValue{ - {bestJustification, encodedJustificaiton}, //nolint + insert := []KeyValue{ + {bestJustification, encodedJustificaiton}, } err = write(insert) if err != nil { @@ -225,15 +219,15 @@ func BestJustification[ N constraints.Unsigned, S comparable, ID AuthorityID, - H Header[Hash, N]]( - store api.AuxStore) (*Justification[Hash, N, S, ID, H], error) { - justification := Justification[Hash, N, S, ID, H]{} + H Header[Hash, N], +](store AuxStore) (*GrandpaJustification[Hash, N, S, ID], error) { + justification := decodeGrandpaJustification[Hash, N, S, ID, H]{} err := loadDecoded(store, bestJustification, &justification) if err != nil { return nil, err } - return &justification, nil + return justification.GrandpaJustification(), nil } // WriteVoterSetState Write voter set state. @@ -244,8 +238,8 @@ func WriteVoterSetState[H comparable, N constraints.Unsigned, ID AuthorityID, Si if err != nil { return err } - insert := []api.KeyValue{ - {setStateKey, encodedVoterSet}, //nolint + insert := []KeyValue{ + {setStateKey, encodedVoterSet}, } err = write(insert) if err != nil { @@ -271,8 +265,8 @@ func WriteConcludedRound[H comparable, N constraints.Unsigned, ID AuthorityID, S return err } - insert := []api.KeyValue{ - {key, encRoundData}, //nolint + insert := []KeyValue{ + {key, encRoundData}, } err = write(insert) if err != nil { diff --git a/client/consensus/grandpa/aux_schema_test.go b/client/consensus/grandpa/aux_schema_test.go index 70a3434139..741552fda9 100644 --- a/client/consensus/grandpa/aux_schema_test.go +++ b/client/consensus/grandpa/aux_schema_test.go @@ -6,7 +6,6 @@ package grandpa import ( "testing" - "github.com/ChainSafe/gossamer/client/api" finalityGrandpa "github.com/ChainSafe/gossamer/pkg/finality-grandpa" "github.com/ChainSafe/gossamer/pkg/scale" "github.com/stretchr/testify/require" @@ -17,15 +16,15 @@ func genesisAuthorities[ID AuthorityID](auths []Authority[ID], err error) getGen return func() ([]Authority[ID], error) { return auths, err } } -func write(store api.AuxStore) writeAux { - return func(insertions []api.KeyValue) error { +func write(store AuxStore) writeAux { + return func(insertions []KeyValue) error { return store.Insert(insertions, nil) } } -type dummyStore []api.KeyValue +type dummyStore []KeyValue -func (client *dummyStore) Insert(insert []api.KeyValue, deleted []api.Key) error { +func (client *dummyStore) Insert(insert []KeyValue, deleted []Key) error { for _, val := range insert { *client = append(*client, val) } @@ -48,7 +47,7 @@ func (client *dummyStore) Insert(insert []api.KeyValue, deleted []api.Key) error } -func (client *dummyStore) Get(key api.Key) (*[]byte, error) { +func (client *dummyStore) Get(key Key) (*[]byte, error) { for _, value := range *client { if slices.Equal(value.Key, key) { return &value.Value, nil @@ -64,15 +63,15 @@ func newDummyStore(t *testing.T) *dummyStore { func TestDummyStore(t *testing.T) { store := newDummyStore(t) - insert := []api.KeyValue{ - {authoritySetKey, scale.MustMarshal([]byte{1})}, //nolint - {setStateKey, scale.MustMarshal([]byte{2})}, //nolint + insert := []KeyValue{ + {authoritySetKey, scale.MustMarshal([]byte{1})}, + {setStateKey, scale.MustMarshal([]byte{2})}, } err := store.Insert(insert, nil) require.NoError(t, err) require.True(t, len(*store) == 2) - del := []api.Key{setStateKey} + del := []Key{setStateKey} err = store.Insert(nil, del) require.NoError(t, err) require.True(t, len(*store) == 1) @@ -152,9 +151,9 @@ func TestLoadPersistentNotGenesis(t *testing.T) { genesisState, err := NewLiveVoterSetState[string, uint, dummyAuthID, uint](0, *genesisSet, *base) require.NoError(t, err) - insert := []api.KeyValue{ - {authoritySetKey, scale.MustMarshal(*genesisSet)}, //nolint - {setStateKey, scale.MustMarshal(genesisState)}, //nolint + insert := []KeyValue{ + {authoritySetKey, scale.MustMarshal(*genesisSet)}, + {setStateKey, scale.MustMarshal(genesisState)}, } err = store.Insert(insert, nil) @@ -176,8 +175,8 @@ func TestLoadPersistentNotGenesis(t *testing.T) { // Auth set written but not set state store = newDummyStore(t) - insert = []api.KeyValue{ - {authoritySetKey, scale.MustMarshal(*genesisSet)}, //nolint + insert = []KeyValue{ + {authoritySetKey, scale.MustMarshal(*genesisSet)}, } err = store.Insert(insert, nil) @@ -380,13 +379,13 @@ func TestWriteJustification(t *testing.T) { precommit := makePrecommit(t, "a", 1, 1) precommits = append(precommits, precommit) - expAncestries := make([]testHeader[string, uint], 0) + expAncestries := make([]Header[string, uint], 0) expAncestries = append(expAncestries, testHeader[string, uint]{ NumberField: 100, ParentHashField: "a", }) - justification := Justification[string, uint, string, dummyAuthID, testHeader[string, uint]]{ + justification := GrandpaJustification[string, uint, string, dummyAuthID]{ Round: 2, Commit: finalityGrandpa.Commit[string, uint, string, dummyAuthID]{ TargetHash: "a", @@ -399,7 +398,7 @@ func TestWriteJustification(t *testing.T) { _, err := BestJustification[string, uint, string, dummyAuthID, testHeader[string, uint]](store) require.ErrorIs(t, err, errValueNotFound) - err = updateBestJustification[string, uint, string, dummyAuthID, testHeader[string, uint]](justification, write(store)) + err = updateBestJustification[string, uint, string, dummyAuthID](justification, write(store)) require.NoError(t, err) bestJust, err := BestJustification[string, uint, string, dummyAuthID, testHeader[string, uint]](store) diff --git a/client/consensus/grandpa/finality_proof.go b/client/consensus/grandpa/finality_proof.go new file mode 100644 index 0000000000..9d1262c8a3 --- /dev/null +++ b/client/consensus/grandpa/finality_proof.go @@ -0,0 +1,246 @@ +// Copyright 2023 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + +package grandpa + +import ( + "errors" + + "github.com/ChainSafe/gossamer/pkg/scale" + "golang.org/x/exp/constraints" +) + +// GRANDPA block finality proof generation and check. +// +// Finality of block B is proved by providing: +// 1) the justification for the descendant block F; +// 2) headers sub-chain (B; F] if B != F; +// 3) proof of GRANDPA::authorities() if the set changes at block F. +// +// Since earliest possible justification is returned, the GRANDPA authorities set +// at the block F is guaranteed to be the same as in the block B (this is because block +// that enacts new GRANDPA authorities set always comes with justification). It also +// means that the `set_id` is the same at blocks B and F. +// +// Let U be the last finalized block known to caller. If authorities set has changed several +// times in the (U; F] interval, multiple finality proof fragments are returned (one for each +// authority set change) and they must be verified in-order. +// +// Finality proof provider can choose how to provide finality proof on its own. The incomplete +// finality proof (that finalises some block C that is ancestor of the B and descendant +// of the U) could be returned. + +var ( + // The requested block has not yet been finalized + errBlockNotYetFinalized = errors.New("block not yet finalized") + // The requested block is not covered by authority set changes. Likely this means the block is + // in the latest authority set, and the subscription API is more appropriate + errBlockNotInAuthoritySetChanges = errors.New("block not covered by authority set changes") +) + +const maxUnknownHeaders = 100_000 + +// FinalityProofProvider Finality proof provider for serving network requests. +type FinalityProofProvider[ + BE Backend[Hash, N, H, B], + Hash constraints.Ordered, + N constraints.Unsigned, + S comparable, + ID AuthorityID, + H Header[Hash, N], + B BlockchainBackend[Hash, N, H], +] struct { + backend BE + sharedAuthoritySet *SharedAuthoritySet[Hash, N, ID] +} + +// NewFinalityProofProvider Create new finality proof provider using: +// +// - backend for accessing blockchain data; +// - authorityProvider for calling and proving runtime methods. +// - sharedAuthoritySet for accessing authority set data +func NewFinalityProofProvider[ + BE Backend[Hash, N, H, B], + Hash constraints.Ordered, + N constraints.Unsigned, + S comparable, + ID AuthorityID, + H Header[Hash, N], + B BlockchainBackend[Hash, N, H], +]( + backend BE, + sharedAuthSet *SharedAuthoritySet[Hash, N, ID]) *FinalityProofProvider[BE, Hash, N, S, ID, H, B] { + return &FinalityProofProvider[BE, Hash, N, S, ID, H, B]{ + backend: backend, + sharedAuthoritySet: sharedAuthSet, + } +} + +// ProveFinality Prove finality for the given block number by returning a Justification for the last block of +// the authority set in bytes. +func (provider FinalityProofProvider[BE, H, N, S, ID, Header, B]) ProveFinality(block N) (*[]byte, error) { + proof, err := provider.proveFinalityProof(block, true) + if err != nil { + return nil, err + } + + if proof != nil { + encodedProof, err := scale.Marshal(*proof) + if err != nil { + return nil, err + } + return &encodedProof, nil + } + + return nil, nil +} + +// Prove finality for the given block number by returning a Justification for the last block of +// the authority set. +// +// If `collectUnknownHeaders` is true, the finality proof will include all headers from the +// requested block until the block the justification refers to. +func (provider FinalityProofProvider[BE, Hash, N, S, ID, H, B]) proveFinalityProof( + block N, + collectUnknownHeaders bool) (*FinalityProof[Hash, N, H], error) { + if provider.sharedAuthoritySet == nil { + return nil, nil + } + + return proveFinality[BE, Hash, N, S, ID, H, B]( + provider.backend, + provider.sharedAuthoritySet.inner.AuthoritySetChanges, + block, + collectUnknownHeaders, + ) +} + +// FinalityProof Finality for block B is proved by providing: +// 1) the justification for the descendant block F; +// 2) headers sub-chain (B; F] if B != F; +type FinalityProof[Hash constraints.Ordered, N constraints.Unsigned, H Header[Hash, N]] struct { + // The hash of block F for which justification is provided + Block Hash + // Justification of the block F + Justification []byte + // The set of headers in the range (B; F] that we believe are unknown to the caller. Ordered. + UnknownHeaders []H +} + +// Prove finality for the given block number by returning a justification for the last block of +// the authority set of which the given block is part of, or a justification for the latest +// finalized block if the given block is part of the current authority set. +// +// If `collectUnknownHeaders` is true, the finality proof will include all headers from the +// requested block until the block the justification refers to. +func proveFinality[ + BE Backend[Hash, N, H, B], + Hash constraints.Ordered, + N constraints.Unsigned, + S comparable, + ID AuthorityID, + H Header[Hash, N], + B BlockchainBackend[Hash, N, H], +]( + backend BE, + authSetChanges AuthoritySetChanges[N], + block N, + collectUnknownHeaders bool, +) (*FinalityProof[Hash, N, H], error) { + // Early-return if we are sure that there are no blocks finalized that cover the requested + // block. + finalizedNumber := backend.Blockchain().Info().FinalizedNumber + if finalizedNumber < block { + logger.Tracef("requested finality proof for descendant of %v while we only have finalized %v", block, finalizedNumber) + return nil, errBlockNotYetFinalized + } + + authSetChangeID, err := authSetChanges.getSetID(block) + if err != nil { + return nil, err + } + + var encJustification []byte + var justBlock N + + switch val := authSetChangeID.(type) { + case authoritySetChangeIDLatest: + justification, err := BestJustification[Hash, N, S, ID, H](backend) + if err != nil && !errors.Is(err, errValueNotFound) { + return nil, err + } + + if justification != nil { + encJustification, err = scale.Marshal(*justification) + if err != nil { + return nil, err + } + justBlock = justification.Target().number + } else { + logger.Trace("No justification found for the authoritySetChangeIDLatest finalized block. Returning empty proof") + return nil, nil + } + case authoritySetChangeIDSet[N]: + lastBlockForSetID, err := backend.Blockchain().ExpectBlockHashFromID(val.inner.BlockNumber) + if err != nil { + return nil, err + } + + // If error or no justifications found, return empty proof + justifications, err := backend.Blockchain().Justifications(lastBlockForSetID) + if err != nil || justifications == nil { + logger.Tracef("getting justifications when making finality proof for %v. Returning empty proof", + block) + return nil, nil //nolint + } + justification := justifications.IntoJustification(GrandpaEngineID) + if justification != nil { + encJustification = *justification + justBlock = val.inner.BlockNumber + } else { + logger.Tracef("No justification found when making finality proof for %v. Returning empty proof", + block) + return nil, nil + } + case authoritySetChangeIDUnknown: + logger.Tracef("authoritySetChanges does not cover the requested block %v due to missing data."+ + " You need to resync to populate AuthoritySetChanges properly", block) + + return nil, errBlockNotInAuthoritySetChanges + default: + panic("authoritySetChangeIDUnknown type for authSetChangeID") + } + + var headers []H + if collectUnknownHeaders { + // Collect all headers from the requested block until the last block of the set + current := block + 1 + for { + if current > justBlock || len(headers) >= maxUnknownHeaders { + break + } + hash, err := backend.Blockchain().ExpectBlockHashFromID(current) + if err != nil { + return nil, err + } + + header, err := backend.Blockchain().ExpectHeader(hash) + if err != nil { + return nil, err + } + headers = append(headers, header) + current += 1 + } + } + + blockHash, err := backend.Blockchain().ExpectBlockHashFromID(justBlock) + if err != nil { + return nil, err + } + + return &FinalityProof[Hash, N, H]{ + Block: blockHash, + Justification: encJustification, + UnknownHeaders: headers, + }, nil +} diff --git a/client/consensus/grandpa/finality_proof_test.go b/client/consensus/grandpa/finality_proof_test.go new file mode 100644 index 0000000000..3204d234b0 --- /dev/null +++ b/client/consensus/grandpa/finality_proof_test.go @@ -0,0 +1,501 @@ +// Copyright 2023 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + +package grandpa + +import ( + "fmt" + "testing" + + finalityGrandpa "github.com/ChainSafe/gossamer/pkg/finality-grandpa" + "github.com/ChainSafe/gossamer/pkg/scale" + "github.com/stretchr/testify/require" + "golang.org/x/exp/constraints" +) + +// Check GRANDPA proof-of-finality for the given block. +// +// Returns the vector of headers that MUST be validated + imported +// AND if at least one of those headers is invalid, all other MUST be considered invalid. +func checkFinalityProof[ + Hash constraints.Ordered, + N constraints.Unsigned, + S comparable, + H Header[Hash, N], + ID AuthorityID, +]( + currentSetID uint64, + currentAuthorities AuthorityList[ID], + remoteProof []byte, +) (FinalityProof[Hash, N, H], error) { + proof := FinalityProof[Hash, N, H]{} + err := scale.Unmarshal(remoteProof, &proof) + if err != nil { + return FinalityProof[Hash, N, H]{}, fmt.Errorf("failed to decode finality proof %s", err) + } + + justification := GrandpaJustification[Hash, N, S, ID]{} + err = scale.Unmarshal(proof.Justification, &justification) + if err != nil { + return FinalityProof[Hash, N, H]{}, fmt.Errorf("error decoding justification for header %s", err) + } + + err = justification.Verify(currentSetID, currentAuthorities) + if err != nil { + return FinalityProof[Hash, N, H]{}, err + } + + return proof, nil +} + +func createCommit( + t *testing.T, + targetHash string, + targetNum uint, + round uint64, + ID dummyAuthID, +) finalityGrandpa.Commit[string, uint, string, dummyAuthID] { + t.Helper() + precommit := finalityGrandpa.Precommit[string, uint]{ + TargetHash: targetHash, + TargetNumber: targetNum, + } + + message := finalityGrandpa.Message[string, uint]{ + Value: precommit, + } + + msg := messageData[string, uint]{ + round, + 1, + message, + } + + encMsg, err := scale.Marshal(msg) + require.NoError(t, err) + + signedPrecommit := finalityGrandpa.SignedPrecommit[string, uint, string, dummyAuthID]{ + Precommit: precommit, + ID: ID, + Signature: string(encMsg), + } + + commit := finalityGrandpa.Commit[string, uint, string, dummyAuthID]{ + TargetHash: targetHash, + TargetNumber: targetNum, + Precommits: []finalityGrandpa.SignedPrecommit[string, uint, string, dummyAuthID]{signedPrecommit}, + } + + return commit +} + +func TestFinalityProof_FailsIfNoMoreLastFinalizedBlocks(t *testing.T) { + dummyInfo := Info[uint]{ + FinalizedNumber: 4, + } + mockBlockchain := NewBlockchainBackendMock[string, uint, testHeader[string, uint]](t) + mockBlockchain.EXPECT().Info().Return(dummyInfo).Once() + + mockBackend := NewBackendMock[ + string, + uint, + testHeader[string, uint], + *BlockchainBackendMock[string, uint, testHeader[string, uint]]](t) + mockBackend.EXPECT().Blockchain().Return(mockBlockchain).Once() + + // The last finalized block is 4, so we cannot provide further justifications. + authoritySetChanges := AuthoritySetChanges[uint]{} + _, err := proveFinality[ + *BackendMock[string, uint, testHeader[string, uint], + *BlockchainBackendMock[string, uint, testHeader[string, uint]]], + string, + uint, + string, + dummyAuthID, + testHeader[string, uint], + *BlockchainBackendMock[string, uint, testHeader[string, uint]], + ]( + mockBackend, + authoritySetChanges, + 5, + true) + require.ErrorIs(t, err, errBlockNotYetFinalized) +} + +func TestFinalityProof_IsNoneIfNoJustificationKnown(t *testing.T) { + dummyInfo := Info[uint]{ + FinalizedNumber: 4, + } + dummyHash := "dummyHash" + mockBlockchain := NewBlockchainBackendMock[string, uint, testHeader[string, uint]](t) + mockBlockchain.EXPECT().Info().Return(dummyInfo).Once() + mockBlockchain.EXPECT().ExpectBlockHashFromID(uint(4)).Return(dummyHash, nil).Once() + mockBlockchain.EXPECT().Justifications(dummyHash).Return(nil, nil).Once() + + mockBackend := NewBackendMock[string, uint, testHeader[string, uint], + *BlockchainBackendMock[string, uint, testHeader[string, uint]]](t) + mockBackend.EXPECT().Blockchain().Return(mockBlockchain).Times(3) + + authoritySetChanges := AuthoritySetChanges[uint]{} + authoritySetChanges.append(0, 4) + + // Block 4 is finalized without justification + // => we can't prove finality of 3 + proofOf3, err := proveFinality[ + *BackendMock[string, uint, testHeader[string, uint], + *BlockchainBackendMock[string, uint, testHeader[string, uint]]], + string, + uint, + string, + dummyAuthID, + testHeader[string, uint], + *BlockchainBackendMock[string, uint, testHeader[string, uint]], + ]( + mockBackend, + authoritySetChanges, + 3, + true, + ) + require.NoError(t, err) + require.Nil(t, proofOf3) +} + +func TestFinalityProof_CheckFailsWhenProofDecodeFails(t *testing.T) { + // When we can't decode proof from Vec + authorityList := AuthorityList[dummyAuthID]{} + _, err := checkFinalityProof[string, uint, string, testHeader[string, uint], dummyAuthID]( + 1, + authorityList, + []byte{42}, + ) + require.NotNil(t, err) + require.ErrorContains(t, err, "failed to decode finality proof") +} + +func TestFinalityProof_CheckFailsWhenProofIsEmpty(t *testing.T) { + // When decoded proof has zero length + authorityList := AuthorityList[dummyAuthID]{} + grandpaJustification := GrandpaJustification[string, + uint, + string, + dummyAuthID, + ]{} + encJustification, err := scale.Marshal(grandpaJustification) + require.NoError(t, err) + _, err = checkFinalityProof[string, uint, string, testHeader[string, uint], dummyAuthID]( + 1, + authorityList, + encJustification, + ) + require.NotNil(t, err) +} + +func TestFinalityProof_CheckFailsWithIncompleteJustification(t *testing.T) { + authorityList := AuthorityList[dummyAuthID]{ + Authority[dummyAuthID]{ + Key: dummyAuthID(1), + Weight: uint64(1), + }, + } + + // Create a commit without precommits + commit := finalityGrandpa.Commit[string, uint, string, dummyAuthID]{ + TargetHash: "hash7", + TargetNumber: uint(7), + } + + grandpaJust := GrandpaJustification[string, uint, string, dummyAuthID]{ + Round: 8, + Commit: commit, + } + + finalityProof := FinalityProof[string, uint, testHeader[string, uint]]{ + Block: "hash2", + Justification: scale.MustMarshal(grandpaJust), + } + + _, err := checkFinalityProof[string, uint, string, testHeader[string, uint], dummyAuthID]( + 1, + authorityList, + scale.MustMarshal(finalityProof), + ) + require.ErrorIs(t, err, errBadJustification) +} + +func TestFinalityProof_CheckWorksWithCorrectJustification(t *testing.T) { + ID := dummyAuthID(1) + targetHash := "target" + targetNum := uint(21) + authorityList := AuthorityList[dummyAuthID]{ + Authority[dummyAuthID]{ + Key: ID, + Weight: uint64(1), + }, + } + + commit := createCommit(t, targetHash, targetNum, 1, ID) + grandpaJust := GrandpaJustification[string, uint, string, dummyAuthID]{ + Round: 8, + Commit: commit, + } + + finalityProof := FinalityProof[string, uint, testHeader[string, uint]]{ + Block: "hash2", + Justification: scale.MustMarshal(grandpaJust), + } + + newFinalityProof, err := checkFinalityProof[string, uint, string, testHeader[string, uint], dummyAuthID]( + 1, + authorityList, + scale.MustMarshal(finalityProof), + ) + require.NoError(t, err) + require.Equal(t, finalityProof, newFinalityProof) +} + +func TestFinalityProof_UsingAuthoritySetChangesFailsWithUndefinedStart(t *testing.T) { + dummyInfo := Info[uint]{ + FinalizedNumber: 8, + } + mockBlockchain := NewBlockchainBackendMock[string, uint, testHeader[string, uint]](t) + mockBlockchain.EXPECT().Info().Return(dummyInfo).Once() + + mockBackend := NewBackendMock[string, uint, testHeader[string, uint], + *BlockchainBackendMock[string, uint, testHeader[string, uint]]](t) + mockBackend.EXPECT().Blockchain().Return(mockBlockchain).Once() + + // We are missing the block for the preceding set the start is not well-defined. + authoritySetChanges := AuthoritySetChanges[uint]{} + authoritySetChanges.append(1, 8) + + _, err := proveFinality[ + *BackendMock[string, uint, testHeader[string, uint], + *BlockchainBackendMock[string, uint, testHeader[string, uint]]], + string, + uint, + string, + dummyAuthID, + testHeader[string, uint], + *BlockchainBackendMock[string, uint, testHeader[string, uint]], + ]( + mockBackend, + authoritySetChanges, + 6, + true, + ) + require.ErrorIs(t, err, errBlockNotInAuthoritySetChanges) +} + +func TestFinalityProof_UsingAuthoritySetChangesWorks(t *testing.T) { + ID := dummyAuthID(1) + header7 := testHeader[string, uint]{ + NumberField: uint(7), + HashField: "hash7", + } + header8 := testHeader[string, uint]{ + NumberField: uint(8), + HashField: "hash8", + ParentHashField: "hash7", + } + + dummyInfo := Info[uint]{ + FinalizedNumber: 8, + } + + round := uint64(8) + commit := createCommit(t, "hash8", uint(8), round, ID) + grandpaJust := GrandpaJustification[string, uint, string, dummyAuthID]{ + Round: round, + Commit: commit, + } + + encJust, err := scale.Marshal(grandpaJust) + require.NoError(t, err) + + justifications := Justifications{Justification{ + EngineID: GrandpaEngineID, + EncodedJustification: encJust, + }} + + mockBlockchain := NewBlockchainBackendMock[string, uint, testHeader[string, uint]](t) + mockBlockchain.EXPECT().Info().Return(dummyInfo).Once() + mockBlockchain.EXPECT().ExpectBlockHashFromID(uint(7)).Return("hash7", nil).Once() + mockBlockchain.EXPECT().ExpectHeader("hash7").Return(header7, nil).Once() + mockBlockchain.EXPECT().ExpectBlockHashFromID(uint(8)).Return("hash8", nil).Times(3) + mockBlockchain.EXPECT().Justifications("hash8").Return(&justifications, nil).Times(1) + mockBlockchain.EXPECT().ExpectHeader("hash8").Return(header8, nil).Once() + + mockBackend := NewBackendMock[string, uint, testHeader[string, uint], + *BlockchainBackendMock[string, uint, testHeader[string, uint]]](t) + mockBackend.EXPECT().Blockchain().Return(mockBlockchain).Times(8) + + // Authority set change at block 8, so the justification stored there will be used in the + // FinalityProof for block 6 + authoritySetChanges := AuthoritySetChanges[uint]{} + authoritySetChanges.append(0, 5) + authoritySetChanges.append(1, 8) + + proofOf6, err := proveFinality[ + *BackendMock[string, uint, testHeader[string, uint], + *BlockchainBackendMock[string, uint, testHeader[string, uint]]], + string, + uint, + string, + dummyAuthID, + testHeader[string, uint], + *BlockchainBackendMock[string, uint, testHeader[string, uint]], + ]( + mockBackend, + authoritySetChanges, + 6, + true, + ) + require.NoError(t, err) + + unknownHeaders := []testHeader[string, uint]{header7, header8} + expFinalityProof := &FinalityProof[string, uint, testHeader[string, uint]]{ + Block: "hash8", + Justification: encJust, + UnknownHeaders: unknownHeaders, + } + require.Equal(t, expFinalityProof, proofOf6) + + mockBlockchain2 := NewBlockchainBackendMock[string, uint, testHeader[string, uint]](t) + mockBlockchain2.EXPECT().Info().Return(dummyInfo).Once() + mockBlockchain2.EXPECT().ExpectBlockHashFromID(uint(8)).Return("hash8", nil).Times(2) + mockBlockchain2.EXPECT().Justifications("hash8").Return(&justifications, nil).Times(1) + + mockBackend2 := NewBackendMock[string, uint, testHeader[string, uint], + *BlockchainBackendMock[string, uint, testHeader[string, uint]]](t) + mockBackend2.EXPECT().Blockchain().Return(mockBlockchain2).Times(4) + + proofOf6WithoutUnknown, err := proveFinality[ + *BackendMock[string, uint, testHeader[string, uint], + *BlockchainBackendMock[string, uint, testHeader[string, uint]]], + string, + uint, + string, + dummyAuthID, + testHeader[string, uint], + *BlockchainBackendMock[string, uint, testHeader[string, uint]], + ]( + mockBackend2, + authoritySetChanges, + 6, + false, + ) + require.NoError(t, err) + + expFinalityProof = &FinalityProof[string, uint, testHeader[string, uint]]{ + Block: "hash8", + Justification: encJust, + } + require.Equal(t, expFinalityProof, proofOf6WithoutUnknown) +} + +func TestFinalityProof_InLastSetFailsWithoutLatest(t *testing.T) { + dummyInfo := Info[uint]{ + FinalizedNumber: 8, + } + mockBlockchain := NewBlockchainBackendMock[string, uint, testHeader[string, uint]](t) + mockBlockchain.EXPECT().Info().Return(dummyInfo).Once() + + mockBackend := NewBackendMock[string, uint, testHeader[string, uint], + *BlockchainBackendMock[string, uint, testHeader[string, uint]]](t) + mockBackend.EXPECT().Blockchain().Return(mockBlockchain).Times(1) + mockBackend.EXPECT().Get(Key("grandpa_best_justification")).Return(nil, nil).Times(1) + + // No recent authority set change, so we are in the authoritySetChangeIDLatest set, and we will try to pickup + // the best stored justification, for which there is none in this case. + authoritySetChanges := AuthoritySetChanges[uint]{} + authoritySetChanges.append(0, 5) + + proof, err := proveFinality[ + *BackendMock[string, uint, testHeader[string, uint], + *BlockchainBackendMock[string, uint, testHeader[string, uint]]], + string, + uint, + string, + dummyAuthID, + testHeader[string, uint], + *BlockchainBackendMock[string, uint, testHeader[string, uint]], + ]( + mockBackend, + authoritySetChanges, + 6, + true, + ) + // When justification is not stored in db, return nil + require.NoError(t, err) + require.Nil(t, proof) +} + +func TestFinalityProof_InLastSetUsingLatestJustificationWorks(t *testing.T) { + ID := dummyAuthID(1) + header7 := testHeader[string, uint]{ + NumberField: uint(7), + HashField: "hash7", + } + header8 := testHeader[string, uint]{ + NumberField: uint(8), + HashField: "hash8", + ParentHashField: "hash7", + } + + dummyInfo := Info[uint]{ + FinalizedNumber: 8, + } + + round := uint64(8) + commit := createCommit(t, "hash8", uint(8), round, ID) + grandpaJust := GrandpaJustification[string, uint, string, dummyAuthID]{ + Round: round, + Commit: commit, + } + + encJust, err := scale.Marshal(grandpaJust) + require.NoError(t, err) + + mockBlockchain := NewBlockchainBackendMock[string, uint, testHeader[string, uint]](t) + mockBlockchain.EXPECT().Info().Return(dummyInfo).Once() + mockBlockchain.EXPECT().ExpectBlockHashFromID(uint(7)).Return("hash7", nil).Once() + mockBlockchain.EXPECT().ExpectHeader("hash7").Return(header7, nil).Once() + mockBlockchain.EXPECT().ExpectBlockHashFromID(uint(8)).Return("hash8", nil).Times(2) + mockBlockchain.EXPECT().ExpectHeader("hash8").Return(header8, nil).Once() + + mockBackend := NewBackendMock[string, uint, testHeader[string, uint], + *BlockchainBackendMock[string, uint, testHeader[string, uint]]](t) + mockBackend.EXPECT().Blockchain().Return(mockBlockchain).Times(6) + mockBackend.EXPECT().Get(Key("grandpa_best_justification")).Return(&encJust, nil).Times(1) + + // No recent authority set change, so we are in the authoritySetChangeIDLatest set, and will pickup the best + // stored justification (via mock get call) + authoritySetChanges := AuthoritySetChanges[uint]{} + authoritySetChanges.append(0, 5) + + proofOf6, err := proveFinality[ + *BackendMock[string, uint, testHeader[string, uint], + *BlockchainBackendMock[string, uint, testHeader[string, uint]]], + string, + uint, + string, + dummyAuthID, + testHeader[string, uint], + *BlockchainBackendMock[string, uint, testHeader[string, uint]], + ]( + mockBackend, + authoritySetChanges, + 6, + true, + ) + require.NoError(t, err) + + unknownHeaders := []testHeader[string, uint]{header7, header8} + + expFinalityProof := &FinalityProof[string, uint, testHeader[string, uint]]{ + Block: "hash8", + Justification: scale.MustMarshal(grandpaJust), + UnknownHeaders: unknownHeaders, + } + require.Equal(t, expFinalityProof, proofOf6) +} diff --git a/client/consensus/grandpa/helpers_test.go b/client/consensus/grandpa/helpers_test.go new file mode 100644 index 0000000000..d329aaff0c --- /dev/null +++ b/client/consensus/grandpa/helpers_test.go @@ -0,0 +1,50 @@ +// Copyright 2023 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + +package grandpa + +import ( + "golang.org/x/exp/constraints" +) + +// //// Fulfils Header interface //// +type testHeader[Hash constraints.Ordered, N constraints.Unsigned] struct { + ParentHashField Hash + NumberField N + StateRoot Hash + ExtrinsicsRoot Hash + HashField Hash +} + +func (s testHeader[Hash, N]) ParentHash() Hash { + return s.ParentHashField +} + +func (s testHeader[Hash, N]) Hash() Hash { + return s.HashField +} + +func (s testHeader[Hash, N]) Number() N { + return s.NumberField +} + +// //// Fulfils HeaderBackend interface ////// +type testHeaderBackend[Hash constraints.Ordered, N constraints.Unsigned] struct { + header *Header[Hash, N] +} + +func (backend testHeaderBackend[Hash, N]) Header(hash Hash) (*Header[Hash, N], error) { + return backend.header, nil +} + +func (backend testHeaderBackend[Hash, N]) Info() Info[N] { + panic("unimplemented") +} + +func (backend testHeaderBackend[Hash, N]) ExpectBlockHashFromID(id N) (Hash, error) { + panic("unimplemented") +} + +func (backend testHeaderBackend[Hash, N]) ExpectHeader(hash Hash) (Header[Hash, N], error) { + panic("unimplemented") +} diff --git a/client/consensus/grandpa/interfaces.go b/client/consensus/grandpa/interfaces.go index b772a26efa..9d4f57dcaa 100644 --- a/client/consensus/grandpa/interfaces.go +++ b/client/consensus/grandpa/interfaces.go @@ -10,13 +10,134 @@ import ( // Telemetry TODO issue #3474 type Telemetry interface{} +/* + Following is from api/backend +*/ + +type Key []byte + +type KeyValue struct { + Key Key + Value []byte +} + +// AuxStore is part of the substrate backend. +// Provides access to an auxiliary database. +// +// This is a simple global database not aware of forks. Can be used for storing auxiliary +// information like total block weight/difficulty for fork resolution purposes as a common use +// case. +// TODO should this just be in Backend? +type AuxStore interface { + // Insert auxiliary data into key-Value store. + // + // Deletions occur after insertions. + Insert(insert []KeyValue, delete []Key) error + // Get Query auxiliary data from key-Value store. + Get(key Key) (*[]byte, error) +} + +// Backend Client backend. +// +// Manages the data layer. +// +// # State Pruning +// +// While an object from `state_at` is alive, the state +// should not be pruned. The backend should internally reference-count +// its state objects. +// +// The same applies for live `BlockImportOperation`s: while an import operation building on a +// parent `P` is alive, the state for `P` should not be pruned. +// +// # Block Pruning +// +// Users can pin blocks in memory by calling `pin_block`. When +// a block would be pruned, its value is kept in an in-memory cache +// until it is unpinned via `unpin_block`. +// +// While a block is pinned, its state is also preserved. +// +// The backend should internally reference count the number of pin / unpin calls. +type Backend[ + Hash constraints.Ordered, + N constraints.Unsigned, + H Header[Hash, N], + B BlockchainBackend[Hash, N, H]] interface { + AuxStore + Blockchain() B +} + +/* + Following is from primitives/blockchain +*/ + +// HeaderBackend Blockchain database header backend. Does not perform any validation. +// primitives/blockchains/src/backend +type HeaderBackend[Hash constraints.Ordered, N constraints.Unsigned, H Header[Hash, N]] interface { + // Header Get block header. Returns None if block is not found. + Header(hash Hash) (*H, error) + // Info Get blockchain info. + Info() Info[N] + // ExpectBlockHashFromID This takes an enum blockID, but for now just using block Number N + ExpectBlockHashFromID(id N) (Hash, error) + // ExpectHeader return Header + ExpectHeader(hash Hash) (H, error) +} + +// Info HeaderBackend blockchain info +type Info[N constraints.Unsigned] struct { + FinalizedNumber N +} + +// BlockchainBackend Blockchain database backend. Does not perform any validation. +// pub trait Backend:HeaderBackend + HeaderMetadata