Initial reconfiguration mechanism for Mir

.gitignore

@@ -53,7 +53,7 @@ bin/tmp/*
 scratchpad
 gen.gen
 
-**/*-wal/
+**/*-wal-*/
 .devcontainer
 /eudico-stats
 *.key

Makefile

@@ -417,3 +417,7 @@ print-%:
 
 circleci:
 	go generate -x ./.circleci
+
+.PHONY: lint
+lint:
+	golangci-lint run ./...

chain/consensus/README.md

@@ -1,14 +1,14 @@
 # Consensus Interface
 
-`Consensus interface` is an interface that is used to agree on the next block using the concrete implementation of a consensus protocol.
-Eudico has several implementations of different BFT-type and Nakamoto-type consensus protocols for research purposes. 
+`Consensus` interface is an interface that is used to agree on the next block using the concrete implementation of a consensus protocol.
+Eudico uses several implementations of different BFT-type and Nakamoto-type consensus protocols for research purposes. 
 
 ## How to Add a New Consensus Protocol to Eudico
-1. Register a consensus type constant in `chain/consensus/hierarchical/types.go`
-2. Instantiate a consensus and the corresponding miner for a subnet via `New()`, `Weight()`, and `Mine()` functions of `chain/consensus/hierarchical/subnet/consensus/consensus.go`
-3. Create a new file with implementations of genesis block functions in `chain/consensus/hierarchical/actors/subnet/` and in `chain/consensus/hierarchical/actors/subnet/genesis.go`
-4. Implement the `Consensus` interface defined in `chain/consensus/iface.go` for the target consensus protocol
-5. Add the corresponding CLI commands in `cmd/eudico/$CONSENSUS.go`
+1. Register a consensus type constant in `chain/consensus/hierarchical/types.go` and add name, finality, checkpoint period values in the corresponding functions.
+2. Instantiate a consensus and the corresponding miner for a subnet via `New()`, `Weight()`, and `Mine()` functions of `chain/consensus/hierarchical/subnet/consensus/consensus.go`.
+3. Create a new file with implementations of genesis block functions in `chain/consensus/hierarchical/actors/subnet/` and in `chain/consensus/hierarchical/actors/subnet/genesis.go`.
+4. Implement the `Consensus` interface defined in `chain/consensus/iface.go` for the target consensus protocol.
+5. Add the corresponding CLI commands in `cmd/eudico/$CONSENSUS.go`.
 6. Adapt [bad blocks cache](https://github.com/filecoin-project/eudico/blob/0306742e553f6bd6260332b501bb65a5bfc16a76/chain/sync.go#L725) for the consensus protocol if needed.
    For example, it may be necessary to not process blocks if consensus RPCs are unreachable.
 7. Add a new consensus code to the consensus usage message in `cmd/eudico/subnet.go`.

chain/consensus/delegcns/mine.go

@@ -41,6 +41,9 @@ func Mine(ctx context.Context, addr address.Address, api v1api.FullNode) error {
 	timer := time.NewTicker(time.Duration(build.BlockDelaySecs) * time.Second)
 	for {
 		select {
+		case <-ctx.Done():
+			log.Debug("Delegcns miner: context closed")
+			return nil
 		case <-timer.C:
 			base, err := api.ChainHead(ctx)
 			if err != nil {
@@ -96,10 +99,7 @@ func Mine(ctx context.Context, addr address.Address, api v1api.FullNode) error {
 				log.Errorw("submitting block failed", "error", err)
 				continue
 			}
-
 			log.Info("delegated mined a block! ", bh.Cid(), " msgs ", len(msgs))
-		case <-ctx.Done():
-			return nil
 		}
 	}
 }

chain/consensus/dummy/dummy.go

@@ -1,6 +1,10 @@
-// Package dummy implements consensus for testing purposes only.
-// Dummy consensus is a centralised consensus, it works on one node only,
-// and fundamentally cannot be extended to run on multiple nodes.
+// Package dummy implements dummy insecure non-robust consensus protocol for testing and demo purposes only.
+// Dummy consensus has the following properties: permissioned, leader-based, round-robin, non-crash tolerant.
+// It works as follows:
+// 1. All nodes have the same (agreed) validator set.
+// 2. The first validator from this set is the permanent leader.
+// 3. The leader proposes blocks, the block miner is assigned using round-robin.
+// 4. Signing is not used.
 package dummy
 
 import (
@@ -36,7 +40,7 @@ import (
 )
 
 const (
-	MaxHeightDrift = 5
+	ValidatorsEnv = "EUDICO_DUMMY_VALIDATORS"
 )
 
 var (
@@ -69,7 +73,7 @@ func NewConsensus(
 	if err != nil {
 		return nil, err
 	}
-	log.Infof("New Dummy consensus for %s subnet", subnetID)
+	log.Infof("New dummy consensus for %s subnet", subnetID)
 
 	return &Dummy{
 		store:    sm.ChainStore(),
@@ -110,7 +114,7 @@ func (bft *Dummy) ValidateBlock(ctx context.Context, b *types.FullBlock) (err er
 		log.Warn("Got block from the future, but within threshold", h.Timestamp, build.Clock.Now().Unix())
 	}
 
-	msgsChecks := common.CheckMsgs(ctx, bft.store, bft.sm, bft.subMgr, bft.resolver, bft.netName, b, baseTs)
+	msgsChecks := common.CheckMsgsWithoutBlockSig(ctx, bft.store, bft.sm, bft.subMgr, bft.resolver, bft.netName, b, baseTs)
 
 	minerCheck := async.Err(func() error {
 		if err := bft.minerIsValid(b.Header.Miner); err != nil {
@@ -170,7 +174,7 @@ func (bft *Dummy) ValidateBlock(ctx context.Context, b *types.FullBlock) (err er
 
 func (bft *Dummy) ValidateBlockPubsub(ctx context.Context, self bool, msg *pubsub.Message) (pubsub.ValidationResult, string) {
 	if self {
-		return common.ValidateLocalBlock(ctx, msg)
+		return validateLocalBlock(ctx, msg)
 	}
 
 	// track validation time
@@ -186,7 +190,7 @@ func (bft *Dummy) ValidateBlockPubsub(ctx context.Context, self bool, msg *pubsu
 		panic(what)
 	}
 
-	blk, what, err := common.DecodeAndCheckBlock(msg)
+	blk, what, err := decodeAndCheckBlock(msg)
 	if err != nil {
 		log.Error("got invalid block over pubsub: ", err)
 		recordFailureFlagPeer(what)
@@ -221,12 +225,7 @@ func (bft *Dummy) minerIsValid(maddr address.Address) error {
 // We are currently using defaults here and not worrying about it.
 // We will consider potential changes of Consensus interface in https://github.com/filecoin-project/eudico/issues/143.
 func (bft *Dummy) IsEpochBeyondCurrMax(epoch abi.ChainEpoch) bool {
-	if bft.genesis == nil {
-		return false
-	}
-
-	now := uint64(build.Clock.Now().Unix())
-	return epoch > (abi.ChainEpoch((now-bft.genesis.MinTimestamp())/build.BlockDelaySecs) + MaxHeightDrift)
+	return false
 }
 
 func (bft *Dummy) Type() hierarchical.ConsensusType {
@@ -242,3 +241,40 @@ func Weight(ctx context.Context, stateBs bstore.Blockstore, ts *types.TipSet) (t
 
 	return big.NewInt(int64(ts.Height() + 1)), nil
 }
+
+func validateLocalBlock(ctx context.Context, msg *pubsub.Message) (pubsub.ValidationResult, string) {
+	stats.Record(ctx, metrics.BlockPublished.M(1))
+
+	if size := msg.Size(); size > 1<<20-1<<15 {
+		log.Errorf("ignoring oversize block (%dB)", size)
+		return pubsub.ValidationIgnore, "oversize_block"
+	}
+
+	blk, what, err := decodeAndCheckBlock(msg)
+	if err != nil {
+		log.Errorf("got invalid local block: %s", err)
+		return pubsub.ValidationIgnore, what
+	}
+
+	msg.ValidatorData = blk
+	stats.Record(ctx, metrics.BlockValidationSuccess.M(1))
+	return pubsub.ValidationAccept, ""
+}
+
+func decodeAndCheckBlock(msg *pubsub.Message) (*types.BlockMsg, string, error) {
+	blk, err := types.DecodeBlockMsg(msg.GetData())
+	if err != nil {
+		return nil, "invalid", xerrors.Errorf("error decoding block: %w", err)
+	}
+
+	if count := len(blk.BlsMessages) + len(blk.SecpkMessages); count > build.BlockMessageLimit {
+		return nil, "too_many_messages", xerrors.Errorf("block contains too many messages (%d)", count)
+	}
+
+	// make sure we have a signature
+	if blk.Header.BlockSig != nil {
+		return nil, "missing_signature", xerrors.Errorf("block with a signature")
+	}
+
+	return blk, "", nil
+}

chain/consensus/dummy/mine.go

@@ -2,12 +2,15 @@ package dummy
 
 import (
 	"context"
+	"fmt"
+	"os"
 	"time"
 
 	"github.com/filecoin-project/go-address"
 	lapi "github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/api/v1api"
 	"github.com/filecoin-project/lotus/chain/consensus/common"
+	"github.com/filecoin-project/lotus/chain/consensus/hierarchical"
 	"github.com/filecoin-project/lotus/chain/consensus/platform/logging"
 	"github.com/filecoin-project/lotus/chain/types"
 )
@@ -24,22 +27,40 @@ func Mine(ctx context.Context, miner address.Address, api v1api.FullNode) error
 		return err
 	}
 
-	submitting := time.NewTicker(300 * time.Millisecond)
-	defer submitting.Stop()
+	validatorsEnv := os.Getenv(ValidatorsEnv)
+	var validators []address.Address
+	validators, err = validatorsFromString(validatorsEnv)
+	if err != nil {
+		return fmt.Errorf("failed to get validators addresses: %w", err)
+	}
+	if len(validators) == 0 {
+		validators = append(validators, miner)
+	}
 
-	for {
-		base, err := api.ChainHead(ctx)
-		if err != nil {
-			log.Errorw("failed to get the head of chain", "error", err)
-			continue
-		}
+	submit := time.NewTicker(300 * time.Millisecond)
+	defer submit.Stop()
+
+	leader := validators[0]
 
+	for {
 		select {
 		case <-ctx.Done():
-			log.Debug("Dummy miner %s: context closed")
+			log.Debug("Dummy miner: context closed")
 			return nil
 
-		case <-submitting.C:
+		case <-submit.C:
+			base, err := api.ChainHead(ctx)
+			if err != nil {
+				log.Errorw("failed to get the head of chain", "error", err)
+				continue
+			}
+
+			if miner != leader {
+				continue
+			}
+
+			epochMiner := validators[int(base.Height())%len(validators)]
+
 			msgs, err := api.MpoolSelect(ctx, base.Key(), 1)
 			if err != nil {
 				log.Errorw("unable to select messages from mempool", "error", err)
@@ -54,7 +75,7 @@ func Mine(ctx context.Context, miner address.Address, api v1api.FullNode) error
 
 			log.Infof("[subnet: %s, epoch: %d] try to create a block", subnetID, base.Height()+1)
 			bh, err := api.MinerCreateBlock(ctx, &lapi.BlockTemplate{
-				Miner:            miner,
+				Miner:            epochMiner,
 				Parents:          base.Key(),
 				BeaconValues:     nil,
 				Ticket:           nil,
@@ -96,10 +117,20 @@ func (bft *Dummy) CreateBlock(ctx context.Context, w lapi.Wallet, bt *lapi.Block
 		return nil, err
 	}
 
-	err = common.SignBlock(ctx, w, b)
-	if err != nil {
-		return nil, err
-	}
-
 	return b, nil
 }
+
+// validatorsFromString parses comma-separated validator addresses string.
+//
+// Examples of the validator string: "t1wpixt5mihkj75lfhrnaa6v56n27epvlgwparujy,t1wpixt5mihkj75lfhrnaa6v56n27epvlgwparujy"
+func validatorsFromString(input string) ([]address.Address, error) {
+	var addrs []address.Address
+	for _, id := range hierarchical.SplitAndTrimEmpty(input, ",", " ") {
+		a, err := address.NewFromString(id)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse %v: %w", id, err)
+		}
+		addrs = append(addrs, a)
+	}
+	return addrs, nil
+}

chain/consensus/hierarchical/actors/subnet/subnet_actor.go

@@ -173,6 +173,7 @@ func (a SubnetActor) Join(rt runtime.Runtime, v *hierarchical.Validator) *abi.Em
 func (a SubnetActor) Leave(rt runtime.Runtime, _ *abi.EmptyValue) *abi.EmptyValue {
 	rt.ValidateImmediateCallerAcceptAny()
 	sourceAddr := rt.Caller()
+	sourceSecpAddr := sca.SecpBLSAddr(rt, rt.Caller())
 
 	var (
 		st         SubnetState
@@ -203,8 +204,13 @@ func (a SubnetActor) Leave(rt runtime.Runtime, _ *abi.EmptyValue) *abi.EmptyValu
 	priorBalance := rt.CurrentBalance()
 	var retFunds abi.TokenAmount
 	rt.StateTransaction(&st, func() {
-		// Remove stake from stake balanace table.
+		// Remove stake from stake balance table.
 		retFunds = st.rmStake(rt, sourceAddr, stakes, minerStake)
+
+		// Remove the validator to signal to other validators on the subnet to run reconfiguration.
+		if st.ValidatorSet != nil {
+			st.ValidatorSet = rmValidator(sourceSecpAddr, st.ValidatorSet)
+		}
 	})
 
 	// Never send back if we don't have enough balance
@@ -484,6 +490,15 @@ func (st *SubnetState) rmStake(rt runtime.Runtime, sourceAddr address.Address, s
 	return retFunds
 }
 
+func rmValidator(validator address.Address, validators []hierarchical.Validator) []hierarchical.Validator {
+	for i, v := range validators {
+		if v.Addr == validator {
+			return append(validators[:i], validators[i+1:]...)
+		}
+	}
+	return validators
+}
+
 func rmMiner(miner address.Address, ls []address.Address) []address.Address {
 	for i, v := range ls {
 		if v == miner {

chain/consensus/hierarchical/actors/subnet/subnet_test.go

@@ -109,8 +109,11 @@ func TestLeaveAndKill(t *testing.T) {
 	rt := getRuntime(t)
 	h.constructAndVerify(t, rt)
 	joiner := tutil.NewIDAddr(t, 102)
+	joinerSecp := tutil.NewSECP256K1Addr(h.t, joiner.String())
 	joiner2 := tutil.NewIDAddr(t, 103)
+	joiner2Secp := tutil.NewSECP256K1Addr(h.t, joiner2.String())
 	joiner3 := tutil.NewIDAddr(t, 104)
+	joiner3Secp := tutil.NewSECP256K1Addr(h.t, joiner3.String())
 	totalStake := abi.NewTokenAmount(0)
 
 	t.Log("first miner joins subnet")
@@ -171,6 +174,7 @@ func TestLeaveAndKill(t *testing.T) {
 	minerStake := getStake(t, rt, joiner2)
 	totalStake = big.Sub(totalStake, minerStake)
 	rt.SetBalance(minerStake)
+	rt.ExpectSend(joiner2, builtin.MethodsAccount.PubkeyAddress, nil, big.Zero(), &joiner2Secp, exitcode.Ok)
 	rt.ExpectSend(hierarchical.SubnetCoordActorAddr, sca.Methods.ReleaseStake, &sca.FundParams{Value: minerStake}, big.Zero(), nil, exitcode.Ok)
 	rt.ExpectSend(joiner2, builtin.MethodSend, nil, big.Div(minerStake, actor.LeavingFeeCoeff), nil, exitcode.Ok)
 	rt.Call(h.SubnetActor.Leave, nil)
@@ -194,6 +198,7 @@ func TestLeaveAndKill(t *testing.T) {
 	minerStake = getStake(t, rt, joiner)
 	totalStake = big.Sub(totalStake, minerStake)
 	rt.SetBalance(minerStake)
+	rt.ExpectSend(joiner, builtin.MethodsAccount.PubkeyAddress, nil, big.Zero(), &joinerSecp, exitcode.Ok)
 	rt.ExpectSend(hierarchical.SubnetCoordActorAddr, sca.Methods.ReleaseStake, &sca.FundParams{Value: minerStake}, big.Zero(), nil, exitcode.Ok)
 	rt.ExpectSend(joiner, builtin.MethodSend, nil, big.Div(minerStake, actor.LeavingFeeCoeff), nil, exitcode.Ok)
 	rt.Call(h.SubnetActor.Leave, nil)
@@ -206,6 +211,7 @@ func TestLeaveAndKill(t *testing.T) {
 
 	t.Log("miner can't leave twice")
 	rt.ExpectValidateCallerAny()
+	rt.ExpectSend(joiner, builtin.MethodsAccount.PubkeyAddress, nil, big.Zero(), &joinerSecp, exitcode.Ok)
 	rt.ExpectAbort(exitcode.ErrForbidden, func() {
 		rt.Call(h.SubnetActor.Leave, nil)
 	})
@@ -223,6 +229,7 @@ func TestLeaveAndKill(t *testing.T) {
 
 	t.Log("subnet can't be killed twice")
 	rt.ExpectValidateCallerAny()
+	rt.ExpectSend(joiner3, builtin.MethodsAccount.PubkeyAddress, nil, big.Zero(), &joiner3Secp, exitcode.Ok)
 	rt.ExpectAbort(exitcode.ErrIllegalState, func() {
 		rt.Call(h.SubnetActor.Kill, nil)
 	})