diff --git a/tests/e2e/actions.go b/tests/e2e/actions.go index a47219b5af..7cec626ec9 100644 --- a/tests/e2e/actions.go +++ b/tests/e2e/actions.go @@ -172,6 +172,7 @@ func (tr *Chain) startChain( cometmockArg = "false" } + chainHome := string(action.Chain) startChainScript := tr.target.GetTestScriptPath(action.IsConsumer, "start-chain.sh") cmd := tr.target.ExecCommand("/bin/bash", startChainScript, chainConfig.BinaryName, string(vals), @@ -183,6 +184,7 @@ func (tr *Chain) startChain( // with short timeout_commit (eg. timeout_commit = 1s) some nodes may miss blocks causing the test run to fail tr.testConfig.tendermintConfigOverride, cometmockArg, + chainHome, ) cmdReader, err := cmd.StdoutPipe() @@ -311,7 +313,7 @@ func (tr Chain) updateConsumerChain(action UpdateConsumerChainAction, verbose bo InitializationParameters: &initParams, PowerShapingParameters: &powerShapingParams, } - tr.UpdateConsumer(action.Chain, action.From, msg) + tr.UpdateConsumer(action.Chain, action.From, msg, verbose) } type CreateConsumerChainAction struct { @@ -333,6 +335,13 @@ type CreateConsumerChainAction struct { // createConsumerChain creates and initializes a consumer chain func (tr Chain) createConsumerChain(action CreateConsumerChainAction, verbose bool) { spawnTime := tr.testConfig.containerConfig.Now.Add(time.Duration(action.SpawnTime) * time.Millisecond) + consumerChainCfg := tr.testConfig.chainConfigs[action.ConsumerChain] + providerChainCfg := tr.testConfig.chainConfigs[action.Chain] + + if consumerChainCfg.ConsumerId != "" { + log.Fatalf("consumer chain already created for '%s'", action.ConsumerChain) + } + params := ccvtypes.DefaultParams() initParams := types.ConsumerInitializationParameters{ InitialHeight: action.InitialHeight, @@ -360,16 +369,20 @@ func (tr Chain) createConsumerChain(action CreateConsumerChainAction, verbose bo } metadata := types.ConsumerMetadata{ - Name: "chain name of " + string(action.Chain), + Name: "chain name of " + string(consumerChainCfg.ChainId), Description: "no description", Metadata: "no metadata", } // create consumer to get a consumer-id - consumerId := tr.CreateConsumer(action.Chain, action.ConsumerChain, action.From, metadata, &initParams, &powerShapingParams) + consumerId := tr.CreateConsumer(providerChainCfg.ChainId, consumerChainCfg.ChainId, action.From, metadata, &initParams, &powerShapingParams) if verbose { - fmt.Println("Create consumer chain", string(action.ConsumerChain), " with consumer-id", string(consumerId)) + fmt.Println("Created consumer chain", string(consumerChainCfg.ChainId), " with consumer-id", string(consumerId)) } + + // Set the new created consumer-id on the chain's config + consumerChainCfg.ConsumerId = consumerId + tr.testConfig.chainConfigs[action.ConsumerChain] = consumerChainCfg } type SubmitConsumerAdditionProposalAction struct { @@ -390,7 +403,7 @@ type SubmitConsumerAdditionProposalAction struct { AllowInactiveVals bool } -func (tr Chain) UpdateConsumer(providerChain ChainID, validator ValidatorID, update types.MsgUpdateConsumer) { +func (tr Chain) UpdateConsumer(providerChain ChainID, validator ValidatorID, update types.MsgUpdateConsumer, verbose bool) { content, err := json.Marshal(update) if err != nil { log.Fatal("failed marshalling MsgUpdateConsumer: ", err.Error()) @@ -419,7 +432,8 @@ func (tr Chain) UpdateConsumer(providerChain ChainID, validator ValidatorID, upd bz, err = cmd.CombinedOutput() if err != nil { - log.Fatal("update consumer failed ", "error: ", err, "output: ", string(bz)) + fmt.Println("command failed: ", cmd) + log.Fatal("update consumer failed error: %w, output: %s", err, string(bz)) } // Check transaction @@ -432,14 +446,19 @@ func (tr Chain) UpdateConsumer(providerChain ChainID, validator ValidatorID, upd if txResponse.Code != 0 { log.Fatalf("sending update-consumer transaction failed with error code %d, Log:'%s'", txResponse.Code, txResponse.RawLog) } + + if verbose { + fmt.Println("running 'update-consumer' returned: ", txResponse) + } + tr.waitBlocks(providerChain, 2, 10*time.Second) } // CreateConsumer creates a consumer chain and returns its consumer-id func (tr Chain) CreateConsumer(providerChain, consumerChain ChainID, validator ValidatorID, metadata types.ConsumerMetadata, initParams *types.ConsumerInitializationParameters, powerShapingParams *types.PowerShapingParameters) ConsumerID { - chainID := string(tr.testConfig.chainConfigs[consumerChain].ChainId) + msg := types.MsgCreateConsumer{ - ChainId: chainID, + ChainId: string(consumerChain), Metadata: metadata, InitializationParameters: initParams, PowerShapingParameters: powerShapingParams, @@ -522,19 +541,7 @@ func (tr Chain) CreateConsumer(providerChain, consumerChain ChainID, validator V log.Fatalf("no consumer-id found in consumer creation transaction events for chain '%s'. events: %v", consumerChain, txResponse.Events) } - cfg, exists := tr.testConfig.chainConfigs[e2e.ChainID(chainID)] - if !exists { - log.Fatal("no chain config found for consumer chain", chainID) - } - if cfg.ConsumerId != "" && cfg.ConsumerId != e2e.ConsumerID(consumerId) { - log.Fatalf("chain '%s'registered already with a different consumer ID '%s'", chainID, consumerId) - } - - // Set the new created consumer-id on the chain's config - cfg.ConsumerId = e2e.ConsumerID(consumerId) - tr.testConfig.chainConfigs[e2e.ChainID(chainID)] = cfg - - return e2e.ConsumerID(consumerId) + return ConsumerID(consumerId) } // submitConsumerAdditionProposal initializes a consumer chain and submits a governance proposal @@ -544,6 +551,8 @@ func (tr Chain) submitConsumerAdditionProposal( ) { params := ccvtypes.DefaultParams() spawnTime := tr.testConfig.containerConfig.Now.Add(time.Duration(action.SpawnTime) * time.Millisecond) + consumerChainCfg := tr.testConfig.chainConfigs[action.ConsumerChain] + providerChainCfg := tr.testConfig.chainConfigs[action.Chain] Metadata := types.ConsumerMetadata{ Name: "chain " + string(action.Chain), @@ -566,8 +575,11 @@ func (tr Chain) submitConsumerAdditionProposal( DistributionTransmissionChannel: action.DistributionChannel, } - consumerId := tr.CreateConsumer(action.Chain, action.ConsumerChain, action.From, Metadata, nil, nil) + consumerId := tr.CreateConsumer(providerChainCfg.ChainId, consumerChainCfg.ChainId, action.From, Metadata, nil, nil) authority := "cosmos10d07y265gmmuvt4z0w9aw880jnsr700j6zn9kn" + // Set the new created consumer-id on the chain's config + consumerChainCfg.ConsumerId = consumerId + tr.testConfig.chainConfigs[action.ConsumerChain] = consumerChainCfg // Update consumer to change owner to governance before submitting the proposal update := &types.MsgUpdateConsumer{ @@ -585,7 +597,7 @@ func (tr Chain) submitConsumerAdditionProposal( AllowInactiveVals: action.AllowInactiveVals, } update.PowerShapingParameters = &powerShapingParameters - tr.UpdateConsumer(action.Chain, action.From, *update) + tr.UpdateConsumer(action.Chain, action.From, *update, verbose) // - set PowerShaping params TopN > 0 for consumer chain update.PowerShapingParameters.Top_N = action.TopN @@ -618,10 +630,10 @@ func (tr Chain) submitConsumerAdditionProposal( tr.testConfig.chainConfigs[action.Chain].BinaryName, "tx", "gov", "submit-proposal", proposalFile, `--from`, `validator`+fmt.Sprint(action.From), - `--chain-id`, string(tr.testConfig.chainConfigs[action.Chain].ChainId), - `--home`, tr.getValidatorHome(action.Chain, action.From), + `--chain-id`, string(providerChainCfg.ChainId), + `--home`, tr.getValidatorHome(providerChainCfg.ChainId, action.From), `--gas`, `900000`, - `--node`, tr.getValidatorNode(action.Chain, action.From), + `--node`, tr.getValidatorNode(providerChainCfg.ChainId, action.From), `--keyring-backend`, `test`, `-o json`, `-y`, @@ -651,7 +663,7 @@ func (tr Chain) submitConsumerAdditionProposal( } // wait for inclusion in a block -> '--broadcast-mode block' is deprecated - tr.waitBlocks(action.Chain, 2, 10*time.Second) + tr.waitBlocks(providerChainCfg.ChainId, 2, 10*time.Second) } func (tr Chain) submitConsumerAdditionLegacyProposal( @@ -2885,8 +2897,10 @@ func (tr Chain) startConsumerEvidenceDetector( } type OptInAction struct { - Chain ChainID - Validator ValidatorID + Chain ChainID + Validator ValidatorID + ExpectError bool + ExpectedError string } func (tr Chain) optIn(action OptInAction, verbose bool) { @@ -2919,14 +2933,21 @@ func (tr Chain) optIn(action OptInAction, verbose bool) { } bz, err := cmd.CombinedOutput() - if err != nil { + if err != nil && !action.ExpectError { log.Fatal(err, "\n", string(bz)) } - if !tr.testConfig.useCometmock { // error report only works with --gas auto, which does not work with CometMock, so ignore + if action.ExpectError && !tr.testConfig.useCometmock { // error report only works with --gas auto, which does not work with CometMock, so ignore if err != nil && verbose { fmt.Printf("got error during opt in | err: %s | output: %s \n", err, string(bz)) } + if err == nil || !strings.Contains(string(bz), action.ExpectedError) { + log.Fatalf("expected error not raised: expected: '%s', got '%s'", action.ExpectedError, (bz)) + } + + if verbose { + fmt.Printf("got expected error during key assignment | err: %s | output: %s \n", err, string(bz)) + } } // wait for inclusion in a block -> '--broadcast-mode block' is deprecated diff --git a/tests/e2e/config.go b/tests/e2e/config.go index f97cfb5df9..65ffab74b7 100644 --- a/tests/e2e/config.go +++ b/tests/e2e/config.go @@ -101,6 +101,7 @@ const ( InactiveValsMintTestCfg TestConfigType = "inactive-vals-mint" MintTestCfg TestConfigType = "mint" InactiveValsExtraValsTestCfg TestConfigType = "inactive-vals-extra-vals" + PermissionlessTestCfg TestConfigType = "permissionless-ics" ) type TestConfig struct { @@ -109,6 +110,7 @@ type TestConfig struct { containerConfig ContainerConfig validatorConfigs map[ValidatorID]ValidatorConfig chainConfigs map[ChainID]ChainConfig + consumerChains map[ConsumerID]ChainConfig providerVersion string consumerVersion string // override config.toml parameters @@ -210,6 +212,8 @@ func GetTestConfig(cfgType TestConfigType, providerVersion, consumerVersion stri testCfg = MintTestConfig() case InactiveValsExtraValsTestCfg: testCfg = InactiveValsExtraValsTestConfig() + case PermissionlessTestCfg: + testCfg = PermissionlessTestConfig() default: panic(fmt.Sprintf("Invalid test config: %s", cfgType)) } @@ -594,6 +598,68 @@ func DemocracyTestConfig(allowReward bool) TestConfig { return tr } +// PermissionlessTestConfig contains a provider chain and 2 cosumer chains with the same chain identifier +func PermissionlessTestConfig() TestConfig { + tr := TestConfig{ + name: string(PermissionlessTestCfg), + containerConfig: e2e.ContainerConfig{ + ContainerName: "interchain-security-container", + InstanceName: "interchain-security-instance", + CcvVersion: "1", + Now: time.Now(), + }, + validatorConfigs: getDefaultValidators(), + chainConfigs: map[ChainID]e2e.ChainConfig{ + "provi": { + ChainId: ChainID("provi"), + AccountPrefix: ProviderAccountPrefix, + BinaryName: "interchain-security-pd", + IpPrefix: "7.7.7", + VotingWaitTime: 20, + GenesisChanges: ".app_state.gov.params.voting_period = \"20s\" | " + + ".app_state.gov.params.expedited_voting_period = \"10s\" | " + + // Custom slashing parameters for testing validator downtime functionality + // See https://docs.cosmos.network/main/modules/slashing/04_begin_block.html#uptime-tracking + ".app_state.slashing.params.signed_blocks_window = \"10\" | " + + ".app_state.slashing.params.min_signed_per_window = \"0.500000000000000000\" | " + + ".app_state.slashing.params.downtime_jail_duration = \"60s\" | " + + ".app_state.slashing.params.slash_fraction_downtime = \"0.010000000000000000\" | " + + ".app_state.provider.params.slash_meter_replenish_fraction = \"1.0\" | " + // This disables slash packet throttling + ".app_state.provider.params.slash_meter_replenish_period = \"3s\" | " + + ".app_state.provider.params.blocks_per_epoch = 3", + }, + "cons1": { + ChainId: ChainID("consu"), + AccountPrefix: ConsumerAccountPrefix, + BinaryName: "interchain-security-cd", + IpPrefix: "7.7.8", + VotingWaitTime: 20, + GenesisChanges: ".app_state.gov.params.voting_period = \"20s\" | " + + ".app_state.slashing.params.signed_blocks_window = \"20\" | " + + ".app_state.slashing.params.min_signed_per_window = \"0.500000000000000000\" | " + + ".app_state.slashing.params.downtime_jail_duration = \"60s\" | " + + ".app_state.slashing.params.slash_fraction_downtime = \"0.010000000000000000\"", + }, + // ChainID needs to be "consu" as previous consumer chain + "cons2": { + ChainId: ChainID("consu"), + AccountPrefix: ConsumerAccountPrefix, + BinaryName: "interchain-security-cd", + IpPrefix: "7.7.9", + VotingWaitTime: 20, + GenesisChanges: ".app_state.gov.params.voting_period = \"20s\" | " + + ".app_state.slashing.params.signed_blocks_window = \"20\" | " + + ".app_state.slashing.params.min_signed_per_window = \"0.500000000000000000\" | " + + ".app_state.slashing.params.downtime_jail_duration = \"60s\" | " + + ".app_state.slashing.params.slash_fraction_downtime = \"0.010000000000000000\"", + }, + }, + tendermintConfigOverride: `s/timeout_commit = "5s"/timeout_commit = "1s"/;` + + `s/peer_gossip_sleep_duration = "100ms"/peer_gossip_sleep_duration = "50ms"/;`, + } + tr.Initialize() + return tr +} func InactiveProviderValsTestConfig() TestConfig { tr := DefaultTestConfig() tr.name = "InactiveValsConfig" @@ -958,11 +1024,11 @@ func (s *TestConfig) validateStringLiterals() { for chainID, chainConfig := range s.chainConfigs { if len(chainID) > 5 { - panic("chain id string literal must be 5 char or less") + panic(fmt.Sprintf("chain id string literal must be 5 char or less: %s", chainID)) } if chainID != chainConfig.ChainId { - panic("chain config is mapped to a chain id that is different than what's stored in the config") + log.Println("chain config is mapped to a chain id that is different than what's stored in the config") } } } @@ -1266,6 +1332,7 @@ func getValidatorConfigFromVersion(providerVersion, consumerVersion string) map[ // If provided version is before v1.6.0 then a configuration based on template for v1.4.x is returned // otherwise the returned configuration is based on template v1.4. func GetHermesConfig(hermesVersion, queryNodeIP string, chainCfg ChainConfig, isConsumer bool) string { + ChainId := chainCfg.ChainId keyName := "query" rpcAddr := "http://" + queryNodeIP + ":26658" diff --git a/tests/e2e/main.go b/tests/e2e/main.go index e8ae6b2166..3a834a482a 100644 --- a/tests/e2e/main.go +++ b/tests/e2e/main.go @@ -246,13 +246,12 @@ var stepChoices = map[string]StepChoice{ description: "test minting without inactive validators as a sanity check", testConfig: MintTestCfg, }, - // TODO PERMISSIONLESS: ADD NEW E2E TEST - /* "permissionless-ics": { + "permissionless-ics": { name: "permissionless-ics", steps: stepsPermissionlessICS(), description: "test permissionless ics", - testConfig: DefaultTestCfg, - }, */ + testConfig: PermissionlessTestCfg, + }, "inactive-vals-outside-max-validators": { name: "inactive-vals-outside-max-validators", steps: stepsInactiveValsTopNReproduce(), @@ -540,6 +539,7 @@ func printReport(runners []TestRunner, duration time.Duration) { } numTotalTests := len(runners) report := ` + ================================================= TEST RESULTS ------------------------------------------------- @@ -577,19 +577,6 @@ Summary: len(remainingTests), numTotalTests, ) - report += fmt.Sprintln("\nFAILED TESTS:") - for _, t := range failedTests { - report += t.Report() - } - report += fmt.Sprintln("\n\nPASSED TESTS:") - for _, t := range passedTests { - report += t.Report() - } - - report += fmt.Sprintln("\n\nREMAINING TESTS:") - for _, t := range remainingTests { - report += t.Report() - } report += "==================================================" fmt.Print(report) } diff --git a/tests/e2e/state.go b/tests/e2e/state.go index f03380be89..4ca4a41f09 100644 --- a/tests/e2e/state.go +++ b/tests/e2e/state.go @@ -316,7 +316,7 @@ func (tr Chain) getValidatorIP(chain ChainID, validator ValidatorID) string { } func (tr Chain) getValidatorHome(chain ChainID, validator ValidatorID) string { - return `/` + string(tr.testConfig.chainConfigs[chain].ChainId) + `/validator` + fmt.Sprint(validator) + return `/` + string(chain) + `/validator` + fmt.Sprint(validator) } func (tr Chain) curlJsonRPCRequest(method, params, address string) { @@ -926,7 +926,7 @@ func (tr Commands) GetHasToValidate( log.Fatal(err, "\n", string(bz)) } - arr := gjson.Get(string(bz), "consumer_chain_ids").Array() + arr := gjson.Get(string(bz), "consumer_ids").Array() chains := []ChainID{} for _, c := range arr { for _, chain := range tr.chainConfigs { diff --git a/tests/e2e/steps_partial_set_security.go b/tests/e2e/steps_partial_set_security.go index f2646c5cd3..dbe32fc26d 100644 --- a/tests/e2e/steps_partial_set_security.go +++ b/tests/e2e/steps_partial_set_security.go @@ -1752,7 +1752,7 @@ func stepsValidatorsDenylistedChain() []Step { ChainID("consu"): ChainState{ ValPowers: &map[ValidatorID]uint{ ValidatorID("alice"): 100, - // "bob" is denylisted and hence does not valiate the consumer chain + // "bob" is denylisted and hence does not validate the consumer chain ValidatorID("bob"): 0, ValidatorID("carol"): 300, }, diff --git a/tests/e2e/steps_permissionless_ics.go b/tests/e2e/steps_permissionless_ics.go new file mode 100644 index 0000000000..1da18cb2e0 --- /dev/null +++ b/tests/e2e/steps_permissionless_ics.go @@ -0,0 +1,211 @@ +package main + +import ( + "time" + + clienttypes "github.com/cosmos/ibc-go/v8/modules/core/02-client/types" + e2e "github.com/cosmos/interchain-security/v6/tests/e2e/testlib" +) + +// stepsPermissionlessICS tests +// - starting multiple permissionless consumer chains with the same chain ID +// - that a validator CANNOT opt-in on two different chains with the same chain ID +// - taking ownership of a consumer chain +func stepsPermissionlessICS() []Step { + s := concatSteps( + []Step{ + // Start the provider chain + { + Action: StartChainAction{ + Chain: ChainID("provi"), + Validators: []StartChainValidator{ + {Id: ValidatorID("alice"), Stake: 100000000, Allocation: 10000000000}, + {Id: ValidatorID("bob"), Stake: 200000000, Allocation: 10000000000}, + {Id: ValidatorID("carol"), Stake: 300000000, Allocation: 10000000000}, + }, + }, + State: State{ + ChainID("provi"): ChainState{ + ValPowers: &map[ValidatorID]uint{ + ValidatorID("alice"): 100, + ValidatorID("bob"): 200, + ValidatorID("carol"): 300, + }, + }, + }, + }, + + // Initialize a permissionless chain with ChainID `consu` + // - create the consumer chain + // - opt-in a validator + // - launch the chain + { + Action: CreateConsumerChainAction{ + Chain: ChainID("provi"), + From: ValidatorID("alice"), + ConsumerChain: ChainID("cons2"), // test chain "cons2" is configured with ChainID "consu" + SpawnTime: uint(time.Minute * 3), + InitialHeight: clienttypes.Height{RevisionNumber: 0, RevisionHeight: 1}, + TopN: 0, + }, + State: State{}, + }, + { + Action: OptInAction{ + Chain: ChainID("cons2"), + Validator: ValidatorID("alice"), + }, + State: State{ + ChainID("provi"): ChainState{ + HasToValidate: &map[ValidatorID][]ChainID{ + ValidatorID("alice"): {}, + ValidatorID("bob"): {}, + ValidatorID("carol"): {}, + }, + }, + }, + }, + }, + // Start another permissionless chain with ChainID `consu` + // test chain "cons1" is configured with ChainID "consu" + stepsStartPermissionlessChain( + "cons1", "consu", + []string{"consu", "consu"}, // show up both consumer chains "consu" as proposed chains + []ValidatorID{ValidatorID("bob")}, 0), + + []Step{ + { + Action: RelayPacketsAction{ + ChainA: ChainID("provi"), + ChainB: ChainID("cons1"), + Port: "provider", + Channel: 0, + }, + State: State{ + ChainID("cons1"): ChainState{ + ValPowers: &map[ValidatorID]uint{ + ValidatorID("alice"): 0, + ValidatorID("bob"): 200, + ValidatorID("carol"): 0, + }, + }, + ChainID("provi"): e2e.ChainState{ + HasToValidate: &map[ValidatorID][]ChainID{ + ValidatorID("alice"): {}, + ValidatorID("bob"): {"consu"}, + ValidatorID("carol"): {}, + }, + }, + }, + }, + }, + + // Test that a validator CANNOT opt-in on a chain with the same ChainID it is already validating + []Step{ + { + Action: OptInAction{ + Chain: ChainID("cons2"), + Validator: ValidatorID("alice"), + ExpectError: true, + ExpectedError: "already opted in to a chain with the same chain id", + }, + State: State{ + ChainID("provi"): ChainState{ + HasToValidate: &map[ValidatorID][]ChainID{ + ValidatorID("alice"): {}, + ValidatorID("bob"): {"consu"}, + ValidatorID("carol"): {}, + }, + }, + ChainID("cons1"): ChainState{ + ValPowers: &map[ValidatorID]uint{ + ValidatorID("alice"): 0, // alice refused to opt in + ValidatorID("bob"): 200, + ValidatorID("carol"): 0, + }, + }, + }, + }, + }, + []Step{ + { + Action: RelayPacketsAction{ + ChainA: ChainID("provi"), + ChainB: ChainID("cons1"), + Port: "provider", + Channel: 0, + }, + State: State{ + ChainID("cons1"): ChainState{ + ValPowers: &map[ValidatorID]uint{ + ValidatorID("alice"): 0, + ValidatorID("bob"): 200, + ValidatorID("carol"): 0, + }, + }, + ChainID("provi"): e2e.ChainState{ + HasToValidate: &map[ValidatorID][]ChainID{ + ValidatorID("alice"): {}, + ValidatorID("bob"): {"consu"}, + ValidatorID("carol"): {}, + }, + }, + }, + }, + }, + // test chain hijacking prevention + []Step{ + // Try to change owner of chain and change deny-/allowlist + { + Action: UpdateConsumerChainAction{ + Chain: ChainID("provi"), + From: ValidatorID("bob"), + ConsumerChain: ChainID("cons1"), + NewOwner: getDefaultValidators()[ValidatorID("carol")].ValconsAddress, + SpawnTime: 0, // launch now + InitialHeight: clienttypes.Height{RevisionNumber: 0, RevisionHeight: 1}, + TopN: 0, + }, + State: State{}, + }, + { + Action: UpdateConsumerChainAction{ + Chain: ChainID("provi"), + From: ValidatorID("carol"), + ConsumerChain: ChainID("cons1"), + SpawnTime: 0, + InitialHeight: clienttypes.Height{RevisionNumber: 0, RevisionHeight: 1}, + TopN: 0, + Allowlist: []string{getDefaultValidators()[ValidatorID("carol")].ValconsAddress}, + Denylist: []string{getDefaultValidators()[ValidatorID("bob")].ValconsAddress}, + }, + State: State{}, + }, + { + Action: RelayPacketsAction{ + ChainA: ChainID("provi"), + ChainB: ChainID("cons1"), + Port: "provider", + Channel: 0, + }, + State: State{ + ChainID("cons1"): ChainState{ + ValPowers: &map[ValidatorID]uint{ + ValidatorID("alice"): 0, + ValidatorID("bob"): 200, // bob is not 'denylisted' + ValidatorID("carol"): 0, + }, + }, + ChainID("provi"): e2e.ChainState{ + HasToValidate: &map[ValidatorID][]ChainID{ + ValidatorID("alice"): {}, + ValidatorID("bob"): {"consu"}, // bob is still a validator on consu chain + ValidatorID("carol"): {}, + }, + }, + }, + }, + }, + ) + return s +} diff --git a/tests/e2e/steps_start_chains.go b/tests/e2e/steps_start_chains.go index 640816aaeb..7f84d5fa37 100644 --- a/tests/e2e/steps_start_chains.go +++ b/tests/e2e/steps_start_chains.go @@ -1,8 +1,11 @@ package main import ( + "time" + gov "github.com/cosmos/cosmos-sdk/x/gov/types/v1" clienttypes "github.com/cosmos/ibc-go/v8/modules/core/02-client/types" + e2e "github.com/cosmos/interchain-security/v6/tests/e2e/testlib" ) func stepStartProviderChain() []Step { @@ -29,6 +32,144 @@ func stepStartProviderChain() []Step { } } +func stepsStartPermissionlessChain(consumerName, consumerChainId string, proposedChains []string, validators []ValidatorID, chainIndex uint) []Step { + s := []Step{ + { + Action: CreateConsumerChainAction{ + Chain: ChainID("provi"), + From: ValidatorID("alice"), + ConsumerChain: ChainID(consumerName), + SpawnTime: uint(time.Minute * 3), + InitialHeight: clienttypes.Height{RevisionNumber: 0, RevisionHeight: 1}, + TopN: 0, + }, + State: State{ + ChainID("provi"): e2e.ChainState{ + ProposedConsumerChains: &proposedChains, + }, + }, + }, + } + + // Assign validator keys + // add a consumer key before the chain starts + // the key will be present in the consumer genesis initial_val_set + for _, valId := range validators { + valCfg := getDefaultValidators()[valId] + // no consumer-key assignment needed for validators using provider's public key + if !valCfg.UseConsumerKey { + continue + } + step := Step{ + Action: AssignConsumerPubKeyAction{ + Chain: ChainID(consumerName), + Validator: valId, + ConsumerPubkey: valCfg.ConsumerValPubKey, + // consumer chain has not started + // we don't need to reconfigure the node + // since it will start with consumer key + ReconfigureNode: false, + }, + State: State{ + ChainID(consumerName): ChainState{ + AssignedKeys: &map[ValidatorID]string{ + valId: valCfg.ConsumerValconsAddressOnProvider, + }, + ProviderKeys: &map[ValidatorID]string{ + valId: valCfg.ValconsAddress, + }, + }, + }, + } + s = append(s, step) + } + + // Opt-in Validators + for _, valId := range validators { + step := Step{ + Action: OptInAction{ + Chain: ChainID(consumerName), + Validator: valId, + }, + State: State{}, + } + s = append(s, step) + } + + // Launch chain + step := Step{ + Action: UpdateConsumerChainAction{ + Chain: ChainID("provi"), + From: ValidatorID("alice"), + ConsumerChain: ChainID(consumerName), + SpawnTime: 0, // launch now + InitialHeight: clienttypes.Height{RevisionNumber: 0, RevisionHeight: 1}, + TopN: 0, + }, + State: State{}, + } + s = append(s, step) + + // Setup validators for chain + startChainVals := []StartChainValidator{} + valBalance := map[ValidatorID]uint{ + ValidatorID("alice"): 0, + ValidatorID("bob"): 0, + ValidatorID("carol"): 0, + } + + for idx, val := range validators { + startChainVals = append(startChainVals, + StartChainValidator{ + Id: val, + Stake: uint(100000000 * (idx + 1)), + Allocation: 10000000000, + }) + valBalance[val] = 10000000000 + } + + // Start the chain + step = Step{ + Action: StartConsumerChainAction{ + ConsumerChain: ChainID(consumerName), + ProviderChain: ChainID("provi"), + Validators: startChainVals, + }, + State: State{ + ChainID(consumerName): ChainState{ + ValBalances: &valBalance, + }, + }, + } + s = append(s, step) + + // Establish IBC connection + steps := []Step{ + { + Action: AddIbcConnectionAction{ + ChainA: ChainID(consumerName), + ChainB: ChainID("provi"), + ClientA: 0, + ClientB: chainIndex, + }, + State: State{}, + }, + { + Action: AddIbcChannelAction{ + ChainA: ChainID(consumerName), + ChainB: ChainID("provi"), + ConnectionA: 0, + PortA: "consumer", + PortB: "provider", + Order: "ordered", + }, + State: State{}, + }, + } + s = append(s, steps...) + return s +} + func stepsStartConsumerChain(consumerName string, proposalIndex, chainIndex uint, setupTransferChans bool) []Step { s := []Step{ { diff --git a/tests/e2e/test_runner.go b/tests/e2e/test_runner.go index c3ced5ab61..e07360954e 100644 --- a/tests/e2e/test_runner.go +++ b/tests/e2e/test_runner.go @@ -72,7 +72,7 @@ func (res *TestResult) Error() { func (tr *TestRunner) Run() error { tr.result = TestResult{} tr.result.Started() - fmt.Printf("\n\n=============== running %s ===============\n", tr.config.name) + fmt.Printf("\n\n=============== running %s ===============\n", tr.stepChoice.name) fmt.Println(tr.Info()) err := tr.checkConfig() if err != nil { @@ -131,11 +131,11 @@ func CreateTestRunner(config TestConfig, stepChoice StepChoice, target Execution // Info returns a header string containing useful information about the test runner func (tr *TestRunner) Info() string { return fmt.Sprintf(` ------------------------------------------- +------------------------------------------------- Test name : %s Config: %s Target: %s -------------------------------------------`, +-------------------------------------------------`, tr.stepChoice.name, tr.config.name, tr.target.Info(), @@ -144,7 +144,7 @@ Target: %s func (tr *TestRunner) Report() string { return fmt.Sprintf(` ------------------------------------------- +------------------------------------------------- Test name : %s Config: %s Target: %s @@ -152,7 +152,7 @@ Target: %s - Result: %s - Duration: %s - StartTime: %s -------------------------------------------`, +-------------------------------------------------`, tr.stepChoice.name, tr.config.name, tr.target.Info(), diff --git a/tests/e2e/testnet-scripts/start-chain.sh b/tests/e2e/testnet-scripts/start-chain.sh index 6ad97b3589..23a5ca3c94 100644 --- a/tests/e2e/testnet-scripts/start-chain.sh +++ b/tests/e2e/testnet-scripts/start-chain.sh @@ -36,6 +36,8 @@ TENDERMINT_CONFIG_TRANSFORM=$7 # whether to use CometMock USE_COMETMOCK=$8 +CHAIN_HOME=$9 + # stores a comma separated list of nodes addresses # needed for CometMock - these are the addresses that the ABCI servers of the apps are listening on NODE_LISTEN_ADDR_STR="" # example value: 7.7.8.6:26655,7.7.8.4:26655,7.7.8.5:26655 @@ -55,18 +57,18 @@ NODES=$(echo "$VALIDATORS" | jq '. | length') # SETUP NETWORK NAMESPACES, see: https://adil.medium.com/container-networking-under-the-hood-network-namespaces-6b2b8fe8dc2a # Create virtual bridge device (acts like a switch) -ip link add name virtual-bridge type bridge || true +ip link add name virtual-bridge type bridge || true for i in $(seq 0 $(($NODES - 1))); do VAL_ID=$(echo "$VALIDATORS" | jq -r ".[$i].val_id") VAL_IP_SUFFIX=$(echo "$VALIDATORS" | jq -r ".[$i].ip_suffix") NET_NAMESPACE_NAME="$CHAIN_ID-$VAL_ID" - IP_ADDR="$CHAIN_IP_PREFIX.$VAL_IP_SUFFIX/24" + IP_ADDR="$CHAIN_IP_PREFIX.$VAL_IP_SUFFIX/24" - # Create network namespace + # Create network namespace ip netns add $NET_NAMESPACE_NAME - # Create virtual ethernet device to connect with bridge + # Create virtual ethernet device to connect with bridge ip link add $NET_NAMESPACE_NAME-in type veth peer name $NET_NAMESPACE_NAME-out # Connect input end of virtual ethernet device to namespace ip link set $NET_NAMESPACE_NAME-in netns $NET_NAMESPACE_NAME @@ -84,14 +86,14 @@ do VAL_ID=$(echo "$VALIDATORS" | jq -r ".[$i].val_id") NET_NAMESPACE_NAME="$CHAIN_ID-$VAL_ID" - # Enable in/out interfaces for the namespace + # Enable in/out interfaces for the namespace ip link set $NET_NAMESPACE_NAME-out up ip netns exec $NET_NAMESPACE_NAME ip link set dev $NET_NAMESPACE_NAME-in up # Enable loopback device ip netns exec $NET_NAMESPACE_NAME ip link set dev lo up done -# Assign IP for bridge, to route between default network namespace and bridge +# Assign IP for bridge, to route between default network namespace and bridge BRIDGE_IP="$CHAIN_IP_PREFIX.254/24" ip addr add $BRIDGE_IP dev virtual-bridge @@ -99,11 +101,11 @@ ip addr add $BRIDGE_IP dev virtual-bridge # the first validator will also collect the gentx's once generated FIRST_VAL_ID=$(echo "$VALIDATORS" | jq -r ".[0].val_id") FIRST_VAL_IP_SUFFIX=$(echo "$VALIDATORS" | jq -r ".[0].ip_suffix") -echo "$VALIDATORS" | jq -r ".[0].mnemonic" | $BIN init --home /$CHAIN_ID/validator$FIRST_VAL_ID --chain-id=$CHAIN_ID validator$FIRST_VAL_ID --recover > /dev/null +echo "$VALIDATORS" | jq -r ".[0].mnemonic" | $BIN init --home /$CHAIN_HOME/validator$FIRST_VAL_ID --chain-id=$CHAIN_ID validator$FIRST_VAL_ID --recover > /dev/null # Apply jq transformations to genesis file -jq "$GENESIS_TRANSFORM" /$CHAIN_ID/validator$FIRST_VAL_ID/config/genesis.json > /$CHAIN_ID/edited-genesis.json -mv /$CHAIN_ID/edited-genesis.json /$CHAIN_ID/genesis.json +jq "$GENESIS_TRANSFORM" /$CHAIN_HOME/validator$FIRST_VAL_ID/config/genesis.json > /$CHAIN_HOME/edited-genesis.json +mv /$CHAIN_HOME/edited-genesis.json /$CHAIN_HOME/genesis.json @@ -120,28 +122,28 @@ do # optionally start validator with a key different from provider chain key if [[ "$CHAIN_ID" != "provi" && "$START_WITH_CONSUMER_KEY" = "true" ]]; then echo "$VALIDATORS" | jq -r ".[$i].consumer_mnemonic" | $BIN keys add validator$VAL_ID \ - --home /$CHAIN_ID/validator$VAL_ID \ + --home /$CHAIN_HOME/validator$VAL_ID \ --keyring-backend test \ --recover > /dev/null else echo "$VALIDATORS" | jq -r ".[$i].mnemonic" | $BIN keys add validator$VAL_ID \ - --home /$CHAIN_ID/validator$VAL_ID \ + --home /$CHAIN_HOME/validator$VAL_ID \ --keyring-backend test \ --recover > /dev/null fi - + # Give validators their initial token allocations # move the genesis in - mv /$CHAIN_ID/genesis.json /$CHAIN_ID/validator$VAL_ID/config/genesis.json - + mv /$CHAIN_HOME/genesis.json /$CHAIN_HOME/validator$VAL_ID/config/genesis.json + # give this validator some money ALLOCATION=$(echo "$VALIDATORS" | jq -r ".[$i].allocation") $BIN genesis add-genesis-account validator$VAL_ID $ALLOCATION \ - --home /$CHAIN_ID/validator$VAL_ID \ + --home /$CHAIN_HOME/validator$VAL_ID \ --keyring-backend test # move the genesis back out - mv /$CHAIN_ID/validator$VAL_ID/config/genesis.json /$CHAIN_ID/genesis.json + mv /$CHAIN_HOME/validator$VAL_ID/config/genesis.json /$CHAIN_HOME/genesis.json done @@ -153,50 +155,50 @@ do VAL_ID=$(echo "$VALIDATORS" | jq -r ".[$i].val_id") # Copy in the genesis.json - cp /$CHAIN_ID/genesis.json /$CHAIN_ID/validator$VAL_ID/config/genesis.json + cp /$CHAIN_HOME/genesis.json /$CHAIN_HOME/validator$VAL_ID/config/genesis.json # Copy in validator state file - echo '{"height": "0","round": 0,"step": 0}' > /$CHAIN_ID/validator$VAL_ID/data/priv_validator_state.json + echo '{"height": "0","round": 0,"step": 0}' > /$CHAIN_HOME/validator$VAL_ID/data/priv_validator_state.json START_WITH_CONSUMER_KEY=$(echo "$VALIDATORS" | jq -r ".[$i].start_with_consumer_key") if [[ "$CHAIN_ID" != "provi" && "$START_WITH_CONSUMER_KEY" = "true" ]]; then # start with assigned consumer key PRIV_VALIDATOR_KEY=$(echo "$VALIDATORS" | jq -r ".[$i].consumer_priv_validator_key") if [[ "$PRIV_VALIDATOR_KEY" ]]; then - echo "$PRIV_VALIDATOR_KEY" > /$CHAIN_ID/validator$VAL_ID/config/priv_validator_key.json + echo "$PRIV_VALIDATOR_KEY" > /$CHAIN_HOME/validator$VAL_ID/config/priv_validator_key.json fi else PRIV_VALIDATOR_KEY=$(echo "$VALIDATORS" | jq -r ".[$i].priv_validator_key") if [[ "$PRIV_VALIDATOR_KEY" ]]; then - echo "$PRIV_VALIDATOR_KEY" > /$CHAIN_ID/validator$VAL_ID/config/priv_validator_key.json + echo "$PRIV_VALIDATOR_KEY" > /$CHAIN_HOME/validator$VAL_ID/config/priv_validator_key.json fi fi NODE_KEY=$(echo "$VALIDATORS" | jq -r ".[$i].node_key") if [[ "$NODE_KEY" ]]; then - echo "$NODE_KEY" > /$CHAIN_ID/validator$VAL_ID/config/node_key.json + echo "$NODE_KEY" > /$CHAIN_HOME/validator$VAL_ID/config/node_key.json fi # Make a gentx (this command also sets up validator state on disk even if we are not going to use the gentx for anything) - if [ "$SKIP_GENTX" = "false" ] ; then + if [ "$SKIP_GENTX" = "false" ] ; then STAKE_AMOUNT=$(echo "$VALIDATORS" | jq -r ".[$i].stake") $BIN genesis gentx validator$VAL_ID "$STAKE_AMOUNT" \ - --home /$CHAIN_ID/validator$VAL_ID \ + --home /$CHAIN_HOME/validator$VAL_ID \ --keyring-backend test \ --moniker validator$VAL_ID \ --chain-id=$CHAIN_ID - # Copy gentxs to the first validator for possible future collection. + # Copy gentxs to the first validator for possible future collection. # Obviously we don't need to copy the first validator's gentx to itself if [ $VAL_ID != $FIRST_VAL_ID ]; then - cp /$CHAIN_ID/validator$VAL_ID/config/gentx/* /$CHAIN_ID/validator$FIRST_VAL_ID/config/gentx/ + cp /$CHAIN_HOME/validator$VAL_ID/config/gentx/* /$CHAIN_HOME/validator$FIRST_VAL_ID/config/gentx/ fi fi # Modify tendermint configs of validator - if [ "$TENDERMINT_CONFIG_TRANSFORM" != "" ] ; then + if [ "$TENDERMINT_CONFIG_TRANSFORM" != "" ] ; then #'s/foo/bar/;s/abc/def/' - sed -i "$TENDERMINT_CONFIG_TRANSFORM" $CHAIN_ID/validator$VAL_ID/config/config.toml + sed -i "$TENDERMINT_CONFIG_TRANSFORM" /$CHAIN_HOME/validator$VAL_ID/config/config.toml fi done @@ -207,16 +209,16 @@ done if [ "$SKIP_GENTX" = "false" ] ; then # make the final genesis.json - $BIN genesis collect-gentxs --home /$CHAIN_ID/validator$FIRST_VAL_ID + $BIN genesis collect-gentxs --home /$CHAIN_HOME/validator$FIRST_VAL_ID - # and copy it to the root - cp /$CHAIN_ID/validator$FIRST_VAL_ID/config/genesis.json /$CHAIN_ID/genesis.json + # and copy it to the root + cp /$CHAIN_HOME/validator$FIRST_VAL_ID/config/genesis.json /$CHAIN_HOME/genesis.json # put the now final genesis.json into the correct folders for i in $(seq 1 $(($NODES - 1))); do VAL_ID=$(echo "$VALIDATORS" | jq -r ".[$i].val_id") - cp /$CHAIN_ID/genesis.json /$CHAIN_ID/validator$VAL_ID/config/genesis.json + cp /$CHAIN_HOME/genesis.json /$CHAIN_HOME/validator$VAL_ID/config/genesis.json done fi @@ -231,7 +233,7 @@ do VAL_IP_SUFFIX=$(echo "$VALIDATORS" | jq -r ".[$i].ip_suffix") NET_NAMESPACE_NAME="$CHAIN_ID-$VAL_ID" - NODE_HOME="/$CHAIN_ID/validator$VAL_ID" + NODE_HOME="/$CHAIN_HOME/validator$VAL_ID" GAIA_HOME="--home $NODE_HOME" NODE_HOMES="$NODE_HOME,$NODE_HOMES" RPC_ADDRESS="--rpc.laddr tcp://$CHAIN_IP_PREFIX.$VAL_IP_SUFFIX:26658" @@ -249,27 +251,27 @@ do do if [ $i -ne $j ]; then PEER_VAL_ID=$(echo "$VALIDATORS" | jq -r ".[$j].val_id") - PEER_VAL_IP_SUFFIX=$(echo "$VALIDATORS" | jq -r ".[$j].ip_suffix") - NODE_ID=$($BIN tendermint show-node-id --home /$CHAIN_ID/validator$PEER_VAL_ID) + PEER_VAL_IP_SUFFIX=$(echo "$VALIDATORS" | jq -r ".[$j].ip_suffix") + NODE_ID=$($BIN tendermint show-node-id --home /$CHAIN_HOME/validator$PEER_VAL_ID) ADDRESS="$NODE_ID@$CHAIN_IP_PREFIX.$PEER_VAL_IP_SUFFIX:26656" - # (jq -r '.body.memo' /$CHAIN_ID/validator$j/config/gentx/*) # Getting the address from the gentx should also work + # (jq -r '.body.memo' /$CHAIN_HOME/validator$j/config/gentx/*) # Getting the address from the gentx should also work PERSISTENT_PEERS="$PERSISTENT_PEERS,$ADDRESS" fi done - + if [ "$PERSISTENT_PEERS" != "" ]; then # Remove leading comma and concat to flag PERSISTENT_PEERS="--p2p.persistent_peers ${PERSISTENT_PEERS:1}" fi - + ARGS="$GAIA_HOME $LISTEN_ADDRESS $RPC_ADDRESS $GRPC_ADDRESS $LOG_LEVEL $P2P_ADDRESS $ENABLE_WEBGRPC $PERSISTENT_PEERS" if [[ "$USE_COMETMOCK" == "true" ]]; then # to start with CometMock, ensure ABCI server uses grpc (--transport=grpc) and the app is started without in-process CometBFT (--with-tendermint=false) - ip netns exec $NET_NAMESPACE_NAME $BIN $ARGS start --transport=grpc --with-tendermint=false &> /$CHAIN_ID/validator$VAL_ID/logs & + ip netns exec $NET_NAMESPACE_NAME $BIN $ARGS start --transport=grpc --with-tendermint=false &> /$CHAIN_HOME/validator$VAL_ID/logs & else - ip netns exec $NET_NAMESPACE_NAME $BIN $ARGS start &> /$CHAIN_ID/validator$VAL_ID/logs & + ip netns exec $NET_NAMESPACE_NAME $BIN $ARGS start &> /$CHAIN_HOME/validator$VAL_ID/logs & fi done @@ -314,13 +316,13 @@ ip netns exec $QUERY_NET_NAMESPACE_NAME ip link set dev lo up ## DONE QUERY NODE ENABLE DEVICE ## INIT QUERY NODE -$BIN init --home /$CHAIN_ID/$QUERY_NODE_ID --chain-id=$CHAIN_ID $QUERY_NODE_ID > /dev/null -cp /$CHAIN_ID/genesis.json /$CHAIN_ID/$QUERY_NODE_ID/config/genesis.json +$BIN init --home /$CHAIN_HOME/$QUERY_NODE_ID --chain-id=$CHAIN_ID $QUERY_NODE_ID > /dev/null +cp /$CHAIN_HOME/genesis.json /$CHAIN_HOME/$QUERY_NODE_ID/config/genesis.json ## DONE INIT QUERY NODE ## START QUERY NODE -QUERY_GAIA_HOME="--home /$CHAIN_ID/$QUERY_NODE_ID" +QUERY_GAIA_HOME="--home /$CHAIN_HOME/$QUERY_NODE_ID" QUERY_RPC_ADDRESS="--rpc.laddr tcp://$CHAIN_IP_PREFIX.$QUERY_IP_SUFFIX:26658" QUERY_GRPC_ADDRESS="--grpc.address $CHAIN_IP_PREFIX.$QUERY_IP_SUFFIX:9091" QUERY_LISTEN_ADDRESS="--address tcp://$CHAIN_IP_PREFIX.$QUERY_IP_SUFFIX:26655" @@ -336,7 +338,7 @@ for j in $(seq 0 $(($NODES - 1))); do PEER_VAL_ID=$(echo "$VALIDATORS" | jq -r ".[$j].val_id") PEER_VAL_IP_SUFFIX=$(echo "$VALIDATORS" | jq -r ".[$j].ip_suffix") - NODE_ID=$($BIN tendermint show-node-id --home /$CHAIN_ID/validator$PEER_VAL_ID) + NODE_ID=$($BIN tendermint show-node-id --home /$CHAIN_HOME/validator$PEER_VAL_ID) ADDRESS="$NODE_ID@$CHAIN_IP_PREFIX.$PEER_VAL_IP_SUFFIX:26656" QUERY_PERSISTENT_PEERS="$QUERY_PERSISTENT_PEERS,$ADDRESS" done @@ -349,7 +351,7 @@ ARGS="$QUERY_GAIA_HOME $QUERY_LISTEN_ADDRESS $QUERY_RPC_ADDRESS $QUERY_GRPC_ADDR # Query node is only started if CometMock is not used - with CometMock, it takes the role of the query node if [[ "$USE_COMETMOCK" != "true" ]]; then - ip netns exec $QUERY_NET_NAMESPACE_NAME $BIN $ARGS start &> /$CHAIN_ID/$QUERY_NODE_ID/logs & + ip netns exec $QUERY_NET_NAMESPACE_NAME $BIN $ARGS start &> /$CHAIN_HOME/$QUERY_NODE_ID/logs & fi ## DONE START NODE @@ -366,7 +368,7 @@ NODE_HOMES=${NODE_HOMES%?} # CometMock takes the role of the query node if [[ "$USE_COMETMOCK" == "true" ]]; then sleep 2 - ip netns exec $QUERY_NET_NAMESPACE_NAME cometmock $NODE_LISTEN_ADDR_STR /$CHAIN_ID/genesis.json tcp://$CHAIN_IP_PREFIX.$QUERY_IP_SUFFIX:26658 $NODE_HOMES grpc &> cometmock_${CHAIN_ID}_out.log & + ip netns exec $QUERY_NET_NAMESPACE_NAME cometmock $NODE_LISTEN_ADDR_STR /$CHAIN_HOME/genesis.json tcp://$CHAIN_IP_PREFIX.$QUERY_IP_SUFFIX:26658 $NODE_HOMES grpc &> cometmock_${CHAIN_ID}_out.log & sleep 3 fi @@ -377,7 +379,7 @@ fi # poll for chain start if [[ "$USE_COMETMOCK" == "true" ]]; then set +e - until $BIN query block --type=height 1--node "tcp://$CHAIN_IP_PREFIX.$QUERY_IP_SUFFIX:26658"; do sleep 0.3 ; done + until $BIN query block --type=height 1 --node "tcp://$CHAIN_IP_PREFIX.$QUERY_IP_SUFFIX:26658"; do sleep 0.3 ; done set -e else set +e