e2e: adding e2e upgrade test for ibc-go/v6 (#2490)

* adding x/group to simapp

* [WIP] initial groups e2e scaffolding

* work in progress

* clean

* adding interchain account address query to ica controller

* adding basic cli query

* satisfy linter, aligning recvr var naming

* initial passing, register account proposal

* successfully passing locally

* updating default genesis state with allow all ica msgs, cleanup

* reinstate num validators and num full nodes

* unpin ibctest

* Update e2e/tests/interchain_accounts/groups_test.go

Co-authored-by: Carlos Rodriguez <carlos@interchain.io>

* updating swagger docs

* updating e2e to ibc-go/v6

* WIP e2e upgrades for ibc-go/v6

* test manual e2e icad from branch

* whoops

* replace transfer logic with ics27 logic

* removing pr number version tags

* upgrade ica dep and tidy

* removing upgrade name const

* update deps, remove genesis code, extract consts

* code comments and consts

* readding intertx query client

* fixing godoc

* removing json entry in test matracies

* refactor upgrades tests

* hardcoding image values

* updating workflows and correcting import

* adding sanity checks and in-line comment

* chore: adapt upgrade e2e workflow for custom images (#2541)

* e2e upgrade test workflows in progress

* adding chain binary to workflow

* adding upgrade to compatibility workflows

* Revert "adding upgrade to compatibility workflows"

This reverts commit 7701999.

* replace v5 tag in upgrade e2e

* updating image tags

* use v5.0.1 release version for upgrades e2e

Co-authored-by: Carlos Rodriguez <carlos@interchain.io>
Co-authored-by: colin axnér <25233464+colin-axner@users.noreply.github.com>

.github/workflows/e2e-manual-icad.yaml

@@ -12,6 +12,7 @@ on:
         options:
           - TestInterTxTestSuite
           - TestIncentivizedInterTxTestSuite
+          - TestUpgradeTestSuite
       chain-image:
         description: 'The image to use for chain A'
         required: true
@@ -24,6 +25,8 @@ on:
         default: master
         options:
           - master
+          - v0.4.0
+          - v0.3.5
           - v0.3.4
           - v0.2.3
           - v0.1.4
@@ -34,6 +37,8 @@ on:
         type: choice
         options:
           - master
+          - v0.4.0
+          - v0.3.5
           - v0.3.4
           - v0.2.3
           - v0.1.4

.github/workflows/e2e-upgrade.yaml

@@ -1,8 +1,5 @@
 name: Tests / E2E Upgrade
-on:
-  workflow_dispatch:
-  schedule:
-    - cron: '0 0 * * *'
+on: workflow_dispatch
 
 jobs:
   upgrade-tests:
@@ -12,6 +9,16 @@ jobs:
       matrix:
         include:
           - test: TestV4ToV5ChainUpgrade
+            chain-image: ghcr.io/cosmos/ibc-go-simd
+            chain-a-tag: v4.1.0
+            chain-b-tag: v4.1.0
+            chain-upgrade-tag: v5.0.1
+          - test: TestV5ToV6ChainUpgrade
+            chain-image: ghcr.io/cosmos/ibc-go-icad
+            chain-binary: icad
+            chain-a-tag: v0.3.5
+            chain-b-tag: v0.3.5
+            chain-upgrade-tag: v0.4.0
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
@@ -21,3 +28,9 @@ jobs:
         run: |
           cd e2e
           make e2e-test entrypoint=TestUpgradeTestSuite test=${{ matrix.test }}
+        env:
+          CHAIN_IMAGE: ${{ matrix.chain-image }}
+          CHAIN_BINARY: ${{ matrix.chain-binary }}
+          CHAIN_A_TAG: ${{ matrix.chain-a-tag }}
+          CHAIN_B_TAG: ${{ matrix.chain-b-tag }}
+          CHAIN_UPGRADE_TAG: ${{ matrix.chain-upgrade-tag }}

e2e/testconfig/testconfig.go

@@ -31,6 +31,8 @@ const (
 	GoRelayerTagEnv = "RLY_TAG"
 	// ChainBinaryEnv binary is the binary that will be used for both chains.
 	ChainBinaryEnv = "CHAIN_BINARY"
+	// ChainUpgradeTagEnv specifies the upgrade version tag
+	ChainUpgradeTagEnv = "CHAIN_UPGRADE_TAG"
 	// defaultBinary is the default binary that will be used by the chains.
 	defaultBinary = "simd"
 	// defaultRlyTag is the tag that will be used if no relayer tag is specified.
@@ -49,6 +51,7 @@ type TestConfig struct {
 	ChainAConfig ChainConfig
 	ChainBConfig ChainConfig
 	RlyTag       string
+	UpgradeTag   string
 }
 
 type ChainConfig struct {
@@ -86,6 +89,11 @@ func FromEnv() TestConfig {
 	}
 	chainBImage := chainAImage
 
+	upgradeTag, ok := os.LookupEnv(ChainUpgradeTagEnv)
+	if !ok {
+		upgradeTag = ""
+	}
+
 	return TestConfig{
 		ChainAConfig: ChainConfig{
 			Image:  chainAImage,
@@ -97,7 +105,8 @@ func FromEnv() TestConfig {
 			Tag:    chainBTag,
 			Binary: chainBinary,
 		},
-		RlyTag: rlyTag,
+		RlyTag:     rlyTag,
+		UpgradeTag: upgradeTag,
 	}
 }
 

e2e/tests/upgrades/upgrade_test.go

@@ -3,25 +3,32 @@ package upgrades
 import (
 	"context"
 	"fmt"
-	"os"
 	"testing"
 	"time"
 
+	sdk "github.com/cosmos/cosmos-sdk/types"
+	banktypes "github.com/cosmos/cosmos-sdk/x/bank/types"
 	upgradetypes "github.com/cosmos/cosmos-sdk/x/upgrade/types"
+	v6upgrades "github.com/cosmos/interchain-accounts/app/upgrades/v6"
+	intertxtypes "github.com/cosmos/interchain-accounts/x/inter-tx/types"
+	ibctest "github.com/strangelove-ventures/ibctest/v6"
 	"github.com/strangelove-ventures/ibctest/v6/chain/cosmos"
 	"github.com/strangelove-ventures/ibctest/v6/ibc"
 	"github.com/strangelove-ventures/ibctest/v6/test"
 	"github.com/stretchr/testify/suite"
+	"golang.org/x/mod/semver"
 
 	"github.com/cosmos/ibc-go/e2e/testconfig"
 	"github.com/cosmos/ibc-go/e2e/testsuite"
 	"github.com/cosmos/ibc-go/e2e/testvalues"
+	icatypes "github.com/cosmos/ibc-go/v6/modules/apps/27-interchain-accounts/types"
+	ibctesting "github.com/cosmos/ibc-go/v6/testing"
+	simappupgrades "github.com/cosmos/ibc-go/v6/testing/simapp/upgrades"
 )
 
 const (
 	haltHeight         = uint64(100)
 	blocksAfterUpgrade = uint64(10)
-	defaultUpgradeName = "normal upgrade"
 )
 
 func TestUpgradeTestSuite(t *testing.T) {
@@ -34,14 +41,14 @@ type UpgradeTestSuite struct {
 
 // UpgradeChain upgrades a chain to a specific version using the planName provided.
 // The software upgrade proposal is broadcast by the provided wallet.
-func (s *UpgradeTestSuite) UpgradeChain(ctx context.Context, chain *cosmos.CosmosChain, wallet *ibc.Wallet, planName, upgradeVersion string) {
-	prevVersion := chain.Nodes()[0].Image.Version
+func (s *UpgradeTestSuite) UpgradeChain(ctx context.Context, chain *cosmos.CosmosChain, wallet *ibc.Wallet, planName, currentVersion, upgradeVersion string) {
 	plan := upgradetypes.Plan{
 		Name:   planName,
 		Height: int64(haltHeight),
-		Info:   fmt.Sprintf("upgrade version test from %s to %s", prevVersion, upgradeVersion),
+		Info:   fmt.Sprintf("upgrade version test from %s to %s", currentVersion, upgradeVersion),
 	}
-	upgradeProposal := upgradetypes.NewSoftwareUpgradeProposal(fmt.Sprintf("upgrade from %s to %s", prevVersion, upgradeVersion), "upgrade chain E2E test", plan)
+
+	upgradeProposal := upgradetypes.NewSoftwareUpgradeProposal(fmt.Sprintf("upgrade from %s to %s", currentVersion, upgradeVersion), "upgrade chain E2E test", plan)
 	s.ExecuteGovProposal(ctx, chain, wallet, upgradeProposal)
 
 	height, err := chain.Height(ctx)
@@ -70,16 +77,12 @@ func (s *UpgradeTestSuite) UpgradeChain(ctx context.Context, chain *cosmos.Cosmo
 	height, err = chain.Height(ctx)
 	s.Require().NoError(err, "error fetching height after upgrade")
 
-	s.Require().GreaterOrEqual(height, haltHeight+blocksAfterUpgrade, "height did not increment enough after upgrade")
+	s.Require().Greater(height, haltHeight, "height did not increment after upgrade")
 }
 
 func (s *UpgradeTestSuite) TestV4ToV5ChainUpgrade() {
 	t := s.T()
-	// TODO: temporarily hard code the version upgrades.
-	oldVersion := "v4.0.0"
-	targetVersion := "pr-2144" // v5 version with upgrade handler, replace with v5.0.0-rc3 when it is cut.
-	s.Require().NoError(os.Setenv(testconfig.ChainATagEnv, oldVersion))
-	s.Require().NoError(os.Setenv(testconfig.ChainBTagEnv, oldVersion))
+	testCfg := testconfig.FromEnv()
 
 	ctx := context.Background()
 	relayer, channelA := s.SetupChainsRelayerAndChannel(ctx)
@@ -136,7 +139,7 @@ func (s *UpgradeTestSuite) TestV4ToV5ChainUpgrade() {
 	s.Require().NoError(test.WaitForBlocks(ctx, 5, chainA, chainB), "failed to wait for blocks")
 
 	t.Run("upgrade chainA", func(t *testing.T) {
-		s.UpgradeChain(ctx, chainA, chainAUpgradeProposalWallet, defaultUpgradeName, targetVersion)
+		s.UpgradeChain(ctx, chainA, chainAUpgradeProposalWallet, simappupgrades.DefaultUpgradeName, testCfg.ChainAConfig.Tag, testCfg.UpgradeTag)
 	})
 
 	t.Run("restart relayer", func(t *testing.T) {
@@ -182,3 +185,175 @@ func (s *UpgradeTestSuite) TestV4ToV5ChainUpgrade() {
 		})
 	})
 }
+
+func (s *UpgradeTestSuite) TestV5ToV6ChainUpgrade() {
+	t := s.T()
+	testCfg := testconfig.FromEnv()
+
+	ctx := context.Background()
+	relayer, _ := s.SetupChainsRelayerAndChannel(ctx)
+	chainA, chainB := s.GetChains()
+
+	// create separate user specifically for the upgrade proposal to more easily verify starting
+	// and end balances of the chainA users.
+	chainAUpgradeProposalWallet := s.CreateUserOnChainA(ctx, testvalues.StartingTokenAmount)
+
+	s.Require().NoError(test.WaitForBlocks(ctx, 1, chainA, chainB), "failed to wait for blocks")
+
+	// setup 2 accounts: controller account on chain A, a second chain B account.
+	// host account will be created when the ICA is registered
+	controllerAccount := s.CreateUserOnChainA(ctx, testvalues.StartingTokenAmount)
+	chainBAccount := s.CreateUserOnChainB(ctx, testvalues.StartingTokenAmount)
+	var hostAccount string
+
+	t.Run("register interchain account", func(t *testing.T) {
+		version := getICAVersion(testconfig.GetChainATag(), testconfig.GetChainBTag())
+		msgRegisterAccount := intertxtypes.NewMsgRegisterAccount(controllerAccount.Bech32Address(chainA.Config().Bech32Prefix), ibctesting.FirstConnectionID, version)
+		err := s.RegisterInterchainAccount(ctx, chainA, controllerAccount, msgRegisterAccount)
+		s.Require().NoError(err)
+	})
+
+	t.Run("start relayer", func(t *testing.T) {
+		s.StartRelayer(relayer)
+	})
+
+	t.Run("verify interchain account", func(t *testing.T) {
+		var err error
+		hostAccount, err = s.QueryInterchainAccount(ctx, chainA, controllerAccount.Bech32Address(chainA.Config().Bech32Prefix), ibctesting.FirstConnectionID)
+		s.Require().NoError(err)
+		s.Require().NotZero(len(hostAccount))
+
+		channels, err := relayer.GetChannels(ctx, s.GetRelayerExecReporter(), chainA.Config().ChainID)
+		s.Require().NoError(err)
+		s.Require().Equal(len(channels), 2)
+	})
+
+	t.Run("interchain account executes a bank transfer on behalf of the corresponding owner account", func(t *testing.T) {
+		t.Run("fund interchain account wallet", func(t *testing.T) {
+			// fund the host account account so it has some $$ to send
+			err := chainB.SendFunds(ctx, ibctest.FaucetAccountKeyName, ibc.WalletAmount{
+				Address: hostAccount,
+				Amount:  testvalues.StartingTokenAmount,
+				Denom:   chainB.Config().Denom,
+			})
+			s.Require().NoError(err)
+		})
+
+		t.Run("broadcast MsgSubmitTx", func(t *testing.T) {
+			// assemble bank transfer message from host account to user account on host chain
+			msgSend := &banktypes.MsgSend{
+				FromAddress: hostAccount,
+				ToAddress:   chainBAccount.Bech32Address(chainB.Config().Bech32Prefix),
+				Amount:      sdk.NewCoins(testvalues.DefaultTransferAmount(chainB.Config().Denom)),
+			}
+
+			// assemble submitMessage tx for intertx
+			msgSubmitTx, err := intertxtypes.NewMsgSubmitTx(
+				msgSend,
+				ibctesting.FirstConnectionID,
+				controllerAccount.Bech32Address(chainA.Config().Bech32Prefix),
+			)
+			s.Require().NoError(err)
+
+			// broadcast submitMessage tx from controller account on chain A
+			// this message should trigger the sending of an ICA packet over channel-1 (channel created between controller and host)
+			// this ICA packet contains the assembled bank transfer message from above, which will be executed by the host account on the host chain.
+			resp, err := s.BroadcastMessages(
+				ctx,
+				chainA,
+				controllerAccount,
+				msgSubmitTx,
+			)
+
+			s.AssertValidTxResponse(resp)
+			s.Require().NoError(err)
+
+			s.Require().NoError(test.WaitForBlocks(ctx, 10, chainA, chainB))
+		})
+
+		t.Run("verify tokens transferred", func(t *testing.T) {
+			balance, err := chainB.GetBalance(ctx, chainBAccount.Bech32Address(chainB.Config().Bech32Prefix), chainB.Config().Denom)
+			s.Require().NoError(err)
+
+			_, err = chainB.GetBalance(ctx, hostAccount, chainB.Config().Denom)
+			s.Require().NoError(err)
+
+			expected := testvalues.IBCTransferAmount + testvalues.StartingTokenAmount
+			s.Require().Equal(expected, balance)
+		})
+	})
+
+	s.Require().NoError(test.WaitForBlocks(ctx, 5, chainA, chainB), "failed to wait for blocks")
+
+	t.Run("upgrade chainA", func(t *testing.T) {
+		s.UpgradeChain(ctx, chainA, chainAUpgradeProposalWallet, v6upgrades.UpgradeName, testCfg.ChainAConfig.Tag, testCfg.UpgradeTag)
+	})
+
+	t.Run("restart relayer", func(t *testing.T) {
+		s.StopRelayer(ctx, relayer)
+		s.StartRelayer(relayer)
+	})
+
+	t.Run("broadcast MsgSubmitTx", func(t *testing.T) {
+		// assemble bank transfer message from host account to user account on host chain
+		msgSend := &banktypes.MsgSend{
+			FromAddress: hostAccount,
+			ToAddress:   chainBAccount.Bech32Address(chainB.Config().Bech32Prefix),
+			Amount:      sdk.NewCoins(testvalues.DefaultTransferAmount(chainB.Config().Denom)),
+		}
+
+		// assemble submitMessage tx for intertx
+		msgSubmitTx, err := intertxtypes.NewMsgSubmitTx(
+			msgSend,
+			ibctesting.FirstConnectionID,
+			controllerAccount.Bech32Address(chainA.Config().Bech32Prefix),
+		)
+		s.Require().NoError(err)
+
+		// broadcast submitMessage tx from controller account on chain A
+		// this message should trigger the sending of an ICA packet over channel-1 (channel created between controller and host)
+		// this ICA packet contains the assembled bank transfer message from above, which will be executed by the host account on the host chain.
+		resp, err := s.BroadcastMessages(
+			ctx,
+			chainA,
+			controllerAccount,
+			msgSubmitTx,
+		)
+
+		s.AssertValidTxResponse(resp)
+		s.Require().NoError(err)
+
+		s.Require().NoError(test.WaitForBlocks(ctx, 10, chainA, chainB))
+	})
+
+	t.Run("verify tokens transferred", func(t *testing.T) {
+		balance, err := chainB.GetBalance(ctx, chainBAccount.Bech32Address(chainB.Config().Bech32Prefix), chainB.Config().Denom)
+		s.Require().NoError(err)
+
+		_, err = chainB.GetBalance(ctx, hostAccount, chainB.Config().Denom)
+		s.Require().NoError(err)
+
+		expected := (testvalues.IBCTransferAmount * 2) + testvalues.StartingTokenAmount
+		s.Require().Equal(expected, balance)
+	})
+}
+
+// RegisterInterchainAccount will attempt to register an interchain account on the counterparty chain.
+func (s *UpgradeTestSuite) RegisterInterchainAccount(ctx context.Context, chain *cosmos.CosmosChain, user *ibc.Wallet, msgRegisterAccount *intertxtypes.MsgRegisterAccount) error {
+	txResp, err := s.BroadcastMessages(ctx, chain, user, msgRegisterAccount)
+	s.Require().NoError(err)
+	s.AssertValidTxResponse(txResp)
+	return err
+}
+
+// getICAVersion returns the version which should be used in the MsgRegisterAccount broadcast from the
+// controller chain.
+func getICAVersion(chainAVersion, chainBVersion string) string {
+	chainBIsGreaterThanOrEqualToChainA := semver.Compare(chainAVersion, chainBVersion) <= 0
+	if chainBIsGreaterThanOrEqualToChainA {
+		// allow version to be specified by the controller chain
+		return ""
+	}
+	// explicitly set the version string because the host chain might not yet support incentivized channels.
+	return icatypes.NewDefaultMetadataString(ibctesting.FirstConnectionID, ibctesting.FirstConnectionID)
+}