refactor(systemtests): Extract system test framework (backport #22578) (

#22655)

Co-authored-by: Alexander Peters <alpe@users.noreply.github.com>
Co-authored-by: Julien Robert <julien@rbrt.fr>

tests/systemtests/.gitignore

@@ -1,2 +1,3 @@
 /testnet
 /binaries
+foo/

tests/systemtests/README.md

@@ -1,61 +1,48 @@
-# Testing
+# System tests
 
-Test framework for system tests. 
-Starts and interacts with a (multi node) blockchain in Go.
-Supports
+Go black box tests that setup and interact with a local blockchain. The system test [framework](../../systemtests) 
+works with the compiled binary of the chain artifact only.
+To get up to speed, checkout the [getting started guide](../../systemtests/getting_started.md).
 
-* CLI
-* Servers
-* Events
-* RPC
+Beside the Go tests and testdata files, this directory can contain the following directories:  
 
-Uses:
+* `binaries` - cache for binary
+* `testnet` - node files
 
-* testify
-* gjson
-* sjson
+Please make sure to not add or push them to git. 
 
-Server and client side are executed on the host machine.
+## Execution
 
-## Developer
+Build a new binary from current branch and copy it to the `tests/systemtests/binaries` folder by running system tests.
+In project root:
 
-### Test strategy
-
-System tests cover the full stack via cli and a running (multi node) network. They are more expensive (in terms of time/ cpu) 
-to run compared to unit or integration tests. 
-Therefore, we focus on the **critical path** and do not cover every condition.
-
-## How to use
-
-Read the [getting_started.md](getting_started.md) guide to get started.
-
-### Execute a single test
-
-```sh
-go test -tags system_test -count=1 -v . --run TestStakeUnstake  -verbose
+```shell
+make test-system
 ```
 
-Test cli parameters
-
-* `-verbose` verbose output
-* `-wait-time` duration - time to wait for chain events (default 30s)
-* `-nodes-count` int - number of nodes in the cluster (default 4)
+Or via manual steps
 
-# Port ranges
+```shell
+make build
+mkdir -p ./tests/systemtests/binaries
+cp ./build/simd ./tests/systemtests/binaries/
+```
 
-With *n* nodes:
+### Manual test run
 
-* `26657` - `26657+n` - RPC
-* `1317` - `1317+n` - API
-* `9090` - `9090+n` - GRPC
-* `16656` - `16656+n` - P2P
+```shell
+go test -v -mod=readonly -failfast -tags='system_test' --run TestStakeUnstake    ./... --verbose
+```
 
-For example Node *3* listens on `26660` for RPC calls
+### Working with macOS
 
-## Resources
+Most tests should function seamlessly. However, the file [upgrade_test.go](upgrade_test.go) includes a **build annotation** for Linux only.
 
-* [gjson query syntax](https://github.com/tidwall/gjson#path-syntax)
+For the system upgrade test, an older version of the binary is utilized to perform a chain upgrade. This artifact is retrieved from a Docker container built for Linux.
 
-## Disclaimer
+To circumvent this limitation locally:
+1. Checkout and build the older version of the artifact from a specific tag for your OS.
+2. Place the built artifact into the `binaries` folder.
+3. Ensure that the filename, including the version, is correct.
 
-This is based on the system test framework in [wasmd](https://github.com/CosmWasm/wasmd) built by Confio.
+With the cached artifact in place, the test will use this file instead of attempting to pull it from Docker.

tests/systemtests/account_test.go

@@ -8,6 +8,8 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/tidwall/gjson"
+
+	systest "cosmossdk.io/systemtests"
 )
 
 func TestAccountCreation(t *testing.T) {
@@ -18,24 +20,24 @@ func TestAccountCreation(t *testing.T) {
 	// when accountB is sending funds to accountA,
 	// AccountB should be created
 
-	sut.ResetChain(t)
-	cli := NewCLIWrapper(t, sut, verbose)
+	systest.Sut.ResetChain(t)
+	cli := systest.NewCLIWrapper(t, systest.Sut, systest.Verbose)
 	// add genesis account with some tokens
 	account1Addr := cli.AddKey("account1")
 	account2Addr := cli.AddKey("account2")
 	require.NotEqual(t, account1Addr, account2Addr)
-	sut.ModifyGenesisCLI(t,
+	systest.Sut.ModifyGenesisCLI(t,
 		[]string{"genesis", "add-genesis-account", account1Addr, "10000000stake"},
 	)
 
-	sut.StartChain(t)
+	systest.Sut.StartChain(t)
 
 	// query account1
 	rsp := cli.CustomQuery("q", "auth", "account", account1Addr)
 	assert.Equal(t, account1Addr, gjson.Get(rsp, "account.value.address").String(), rsp)
 
 	rsp1 := cli.RunAndWait("tx", "bank", "send", account1Addr, account2Addr, "5000stake", "--from="+account1Addr, "--fees=1stake")
-	RequireTxSuccess(t, rsp1)
+	systest.RequireTxSuccess(t, rsp1)
 
 	// query account2
 	assertNotFound := func(t assert.TestingT, err error, msgAndArgs ...interface{}) (ok bool) {
@@ -44,7 +46,7 @@ func TestAccountCreation(t *testing.T) {
 	_ = cli.WithRunErrorMatcher(assertNotFound).CustomQuery("q", "auth", "account", account2Addr)
 
 	rsp3 := cli.RunAndWait("tx", "bank", "send", account2Addr, account1Addr, "1000stake", "--from="+account2Addr, "--fees=1stake")
-	RequireTxSuccess(t, rsp3)
+	systest.RequireTxSuccess(t, rsp3)
 
 	// query account2 to make sure its created
 	rsp4 := cli.CustomQuery("q", "auth", "account", account2Addr)
@@ -54,19 +56,19 @@ func TestAccountCreation(t *testing.T) {
 }
 
 func TestAccountsMigration(t *testing.T) {
-	sut.ResetChain(t)
-	cli := NewCLIWrapper(t, sut, verbose)
+	systest.Sut.ResetChain(t)
+	cli := systest.NewCLIWrapper(t, systest.Sut, systest.Verbose)
 
-	legacyAddress := cli.GetKeyAddr(defaultSrcAddr)
+	legacyAddress := cli.GetKeyAddr("node0")
 	// Create a receiver account
 	receiverName := "receiver-account"
 	receiverAddress := cli.AddKey(receiverName)
 	require.NotEmpty(t, receiverAddress)
-	sut.ModifyGenesisCLI(t,
+	systest.Sut.ModifyGenesisCLI(t,
 		[]string{"genesis", "add-genesis-account", receiverAddress, "1000000stake"},
 	)
 
-	sut.StartChain(t)
+	systest.Sut.StartChain(t)
 
 	// Get pubkey
 	pubKeyValue := cli.GetPubKeyByCustomField(legacyAddress, "address")
@@ -99,7 +101,7 @@ func TestAccountsMigration(t *testing.T) {
 		fmt.Sprintf("--account-init-msg=%s", accountInitMsg),
 		fmt.Sprintf("--from=%s", legacyAddress),
 		"--fees=1stake")
-	RequireTxSuccess(t, rsp)
+	systest.RequireTxSuccess(t, rsp)
 
 	// 3. Now the account should be existed, query the account Sequence
 	rsp = cli.CustomQuery("q", "accounts", "query", legacyAddress, "cosmos.accounts.defaults.base.v1.QuerySequence", "{}")
@@ -121,7 +123,7 @@ func TestAccountsMigration(t *testing.T) {
 		transferAmount+"stake",
 		fmt.Sprintf("--from=%s", legacyAddress),
 		"--fees=1stake")
-	RequireTxSuccess(t, rsp)
+	systest.RequireTxSuccess(t, rsp)
 
 	// Verify the balances after the transaction
 	newLegacyBalance := cli.QueryBalance(legacyAddress, "stake")
@@ -151,5 +153,5 @@ func TestAccountsMigration(t *testing.T) {
 	rsp = cli.RunAndWait("tx", "accounts", "execute", legacyAddress, "cosmos.accounts.defaults.base.v1.MsgSwapPubKey", swapKeyMsg,
 		fmt.Sprintf("--from=%s", legacyAddress),
 		"--fees=1stake")
-	RequireTxSuccess(t, rsp)
+	systest.RequireTxSuccess(t, rsp)
 }