feat(zombienet): add zombienet testing to github workflow

.github/workflows/zombienet.yml

@@ -0,0 +1,56 @@
+name: zombienet
+run-name: Zombienet tests
+on:
+  pull_request:
+
+jobs:
+  zombienet-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+          stable: true
+          check-latest: true
+
+      - name: Set cache variables
+        id: go-cache-paths
+        run: |
+          echo "::set-output name=go-build::$(go env GOCACHE)"
+          echo "::set-output name=go-mod::$(go env GOMODCACHE)"
+      - uses: actions/checkout@v3
+
+      - name: Go build cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ steps.go-cache-paths.outputs.go-build }}
+          key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ runner.os }}-go-build
+
+      - name: Go modules cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ steps.go-cache-paths.outputs.go-mod }}
+          key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ runner.os }}-go-mod
+
+      - name: Run build
+        run: |
+          make build
+          echo "$HOME/work/gossamer/gossamer/bin" >> $GITHUB_PATH
+
+      - name: Install Zombienet
+        run: |
+          if [ "$RUNNER_OS" == "Linux" ]; then
+            wget -O /usr/local/bin/zombienet https://github.com/paritytech/zombienet/releases/download/v1.3.41/zombienet-linux-x64
+          elif [ "$RUNNER_OS" == "macOS" ]; then
+            wget -O /usr/local/bin/zombienet https://github.com/paritytech/zombienet/releases/download/v1.3.41/zombienet-macos
+          else
+            echo "Zombienet for $RUNNER_OS is not supported"
+            exit 1
+          fi
+          chmod +x /usr/local/bin/zombienet
+
+      - name: Zombienet test
+        run: |
+          zombienet test -p native zombienet_tests/functional/0001-basic-network.zndsl

chain/resources.go

@@ -0,0 +1,10 @@
+// Copyright 2023 ChainSafe Systems (ON)
+// SPDX-License-Identifier: LGPL-3.0-only
+package chain
+
+import "embed"
+
+// DefaultConfigTomlFiles is the embedded file system containing the default toml configurations.
+//
+//go:embed */*.toml
+var DefaultConfigTomlFiles embed.FS

cmd/gossamer/config.go

@@ -26,18 +26,18 @@ import (
 var (
 	// DefaultCfg is the default configuration for the node.
 	DefaultCfg                  = dot.PolkadotConfig
-	defaultKusamaConfigPath     = "./chain/kusama/config.toml"
-	defaultPolkadotConfigPath   = "./chain/polkadot/config.toml"
-	defaultWestendDevConfigPath = "./chain/westend-dev/config.toml"
-	defaultWestendConfigPath    = "./chain/westend/config.toml"
+	defaultKusamaConfigPath     = "kusama/config.toml"
+	defaultPolkadotConfigPath   = "polkadot/config.toml"
+	defaultWestendDevConfigPath = "westend-dev/config.toml"
+	defaultWestendConfigPath    = "westend/config.toml"
 )
 
 // loadConfigFile loads a default config file if --chain is specified, a specific
 // config if --config is specified, or the default gossamer config otherwise.
 func loadConfigFile(ctx *cli.Context, cfg *ctoml.Config) (err error) {
 	cfgPath := ctx.String(ConfigFlag.Name)
 	if cfgPath == "" {
-		return loadConfig(cfg, defaultPolkadotConfigPath)
+		return loadConfigFromResource(cfg, defaultPolkadotConfigPath)
 	}
 
 	logger.Info("loading toml configuration from " + cfgPath + "...")
@@ -48,7 +48,7 @@ func loadConfigFile(ctx *cli.Context, cfg *ctoml.Config) (err error) {
 			"overwriting default configuration with id " + cfg.Global.ID +
 				" with toml configuration values from " + cfgPath)
 	}
-	return loadConfig(cfg, cfgPath)
+	return loadConfigFromFile(cfg, cfgPath)
 }
 
 func setupConfigFromChain(ctx *cli.Context) (*ctoml.Config, *dot.Config, error) {
@@ -68,22 +68,22 @@ func setupConfigFromChain(ctx *cli.Context) (*ctoml.Config, *dot.Config, error)
 			logger.Info("loading toml configuration from " + defaultKusamaConfigPath + "...")
 			tomlCfg = &ctoml.Config{}
 			cfg = dot.KusamaConfig()
-			err = loadConfig(tomlCfg, defaultKusamaConfigPath)
+			err = loadConfigFromResource(tomlCfg, defaultKusamaConfigPath)
 		case "polkadot":
 			logger.Info("loading toml configuration from " + defaultPolkadotConfigPath + "...")
 			tomlCfg = &ctoml.Config{}
 			cfg = dot.PolkadotConfig()
-			err = loadConfig(tomlCfg, defaultPolkadotConfigPath)
+			err = loadConfigFromResource(tomlCfg, defaultPolkadotConfigPath)
 		case "westend-dev":
 			logger.Info("loading toml configuration from " + defaultWestendDevConfigPath + "...")
 			tomlCfg = &ctoml.Config{}
 			cfg = dot.WestendDevConfig()
-			err = loadConfig(tomlCfg, defaultWestendDevConfigPath)
+			err = loadConfigFromResource(tomlCfg, defaultWestendDevConfigPath)
 		case "westend":
 			logger.Info("loading toml configuration from " + defaultWestendConfigPath + "...")
 			tomlCfg = &ctoml.Config{}
 			cfg = dot.WestendConfig()
-			err = loadConfig(tomlCfg, defaultWestendConfigPath)
+			err = loadConfigFromResource(tomlCfg, defaultWestendConfigPath)
 		default:
 			logger.Info("loading chain config from " + id + "...")
 			fileInfo, err := os.Stat(id)
@@ -909,10 +909,7 @@ func updateDotConfigFromGenesisData(ctx *cli.Context, cfg *dot.Config) error {
 		return fmt.Errorf("failed to load genesis data: %s", err)
 	}
 
-	// check genesis id and use genesis id if --chain flag not set
-	if !ctx.IsSet(ChainFlag.Name) {
-		cfg.Global.ID = gen.ID
-	}
+	cfg.Global.ID = gen.ID
 
 	// check genesis bootnodes and use genesis --bootnodes if name flag not set
 	if !ctx.IsSet(BootnodesFlag.Name) {

cmd/gossamer/config_test.go

@@ -845,7 +845,7 @@ func TestGlobalNodeName_WhenNodeAlreadyHasStoredName(t *testing.T) {
 	cfg := newTestConfig(t, westendDevConfig)
 	cfg.Global.Name = globalName
 
-	runtimeFilePath, err := runtime.GetRuntime(context.Background(), runtime.NODE_RUNTIME)
+	runtimeFilePath, err := runtime.GetRuntime(context.Background(), runtime.WESTEND_RUNTIME_v0929)
 	require.NoError(t, err)
 	runtimeData, err := os.ReadFile(runtimeFilePath)
 	require.NoError(t, err)

cmd/gossamer/export_test.go

@@ -8,7 +8,6 @@ import (
 	"testing"
 
 	"github.com/ChainSafe/gossamer/dot"
-
 	ctoml "github.com/ChainSafe/gossamer/dot/config/toml"
 	"github.com/ChainSafe/gossamer/internal/log"
 	"github.com/stretchr/testify/require"
@@ -165,7 +164,7 @@ func TestExportCommand(t *testing.T) {
 			config := ctx.String(ConfigFlag.Name)
 
 			cfg := new(ctoml.Config)
-			err = loadConfig(cfg, config)
+			err = loadConfigFromFile(cfg, config)
 			require.NoError(t, err)
 
 			require.Equal(t, dotConfigToToml(c.expected), cfg)

cmd/gossamer/main_test.go

@@ -192,6 +192,8 @@ func runTestGossamer(t *testing.T, args ...string) *testgossamer {
 	return tt
 }
 
+var testWestendDevConfigPath string
+
 func TestMain(m *testing.M) {
 	if reexec.Init() {
 		return
@@ -202,10 +204,7 @@ func TestMain(m *testing.M) {
 		panic(err)
 	}
 
-	defaultKusamaConfigPath = filepath.Join(rootPath, "./chain/kusama/config.toml")
-	defaultPolkadotConfigPath = filepath.Join(rootPath, "./chain/polkadot/config.toml")
-	defaultWestendDevConfigPath = filepath.Join(rootPath, "./chain/westend-dev/config.toml")
-
+	testWestendDevConfigPath = filepath.Join(rootPath, "./chain/westend-dev/config.toml")
 	os.Exit(m.Run())
 }
 
@@ -234,7 +233,7 @@ func TestInitCommand_RenameNodeWhenCalled(t *testing.T) {
 		"--basepath", tempDir,
 		"--genesis", genesisPath,
 		"--name", nodeName,
-		"--config", defaultWestendDevConfigPath,
+		"--config", testWestendDevConfigPath,
 		"--force",
 	)
 
@@ -249,7 +248,7 @@ func TestInitCommand_RenameNodeWhenCalled(t *testing.T) {
 		"init",
 		"--basepath", tempDir,
 		"--genesis", genesisPath,
-		"--config", defaultWestendDevConfigPath,
+		"--config", testWestendDevConfigPath,
 		"--force",
 	)
 

cmd/gossamer/toml_config.go

@@ -11,25 +11,35 @@ import (
 	"reflect"
 	"unicode"
 
-	"github.com/naoina/toml"
-
+	"github.com/ChainSafe/gossamer/chain"
 	ctoml "github.com/ChainSafe/gossamer/dot/config/toml"
+	"github.com/naoina/toml"
 )
 
-// loadConfig loads the values from the toml configuration file into the provided configuration
-func loadConfig(cfg *ctoml.Config, fp string) error {
+func loadConfigFromResource(cfg *ctoml.Config, resourcePath string) error {
+	file, err := chain.DefaultConfigTomlFiles.Open(resourcePath)
+	if err != nil {
+		return fmt.Errorf("opening toml configuration file: %w", err)
+	}
+	return loadConfig(cfg, file)
+}
+
+func loadConfigFromFile(cfg *ctoml.Config, fp string) error {
 	fp, err := filepath.Abs(fp)
 	if err != nil {
 		logger.Errorf("failed to create absolute path for toml configuration file: %s", err)
 		return err
 	}
-
 	file, err := os.Open(filepath.Clean(fp))
 	if err != nil {
 		logger.Errorf("failed to open toml configuration file: %s", err)
 		return err
 	}
+	return loadConfig(cfg, file)
+}
 
+// loadConfig loads the values from the toml configuration file into the provided configuration
+func loadConfig(cfg *ctoml.Config, file fs.File) (err error) {
 	var tomlSettings = toml.Config{
 		NormFieldName: func(rt reflect.Type, key string) string {
 			return key

cmd/gossamer/toml_config_test.go

@@ -25,7 +25,7 @@ func TestLoadConfig(t *testing.T) {
 	err := dot.InitNode(cfg)
 	require.NoError(t, err)
 
-	err = loadConfig(dotConfigToToml(cfg), cfgFile)
+	err = loadConfigFromFile(dotConfigToToml(cfg), cfgFile)
 	require.NoError(t, err)
 }
 
@@ -43,7 +43,7 @@ func TestLoadConfigWestendDev(t *testing.T) {
 	projectRootPath := utils.GetProjectRootPathTest(t)
 	configPath := filepath.Join(projectRootPath, "./chain/westend-dev/config.toml")
 
-	err = loadConfig(dotConfigToToml(cfg), configPath)
+	err = loadConfigFromFile(dotConfigToToml(cfg), configPath)
 	require.NoError(t, err)
 }
 
@@ -60,6 +60,6 @@ func TestLoadConfigKusama(t *testing.T) {
 	projectRootPath := utils.GetProjectRootPathTest(t)
 	kusamaConfigPath := filepath.Join(projectRootPath, "./chain/kusama/config.toml")
 
-	err = loadConfig(dotConfigToToml(cfg), kusamaConfigPath)
+	err = loadConfigFromFile(dotConfigToToml(cfg), kusamaConfigPath)
 	require.NoError(t, err)
 }

dot/core/service_integration_test.go

@@ -288,7 +288,7 @@ func TestHandleChainReorg_WithReorg_Transactions(t *testing.T) {
 	t.Skip() // need to update this test to use a valid transaction
 
 	cfg := &Config{
-		Runtime: wasmer.NewTestInstance(t, runtime.NODE_RUNTIME),
+		Runtime: wasmer.NewTestInstance(t, runtime.WESTEND_RUNTIME_v0929),
 	}
 
 	s := NewTestService(t, cfg)

dot/network/service.go

@@ -87,6 +87,12 @@ var (
 		Name:      "outbound_total",
 		Help:      "total number of outbound streams",
 	})
+	processStartTimeGauge = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: "substrate",
+		Name:      "process_start_time_seconds",
+		Help: "gossamer process start seconds unix timestamp, " +
+			"using substrate namespace so zombienet detects node start",
+	})
 )
 
 type (
@@ -352,6 +358,7 @@ func (s *Service) updateMetrics() {
 		case <-s.ctx.Done():
 			return
 		case <-ticker.C:
+			processStartTimeGauge.Set(float64(time.Now().Unix()))
 			peerCountGauge.Set(float64(s.host.peerCount()))
 			connectionsGauge.Set(float64(len(s.host.p2pHost.Network().Conns())))
 			nodeLatencyGauge.Set(float64(

dot/node.go

@@ -241,7 +241,7 @@ func newNode(cfg *Config,
 	}
 
 	logger.Patch(log.SetLevel(cfg.Global.LogLvl))
-
+	logger.Debugf("node config %+v\n", cfg)
 	logger.Infof(
 		"🕸️ initialising node services with global configuration name %s, id %s and base path %s...",
 		cfg.Global.Name, cfg.Global.ID, cfg.Global.BasePath)

dot/node_integration_test.go

@@ -458,7 +458,7 @@ func TestNode_PersistGlobalName_WhenInitialize(t *testing.T) {
 // newTestGenesisAndRuntime create a new test runtime and a new test genesis
 // file with the test runtime stored in raw data and returns the genesis file
 func newTestGenesisAndRuntime(t *testing.T) (filename string) {
-	runtimeFilePath, err := runtime.GetRuntime(context.Background(), runtime.NODE_RUNTIME)
+	runtimeFilePath, err := runtime.GetRuntime(context.Background(), runtime.WESTEND_RUNTIME_v0929)
 	require.NoError(t, err)
 	runtimeData, err := os.ReadFile(runtimeFilePath)
 	require.NoError(t, err)

dot/rpc/modules/dev_integration_test.go

@@ -57,7 +57,7 @@ func newBABEService(t *testing.T) *babe.Service {
 
 	bs, es := newState(t)
 	tt := trie.NewEmptyTrie()
-	rt := wasmer.NewTestInstanceWithTrie(t, runtime.NODE_RUNTIME, tt)
+	rt := wasmer.NewTestInstanceWithTrie(t, runtime.WESTEND_RUNTIME_v0929, tt)
 	bs.StoreRuntime(bs.GenesisHash(), rt)
 	tt.Put(
 		common.MustHexToBytes("0x886726f904d8372fdabb7707870c2fad"),

dot/rpc/modules/state_integration_test.go

@@ -28,29 +28,35 @@ const (
 )
 
 func TestStateModule_GetRuntimeVersion(t *testing.T) {
-	// expected results based on responses from prior tests
+	/* expected results based on responses from prior tests
+	We can get this data from polkadot runtime release source code
+	https://github.com/paritytech/polkadot/blob/v0.9.29/runtime/westend/src/lib.rs#L105-L117
+	*/
 	expected := StateRuntimeVersionResponse{
-		SpecName:         "node",
-		ImplName:         "substrate-node",
-		AuthoringVersion: 10,
-		SpecVersion:      264,
+		SpecName:         "westend",
+		ImplName:         "parity-westend",
+		AuthoringVersion: 2,
+		SpecVersion:      9290,
 		ImplVersion:      0,
 		Apis: []interface{}{
-			[]interface{}{"0xdf6acb689907609b", uint32(3)},
+			[]interface{}{"0xdf6acb689907609b", uint32(4)},
 			[]interface{}{"0x37e397fc7c91f5e4", uint32(1)},
-			[]interface{}{"0x40fe3ad401f8959a", uint32(4)},
-			[]interface{}{"0xd2bc9897eed08f15", uint32(2)},
+			[]interface{}{"0x40fe3ad401f8959a", uint32(6)},
+			[]interface{}{"0xd2bc9897eed08f15", uint32(3)},
 			[]interface{}{"0xf78b278be53f454c", uint32(2)},
-			[]interface{}{"0xed99c5acb25eedf5", uint32(2)},
+			[]interface{}{"0xaf2c0297a23e6d3d", uint32(2)},
+			[]interface{}{"0x49eaaf1b548a0cb0", uint32(1)},
+			[]interface{}{"0x91d5df18b0d2cf58", uint32(1)},
+			[]interface{}{"0xed99c5acb25eedf5", uint32(3)},
 			[]interface{}{"0xcbca25e39f142387", uint32(2)},
 			[]interface{}{"0x687ad44ad37f03c2", uint32(1)},
+			[]interface{}{"0xab3c0572291feb8b", uint32(1)},
 			[]interface{}{"0xbc9d89904f5b923f", uint32(1)},
-			[]interface{}{"0x68b66ba122c93fa7", uint32(1)},
 			[]interface{}{"0x37c8bb1350a9a2a8", uint32(1)},
-			[]interface{}{"0x91d5df18b0d2cf58", uint32(1)},
-			[]interface{}{"0xab3c0572291feb8b", uint32(1)},
+			[]interface{}{"0xf3ff14d5ab527059", uint32(1)},
+			[]interface{}{"0x17a6bc0d0062aeb3", uint32(1)},
 		},
-		TransactionVersion: 2,
+		TransactionVersion: 12,
 	}
 
 	sm, hash, _ := setupStateModule(t)

dot/rpc/modules/system_integration_test.go

@@ -354,7 +354,7 @@ func setupSystemModule(t *testing.T) *SystemModule {
 func newCoreService(t *testing.T, srvc *state.Service) *core.Service {
 	// setup service
 	tt := trie.NewEmptyTrie()
-	rt := wasmer.NewTestInstanceWithTrie(t, runtime.NODE_RUNTIME, tt)
+	rt := wasmer.NewTestInstanceWithTrie(t, runtime.WESTEND_RUNTIME_v0929, tt)
 	ks := keystore.NewGlobalKeystore()
 	t.Cleanup(func() {
 		rt.Stop()