From 464c4be629f4333d9aa3bf946a7bcf54393182e0 Mon Sep 17 00:00:00 2001 From: Dhia Ayachi Date: Wed, 5 Apr 2023 13:54:12 -0400 Subject: [PATCH 1/7] add ability to start container tests in debug mode and attach a debugger to consul while running it. --- GNUmakefile | 13 +++++++++++++ build-support/docker/Consul-Dev-Dbg.dockerfile | 10 ++++++++++ .../consul-container/libs/cluster/builder.go | 8 ++++++-- .../consul-container/libs/cluster/container.go | 3 +++ .../consul-container/libs/utils/version.go | 17 ++++++++++++++--- .../test/snapshot/snapshot_restore_test.go | 4 ++-- .../consul-container/test/upgrade/basic_test.go | 2 +- .../test/upgrade/fullstopupgrade_test.go | 2 +- .../test/upgrade/healthcheck_test.go | 2 +- 9 files changed, 51 insertions(+), 10 deletions(-) create mode 100644 build-support/docker/Consul-Dev-Dbg.dockerfile diff --git a/GNUmakefile b/GNUmakefile index 9a68a484c116..b480a3dd0a46 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -163,6 +163,19 @@ dev-build: rm -f ./bin/consul cp ${MAIN_GOPATH}/bin/consul ./bin/consul + +dev-docker-dbg: dev-docker linux dev-build + @echo "Pulling consul container image - $(CONSUL_IMAGE_VERSION)" + @docker pull consul:$(CONSUL_IMAGE_VERSION) >/dev/null + @echo "Building Consul Development container - $(CONSUL_DEV_IMAGE)" + @# 'consul:local' tag is needed to run the integration tests + @# 'consul-dev:latest' is needed by older workflows + @docker buildx use default && docker buildx build -t 'consul-dbg:local' -t '$(CONSUL_DEV_IMAGE)' \ + --platform linux/$(GOARCH) \ + --build-arg CONSUL_IMAGE_VERSION=$(CONSUL_IMAGE_VERSION) \ + --load \ + -f $(CURDIR)/build-support/docker/Consul-Dev-Dbg.dockerfile $(CURDIR)/pkg/bin/ + dev-docker: linux dev-build @echo "Pulling consul container image - $(CONSUL_IMAGE_VERSION)" @docker pull consul:$(CONSUL_IMAGE_VERSION) >/dev/null diff --git a/build-support/docker/Consul-Dev-Dbg.dockerfile b/build-support/docker/Consul-Dev-Dbg.dockerfile new file mode 100644 index 000000000000..179e12b99a12 --- /dev/null +++ b/build-support/docker/Consul-Dev-Dbg.dockerfile @@ -0,0 +1,10 @@ +FROM consul:local +EXPOSE 4000 + +COPY --from=golang:1.20-alpine /usr/local/go/ /usr/local/go/ + +ENV PATH="/usr/local/go/bin:${PATH}" + +RUN CGO_ENABLED=0 go install -ldflags "-s -w -extldflags '-static'" github.com/go-delve/delve/cmd/dlv@latest + +CMD [ "/root/go/bin/dlv", "exec", "/bin/consul", "--listen=:4000", "--headless=true", "", "--accept-multiclient", "--continue", "--api-version=2", "--", "agent", "--advertise=0.0.0.0"] \ No newline at end of file diff --git a/test/integration/consul-container/libs/cluster/builder.go b/test/integration/consul-container/libs/cluster/builder.go index cad88515f153..432578140859 100644 --- a/test/integration/consul-container/libs/cluster/builder.go +++ b/test/integration/consul-container/libs/cluster/builder.go @@ -121,7 +121,7 @@ func NewBuildContext(t *testing.T, opts BuildOptions) *BuildContext { } if ctx.consulImageName == "" { - ctx.consulImageName = utils.TargetImageName + ctx.consulImageName = utils.GetTargetImageName() } if ctx.consulVersion == "" { ctx.consulVersion = utils.TargetVersion @@ -306,11 +306,15 @@ func (b *Builder) ToAgentConfig(t *testing.T) *Config { confCopy, err := b.conf.Clone() require.NoError(t, err) + cmd := []string{"agent"} + if utils.Debug { + cmd = []string{"/root/go/bin/dlv", "exec", "/bin/consul", "--listen=:4000", "--headless=true", "", "--accept-multiclient", "--continue", "--api-version=2", "--", "agent", "--config-file=/consul/config/config.json"} + } return &Config{ JSON: string(out), ConfigBuilder: confCopy, - Cmd: []string{"agent"}, + Cmd: cmd, Image: b.context.consulImageName, Version: b.context.consulVersion, diff --git a/test/integration/consul-container/libs/cluster/container.go b/test/integration/consul-container/libs/cluster/container.go index 687f06bcf318..a99938bead29 100644 --- a/test/integration/consul-container/libs/cluster/container.go +++ b/test/integration/consul-container/libs/cluster/container.go @@ -565,6 +565,9 @@ func newContainerRequest(config Config, opts containerOpts, ports ...int) (podRe for _, port := range ports { pod.ExposedPorts = append(pod.ExposedPorts, fmt.Sprintf("%d/tcp", port)) } + if utils.Debug { + pod.ExposedPorts = append(pod.ExposedPorts, "4000/tcp") + } // For handshakes like auto-encrypt, it can take 10's of seconds for the agent to become "ready". // If we only wait until the log stream starts, subsequent commands to agents will fail. diff --git a/test/integration/consul-container/libs/utils/version.go b/test/integration/consul-container/libs/utils/version.go index 62d22e81b33c..4d1ae5af0928 100644 --- a/test/integration/consul-container/libs/utils/version.go +++ b/test/integration/consul-container/libs/utils/version.go @@ -10,13 +10,15 @@ import ( ) var ( - TargetImageName string + targetImageName string TargetVersion string LatestImageName string LatestVersion string - FollowLog bool + FollowLog bool + + Debug bool Version_1_14, _ = version.NewVersion("1.14") ) @@ -27,13 +29,22 @@ const ( ) func init() { - flag.StringVar(&TargetImageName, "target-image", defaultImageName, "docker image name to be used under test (Default: "+defaultImageName+")") + flag.BoolVar(&Debug, "debug", false, "run consul with dlv to enable live debugging") + flag.StringVar(&targetImageName, "target-image", defaultImageName, "docker image name to be used under test (Default: "+defaultImageName+")") flag.StringVar(&TargetVersion, "target-version", "local", "docker image version to be used as UUT (unit under test)") flag.StringVar(&LatestImageName, "latest-image", defaultImageName, "docker image name to be used under test (Default: "+defaultImageName+")") flag.StringVar(&LatestVersion, "latest-version", "latest", "docker image to be used as latest") flag.BoolVar(&FollowLog, "follow-log", true, "follow container log in output (Default: true)") + +} + +func GetTargetImageName() string { + if Debug { + return targetImageName + "-dbg" + } + return targetImageName } func DockerImage(image, version string) string { diff --git a/test/integration/consul-container/test/snapshot/snapshot_restore_test.go b/test/integration/consul-container/test/snapshot/snapshot_restore_test.go index 3e989bc7df1f..d472f6efc7a2 100644 --- a/test/integration/consul-container/test/snapshot/snapshot_restore_test.go +++ b/test/integration/consul-container/test/snapshot/snapshot_restore_test.go @@ -38,7 +38,7 @@ func testSnapShotRestoreForLogStore(t *testing.T, logStore libcluster.LogStore) NumClients: 0, BuildOpts: &libcluster.BuildOptions{ Datacenter: "dc1", - ConsulImageName: utils.TargetImageName, + ConsulImageName: utils.GetTargetImageName(), ConsulVersion: utils.TargetVersion, LogStore: logStore, }, @@ -67,7 +67,7 @@ func testSnapShotRestoreForLogStore(t *testing.T, logStore libcluster.LogStore) NumClients: 0, BuildOpts: &libcluster.BuildOptions{ Datacenter: "dc1", - ConsulImageName: utils.TargetImageName, + ConsulImageName: utils.GetTargetImageName(), ConsulVersion: utils.TargetVersion, LogStore: logStore, }, diff --git a/test/integration/consul-container/test/upgrade/basic_test.go b/test/integration/consul-container/test/upgrade/basic_test.go index e65b7a1b2369..13b98587e0b4 100644 --- a/test/integration/consul-container/test/upgrade/basic_test.go +++ b/test/integration/consul-container/test/upgrade/basic_test.go @@ -19,7 +19,7 @@ func TestBasic(t *testing.T) { t.Parallel() configCtx := libcluster.NewBuildContext(t, libcluster.BuildOptions{ - ConsulImageName: utils.TargetImageName, + ConsulImageName: utils.GetTargetImageName(), ConsulVersion: utils.LatestVersion, }) diff --git a/test/integration/consul-container/test/upgrade/fullstopupgrade_test.go b/test/integration/consul-container/test/upgrade/fullstopupgrade_test.go index 2539a66f9dd9..dfe6e1a13510 100644 --- a/test/integration/consul-container/test/upgrade/fullstopupgrade_test.go +++ b/test/integration/consul-container/test/upgrade/fullstopupgrade_test.go @@ -51,7 +51,7 @@ func TestStandardUpgradeToTarget_fromLatest(t *testing.T) { run := func(t *testing.T, tc testcase) { configCtx := libcluster.NewBuildContext(t, libcluster.BuildOptions{ - ConsulImageName: utils.TargetImageName, + ConsulImageName: utils.GetTargetImageName(), ConsulVersion: tc.oldVersion, }) diff --git a/test/integration/consul-container/test/upgrade/healthcheck_test.go b/test/integration/consul-container/test/upgrade/healthcheck_test.go index da7572511fc6..1a6116e095c5 100644 --- a/test/integration/consul-container/test/upgrade/healthcheck_test.go +++ b/test/integration/consul-container/test/upgrade/healthcheck_test.go @@ -89,7 +89,7 @@ func testMixedServersGAClient(t *testing.T, majorityIsTarget bool) { ConsulVersion: utils.LatestVersion, } targetOpts = libcluster.BuildOptions{ - ConsulImageName: utils.TargetImageName, + ConsulImageName: utils.GetTargetImageName(), ConsulVersion: utils.TargetVersion, } From f9aa7738e2bfdcf2775ab84448b8bbc32d3a8d2b Mon Sep 17 00:00:00 2001 From: Dhia Ayachi Date: Thu, 13 Apr 2023 11:07:57 -0400 Subject: [PATCH 2/7] add a debug message with the debug port --- .../consul-container/libs/cluster/container.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/integration/consul-container/libs/cluster/container.go b/test/integration/consul-container/libs/cluster/container.go index a99938bead29..939ded740c0f 100644 --- a/test/integration/consul-container/libs/cluster/container.go +++ b/test/integration/consul-container/libs/cluster/container.go @@ -32,6 +32,7 @@ const disableRYUKEnv = "TESTCONTAINERS_RYUK_DISABLED" const MaxEnvoyOnNode = 10 // the max number of Envoy sidecar can run along with the agent, base is 19000 const ServiceUpstreamLocalBindPort = 5000 // local bind Port of service's upstream const ServiceUpstreamLocalBindPort2 = 5001 // local bind Port of service's upstream, for services with 2 upstreams +const debugPort = "4000/tcp" // consulContainerNode implements the Agent interface by running a Consul agent // in a container. @@ -441,6 +442,13 @@ func (c *consulContainerNode) Upgrade(ctx context.Context, config Config) error return err } + if utils.Debug { + port, err := container.MappedPort(ctx, debugPort) + if err != nil { + return err + } + fmt.Printf("\nDebug endpoint:: 0.0.0.0:%s\n", port) + } if utils.FollowLog { if err := container.StartLogProducer(ctx); err != nil { return err @@ -566,7 +574,7 @@ func newContainerRequest(config Config, opts containerOpts, ports ...int) (podRe pod.ExposedPorts = append(pod.ExposedPorts, fmt.Sprintf("%d/tcp", port)) } if utils.Debug { - pod.ExposedPorts = append(pod.ExposedPorts, "4000/tcp") + pod.ExposedPorts = append(pod.ExposedPorts, debugPort) } // For handshakes like auto-encrypt, it can take 10's of seconds for the agent to become "ready". From 6c9ad62f24886d28c8bc14848862203f059515ca Mon Sep 17 00:00:00 2001 From: Dhia Ayachi Date: Thu, 13 Apr 2023 13:47:46 -0400 Subject: [PATCH 3/7] use pod to get the right port --- .../libs/cluster/container.go | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/test/integration/consul-container/libs/cluster/container.go b/test/integration/consul-container/libs/cluster/container.go index 939ded740c0f..5eb731e2757b 100644 --- a/test/integration/consul-container/libs/cluster/container.go +++ b/test/integration/consul-container/libs/cluster/container.go @@ -157,6 +157,20 @@ func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster, po info AgentInfo ) + if utils.Debug { + for i := 0; i < 10; i++ { + uri, err := podContainer.PortEndpoint(ctx, "4000", "tcp") + if err != nil { + time.Sleep(500 * time.Millisecond) + continue + } + fmt.Printf("\nDebug endpoint:: %s\n\n", uri) + break + } + if err != nil { + return nil, err + } + } if httpPort > 0 { for i := 0; i < 10; i++ { uri, err := podContainer.PortEndpoint(ctx, "8500", "http") @@ -442,13 +456,6 @@ func (c *consulContainerNode) Upgrade(ctx context.Context, config Config) error return err } - if utils.Debug { - port, err := container.MappedPort(ctx, debugPort) - if err != nil { - return err - } - fmt.Printf("\nDebug endpoint:: 0.0.0.0:%s\n", port) - } if utils.FollowLog { if err := container.StartLogProducer(ctx); err != nil { return err From 94358e9279f8f1d7def11ec8c170faf3cd99124e Mon Sep 17 00:00:00 2001 From: Dhia Ayachi Date: Thu, 13 Apr 2023 14:00:54 -0400 Subject: [PATCH 4/7] fix image used in basic test --- test/integration/consul-container/test/upgrade/basic_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/consul-container/test/upgrade/basic_test.go b/test/integration/consul-container/test/upgrade/basic_test.go index 13b98587e0b4..bd62caae94bb 100644 --- a/test/integration/consul-container/test/upgrade/basic_test.go +++ b/test/integration/consul-container/test/upgrade/basic_test.go @@ -20,7 +20,7 @@ func TestBasic(t *testing.T) { configCtx := libcluster.NewBuildContext(t, libcluster.BuildOptions{ ConsulImageName: utils.GetTargetImageName(), - ConsulVersion: utils.LatestVersion, + ConsulVersion: utils.TargetVersion, }) const numServers = 1 @@ -29,7 +29,7 @@ func TestBasic(t *testing.T) { Bootstrap(numServers). ToAgentConfig(t) t.Logf("Cluster config:\n%s", serverConf.JSON) - require.Equal(t, utils.LatestVersion, serverConf.Version) // TODO: remove + require.Equal(t, utils.TargetVersion, serverConf.Version) // TODO: remove cluster, err := libcluster.NewN(t, *serverConf, numServers) require.NoError(t, err) From c67dac84e33c843bc14301566b38967130a7558b Mon Sep 17 00:00:00 2001 From: Dhia Ayachi Date: Fri, 14 Apr 2023 10:28:44 -0400 Subject: [PATCH 5/7] add more data to identify which container to debug. --- .../consul-container/libs/cluster/agent.go | 1 + .../consul-container/libs/cluster/cluster.go | 25 +++++++++++++++++++ .../libs/cluster/container.go | 24 ++++++++++-------- 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/test/integration/consul-container/libs/cluster/agent.go b/test/integration/consul-container/libs/cluster/agent.go index 963133c6de74..d4de6f465508 100644 --- a/test/integration/consul-container/libs/cluster/agent.go +++ b/test/integration/consul-container/libs/cluster/agent.go @@ -79,4 +79,5 @@ type AgentInfo struct { CACertFile string UseTLSForAPI bool UseTLSForGRPC bool + DebugUri string } diff --git a/test/integration/consul-container/libs/cluster/cluster.go b/test/integration/consul-container/libs/cluster/cluster.go index 9b3620e021c4..2f307892b08d 100644 --- a/test/integration/consul-container/libs/cluster/cluster.go +++ b/test/integration/consul-container/libs/cluster/cluster.go @@ -8,6 +8,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" "os" "path/filepath" "strconv" @@ -170,6 +171,10 @@ func (c *Cluster) Add(configs []Config, serfJoin bool, ports ...int) (xe error) } } + if utils.Debug { + c.PrintDebugInfo(agents) + } + return nil } @@ -661,6 +666,26 @@ func (c *Cluster) ConfigEntryDelete(entry api.ConfigEntry) error { return err } +func (c *Cluster) PrintDebugInfo(agents []Agent) { + for _, a := range agents { + uri := a.GetInfo().DebugUri + n := a.GetAgentName() + s := a.IsServer() + l := "NA" + if s { + leader, err := c.Leader() + if err == nil { + if leader == a { + l = "true" + } else { + l = "false" + } + } + } + fmt.Printf("\ndebug info:: n=%s,s=%t,l=%s,uri=%s\n\n", n, s, l, uri) + } +} + func extractSecretIDFrom(tokenOutput string) (string, error) { lines := strings.Split(tokenOutput, "\n") for _, line := range lines { diff --git a/test/integration/consul-container/libs/cluster/container.go b/test/integration/consul-container/libs/cluster/container.go index 5eb731e2757b..1b48e0c0afdc 100644 --- a/test/integration/consul-container/libs/cluster/container.go +++ b/test/integration/consul-container/libs/cluster/container.go @@ -157,19 +157,21 @@ func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster, po info AgentInfo ) + debugUri := "" if utils.Debug { - for i := 0; i < 10; i++ { - uri, err := podContainer.PortEndpoint(ctx, "4000", "tcp") - if err != nil { - time.Sleep(500 * time.Millisecond) - continue - } - fmt.Printf("\nDebug endpoint:: %s\n\n", uri) - break - } - if err != nil { - return nil, err + if err := goretry.Do( + func() (err error) { + debugUri, err = podContainer.PortEndpoint(ctx, "4000", "tcp") + return err + }, + goretry.Delay(10*time.Second), + goretry.RetryIf(func(err error) bool { + return err != nil + }), + ); err != nil { + return nil, fmt.Errorf("container creating: %s", err) } + info.DebugUri = debugUri } if httpPort > 0 { for i := 0; i < 10; i++ { From aa4d2140843b6e84f439f570026545cf3e8e03dd Mon Sep 17 00:00:00 2001 From: Dhia Ayachi Date: Mon, 17 Apr 2023 12:57:08 -0400 Subject: [PATCH 6/7] fix comment Co-authored-by: Evan Culver --- GNUmakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GNUmakefile b/GNUmakefile index b480a3dd0a46..815d56d204d6 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -168,7 +168,7 @@ dev-docker-dbg: dev-docker linux dev-build @echo "Pulling consul container image - $(CONSUL_IMAGE_VERSION)" @docker pull consul:$(CONSUL_IMAGE_VERSION) >/dev/null @echo "Building Consul Development container - $(CONSUL_DEV_IMAGE)" - @# 'consul:local' tag is needed to run the integration tests + @# 'consul-dbg:local' tag is needed to run the integration tests @# 'consul-dev:latest' is needed by older workflows @docker buildx use default && docker buildx build -t 'consul-dbg:local' -t '$(CONSUL_DEV_IMAGE)' \ --platform linux/$(GOARCH) \ From 8fb470d6e3e42e3da30fc6cb1698ebf56f77f8bb Mon Sep 17 00:00:00 2001 From: Dhia Ayachi Date: Mon, 17 Apr 2023 12:57:31 -0400 Subject: [PATCH 7/7] rename debugUri to debugURI --- test/integration/consul-container/libs/cluster/agent.go | 2 +- test/integration/consul-container/libs/cluster/cluster.go | 2 +- test/integration/consul-container/libs/cluster/container.go | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/integration/consul-container/libs/cluster/agent.go b/test/integration/consul-container/libs/cluster/agent.go index d4de6f465508..ceef8996523c 100644 --- a/test/integration/consul-container/libs/cluster/agent.go +++ b/test/integration/consul-container/libs/cluster/agent.go @@ -79,5 +79,5 @@ type AgentInfo struct { CACertFile string UseTLSForAPI bool UseTLSForGRPC bool - DebugUri string + DebugURI string } diff --git a/test/integration/consul-container/libs/cluster/cluster.go b/test/integration/consul-container/libs/cluster/cluster.go index 2f307892b08d..98eb2419daaf 100644 --- a/test/integration/consul-container/libs/cluster/cluster.go +++ b/test/integration/consul-container/libs/cluster/cluster.go @@ -668,7 +668,7 @@ func (c *Cluster) ConfigEntryDelete(entry api.ConfigEntry) error { func (c *Cluster) PrintDebugInfo(agents []Agent) { for _, a := range agents { - uri := a.GetInfo().DebugUri + uri := a.GetInfo().DebugURI n := a.GetAgentName() s := a.IsServer() l := "NA" diff --git a/test/integration/consul-container/libs/cluster/container.go b/test/integration/consul-container/libs/cluster/container.go index 1b48e0c0afdc..170d15a0cef9 100644 --- a/test/integration/consul-container/libs/cluster/container.go +++ b/test/integration/consul-container/libs/cluster/container.go @@ -157,11 +157,11 @@ func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster, po info AgentInfo ) - debugUri := "" + debugURI := "" if utils.Debug { if err := goretry.Do( func() (err error) { - debugUri, err = podContainer.PortEndpoint(ctx, "4000", "tcp") + debugURI, err = podContainer.PortEndpoint(ctx, "4000", "tcp") return err }, goretry.Delay(10*time.Second), @@ -171,7 +171,7 @@ func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster, po ); err != nil { return nil, fmt.Errorf("container creating: %s", err) } - info.DebugUri = debugUri + info.DebugURI = debugURI } if httpPort > 0 { for i := 0; i < 10; i++ {