Add traffic permissions integration tests.

.github/workflows/test-integrations.yml

@@ -116,7 +116,7 @@ jobs:
             --packages="./command/agent/consul" \
             --junitfile $TEST_RESULTS_DIR/results.xml -- \
             -run TestConsul
-      
+
       # NOTE: ENT specific step as we store secrets in Vault.
       - name: Authenticate to Vault
         if: ${{ endsWith(github.repository, '-enterprise') }}
@@ -257,7 +257,7 @@ jobs:
       - name: Generate Envoy Job Matrix
         id: set-matrix
         env:
-          # this is further going to multiplied in envoy-integration tests by the 
+          # this is further going to multiplied in envoy-integration tests by the
           # other dimensions in the matrix.  Currently TOTAL_RUNNERS would be
           # multiplied by 8 based on these values:
           # envoy-version: ["1.24.10", "1.25.9", "1.26.4", "1.27.0"]
@@ -281,7 +281,7 @@ jobs:
               | jq --raw-input --argjson runnercount "$NUM_RUNNERS" "$JQ_SLICER" \
               | jq --compact-output 'map(join("|"))'
           } >> "$GITHUB_OUTPUT"
-  
+
   envoy-integration-test:
     runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }}
     needs:
@@ -384,7 +384,7 @@ jobs:
       contents: read
     env:
       ENVOY_VERSION: "1.25.4"
-      CONSUL_DATAPLANE_IMAGE: "docker.io/hashicorppreview/consul-dataplane:1.3-dev"
+      CONSUL_DATAPLANE_IMAGE: "docker.io/hashicorppreview/consul-dataplane:1.3-dev-ubi"
     steps:
       - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
       # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos.
@@ -417,7 +417,7 @@ jobs:
         if: steps.buildConsulEnvoyImage.outcome == 'failure'
         run: docker build -t consul-envoy:target-version --build-arg CONSUL_IMAGE=${{ env.CONSUL_LATEST_IMAGE_NAME }}:local --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets
       - name: Build consul-dataplane:local image
-        run: docker build -t consul-dataplane:local --build-arg CONSUL_DATAPLANE_IMAGE=${{ env.CONSUL_DATAPLANE_IMAGE }} -f ./test/integration/consul-container/assets/Dockerfile-consul-dataplane ./test/integration/consul-container/assets
+        run: docker build -t consul-dataplane:local --build-arg CONSUL_IMAGE=${{ env.CONSUL_LATEST_IMAGE_NAME }}:local --build-arg CONSUL_DATAPLANE_IMAGE=${{ env.CONSUL_DATAPLANE_IMAGE }} -f ./test/integration/consul-container/assets/Dockerfile-consul-dataplane ./test/integration/consul-container/assets
       - name: Configure GH workaround for ipv6 loopback
         if: ${{ !endsWith(github.repository, '-enterprise') }}
         run: |

Makefile

@@ -66,7 +66,7 @@ UI_BUILD_TAG?=consul-build-ui
 BUILD_CONTAINER_NAME?=consul-builder
 CONSUL_IMAGE_VERSION?=latest
 ENVOY_VERSION?='1.25.4'
-CONSUL_DATAPLANE_IMAGE := $(or $(CONSUL_DATAPLANE_IMAGE),"docker.io/hashicorppreview/consul-dataplane:1.3-dev")
+CONSUL_DATAPLANE_IMAGE := $(or $(CONSUL_DATAPLANE_IMAGE),"docker.io/hashicorppreview/consul-dataplane:1.3-dev-ubi")
 
 CONSUL_VERSION?=$(shell cat version/VERSION)
 
@@ -349,7 +349,7 @@ test-compat-integ-setup: dev-docker
 	@docker run --rm -t $(CONSUL_COMPAT_TEST_IMAGE):local consul version
 	@#  'consul-envoy:target-version' is needed by compatibility integ test
 	@docker build -t consul-envoy:target-version --build-arg CONSUL_IMAGE=$(CONSUL_COMPAT_TEST_IMAGE):local --build-arg ENVOY_VERSION=${ENVOY_VERSION} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets
-	@docker build -t consul-dataplane:local --build-arg CONSUL_DATAPLANE_IMAGE=${CONSUL_DATAPLANE_IMAGE} -f ./test/integration/consul-container/assets/Dockerfile-consul-dataplane ./test/integration/consul-container/assets
+	@docker build -t consul-dataplane:local --build-arg CONSUL_IMAGE=$(CONSUL_COMPAT_TEST_IMAGE):local --build-arg CONSUL_DATAPLANE_IMAGE=${CONSUL_DATAPLANE_IMAGE} -f ./test/integration/consul-container/assets/Dockerfile-consul-dataplane ./test/integration/consul-container/assets
 
 .PHONY: test-compat-integ
 test-compat-integ: test-compat-integ-setup ## Test compat integ

internal/resource/resourcetest/builder.go

@@ -4,6 +4,7 @@
 package resourcetest
 
 import (
+	"context"
 	"strings"
 
 	"github.com/oklog/ulid/v2"
@@ -134,7 +135,14 @@ func (b *resourceBuilder) ReferenceNoSection() *pbresource.Reference {
 func (b *resourceBuilder) Write(t T, client pbresource.ResourceServiceClient) *pbresource.Resource {
 	t.Helper()
 
-	ctx := testutil.TestContext(t)
+	var ctx context.Context
+	rtestClient, ok := client.(*Client)
+	if ok {
+		ctx = rtestClient.Context(t)
+	} else {
+		ctx = testutil.TestContext(t)
+		rtestClient = NewClient(client)
+	}
 
 	res := b.resource
 
@@ -170,7 +178,7 @@ func (b *resourceBuilder) Write(t T, client pbresource.ResourceServiceClient) *p
 		id := proto.Clone(rsp.Resource.Id).(*pbresource.ID)
 		id.Uid = ""
 		t.Cleanup(func() {
-			NewClient(client).MustDelete(t, id)
+			rtestClient.MustDelete(t, id)
 		})
 	}
 

internal/resource/resourcetest/client.go

@@ -4,13 +4,15 @@
 package resourcetest
 
 import (
+	"context"
 	"fmt"
 	"math/rand"
 	"time"
 
 	"github.com/stretchr/testify/require"
 	"golang.org/x/exp/slices"
 	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
 
 	"github.com/hashicorp/consul/internal/resource"
@@ -24,13 +26,19 @@ type Client struct {
 
 	timeout time.Duration
 	wait    time.Duration
+	token   string
 }
 
 func NewClient(client pbresource.ResourceServiceClient) *Client {
+	return NewClientWithACLToken(client, "")
+}
+
+func NewClientWithACLToken(client pbresource.ResourceServiceClient, token string) *Client {
 	return &Client{
 		ResourceServiceClient: client,
 		timeout:               7 * time.Second,
 		wait:                  25 * time.Millisecond,
+		token:                 token,
 	}
 }
 
@@ -46,7 +54,7 @@ func (client *Client) retry(t T, fn func(r *retry.R)) {
 }
 
 func (client *Client) PublishResources(t T, resources []*pbresource.Resource) {
-	ctx := testutil.TestContext(t)
+	ctx := client.Context(t)
 
 	// Randomize the order of insertion. Generally insertion order shouldn't matter as the
 	// controllers should eventually converge on the desired state. The exception to this
@@ -111,10 +119,23 @@ func (client *Client) PublishResources(t T, resources []*pbresource.Resource) {
 	require.Empty(t, resources, "Could not publish all resources - some resources have invalid owner references")
 }
 
+func (client *Client) Context(t T) context.Context {
+	ctx := testutil.TestContext(t)
+
+	if client.token != "" {
+		md := metadata.New(map[string]string{
+			"x-consul-token": client.token,
+		})
+		ctx = metadata.NewOutgoingContext(ctx, md)
+	}
+
+	return ctx
+}
+
 func (client *Client) RequireResourceNotFound(t T, id *pbresource.ID) {
 	t.Helper()
 
-	rsp, err := client.Read(testutil.TestContext(t), &pbresource.ReadRequest{Id: id})
+	rsp, err := client.Read(client.Context(t), &pbresource.ReadRequest{Id: id})
 	require.Error(t, err)
 	require.Equal(t, codes.NotFound, status.Code(err))
 	require.Nil(t, rsp)
@@ -123,7 +144,7 @@ func (client *Client) RequireResourceNotFound(t T, id *pbresource.ID) {
 func (client *Client) RequireResourceExists(t T, id *pbresource.ID) *pbresource.Resource {
 	t.Helper()
 
-	rsp, err := client.Read(testutil.TestContext(t), &pbresource.ReadRequest{Id: id})
+	rsp, err := client.Read(client.Context(t), &pbresource.ReadRequest{Id: id})
 	require.NoError(t, err, "error reading %s with type %s", id.Name, resource.ToGVK(id.Type))
 	require.NotNil(t, rsp)
 	return rsp.Resource
@@ -261,7 +282,7 @@ func (client *Client) ResolveResourceID(t T, id *pbresource.ID) *pbresource.ID {
 
 func (client *Client) MustDelete(t T, id *pbresource.ID) {
 	t.Helper()
-	ctx := testutil.TestContext(t)
+	ctx := client.Context(t)
 
 	client.retry(t, func(r *retry.R) {
 		_, err := client.Delete(ctx, &pbresource.DeleteRequest{Id: id})

test/integration/consul-container/assets/Dockerfile-consul-dataplane

@@ -2,7 +2,30 @@
 # SPDX-License-Identifier: BUSL-1.1
 
 ARG CONSUL_DATAPLANE_IMAGE
+ARG CONSUL_IMAGE
+
+# Docker doesn't support expansion in COPY --copy, so we need to create an intermediate image.
+FROM ${CONSUL_IMAGE} as consul
 
 FROM ${CONSUL_DATAPLANE_IMAGE} as consuldataplane
-COPY --from=busybox:uclibc /bin/sh /bin/sh
-COPY --from=ghcr.io/tarampampam/curl:latest /bin/curl /bin/curl
+
+USER root
+
+# On Mac M1s when TProxy is enabled, consul-dataplane that are spawned from this image
+# (only used in consul-container integration tests) will terminate with the below error.
+# It is related to tproxy-startup.sh calling iptables SDK which then calls the underly
+# iptables. We are investigating how this works on M1s with consul-envoy images which
+# do not have this problem. For the time being tproxy tests on Mac M1s will fail locally
+# but pass in CI.
+#
+# Error setting up traffic redirection rules: failed to run command: /sbin/iptables -t nat -N CONSUL_PROXY_INBOUND, err: exit status 1, output: iptables: Failed to initialize nft: Protocol not supported
+RUN microdnf install -y iptables sudo nc \
+ 	&& usermod -a -G wheel consul-dataplane \
+ 	&& echo 'consul-dataplane ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+COPY --from=consul /bin/consul /bin/consul
+
+COPY tproxy-startup.sh /bin/tproxy-startup.sh
+RUN chmod +x /bin/tproxy-startup.sh && chown root:root /bin/tproxy-startup.sh
+
+USER 100

test/integration/consul-container/libs/cluster/builder.go

@@ -273,6 +273,11 @@ func (b *Builder) Peering(enable bool) *Builder {
 	return b
 }
 
+func (b *Builder) SetACLToken(token string) *Builder {
+	b.conf.Set("acl.tokens.agent", token)
+	return b
+}
+
 func (b *Builder) NodeID(nodeID string) *Builder {
 	b.conf.Set("node_id", nodeID)
 	return b

test/integration/consul-container/libs/cluster/dataplane.go

@@ -6,11 +6,14 @@ package cluster
 import (
 	"context"
 	"fmt"
+	"io"
+	"strconv"
+	"strings"
+	"time"
+
 	"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
 	"github.com/testcontainers/testcontainers-go"
 	"github.com/testcontainers/testcontainers-go/wait"
-	"strconv"
-	"time"
 )
 
 type ConsulDataplaneContainer struct {
@@ -27,6 +30,10 @@ func (g ConsulDataplaneContainer) GetAddr() (string, int) {
 	return g.ip, g.appPort[0]
 }
 
+func (g ConsulDataplaneContainer) GetServiceName() string {
+	return g.serviceName
+}
+
 // GetAdminAddr returns the external admin port
 func (g ConsulDataplaneContainer) GetAdminAddr() (string, int) {
 	return "localhost", g.externalAdminPort
@@ -36,13 +43,28 @@ func (c ConsulDataplaneContainer) Terminate() error {
 	return TerminateContainer(c.ctx, c.container, true)
 }
 
+func (g ConsulDataplaneContainer) Exec(ctx context.Context, cmd []string) (string, error) {
+	exitCode, reader, err := g.container.Exec(ctx, cmd)
+	if err != nil {
+		return "", fmt.Errorf("exec with error %s", err)
+	}
+	if exitCode != 0 {
+		return "", fmt.Errorf("exec with exit code %d", exitCode)
+	}
+	buf, err := io.ReadAll(reader)
+	if err != nil {
+		return "", fmt.Errorf("error reading from exec output: %w", err)
+	}
+	return string(buf), nil
+}
+
 func (g ConsulDataplaneContainer) GetStatus() (string, error) {
 	state, err := g.container.State(g.ctx)
 	return state.Status, err
 }
 
 func NewConsulDataplane(ctx context.Context, proxyID string, serverAddresses string, grpcPort int, serviceBindPorts []int,
-	node Agent, containerArgs ...string) (*ConsulDataplaneContainer, error) {
+	node Agent, tproxy bool, bootstrapToken string, containerArgs ...string) (*ConsulDataplaneContainer, error) {
 	namePrefix := fmt.Sprintf("%s-consul-dataplane-%s", node.GetDatacenter(), proxyID)
 	containerName := utils.RandName(namePrefix)
 
@@ -70,7 +92,39 @@ func NewConsulDataplane(ctx context.Context, proxyID string, serverAddresses str
 	copy(exposedPorts, appPortStrs)
 	exposedPorts = append(exposedPorts, adminPortStr)
 
-	command := []string{
+	req := testcontainers.ContainerRequest{
+		Image:      "consul-dataplane:local",
+		WaitingFor: wait.ForLog("").WithStartupTimeout(60 * time.Second),
+		AutoRemove: false,
+		Name:       containerName,
+		Env:        map[string]string{},
+	}
+
+	var command []string
+
+	if tproxy {
+		req.Entrypoint = []string{"sh", "/bin/tproxy-startup.sh"}
+		req.Env["REDIRECT_TRAFFIC_ARGS"] = strings.Join(
+			[]string{
+				// TODO once we run this on a different pod from Consul agents, we can eliminate most of this.
+				"-exclude-inbound-port", fmt.Sprint(internalAdminPort),
+				"-exclude-inbound-port", "8300",
+				"-exclude-inbound-port", "8301",
+				"-exclude-inbound-port", "8302",
+				"-exclude-inbound-port", "8500",
+				"-exclude-inbound-port", "8502",
+				"-exclude-inbound-port", "8600",
+				"-proxy-inbound-port", "20000",
+				"-consul-dns-ip", "127.0.0.1",
+				"-consul-dns-port", "8600",
+			},
+			" ",
+		)
+		req.CapAdd = append(req.CapAdd, "NET_ADMIN")
+		command = append(command, "consul-dataplane")
+	}
+
+	command = append(command,
 		"-addresses", serverAddresses,
 		fmt.Sprintf("-grpc-port=%d", grpcPort),
 		fmt.Sprintf("-proxy-id=%s", proxyID),
@@ -81,19 +135,16 @@ func NewConsulDataplane(ctx context.Context, proxyID string, serverAddresses str
 		"-envoy-concurrency=2",
 		"-tls-disabled",
 		fmt.Sprintf("-envoy-admin-bind-port=%d", internalAdminPort),
-	}
-
-	command = append(command, containerArgs...)
+	)
 
-	req := testcontainers.ContainerRequest{
-		Image:      "consul-dataplane:local",
-		WaitingFor: wait.ForLog("").WithStartupTimeout(60 * time.Second),
-		AutoRemove: false,
-		Name:       containerName,
-		Cmd:        command,
-		Env:        map[string]string{},
+	if bootstrapToken != "" {
+		command = append(command,
+			"-credential-type=static",
+			fmt.Sprintf("-static-token=%s", bootstrapToken))
 	}
 
+	req.Cmd = append(command, containerArgs...)
+
 	info, err := LaunchContainerOnNode(ctx, node, req, exposedPorts)
 	if err != nil {
 		return nil, err

test/integration/consul-container/libs/topology/peering_topology.go

@@ -272,6 +272,9 @@ func NewClusterWithConfig(
 		Client().
 		Peering(true).
 		RetryJoin(retryJoin...)
+	if cluster.TokenBootstrap != "" {
+		configBuilder.SetACLToken(cluster.TokenBootstrap)
+	}
 	clientConf := configBuilder.ToAgentConfig(t)
 	t.Logf("%s client config: \n%s", opts.Datacenter, clientConf.JSON)
 	if clientHclConfig != "" {

test/integration/consul-container/test/multiport/explicit_destination_test.go

@@ -87,7 +87,7 @@ func createServiceAndDataplane(t *testing.T, node libcluster.Agent, proxyID, ser
 	})
 
 	// Create Consul Dataplane
-	dp, err := libcluster.NewConsulDataplane(context.Background(), proxyID, "0.0.0.0", 8502, serviceBindPorts, node)
+	dp, err := libcluster.NewConsulDataplane(context.Background(), proxyID, "0.0.0.0", 8502, serviceBindPorts, node, false, "")
 	require.NoError(t, err)
 	deferClean.Add(func() {
 		_ = dp.Terminate()