Backport of testing/deployer: update deployer to use v2 catalog constructs when requested into release/1.17.x

.github/workflows/test-integrations.yml

@@ -497,6 +497,8 @@ jobs:
       contents: read
     strategy:
       fail-fast: false
+    env:
+      DEPLOYER_CONSUL_DATAPLANE_IMAGE: "docker.mirror.hashicorp.services/hashicorppreview/consul-dataplane:1.3-dev"
     steps:
       - name: Checkout code
         uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
@@ -514,20 +516,22 @@ jobs:
         run: |
           mkdir -p "${{ env.TEST_RESULTS_DIR }}"
           export NOLOGBUFFER=1
-          cd ./test-integ/connect
+          cd ./test-integ
           go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
             --raw-command \
             --format=standard-verbose \
             --debug \
-            --packages="./..." \
             -- \
             go test \
             -tags "${{ env.GOTAGS }}" \
             -timeout=20m \
             -parallel=2 \
-            -json . \
+            -json \
+            `go list -tags "${{ env.GOTAGS }}" ./... | grep -v peering_commontopo` \
             --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
-            --target-version local
+            --target-version local \
+            --latest-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
+            --latest-version latest
         env:
           # this is needed because of incompatibility between RYUK container and GHA
           GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml

test-integ/Makefile

@@ -0,0 +1,42 @@
+SHELL := /bin/bash
+
+.PHONY: noop
+noop:
+
+##@ Build
+
+.PHONY: tidy
+tidy: ## Run go mod tidy.
+	go mod tidy
+
+##@ Checks
+
+.PHONY: format
+format: ## Format the go files.
+	@for f in $$(find . -name '*.go' -print); do \
+		gofmt -s -w $$f ; \
+	done
+
+.PHONY: lint
+lint: ## Run the full linting rules.
+	golangci-lint run -v
+
+.PHONY: vet
+vet: ## Run go vet.
+	go vet ./...
+
+##@ Help
+
+# The help target prints out all targets with their descriptions organized
+# beneath their categories. The categories are represented by '##@' and the
+# target descriptions by '##'. The awk commands is responsible for reading the
+# entire set of makefiles included in this invocation, looking for lines of the
+# file as xyz: ## something, and then pretty-format the target and help. Then,
+# if there's a line with ##@ something, that gets pretty-printed as a category.
+# More info on the usage of ANSI control characters for terminal formatting:
+# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters
+# More info on the awk command:
+# http://linuxcommand.org/lc3_adv_awk.php
+.PHONY: help
+help: ## Display this help.
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)

test-integ/catalogv2/explicit_destinations_test.go

@@ -0,0 +1,284 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package catalogv2
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	pbauth "github.com/hashicorp/consul/proto-public/pbauth/v2beta1"
+	"github.com/hashicorp/consul/proto-public/pbresource"
+	libassert "github.com/hashicorp/consul/test/integration/consul-container/libs/assert"
+	"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
+	"github.com/hashicorp/consul/testing/deployer/sprawl/sprawltest"
+	"github.com/hashicorp/consul/testing/deployer/topology"
+
+	"github.com/hashicorp/consul/test-integ/topoutil"
+)
+
+// TestBasicL4ExplicitDestination sets up the following:
+//
+// - 1 cluster (no peering / no wanfed)
+// - 3 servers in that cluster
+// - v2 arch is activated
+// - for each tenancy, only using v2 constructs:
+//   - a client with one explicit destination to a single port service
+//   - a client with multiple explicit destinations to multiple ports of the
+//     same multiport service
+//
+// When this test is executed in CE it will only use the default/default
+// tenancy.
+//
+// When this test is executed in Enterprise it will additionally test the same
+// things within these tenancies:
+//
+// - part1/default
+// - default/nsa
+// - part1/nsa
+func TestBasicL4ExplicitDestination(t *testing.T) {
+	cfg := testBasicL4ExplicitDestinationCreator{}.NewConfig(t)
+
+	sp := sprawltest.Launch(t, cfg)
+
+	var (
+		asserter = topoutil.NewAsserter(sp)
+
+		topo    = sp.Topology()
+		cluster = topo.Clusters["dc1"]
+
+		ships = topo.ComputeRelationships()
+	)
+
+	clientV2 := sp.ResourceServiceClientForCluster(cluster.Name)
+
+	t.Log(topology.RenderRelationships(ships))
+
+	// Make sure things are in v2.
+	for _, name := range []string{
+		"single-server",
+		"single-client",
+		"multi-server",
+		"multi-client",
+	} {
+		libassert.CatalogV2ServiceHasEndpointCount(t, clientV2, name, nil, 1)
+	}
+
+	// Check relationships
+	for _, ship := range ships {
+		t.Run("relationship: "+ship.String(), func(t *testing.T) {
+			var (
+				svc           = ship.Caller
+				u             = ship.Upstream
+				clusterPrefix string
+			)
+
+			if u.Peer == "" {
+				if u.ID.PartitionOrDefault() == "default" {
+					clusterPrefix = strings.Join([]string{u.PortName, u.ID.Name, u.ID.Namespace, u.Cluster, "internal"}, ".")
+				} else {
+					clusterPrefix = strings.Join([]string{u.PortName, u.ID.Name, u.ID.Namespace, u.ID.Partition, u.Cluster, "internal-v1"}, ".")
+				}
+			} else {
+				clusterPrefix = strings.Join([]string{u.ID.Name, u.ID.Namespace, u.Peer, "external"}, ".")
+			}
+
+			asserter.UpstreamEndpointStatus(t, svc, clusterPrefix+".", "HEALTHY", 1)
+			asserter.HTTPServiceEchoes(t, svc, u.LocalPort, "")
+			asserter.FortioFetch2FortioName(t, svc, u, cluster.Name, u.ID)
+		})
+	}
+}
+
+type testBasicL4ExplicitDestinationCreator struct{}
+
+func (c testBasicL4ExplicitDestinationCreator) NewConfig(t *testing.T) *topology.Config {
+	const clusterName = "dc1"
+
+	servers := topoutil.NewTopologyServerSet(clusterName+"-server", 3, []string{clusterName, "wan"}, nil)
+
+	cluster := &topology.Cluster{
+		Enterprise: utils.IsEnterprise(),
+		Name:       clusterName,
+		Nodes:      servers,
+	}
+
+	lastNode := 0
+	nodeName := func() string {
+		lastNode++
+		return fmt.Sprintf("%s-box%d", clusterName, lastNode)
+	}
+
+	c.topologyConfigAddNodes(t, cluster, nodeName, "default", "default")
+	if cluster.Enterprise {
+		c.topologyConfigAddNodes(t, cluster, nodeName, "part1", "default")
+		c.topologyConfigAddNodes(t, cluster, nodeName, "part1", "nsa")
+		c.topologyConfigAddNodes(t, cluster, nodeName, "default", "nsa")
+	}
+
+	return &topology.Config{
+		Images: topoutil.TargetImages(),
+		Networks: []*topology.Network{
+			{Name: clusterName},
+			{Name: "wan", Type: "wan"},
+		},
+		Clusters: []*topology.Cluster{
+			cluster,
+		},
+	}
+}
+
+func (c testBasicL4ExplicitDestinationCreator) topologyConfigAddNodes(
+	t *testing.T,
+	cluster *topology.Cluster,
+	nodeName func() string,
+	partition,
+	namespace string,
+) {
+	clusterName := cluster.Name
+
+	newServiceID := func(name string) topology.ServiceID {
+		return topology.ServiceID{
+			Partition: partition,
+			Namespace: namespace,
+			Name:      name,
+		}
+	}
+
+	tenancy := &pbresource.Tenancy{
+		Partition: partition,
+		Namespace: namespace,
+		PeerName:  "local",
+	}
+
+	singleportServerNode := &topology.Node{
+		Kind:      topology.NodeKindDataplane,
+		Version:   topology.NodeVersionV2,
+		Partition: partition,
+		Name:      nodeName(),
+		Services: []*topology.Service{
+			topoutil.NewFortioServiceWithDefaults(
+				clusterName,
+				newServiceID("single-server"),
+				topology.NodeVersionV2,
+				nil,
+			),
+		},
+	}
+	singleportClientNode := &topology.Node{
+		Kind:      topology.NodeKindDataplane,
+		Version:   topology.NodeVersionV2,
+		Partition: partition,
+		Name:      nodeName(),
+		Services: []*topology.Service{
+			topoutil.NewFortioServiceWithDefaults(
+				clusterName,
+				newServiceID("single-client"),
+				topology.NodeVersionV2,
+				func(svc *topology.Service) {
+					delete(svc.Ports, "grpc")     // v2 mode turns this on, so turn it off
+					delete(svc.Ports, "http-alt") // v2 mode turns this on, so turn it off
+					svc.Upstreams = []*topology.Upstream{{
+						ID:           newServiceID("single-server"),
+						PortName:     "http",
+						LocalAddress: "0.0.0.0", // needed for an assertion
+						LocalPort:    5000,
+					}}
+				},
+			),
+		},
+	}
+	singleportTrafficPerms := sprawltest.MustSetResourceData(t, &pbresource.Resource{
+		Id: &pbresource.ID{
+			Type:    pbauth.TrafficPermissionsType,
+			Name:    "single-server-perms",
+			Tenancy: tenancy,
+		},
+	}, &pbauth.TrafficPermissions{
+		Destination: &pbauth.Destination{
+			IdentityName: "single-server",
+		},
+		Action: pbauth.Action_ACTION_ALLOW,
+		Permissions: []*pbauth.Permission{{
+			Sources: []*pbauth.Source{{
+				IdentityName: "single-client",
+				Namespace:    namespace,
+			}},
+		}},
+	})
+
+	multiportServerNode := &topology.Node{
+		Kind:      topology.NodeKindDataplane,
+		Version:   topology.NodeVersionV2,
+		Partition: partition,
+		Name:      nodeName(),
+		Services: []*topology.Service{
+			topoutil.NewFortioServiceWithDefaults(
+				clusterName,
+				newServiceID("multi-server"),
+				topology.NodeVersionV2,
+				nil,
+			),
+		},
+	}
+	multiportClientNode := &topology.Node{
+		Kind:      topology.NodeKindDataplane,
+		Version:   topology.NodeVersionV2,
+		Partition: partition,
+		Name:      nodeName(),
+		Services: []*topology.Service{
+			topoutil.NewFortioServiceWithDefaults(
+				clusterName,
+				newServiceID("multi-client"),
+				topology.NodeVersionV2,
+				func(svc *topology.Service) {
+					svc.Upstreams = []*topology.Upstream{
+						{
+							ID:           newServiceID("multi-server"),
+							PortName:     "http",
+							LocalAddress: "0.0.0.0", // needed for an assertion
+							LocalPort:    5000,
+						},
+						{
+							ID:           newServiceID("multi-server"),
+							PortName:     "http-alt",
+							LocalAddress: "0.0.0.0", // needed for an assertion
+							LocalPort:    5001,
+						},
+					}
+				},
+			),
+		},
+	}
+	multiportTrafficPerms := sprawltest.MustSetResourceData(t, &pbresource.Resource{
+		Id: &pbresource.ID{
+			Type:    pbauth.TrafficPermissionsType,
+			Name:    "multi-server-perms",
+			Tenancy: tenancy,
+		},
+	}, &pbauth.TrafficPermissions{
+		Destination: &pbauth.Destination{
+			IdentityName: "multi-server",
+		},
+		Action: pbauth.Action_ACTION_ALLOW,
+		Permissions: []*pbauth.Permission{{
+			Sources: []*pbauth.Source{{
+				IdentityName: "multi-client",
+				Namespace:    namespace,
+			}},
+		}},
+	})
+
+	cluster.Nodes = append(cluster.Nodes,
+		singleportClientNode,
+		singleportServerNode,
+		multiportClientNode,
+		multiportServerNode,
+	)
+
+	cluster.InitialResources = append(cluster.InitialResources,
+		singleportTrafficPerms,
+		multiportTrafficPerms,
+	)
+}

test-integ/go.mod

@@ -4,6 +4,7 @@ go 1.20
 
 require (
 	github.com/hashicorp/consul/api v1.24.0
+	github.com/hashicorp/consul/proto-public v0.4.1
 	github.com/hashicorp/consul/sdk v0.14.1
 	github.com/hashicorp/consul/test/integration/consul-container v0.0.0-20230628201853-bdf4fad7c5a5
 	github.com/hashicorp/consul/testing/deployer v0.0.0-20230811171106-4a0afb5d1373
@@ -93,8 +94,8 @@ require (
 	github.com/hashicorp/consul v1.16.1 // indirect
 	github.com/hashicorp/consul-awsauth v0.0.0-20220713182709-05ac1c5c2706 // indirect
 	github.com/hashicorp/consul-net-rpc v0.0.0-20221205195236-156cfab66a69 // indirect
+	github.com/hashicorp/consul-server-connection-manager v0.1.4 // indirect
 	github.com/hashicorp/consul/envoyextensions v0.4.1 // indirect
-	github.com/hashicorp/consul/proto-public v0.4.1 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-bexpr v0.1.2 // indirect
 	github.com/hashicorp/go-connlimit v0.3.0 // indirect
@@ -104,6 +105,7 @@ require (
 	github.com/hashicorp/go-msgpack v1.1.5 // indirect
 	github.com/hashicorp/go-msgpack/v2 v2.0.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/go-netaddrs v0.1.0 // indirect
 	github.com/hashicorp/go-plugin v1.4.5 // indirect
 	github.com/hashicorp/go-raftchunking v0.7.0 // indirect
 	github.com/hashicorp/go-retryablehttp v0.6.7 // indirect