Merge branch 'main' into NET-4519

.changelog/18813.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+acl: Use templated policy to generate synthetic policies for tokens/roles with node and/or service identities
+```

.changelog/18816.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+cli: Add `consul acl templated-policy` commands to read, list and preview templated policies. 
+```

agent/acl_endpoint.go

@@ -1134,12 +1134,6 @@ func (s *HTTPHandlers) ACLAuthorize(resp http.ResponseWriter, req *http.Request)
 	return responses, nil
 }
 
-type ACLTemplatedPolicyResponse struct {
-	TemplateName string
-	Schema       string
-	Template     string
-}
-
 func (s *HTTPHandlers) ACLTemplatedPoliciesList(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
 	if s.checkACLDisabled() {
 		return nil, aclDisabled
@@ -1165,10 +1159,10 @@ func (s *HTTPHandlers) ACLTemplatedPoliciesList(resp http.ResponseWriter, req *h
 		return nil, err
 	}
 
-	templatedPolicies := make(map[string]ACLTemplatedPolicyResponse)
+	templatedPolicies := make(map[string]api.ACLTemplatedPolicyResponse)
 
 	for tp, tmpBase := range structs.GetACLTemplatedPolicyList() {
-		templatedPolicies[tp] = ACLTemplatedPolicyResponse{
+		templatedPolicies[tp] = api.ACLTemplatedPolicyResponse{
 			TemplateName: tmpBase.TemplateName,
 			Schema:       tmpBase.Schema,
 			Template:     tmpBase.Template,
@@ -1213,7 +1207,7 @@ func (s *HTTPHandlers) ACLTemplatedPolicyRead(resp http.ResponseWriter, req *htt
 		return nil, HTTPError{StatusCode: http.StatusBadRequest, Reason: fmt.Sprintf("Invalid templated policy Name: %s", templateName)}
 	}
 
-	return ACLTemplatedPolicyResponse{
+	return api.ACLTemplatedPolicyResponse{
 		TemplateName: baseTemplate.TemplateName,
 		Schema:       baseTemplate.Schema,
 		Template:     baseTemplate.Template,

agent/acl_endpoint_test.go

@@ -1372,11 +1372,11 @@ func TestACL_HTTP(t *testing.T) {
 
 			require.Equal(t, http.StatusOK, resp.Code)
 
-			var list map[string]ACLTemplatedPolicyResponse
+			var list map[string]api.ACLTemplatedPolicyResponse
 			require.NoError(t, json.NewDecoder(resp.Body).Decode(&list))
 			require.Len(t, list, 3)
 
-			require.Equal(t, ACLTemplatedPolicyResponse{
+			require.Equal(t, api.ACLTemplatedPolicyResponse{
 				TemplateName: api.ACLTemplatedPolicyServiceName,
 				Schema:       structs.ACLTemplatedPolicyIdentitiesSchema,
 				Template:     structs.ACLTemplatedPolicyService,
@@ -1399,7 +1399,7 @@ func TestACL_HTTP(t *testing.T) {
 				a.srv.h.ServeHTTP(resp, req)
 				require.Equal(t, http.StatusOK, resp.Code)
 
-				var templatedPolicy ACLTemplatedPolicyResponse
+				var templatedPolicy api.ACLTemplatedPolicyResponse
 				require.NoError(t, json.NewDecoder(resp.Body).Decode(&templatedPolicy))
 				require.Equal(t, structs.ACLTemplatedPolicyDNSSchema, templatedPolicy.Schema)
 				require.Equal(t, api.ACLTemplatedPolicyDNSName, templatedPolicy.TemplateName)

agent/agent_test.go

@@ -354,10 +354,10 @@ func TestAgent_HTTPMaxHeaderBytes(t *testing.T) {
 			require.NoError(t, err)
 
 			a, err := New(bd)
+			require.NoError(t, err)
 			mockDelegate := delegateMock{}
 			mockDelegate.On("LicenseCheck").Return()
 			a.delegate = &mockDelegate
-			require.NoError(t, err)
 
 			a.startLicenseManager(testutil.TestContext(t))
 
@@ -392,6 +392,8 @@ func TestAgent_HTTPMaxHeaderBytes(t *testing.T) {
 				resp, err := client.Do(req.WithContext(ctx))
 				require.NoError(t, err)
 				require.Equal(t, tt.expectedHTTPResponse, resp.StatusCode, "expected a '%d' http response, got '%d'", tt.expectedHTTPResponse, resp.StatusCode)
+				resp.Body.Close()
+				s.Shutdown(ctx)
 			}
 		})
 	}

agent/consul/type_registry.go

@@ -4,6 +4,7 @@
 package consul
 
 import (
+	"github.com/hashicorp/consul/internal/auth"
 	"github.com/hashicorp/consul/internal/catalog"
 	"github.com/hashicorp/consul/internal/mesh"
 	"github.com/hashicorp/consul/internal/resource"
@@ -23,6 +24,7 @@ func NewTypeRegistry() resource.Registry {
 	demo.RegisterTypes(registry)
 	mesh.RegisterTypes(registry)
 	catalog.RegisterTypes(registry)
+	auth.RegisterTypes(registry)
 
 	return registry
 }

agent/hcp/telemetry/otel_sink_test.go

@@ -334,7 +334,7 @@ func TestLabelsToAttributes(t *testing.T) {
 			sink, err := NewOTELSink(ctx, opts)
 			require.NoError(t, err)
 
-			require.Equal(t, test.expectedOTELAttributes, sink.labelsToAttributes(test.goMetricsLabels))
+			require.ElementsMatch(t, test.expectedOTELAttributes, sink.labelsToAttributes(test.goMetricsLabels))
 		})
 	}
 }

agent/structs/acl.go

@@ -9,11 +9,11 @@ import (
 	"errors"
 	"fmt"
 	"hash"
-	"hash/fnv"
 	"sort"
 	"strings"
 	"time"
 
+	"github.com/hashicorp/consul/api"
 	"github.com/hashicorp/consul/lib/stringslice"
 
 	"golang.org/x/crypto/blake2b"
@@ -182,22 +182,20 @@ func (s *ACLServiceIdentity) EstimateSize() int {
 }
 
 func (s *ACLServiceIdentity) SyntheticPolicy(entMeta *acl.EnterpriseMeta) *ACLPolicy {
+	// use templated policy to generate synthetic policy
+	templatedPolicy := ACLTemplatedPolicy{
+		TemplateID:   ACLTemplatedPolicyServiceID,
+		TemplateName: api.ACLTemplatedPolicyServiceName,
+		Datacenters:  s.Datacenters,
+		TemplateVariables: &ACLTemplatedPolicyVariables{
+			Name: s.ServiceName,
+		},
+	}
+
 	// Given that we validate this string name before persisting, we do not
-	// have to escape it before doing the following interpolation.
-	rules := aclServiceIdentityRules(s.ServiceName, entMeta)
-
-	hasher := fnv.New128a()
-	hashID := fmt.Sprintf("%x", hasher.Sum([]byte(rules)))
-
-	policy := &ACLPolicy{}
-	policy.ID = hashID
-	policy.Name = fmt.Sprintf("synthetic-policy-%s", hashID)
-	sn := NewServiceName(s.ServiceName, entMeta)
-	policy.Description = fmt.Sprintf("synthetic policy for service identity %q", sn.String())
-	policy.Rules = rules
-	policy.Datacenters = s.Datacenters
-	policy.EnterpriseMeta.Merge(entMeta)
-	policy.SetHash(true)
+	// expect any errors from generating the synthetic policy
+	policy, _ := templatedPolicy.SyntheticPolicy(entMeta)
+
 	return policy
 }
 
@@ -254,21 +252,20 @@ func (s *ACLNodeIdentity) EstimateSize() int {
 }
 
 func (s *ACLNodeIdentity) SyntheticPolicy(entMeta *acl.EnterpriseMeta) *ACLPolicy {
+	// use templated policy to generate synthetic policy
+	templatedPolicy := ACLTemplatedPolicy{
+		TemplateID:   ACLTemplatedPolicyNodeID,
+		TemplateName: api.ACLTemplatedPolicyNodeName,
+		Datacenters:  []string{s.Datacenter},
+		TemplateVariables: &ACLTemplatedPolicyVariables{
+			Name: s.NodeName,
+		},
+	}
+
 	// Given that we validate this string name before persisting, we do not
-	// have to escape it before doing the following interpolation.
-	rules := aclNodeIdentityRules(s.NodeName, entMeta)
-
-	hasher := fnv.New128a()
-	hashID := fmt.Sprintf("%x", hasher.Sum([]byte(rules)))
-
-	policy := &ACLPolicy{}
-	policy.ID = hashID
-	policy.Name = fmt.Sprintf("synthetic-policy-%s", hashID)
-	policy.Description = fmt.Sprintf("synthetic policy for node identity %q", s.NodeName)
-	policy.Rules = rules
-	policy.Datacenters = []string{s.Datacenter}
-	policy.EnterpriseMeta.Merge(entMeta)
-	policy.SetHash(true)
+	// expect any errors from generating the synthetic policy
+	policy, _ := templatedPolicy.SyntheticPolicy(entMeta)
+
 	return policy
 }
 

agent/structs/acl_templated_policy.go

@@ -23,19 +23,20 @@ type ACLTemplatedPolicies []*ACLTemplatedPolicy
 const (
 	ACLTemplatedPolicyNodeID           = "00000000-0000-0000-0000-000000000004"
 	ACLTemplatedPolicyServiceID        = "00000000-0000-0000-0000-000000000003"
-	ACLTemplatedPolicyIdentitiesSchema = `{
-		"type": "object",
-		"properties": {
-			"name": { "type": "string", "$ref": "#/definitions/min-length-one" }
-		},
-		"required": ["name"],
-		"definitions": {
-			"min-length-one": {
-					"type": "string",
-					"minLength": 1
-			}
+	ACLTemplatedPolicyIdentitiesSchema = `
+{
+	"type": "object",
+	"properties": {
+		"name": { "type": "string", "$ref": "#/definitions/min-length-one" }
+	},
+	"required": ["name"],
+	"definitions": {
+		"min-length-one": {
+				"type": "string",
+				"minLength": 1
 		}
-	}`
+	}
+}`
 
 	ACLTemplatedPolicyDNSID     = "00000000-0000-0000-0000-000000000005"
 	ACLTemplatedPolicyDNSSchema = "" // empty schema as it does not require variables
@@ -51,8 +52,9 @@ type ACLTemplatedPolicyBase struct {
 }
 
 var (
-	// TODO(Ronald): add other templates
 	// This supports: node, service and dns templates
+	// Note: when adding a new builtin template, ensure you update `command/acl/templatedpolicy/formatter.go`
+	// to handle the new templates required variables and schema.
 	aclTemplatedPoliciesList = map[string]*ACLTemplatedPolicyBase{
 		api.ACLTemplatedPolicyServiceName: {
 			TemplateID:   ACLTemplatedPolicyServiceID,

agent/xds/proxystateconverter/listeners.go

@@ -14,9 +14,8 @@ import (
 	"time"
 
 	envoy_http_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
-	"github.com/hashicorp/go-uuid"
-
 	"github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/go-uuid"
 	"google.golang.org/protobuf/types/known/durationpb"
 	"google.golang.org/protobuf/types/known/wrapperspb"
 
@@ -1443,7 +1442,11 @@ func makeL4Destination(opts destinationOpts) (*pbproxystate.L4Destination, error
 
 	l4Dest := &pbproxystate.L4Destination{
 		//AccessLog:        accessLogs,
-		Name:       opts.cluster,
+		Destination: &pbproxystate.L4Destination_Cluster{
+			Cluster: &pbproxystate.DestinationCluster{
+				Name: opts.cluster,
+			},
+		},
 		StatPrefix: makeStatPrefix(opts.statPrefix, opts.filterName),
 	}
 	return l4Dest, nil

agent/xdsv2/cluster_resources.go

@@ -12,11 +12,11 @@ import (
 	envoy_aggregate_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/aggregate/v3"
 	envoy_upstreams_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3"
 	envoy_type_v3 "github.com/envoyproxy/go-control-plane/envoy/type/v3"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/anypb"
 
 	"github.com/hashicorp/consul/envoyextensions/xdscommon"
 	"github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1/pbproxystate"
-	"google.golang.org/protobuf/proto"
-	"google.golang.org/protobuf/types/known/anypb"
 )
 
 func (pr *ProxyResources) doesEnvoyClusterAlreadyExist(name string) bool {
@@ -174,9 +174,11 @@ func (pr *ProxyResources) makeEnvoyStaticCluster(name string, protocol string, s
 	}
 	return cluster, nil
 }
+
 func (pr *ProxyResources) makeEnvoyDnsCluster(name string, protocol string, dns *pbproxystate.DNSEndpointGroup) (*envoy_cluster_v3.Cluster, error) {
 	return nil, nil
 }
+
 func (pr *ProxyResources) makeEnvoyPassthroughCluster(name string, protocol string, passthrough *pbproxystate.PassthroughEndpointGroup) (*envoy_cluster_v3.Cluster, error) {
 	cluster := &envoy_cluster_v3.Cluster{
 		Name:                 name,
@@ -343,6 +345,6 @@ func addEnvoyLBToCluster(dynamicConfig *pbproxystate.DynamicEndpointGroupConfig,
 }
 
 // TODO(proxystate): In a future PR this will create clusters and add it to ProxyResources.proxyState
-func (pr *ProxyResources) makeEnvoyClusterFromL4Destination(name string) error {
+func (pr *ProxyResources) makeEnvoyClusterFromL4Destination(l4 *pbproxystate.L4Destination) error {
 	return nil
 }

agent/xdsv2/listener_resources.go

@@ -304,7 +304,7 @@ func (pr *ProxyResources) makeEnvoyResourcesForSNIDestination(sni *pbproxystate.
 }
 
 func (pr *ProxyResources) makeEnvoyResourcesForL4Destination(l4 *pbproxystate.Router_L4) ([]*envoy_listener_v3.Filter, error) {
-	err := pr.makeEnvoyClusterFromL4Destination(l4.L4.Name)
+	err := pr.makeEnvoyClusterFromL4Destination(l4.L4)
 	if err != nil {
 		return nil, err
 	}
@@ -355,9 +355,30 @@ func makeL4Filters(defaultAllow bool, l4 *pbproxystate.L4Destination) ([]*envoy_
 
 		// Add tcp proxy filter
 		tcp := &envoy_tcp_proxy_v3.TcpProxy{
-			ClusterSpecifier: &envoy_tcp_proxy_v3.TcpProxy_Cluster{Cluster: l4.Name},
-			StatPrefix:       l4.StatPrefix,
+			StatPrefix: l4.StatPrefix,
+		}
+
+		switch dest := l4.Destination.(type) {
+		case *pbproxystate.L4Destination_Cluster:
+			tcp.ClusterSpecifier = &envoy_tcp_proxy_v3.TcpProxy_Cluster{Cluster: dest.Cluster.Name}
+		case *pbproxystate.L4Destination_WeightedClusters:
+			clusters := make([]*envoy_tcp_proxy_v3.TcpProxy_WeightedCluster_ClusterWeight, 0, len(dest.WeightedClusters.Clusters))
+			for _, cluster := range dest.WeightedClusters.Clusters {
+				clusters = append(clusters, &envoy_tcp_proxy_v3.TcpProxy_WeightedCluster_ClusterWeight{
+					Name:   cluster.Name,
+					Weight: cluster.Weight.GetValue(),
+				})
+			}
+
+			tcp.ClusterSpecifier = &envoy_tcp_proxy_v3.TcpProxy_WeightedClusters{
+				WeightedClusters: &envoy_tcp_proxy_v3.TcpProxy_WeightedCluster{
+					Clusters: clusters,
+				},
+			}
+		default:
+			return nil, fmt.Errorf("unexpected l4 destination type: %T", l4.Destination)
 		}
+
 		tcpFilter, err := makeEnvoyFilter(envoyNetworkFilterName, tcp)
 		if err != nil {
 			return nil, err