Permissive mTLS (#17035)

This implements permissive mTLS , which allows toggling services into "permissive" mTLS mode.
Permissive mTLS mode allows incoming "non Consul-mTLS" traffic to be forward unmodified to the application.

* Update service-defaults and proxy-defaults config entries with a MutualTLSMode field
* Update the mesh config entry with an AllowEnablingPermissiveMutualTLS field and implement the necessary validation. AllowEnablingPermissiveMutualTLS must be true to allow changing to MutualTLSMode=permissive, but this does not require that all proxy-defaults and service-defaults are currently in strict mode.
* Update xDS listener config to add a "permissive filter chain" when MutualTLSMode=permissive for a particular service. The permissive filter chain matches incoming traffic by the destination port. If the destination port matches the service port from the catalog, then no mTLS is required and the traffic sent is forwarded unmodified to the application.

.changelog/17035.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+mesh: Add new permissive mTLS mode that allows sidecar proxies to forward incoming traffic unmodified to the application. This adds `AllowEnablingPermissiveMutualTLS` setting to the mesh config entry and the `MutualTLSMode` setting to proxy-defaults and service-defaults.
+```

agent/configentry/merge_service_config.go

@@ -154,6 +154,10 @@ func MergeServiceConfig(defaults *structs.ServiceConfigResponse, service *struct
 		ns.Proxy.TransparentProxy.DialedDirectly = defaults.TransparentProxy.DialedDirectly
 	}
 
+	if ns.Proxy.MutualTLSMode == structs.MutualTLSModeDefault {
+		ns.Proxy.MutualTLSMode = defaults.MutualTLSMode
+	}
+
 	// remoteUpstreams contains synthetic Upstreams generated from central config (service-defaults.UpstreamConfigs).
 	remoteUpstreams := make(map[structs.PeeredServiceName]structs.Upstream)
 

agent/configentry/merge_service_config_test.go

@@ -35,6 +35,7 @@ func Test_MergeServiceConfig_TransparentProxy(t *testing.T) {
 					ProxyConfig: map[string]interface{}{
 						"foo": "bar",
 					},
+					MutualTLSMode: structs.MutualTLSModePermissive,
 					Expose: structs.ExposeConfig{
 						Checks: true,
 						Paths: []structs.ExposePath{
@@ -76,6 +77,7 @@ func Test_MergeServiceConfig_TransparentProxy(t *testing.T) {
 						OutboundListenerPort: 10101,
 						DialedDirectly:       true,
 					},
+					MutualTLSMode: structs.MutualTLSModePermissive,
 					Config: map[string]interface{}{
 						"foo": "bar",
 					},

agent/configentry/resolve.go

@@ -47,6 +47,7 @@ func ComputeResolvedServiceConfig(
 		thisReply.ProxyConfig = mapCopy.(map[string]interface{})
 		thisReply.Mode = proxyConf.Mode
 		thisReply.TransparentProxy = proxyConf.TransparentProxy
+		thisReply.MutualTLSMode = proxyConf.MutualTLSMode
 		thisReply.MeshGateway = proxyConf.MeshGateway
 		thisReply.Expose = proxyConf.Expose
 		thisReply.EnvoyExtensions = proxyConf.EnvoyExtensions
@@ -120,6 +121,10 @@ func ComputeResolvedServiceConfig(
 			thisReply.ProxyConfig = proxyConf
 		}
 
+		if serviceConf.MutualTLSMode != structs.MutualTLSModeDefault {
+			thisReply.MutualTLSMode = serviceConf.MutualTLSMode
+		}
+
 		thisReply.Meta = serviceConf.Meta
 		// Service defaults' envoy extensions are appended to the proxy defaults extensions so that proxy defaults
 		// extensions are applied first.

agent/configentry/resolve_test.go

@@ -494,6 +494,50 @@ func Test_ComputeResolvedServiceConfig(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "service-defaults inherits mutual_tls_mode from proxy-defaults",
+			args: args{
+				scReq: &structs.ServiceConfigRequest{
+					Name: "sid",
+				},
+				entries: &ResolvedServiceConfigSet{
+					ProxyDefaults: map[string]*structs.ProxyConfigEntry{
+						acl.DefaultEnterpriseMeta().PartitionOrDefault(): {
+							MutualTLSMode: structs.MutualTLSModePermissive,
+						},
+					},
+					ServiceDefaults: map[structs.ServiceID]*structs.ServiceConfigEntry{
+						sid: {},
+					},
+				},
+			},
+			want: &structs.ServiceConfigResponse{
+				MutualTLSMode: structs.MutualTLSModePermissive,
+			},
+		},
+		{
+			name: "service-defaults overrides mutual_tls_mode in proxy-defaults",
+			args: args{
+				scReq: &structs.ServiceConfigRequest{
+					Name: "sid",
+				},
+				entries: &ResolvedServiceConfigSet{
+					ProxyDefaults: map[string]*structs.ProxyConfigEntry{
+						acl.DefaultEnterpriseMeta().PartitionOrDefault(): {
+							MutualTLSMode: structs.MutualTLSModeStrict,
+						},
+					},
+					ServiceDefaults: map[structs.ServiceID]*structs.ServiceConfigEntry{
+						sid: {
+							MutualTLSMode: structs.MutualTLSModePermissive,
+						},
+					},
+				},
+			},
+			want: &structs.ServiceConfigResponse{
+				MutualTLSMode: structs.MutualTLSModePermissive,
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

agent/consul/state/config_entry.go

@@ -18,6 +18,10 @@ import (
 	"github.com/hashicorp/consul/lib/maps"
 )
 
+var (
+	permissiveModeNotAllowedError = errors.New("cannot set MutualTLSMode=permissive because AllowEnablingPermissiveMutualTLS=false in the mesh config entry")
+)
+
 type ConfigEntryLinkIndex struct {
 }
 
@@ -229,14 +233,16 @@ func ensureConfigEntryTxn(tx WriteTxn, idx uint64, statusUpdate bool, conf struc
 		return fmt.Errorf("failed configuration lookup: %s", err)
 	}
 
+	var existingConf structs.ConfigEntry
 	raftIndex := conf.GetRaftIndex()
 	if existing != nil {
-		existingIdx := existing.(structs.ConfigEntry).GetRaftIndex()
+		existingConf = existing.(structs.ConfigEntry)
+		existingIdx := existingConf.GetRaftIndex()
 		raftIndex.CreateIndex = existingIdx.CreateIndex
 
 		// Handle optional upsert logic.
 		if updatableConf, ok := conf.(structs.UpdatableConfigEntry); ok {
-			if err := updatableConf.UpdateOver(existing.(structs.ConfigEntry)); err != nil {
+			if err := updatableConf.UpdateOver(existingConf); err != nil {
 				return err
 			}
 		}
@@ -256,7 +262,7 @@ func ensureConfigEntryTxn(tx WriteTxn, idx uint64, statusUpdate bool, conf struc
 	}
 	raftIndex.ModifyIndex = idx
 
-	err = validateProposedConfigEntryInGraph(tx, q, conf)
+	err = validateProposedConfigEntryInGraph(tx, q, conf, existingConf)
 	if err != nil {
 		return err // Err is already sufficiently decorated.
 	}
@@ -445,7 +451,7 @@ func deleteConfigEntryTxn(tx WriteTxn, idx uint64, kind, name string, entMeta *a
 		}
 	}
 
-	err = validateProposedConfigEntryInGraph(tx, q, nil)
+	err = validateProposedConfigEntryInGraph(tx, q, nil, c)
 	if err != nil {
 		return err // Err is already sufficiently decorated.
 	}
@@ -544,15 +550,33 @@ func insertConfigEntryWithTxn(tx WriteTxn, idx uint64, conf structs.ConfigEntry)
 func validateProposedConfigEntryInGraph(
 	tx ReadTxn,
 	kindName configentry.KindName,
-	newEntry structs.ConfigEntry,
+	newEntry, existingEntry structs.ConfigEntry,
 ) error {
 	switch kindName.Kind {
 	case structs.ProxyDefaults:
 		// TODO: why handle an invalid case?
 		if kindName.Name != structs.ProxyConfigGlobal {
 			return nil
 		}
+		if newPD, ok := newEntry.(*structs.ProxyConfigEntry); ok && newPD != nil {
+			var existingMode structs.MutualTLSMode
+			if existingPD, ok := existingEntry.(*structs.ProxyConfigEntry); ok && existingPD != nil {
+				existingMode = existingPD.MutualTLSMode
+			}
+			if err := checkMutualTLSMode(tx, kindName, newPD.MutualTLSMode, existingMode); err != nil {
+				return err
+			}
+		}
 	case structs.ServiceDefaults:
+		if newSD, ok := newEntry.(*structs.ServiceConfigEntry); ok && newSD != nil {
+			var existingMode structs.MutualTLSMode
+			if existingSD, ok := existingEntry.(*structs.ServiceConfigEntry); ok && existingSD != nil {
+				existingMode = existingSD.MutualTLSMode
+			}
+			if err := checkMutualTLSMode(tx, kindName, newSD.MutualTLSMode, existingMode); err != nil {
+				return err
+			}
+		}
 	case structs.ServiceRouter:
 	case structs.ServiceSplitter:
 	case structs.ServiceResolver:
@@ -583,6 +607,54 @@ func validateProposedConfigEntryInGraph(
 	return validateProposedConfigEntryInServiceGraph(tx, kindName, newEntry)
 }
 
+// checkMutualTLSMode validates the MutualTLSMode (in proxy-defaults or
+// service-defaults) against the AllowEnablingPermissiveMutualTLS setting in the
+// mesh config entry, as follows:
+//
+// - If AllowEnablingPermissiveMutualTLS=true, any value of MutualTLSMode is allowed.
+// - If AllowEnablingPermissiveMutualTLS=false, *changing* to MutualTLSMode=permissive is not allowed
+//
+// If MutualTLSMode=permissive is already stored, but the setting is not being changed
+// by this transaction, then the permissive setting is allowed (does not cause a validation error).
+func checkMutualTLSMode(tx ReadTxn, kindName configentry.KindName, newMode, existingMode structs.MutualTLSMode) error {
+	// Setting the mode to something not permissive is always allowed.
+	if newMode != structs.MutualTLSModePermissive {
+		return nil
+	}
+
+	// If the MutualTLSMode has not been changed, then do not error. This allows
+	// remaining in MutualTLSMode=permissive without causing validation failures
+	// after AllowEnablingPermissiveMutualTLS=false is set.
+	if existingMode == newMode {
+		return nil
+	}
+
+	// The mesh config entry exists in the default namespace in the given partition.
+	metaInDefaultNS := acl.NewEnterpriseMetaWithPartition(
+		kindName.EnterpriseMeta.PartitionOrDefault(),
+		acl.DefaultNamespaceName,
+	)
+	_, mesh, err := configEntryTxn(tx, nil, structs.MeshConfig, structs.MeshConfigMesh, &metaInDefaultNS)
+	if err != nil {
+		return fmt.Errorf("unable to validate MutualTLSMode against mesh config entry: %w", err)
+	}
+
+	permissiveAllowed := false
+	if mesh != nil {
+		meshConfig, ok := mesh.(*structs.MeshConfigEntry)
+		if !ok {
+			return fmt.Errorf("unable to validate MutualTLSMode: invalid type from mesh config entry lookup: %T", mesh)
+		}
+		permissiveAllowed = meshConfig.AllowEnablingPermissiveMutualTLS
+	}
+
+	// If permissive is not allowed, then any value for MutualTLSMode is allowed.
+	if !permissiveAllowed && newMode == structs.MutualTLSModePermissive {
+		return permissiveModeNotAllowedError
+	}
+	return nil
+}
+
 func checkGatewayClash(tx ReadTxn, kindName configentry.KindName, otherKind string) error {
 	_, entry, err := configEntryTxn(tx, nil, otherKind, kindName.Name, &kindName.EnterpriseMeta)
 	if err != nil {