transparent proxy: add jobspec support

Add a transparent proxy block to the existing Connect sidecar service proxy
block. This changeset is plumbing required to support transparent proxy
configuration on the client.

Ref: #10628

api/consul.go

@@ -157,12 +157,17 @@ func (st *SidecarTask) Canonicalize() {
 
 // ConsulProxy represents a Consul Connect sidecar proxy jobspec block.
 type ConsulProxy struct {
-	LocalServiceAddress string                 `mapstructure:"local_service_address" hcl:"local_service_address,optional"`
-	LocalServicePort    int                    `mapstructure:"local_service_port" hcl:"local_service_port,optional"`
-	Expose              *ConsulExposeConfig    `mapstructure:"expose" hcl:"expose,block"`
-	ExposeConfig        *ConsulExposeConfig    // Deprecated: only to maintain backwards compatibility. Use Expose instead.
-	Upstreams           []*ConsulUpstream      `hcl:"upstreams,block"`
-	Config              map[string]interface{} `hcl:"config,block"`
+	LocalServiceAddress string              `mapstructure:"local_service_address" hcl:"local_service_address,optional"`
+	LocalServicePort    int                 `mapstructure:"local_service_port" hcl:"local_service_port,optional"`
+	Expose              *ConsulExposeConfig `mapstructure:"expose" hcl:"expose,block"`
+	ExposeConfig        *ConsulExposeConfig // Deprecated: only to maintain backwards compatibility. Use Expose instead.
+	Upstreams           []*ConsulUpstream   `hcl:"upstreams,block"`
+
+	// TransparentProxy configures the Envoy sidecar to use "transparent
+	// proxying", which creates IP tables rules inside the network namespace to
+	// ensure traffic flows thru the Envoy proxy
+	TransparentProxy *ConsulTransparentProxy `mapstructure:"transparent_proxy" hcl:"transparent_proxy,block"`
+	Config           map[string]interface{}  `hcl:"config,block"`
 }
 
 func (cp *ConsulProxy) Canonicalize() {
@@ -176,6 +181,8 @@ func (cp *ConsulProxy) Canonicalize() {
 		cp.Upstreams = nil
 	}
 
+	cp.TransparentProxy.Canonicalize()
+
 	for _, upstream := range cp.Upstreams {
 		upstream.Canonicalize()
 	}
@@ -263,6 +270,63 @@ func (cu *ConsulUpstream) Canonicalize() {
 	}
 }
 
+type PortNumber = uint16
+
+// ConsulTransparentProxy is used to configure the Envoy sidecar for
+// "transparent proxying", which creates IP tables rules inside the network
+// namespace to ensure traffic flows thru the Envoy proxy
+type ConsulTransparentProxy struct {
+	// UID of the Envoy proxy. Defaults to the default Envoy proxy container
+	// image user.
+	UID string `mapstructure:"uid" hcl:"uid,optional"`
+
+	// OutboundPort is the Envoy proxy's outbound listener port. Inbound TCP
+	// traffic hitting the PROXY_IN_REDIRECT chain will be redirected here.
+	// Defaults to 15001.
+	OutboundPort PortNumber `mapstructure:"outbound_port" hcl:"outbound_port,optional"`
+
+	// ExcludeInboundPorts is an additional set of ports will be excluded from
+	// redirection to the Envoy proxy. Can be Port.Label or Port.Value. This set
+	// will be added to the ports automatically excluded for the Expose.Port and
+	// Check.Expose fields.
+	ExcludeInboundPorts []string `mapstructure:"exclude_inbound_ports" hcl:"exclude_inbound_ports,optional"`
+
+	// ExcludeOutboundPorts is a set of outbound ports that will not be
+	// redirected to the Envoy proxy, specified as port numbers.
+	ExcludeOutboundPorts []PortNumber `mapstructure:"exclude_outbound_ports" hcl:"exclude_outbound_ports,optional"`
+
+	// ExcludeOutboundCIDRs is a set of outbound CIDR blocks that will not be
+	// redirected to the Envoy proxy.
+	ExcludeOutboundCIDRs []string `mapstructure:"exclude_outbound_cidrs" hcl:"exclude_outbound_cidrs,optional"`
+
+	// ExcludeUIDs is a set of user IDs whose network traffic will not be
+	// redirected through the Envoy proxy.
+	ExcludeUIDs []string `mapstructure:"exclude_uids" hcl:"exclude_uids,optional"`
+
+	// NoDNS disables redirection of DNS traffic to Consul DNS. By default NoDNS
+	// is false and transparent proxy will direct DNS traffic to Consul DNS if
+	// available on the client.
+	NoDNS bool `mapstructure:"no_dns" hcl:"no_dns,optional"`
+}
+
+func (tp *ConsulTransparentProxy) Canonicalize() {
+	if tp == nil {
+		return
+	}
+	if len(tp.ExcludeInboundPorts) == 0 {
+		tp.ExcludeInboundPorts = nil
+	}
+	if len(tp.ExcludeOutboundCIDRs) == 0 {
+		tp.ExcludeOutboundCIDRs = nil
+	}
+	if len(tp.ExcludeOutboundPorts) == 0 {
+		tp.ExcludeOutboundPorts = nil
+	}
+	if len(tp.ExcludeUIDs) == 0 {
+		tp.ExcludeUIDs = nil
+	}
+}
+
 type ConsulExposeConfig struct {
 	Paths []*ConsulExposePath `mapstructure:"path" hcl:"path,block"`
 	Path  []*ConsulExposePath // Deprecated: only to maintain backwards compatibility. Use Paths instead.

command/agent/consul/connect.go

@@ -145,8 +145,13 @@ func connectSidecarProxy(info structs.AllocInfo, proxy *structs.ConsulProxy, cPo
 	if err != nil {
 		return nil, err
 	}
+	mode := api.ProxyModeDefault
+	if proxy.TransparentProxy != nil {
+		mode = api.ProxyModeTransparent
+	}
 
 	return &api.AgentServiceConnectProxyConfig{
+		Mode:                mode,
 		LocalServiceAddress: proxy.LocalServiceAddress,
 		LocalServicePort:    proxy.LocalServicePort,
 		Config:              connectProxyConfig(proxy.Config, cPort, info),

command/agent/job_endpoint.go

@@ -1867,6 +1867,7 @@ func apiConnectSidecarServiceProxyToStructs(in *api.ConsulProxy) *structs.Consul
 		LocalServicePort:    in.LocalServicePort,
 		Upstreams:           apiUpstreamsToStructs(in.Upstreams),
 		Expose:              apiConsulExposeConfigToStructs(expose),
+		TransparentProxy:    apiConnectTransparentProxyToStructs(in.TransparentProxy),
 		Config:              maps.Clone(in.Config),
 	}
 }
@@ -1918,6 +1919,21 @@ func apiConsulExposeConfigToStructs(in *api.ConsulExposeConfig) *structs.ConsulE
 	}
 }
 
+func apiConnectTransparentProxyToStructs(in *api.ConsulTransparentProxy) *structs.ConsulTransparentProxy {
+	if in == nil {
+		return nil
+	}
+	return &structs.ConsulTransparentProxy{
+		UID:                  in.UID,
+		OutboundPort:         in.OutboundPort,
+		ExcludeInboundPorts:  in.ExcludeInboundPorts,
+		ExcludeOutboundPorts: in.ExcludeOutboundPorts,
+		ExcludeOutboundCIDRs: in.ExcludeOutboundCIDRs,
+		ExcludeUIDs:          in.ExcludeUIDs,
+		NoDNS:                in.NoDNS,
+	}
+}
+
 func apiConsulExposePathsToStructs(in []*api.ConsulExposePath) []structs.ConsulExposePath {
 	if len(in) == 0 {
 		return nil

nomad/job_endpoint_hook_connect.go

@@ -561,33 +561,68 @@ func groupConnectValidate(g *structs.TaskGroup) error {
 		}
 	}
 
-	if err := groupConnectUpstreamsValidate(g.Name, g.Services); err != nil {
+	if err := groupConnectUpstreamsValidate(g, g.Services); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-func groupConnectUpstreamsValidate(group string, services []*structs.Service) error {
+func groupConnectUpstreamsValidate(g *structs.TaskGroup, services []*structs.Service) error {
 	listeners := make(map[string]string) // address -> service
 
+	var connectBlockCount int
+	var hasTproxy bool
+
 	for _, service := range services {
+		if service.Connect != nil {
+			connectBlockCount++
+		}
 		if service.Connect.HasSidecar() && service.Connect.SidecarService.Proxy != nil {
 			for _, up := range service.Connect.SidecarService.Proxy.Upstreams {
 				listener := net.JoinHostPort(up.LocalBindAddress, strconv.Itoa(up.LocalBindPort))
 				if s, exists := listeners[listener]; exists {
 					return fmt.Errorf(
 						"Consul Connect services %q and %q in group %q using same address for upstreams (%s)",
-						service.Name, s, group, listener,
+						service.Name, s, g.Name, listener,
 					)
 				}
 				listeners[listener] = service.Name
 			}
+
+			if tp := service.Connect.SidecarService.Proxy.TransparentProxy; tp != nil {
+				hasTproxy = true
+				for _, portLabel := range tp.ExcludeInboundPorts {
+					if !transparentProxyPortLabelValidate(g, portLabel) {
+						return fmt.Errorf(
+							"Consul Connect transparent proxy port %q must be numeric or one of network.port labels", portLabel)
+					}
+				}
+			}
+
 		}
 	}
+	if hasTproxy && connectBlockCount > 1 {
+		return fmt.Errorf("Consul Connect transparent proxy requires there is only one connect block")
+	}
 	return nil
 }
 
+func transparentProxyPortLabelValidate(g *structs.TaskGroup, portLabel string) bool {
+	if _, err := strconv.ParseUint(portLabel, 10, 64); err == nil {
+		return true
+	}
+
+	for _, network := range g.Networks {
+		for _, reservedPort := range network.ReservedPorts {
+			if reservedPort.Label == portLabel {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 func groupConnectSidecarValidate(g *structs.TaskGroup, s *structs.Service) error {
 	if n := len(g.Networks); n != 1 {
 		return fmt.Errorf("Consul Connect sidecars require exactly 1 network, found %d in group %q", n, g.Name)

nomad/job_endpoint_hook_connect_test.go

@@ -548,13 +548,15 @@ func TestJobEndpointConnect_groupConnectUpstreamsValidate(t *testing.T) {
 	ci.Parallel(t)
 
 	t.Run("no connect services", func(t *testing.T) {
-		err := groupConnectUpstreamsValidate("group",
+		tg := &structs.TaskGroup{Name: "group"}
+		err := groupConnectUpstreamsValidate(tg,
 			[]*structs.Service{{Name: "s1"}, {Name: "s2"}})
-		require.NoError(t, err)
+		must.NoError(t, err)
 	})
 
 	t.Run("connect services no overlap", func(t *testing.T) {
-		err := groupConnectUpstreamsValidate("group",
+		tg := &structs.TaskGroup{Name: "group"}
+		err := groupConnectUpstreamsValidate(tg,
 			[]*structs.Service{
 				{
 					Name: "s1",
@@ -589,11 +591,12 @@ func TestJobEndpointConnect_groupConnectUpstreamsValidate(t *testing.T) {
 					},
 				},
 			})
-		require.NoError(t, err)
+		must.NoError(t, err)
 	})
 
 	t.Run("connect services overlap port", func(t *testing.T) {
-		err := groupConnectUpstreamsValidate("group",
+		tg := &structs.TaskGroup{Name: "group"}
+		err := groupConnectUpstreamsValidate(tg,
 			[]*structs.Service{
 				{
 					Name: "s1",
@@ -628,7 +631,55 @@ func TestJobEndpointConnect_groupConnectUpstreamsValidate(t *testing.T) {
 					},
 				},
 			})
-		require.EqualError(t, err, `Consul Connect services "s2" and "s1" in group "group" using same address for upstreams (127.0.0.1:9002)`)
+		must.EqError(t, err, `Consul Connect services "s2" and "s1" in group "group" using same address for upstreams (127.0.0.1:9002)`)
+	})
+
+	t.Run("connect tproxy excludes invalid port", func(t *testing.T) {
+		tg := &structs.TaskGroup{Name: "group", Networks: structs.Networks{
+			{
+				ReservedPorts: []structs.Port{{
+					Label: "www",
+				}},
+			},
+		}}
+		err := groupConnectUpstreamsValidate(tg,
+			[]*structs.Service{
+				{
+					Name: "s1",
+					Connect: &structs.ConsulConnect{
+						SidecarService: &structs.ConsulSidecarService{
+							Proxy: &structs.ConsulProxy{
+								TransparentProxy: &structs.ConsulTransparentProxy{
+									ExcludeInboundPorts: []string{"www", "9000", "no-such-label"},
+								},
+							},
+						},
+					},
+				},
+			})
+		must.EqError(t, err, `Consul Connect transparent proxy port "no-such-label" must be numeric or one of network.port labels`)
+	})
+
+	t.Run("Consul Connect transparent proxy allows only one Connect block", func(t *testing.T) {
+		tg := &structs.TaskGroup{Name: "group"}
+		err := groupConnectUpstreamsValidate(tg,
+			[]*structs.Service{
+				{
+					Name:    "s1",
+					Connect: &structs.ConsulConnect{},
+				},
+				{
+					Name: "s2",
+					Connect: &structs.ConsulConnect{
+						SidecarService: &structs.ConsulSidecarService{
+							Proxy: &structs.ConsulProxy{
+								TransparentProxy: &structs.ConsulTransparentProxy{},
+							},
+						},
+					},
+				},
+			})
+		must.EqError(t, err, `Consul Connect transparent proxy requires there is only one connect block`)
 	})
 }