backport of commit 505b8e1

.changelog/16257.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+peering: Fix issue where mesh gateways would use the wrong address when contacting a remote peer with the same datacenter name.
+```

.changelog/16263.txt

@@ -0,0 +1,4 @@
+```release-note:security
+Upgrade to use Go 1.20.1. 
+This resolves vulnerabilities [CVE-2022-41724](https://go.dev/issue/58001) in `crypto/tls` and [CVE-2022-41723](https://go.dev/issue/57855) in `net/http`.
+```

.changelog/16284.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+cli: adds new CLI commands `consul troubleshoot upstreams` and `consul troubleshoot proxy` to troubleshoot Consul's service mesh configuration and network issues.
+```

.changelog/16301.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+agent configuration: Fix issue of using unix socket when https is used.
+```

.changelog/16339.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+peering: Fix bug where services were incorrectly imported as connect-enabled.
+```

.changelog/16358.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+container: Upgrade container image to use to Alpine 3.17.
+```

.changelog/16369.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+**API Gateway (Beta)** This version adds support for API gateway on VMs. API gateway provides a highly-configurable ingress for requests coming into a Consul network. For more information, refer to the [API gateway](https://developer.hashicorp.com/consul/docs/connect/gateways/api-gateway) documentation.
+```

.circleci/config.yml

@@ -21,12 +21,12 @@ references:
  GIT_COMMITTER_NAME: circleci-consul
  S3_ARTIFACT_BUCKET: consul-dev-artifacts-v2
  BASH_ENV: .circleci/bash_env.sh
- GO_VERSION: 1.19.4
+ GO_VERSION: 1.20.1
  envoy-versions: &supported_envoy_versions
- - &default_envoy_version "1.22.7"
- - "1.23.4"
- - "1.24.2"
- - "1.25.1"
+ - &default_envoy_version "1.21.5"
+ - "1.22.5"
+ - "1.23.2"
+ - "1.24.0"
  nomad-versions: &supported_nomad_versions
  - &default_nomad_version "1.3.3"
  - "1.2.10"
@@ -39,7 +39,7 @@ references:
  images:
  # When updating the Go version, remember to also update the versions in the
  # workflows section for go-test-lib jobs.
- go: &GOLANG_IMAGE docker.mirror.hashicorp.services/cimg/go:1.19.4
+ go: &GOLANG_IMAGE docker.mirror.hashicorp.services/cimg/go:1.20.1
  ember: &EMBER_IMAGE docker.mirror.hashicorp.services/circleci/node:14-browsers
  ubuntu: &UBUNTU_CI_IMAGE ubuntu-2004:202201-02
  cache:
@@ -613,7 +613,7 @@ jobs:
  - run: *notify-slack-failure
  nomad-integration-test: &NOMAD_TESTS
  docker:
- - image: docker.mirror.hashicorp.services/cimg/go:1.19
+ - image: docker.mirror.hashicorp.services/cimg/go:1.20
  parameters:
  nomad-version:
  type: enum
@@ -1110,34 +1110,34 @@ workflows:
  - go-test-lib:
  name: "go-test-envoyextensions"
  path: envoyextensions
- go-version: "1.19"
+ go-version: "1.20"
  requires: [dev-build]
  <<: *filter-ignore-non-go-branches
  - go-test-lib:
  name: "go-test-troubleshoot"
  path: troubleshoot
- go-version: "1.19"
+ go-version: "1.20"
  requires: [dev-build]
  <<: *filter-ignore-non-go-branches
  - go-test-lib:
- name: "go-test-api go1.18"
+ name: "go-test-api go1.19"
  path: api
- go-version: "1.18"
+ go-version: "1.19"
  requires: [dev-build]
  - go-test-lib:
- name: "go-test-api go1.19"
+ name: "go-test-api go1.20"
  path: api
- go-version: "1.19"
+ go-version: "1.20"
  requires: [dev-build]
  - go-test-lib:
- name: "go-test-sdk go1.18"
+ name: "go-test-sdk go1.19"
  path: sdk
- go-version: "1.18"
+ go-version: "1.19"
  <<: *filter-ignore-non-go-branches
  - go-test-lib:
- name: "go-test-sdk go1.19"
+ name: "go-test-sdk go1.20"
  path: sdk
- go-version: "1.19"
+ go-version: "1.20"
  <<: *filter-ignore-non-go-branches
  - go-test-race: *filter-ignore-non-go-branches
  - go-test-32bit: *filter-ignore-non-go-branches

.github/workflows/build.yml

@@ -79,15 +79,15 @@ jobs:
  strategy:
  matrix:
  include:
- - {go: "1.19.4", goos: "linux", goarch: "386"}
- - {go: "1.19.4", goos: "linux", goarch: "amd64"}
- - {go: "1.19.4", goos: "linux", goarch: "arm"}
- - {go: "1.19.4", goos: "linux", goarch: "arm64"}
- - {go: "1.19.4", goos: "freebsd", goarch: "386"}
- - {go: "1.19.4", goos: "freebsd", goarch: "amd64"}
- - {go: "1.19.4", goos: "windows", goarch: "386"}
- - {go: "1.19.4", goos: "windows", goarch: "amd64"}
- - {go: "1.19.4", goos: "solaris", goarch: "amd64"}
+ - {go: "1.20.1", goos: "linux", goarch: "386"}
+ - {go: "1.20.1", goos: "linux", goarch: "amd64"}
+ - {go: "1.20.1", goos: "linux", goarch: "arm"}
+ - {go: "1.20.1", goos: "linux", goarch: "arm64"}
+ - {go: "1.20.1", goos: "freebsd", goarch: "386"}
+ - {go: "1.20.1", goos: "freebsd", goarch: "amd64"}
+ - {go: "1.20.1", goos: "windows", goarch: "386"}
+ - {go: "1.20.1", goos: "windows", goarch: "amd64"}
+ - {go: "1.20.1", goos: "solaris", goarch: "amd64"}
  fail-fast: true
 
  name: Go ${{ matrix.go }} ${{ matrix.goos }} ${{ matrix.goarch }} build
@@ -176,7 +176,7 @@ jobs:
  matrix:
  goos: [ darwin ]
  goarch: [ "amd64", "arm64" ]
- go: [ "1.19.4" ]
+ go: [ "1.20.1" ]
  fail-fast: true
 
  name: Go ${{ matrix.go }} ${{ matrix.goos }} ${{ matrix.goarch }} build

.github/workflows/nightly-test-1.15.x.yaml → .github/workflows/nightly-test-1.11.x.yaml

@@ -1,13 +1,13 @@
-name: Nightly Test 1.15.x
+name: Nightly Test 1.11.x
 on:
  schedule:
  - cron: '0 4 * * *'
  workflow_dispatch: {}
 
 env:
  EMBER_PARTITION_TOTAL: 4 # Has to be changed in tandem with the matrix.partition
- BRANCH: "release/1.15.x"
- BRANCH_NAME: "release/1.15.x" # Used for naming artifacts
+ BRANCH: "release/1.11.x"
+ BRANCH_NAME: "release-1.11.x" # Used for naming artifacts
 
 jobs:
  frontend-test-workspace-node:

Dockerfile

@@ -13,7 +13,7 @@
 # Official docker image that includes binaries from releases.hashicorp.com. This
 # downloads the release from releases.hashicorp.com and therefore requires that
 # the release is published before building the Docker image.
-FROM docker.mirror.hashicorp.services/alpine:3.15 as official
+FROM docker.mirror.hashicorp.services/alpine:3.17 as official
 
 # This is the release of Consul to pull in.
 ARG VERSION
@@ -109,7 +109,7 @@ CMD ["agent", "-dev", "-client", "0.0.0.0"]
 
 # Production docker image that uses CI built binaries.
 # Remember, this image cannot be built locally.
-FROM docker.mirror.hashicorp.services/alpine:3.15 as default
+FROM docker.mirror.hashicorp.services/alpine:3.17 as default
 
 ARG PRODUCT_VERSION
 ARG BIN_NAME

GNUmakefile

@@ -7,11 +7,11 @@ SHELL = bash
 # These version variables can either be a valid string for "go install <module>@<version>"
 # or the string @DEV to imply use what is currently installed locally.
 ###
-GOLANGCI_LINT_VERSION='v1.50.1'
-MOCKERY_VERSION='v2.15.0'
+GOLANGCI_LINT_VERSION='v1.51.1'
+MOCKERY_VERSION='v2.20.0'
 BUF_VERSION='v1.4.0'
 PROTOC_GEN_GO_GRPC_VERSION="v1.2.0"
-MOG_VERSION='v0.3.0'
+MOG_VERSION='v0.4.0'
 PROTOC_GO_INJECT_TAG_VERSION='v1.3.0'
 PROTOC_GEN_GO_BINARY_VERSION="v0.1.0"
 DEEP_COPY_VERSION='bc3f5aa5735d8a54961580a3a24422c308c831c2'

agent/agent.go

@@ -1051,7 +1051,8 @@ func (a *Agent) listenHTTP() ([]apiServer, error) {
  for _, l := range listeners {
  var tlscfg *tls.Config
  _, isTCP := l.(*tcpKeepAliveListener)
- if isTCP && proto == "https" {
+ isUnix := l.Addr().Network() == "unix"
+ if (isTCP || isUnix) && proto == "https" {
  tlscfg = a.tlsConfigurator.IncomingHTTPSConfig()
  l = tls.NewListener(l, tlscfg)
  }

agent/agent_test.go

@@ -4,12 +4,13 @@ import (
  "bytes"
  "context"
  "crypto/md5"
+ "crypto/rand"
  "crypto/tls"
  "crypto/x509"
  "encoding/base64"
  "encoding/json"
  "fmt"
- "math/rand"
+ mathrand "math/rand"
  "net"
  "net/http"
  "net/http/httptest"
@@ -752,7 +753,7 @@ func testAgent_AddServices_AliasUpdateCheckNotReverted(t *testing.T, extraHCL st
 
 func test_createAlias(t *testing.T, agent *TestAgent, chk *structs.CheckType, expectedResult string) func(r *retry.R) {
  t.Helper()
- serviceNum := rand.Int()
+ serviceNum := mathrand.Int()
  srv := &structs.NodeService{
  Service: fmt.Sprintf("serviceAlias-%d", serviceNum),
  Tags: []string{"tag1"},

agent/consul/auto_config_endpoint_test.go

@@ -3,12 +3,11 @@ package consul
 import (
  "bytes"
  "crypto"
- crand "crypto/rand"
+ "crypto/rand"
  "crypto/x509"
  "encoding/base64"
  "encoding/pem"
  "fmt"
- "math/rand"
  "net"
  "net/url"
  "os"
@@ -884,7 +883,7 @@ func TestAutoConfig_parseAutoConfigCSR(t *testing.T) {
  // customizations to allow for better unit testing.
  createCSR := func(tmpl *x509.CertificateRequest, privateKey crypto.Signer) (string, error) {
  connect.HackSANExtensionForCSR(tmpl)
- bs, err := x509.CreateCertificateRequest(crand.Reader, tmpl, privateKey)
+ bs, err := x509.CreateCertificateRequest(rand.Reader, tmpl, privateKey)
  require.NoError(t, err)
  var csrBuf bytes.Buffer
  err = pem.Encode(&csrBuf, &pem.Block{Type: "CERTIFICATE REQUEST", Bytes: bs})

agent/consul/discoverychain/gateway.go

@@ -5,6 +5,7 @@ import (
  "hash/crc32"
  "sort"
  "strconv"
+ "strings"
 
  "github.com/hashicorp/consul/agent/configentry"
  "github.com/hashicorp/consul/agent/structs"
@@ -17,6 +18,7 @@ type GatewayChainSynthesizer struct {
  trustDomain string
  suffix string
  gateway *structs.APIGatewayConfigEntry
+ hostname string
  matchesByHostname map[string][]hostnameMatch
  tcpRoutes []structs.TCPRouteConfigEntry
 }
@@ -44,17 +46,17 @@ func (l *GatewayChainSynthesizer) AddTCPRoute(route structs.TCPRouteConfigEntry)
  l.tcpRoutes = append(l.tcpRoutes, route)
 }
 
+// SetHostname sets the base hostname for a listener that this is being synthesized for
+func (l *GatewayChainSynthesizer) SetHostname(hostname string) {
+ l.hostname = hostname
+}
+
 // AddHTTPRoute takes a new route and flattens its rule matches out per hostname.
 // This is required since a single route can specify multiple hostnames, and a
 // single hostname can be specified in multiple routes. Routing for a given
 // hostname must behave based on the aggregate of all rules that apply to it.
 func (l *GatewayChainSynthesizer) AddHTTPRoute(route structs.HTTPRouteConfigEntry) {
- hostnames := route.Hostnames
- if len(route.Hostnames) == 0 {
- // add a wildcard if there are no explicit hostnames set
- hostnames = append(hostnames, "*")
- }
-
+ hostnames := route.FilteredHostnames(l.hostname)
  for _, host := range hostnames {
  matches, ok := l.matchesByHostname[host]
  if !ok {
@@ -125,6 +127,23 @@ func (l *GatewayChainSynthesizer) Synthesize(chains ...*structs.CompiledDiscover
  if err != nil {
  return nil, nil, err
  }
+
+ // fix up the nodes for the terminal targets to either be a splitter or resolver if there is no splitter present
+ for name, node := range compiled.Nodes {
+ switch node.Type {
+ // we should only have these two types
+ case structs.DiscoveryGraphNodeTypeRouter:
+ for i, route := range node.Routes {
+ node.Routes[i].NextNode = targetForResolverNode(route.NextNode, chains)
+ }
+ case structs.DiscoveryGraphNodeTypeSplitter:
+ for i, split := range node.Splits {
+ node.Splits[i].NextNode = targetForResolverNode(split.NextNode, chains)
+ }
+ }
+ compiled.Nodes[name] = node
+ }
+
  for _, c := range chains {
  for id, target := range c.Targets {
  compiled.Targets[id] = target
@@ -176,6 +195,27 @@ func (l *GatewayChainSynthesizer) consolidateHTTPRoutes() []structs.HTTPRouteCon
  return routes
 }
 
+func targetForResolverNode(nodeName string, chains []*structs.CompiledDiscoveryChain) string {
+ resolverPrefix := structs.DiscoveryGraphNodeTypeResolver + ":"
+ splitterPrefix := structs.DiscoveryGraphNodeTypeSplitter + ":"
+
+ if !strings.HasPrefix(nodeName, resolverPrefix) {
+ return nodeName
+ }
+
+ splitterName := splitterPrefix + strings.TrimPrefix(nodeName, resolverPrefix)
+
+ for _, c := range chains {
+ for name, node := range c.Nodes {
+ if node.IsSplitter() && strings.HasPrefix(splitterName, name) {
+ return name
+ }
+ }
+ }
+
+ return nodeName
+}
+
 func hostsKey(hosts ...string) string {
  sort.Strings(hosts)
  hostsHash := crc32.NewIEEE()

agent/consul/discoverychain/gateway_httproute.go

@@ -77,15 +77,14 @@ func httpRouteToDiscoveryChain(route structs.HTTPRouteConfigEntry) (*structs.Ser
 
  for idx, rule := range route.Rules {
  modifier := httpRouteFiltersToServiceRouteHeaderModifier(rule.Filters.Headers)
- prefixRewrite := httpRouteFiltersToDestinationPrefixRewrite(rule.Filters.URLRewrites)
+ prefixRewrite := httpRouteFiltersToDestinationPrefixRewrite(rule.Filters.URLRewrite)
 
  var destination structs.ServiceRouteDestination
  if len(rule.Services) == 1 {
- // TODO open question: is there a use case where someone might want to set the rewrite to ""?
  service := rule.Services[0]
 
- servicePrefixRewrite := httpRouteFiltersToDestinationPrefixRewrite(service.Filters.URLRewrites)
- if servicePrefixRewrite == "" {
+ servicePrefixRewrite := httpRouteFiltersToDestinationPrefixRewrite(service.Filters.URLRewrite)
+ if service.Filters.URLRewrite == nil {
  servicePrefixRewrite = prefixRewrite
  }
  serviceModifier := httpRouteFiltersToServiceRouteHeaderModifier(service.Filters.Headers)
@@ -176,13 +175,11 @@ func httpRouteToDiscoveryChain(route structs.HTTPRouteConfigEntry) (*structs.Ser
  return router, splitters, defaults
 }
 
-func httpRouteFiltersToDestinationPrefixRewrite(rewrites []structs.URLRewrite) string {
- for _, rewrite := range rewrites {
- if rewrite.Path != "" {
- return rewrite.Path
- }
+func httpRouteFiltersToDestinationPrefixRewrite(rewrite *structs.URLRewrite) string {
+ if rewrite == nil {
+ return ""
  }
- return ""
+ return rewrite.Path
 }
 
 // httpRouteFiltersToServiceRouteHeaderModifier will consolidate a list of HTTP filters