Merge branch 'main' into docs/capigw-v0.5-peered-services-support

.changelog/15253.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+connect: Fixed issue where using Vault 1.11+ as CA provider would eventually break Intermediate CAs [[GH-15217](https://github.com/hashicorp/consul/issues/15217)]
+```

agent/connect/ca/provider_vault.go

@@ -363,7 +363,8 @@ func (v *VaultProvider) GenerateIntermediateCSR() (string, error) {
 			"cannot generate an intermediate CSR")
 	}
 
-	return v.generateIntermediateCSR()
+	csr, _, err := v.generateIntermediateCSR()
+	return csr, err
 }
 
 func (v *VaultProvider) setupIntermediatePKIPath() error {
@@ -406,11 +407,13 @@ func (v *VaultProvider) setupIntermediatePKIPath() error {
 	return err
 }
 
-func (v *VaultProvider) generateIntermediateCSR() (string, error) {
+// generateIntermediateCSR returns the CSR and key_id (only present in
+// Vault 1.11+) or any errors encountered.
+func (v *VaultProvider) generateIntermediateCSR() (string, string, error) {
 	// Generate a new intermediate CSR for the root to sign.
 	uid, err := connect.CompactUID()
 	if err != nil {
-		return "", err
+		return "", "", err
 	}
 	data, err := v.writeNamespaced(v.config.IntermediatePKINamespace, v.config.IntermediatePKIPath+"intermediate/generate/internal", map[string]interface{}{
 		"common_name": connect.CACN("vault", uid, v.clusterID, v.isPrimary),
@@ -419,17 +422,26 @@ func (v *VaultProvider) generateIntermediateCSR() (string, error) {
 		"uri_sans":    v.spiffeID.URI().String(),
 	})
 	if err != nil {
-		return "", err
+		return "", "", err
 	}
 	if data == nil || data.Data["csr"] == "" {
-		return "", fmt.Errorf("got empty value when generating intermediate CSR")
+		return "", "", fmt.Errorf("got empty value when generating intermediate CSR")
 	}
 	csr, ok := data.Data["csr"].(string)
 	if !ok {
-		return "", fmt.Errorf("csr result is not a string")
+		return "", "", fmt.Errorf("csr result is not a string")
 	}
-
-	return csr, nil
+	// Vault 1.11+ will return a "key_id" field which helps
+	// identify the correct issuer to set as default.
+	// https://github.com/hashicorp/vault/blob/e445c8b4f58dc20a0316a7fd1b5725b401c3b17a/builtin/logical/pki/path_intermediate.go#L154
+	if rawkeyId, ok := data.Data["key_id"]; ok {
+		keyId, ok := rawkeyId.(string)
+		if !ok {
+			return "", "", fmt.Errorf("key_id is not a string")
+		}
+		return csr, keyId, nil
+	}
+	return csr, "", nil
 }
 
 // SetIntermediate writes the incoming intermediate and root certificates to the
@@ -531,7 +543,7 @@ func (v *VaultProvider) getCAChain(namespace, path string) (string, error) {
 // necessary, then generates and signs a new CA CSR using the root PKI backend
 // and updates the intermediate backend to use that new certificate.
 func (v *VaultProvider) GenerateIntermediate() (string, error) {
-	csr, err := v.generateIntermediateCSR()
+	csr, keyId, err := v.generateIntermediateCSR()
 	if err != nil {
 		return "", err
 	}
@@ -551,16 +563,81 @@ func (v *VaultProvider) GenerateIntermediate() (string, error) {
 	}
 
 	// Set the intermediate backend to use the new certificate.
-	_, err = v.writeNamespaced(v.config.IntermediatePKINamespace, v.config.IntermediatePKIPath+"intermediate/set-signed", map[string]interface{}{
+	importResp, err := v.writeNamespaced(v.config.IntermediatePKINamespace, v.config.IntermediatePKIPath+"intermediate/set-signed", map[string]interface{}{
 		"certificate": intermediate.Data["certificate"],
 	})
 	if err != nil {
 		return "", err
 	}
 
+	// Vault 1.11+ will return a non-nil response from intermediate/set-signed
+	if importResp != nil {
+		err := v.setDefaultIntermediateIssuer(importResp, keyId)
+		if err != nil {
+			return "", fmt.Errorf("failed to update default intermediate issuer: %w", err)
+		}
+	}
+
 	return v.ActiveIntermediate()
 }
 
+// setDefaultIntermediateIssuer updates the default issuer for
+// intermediate CA since Vault, as part of its 1.11+ support for
+// multiple issuers, no longer overwrites the default issuer when
+// generateIntermediateCSR (intermediate/generate/internal) is called.
+//
+// The response we get from calling [/intermediate/set-signed]
+// should contain a "mapping" data field we can use to cross-reference
+// with the keyId returned when calling [/intermediate/generate/internal].
+//
+// [/intermediate/set-signed]: https://developer.hashicorp.com/vault/api-docs/secret/pki#import-ca-certificates-and-keys
+// [/intermediate/generate/internal]: https://developer.hashicorp.com/vault/api-docs/secret/pki#generate-intermediate-csr
+func (v *VaultProvider) setDefaultIntermediateIssuer(vaultResp *vaultapi.Secret, keyId string) error {
+	if vaultResp.Data["mapping"] == nil {
+		return fmt.Errorf("expected Vault response data to have a 'mapping' key")
+	}
+	if keyId == "" {
+		return fmt.Errorf("expected non-empty keyId")
+	}
+
+	mapping, ok := vaultResp.Data["mapping"].(map[string]any)
+	if !ok {
+		return fmt.Errorf("unexpected type for 'mapping' value in Vault response")
+	}
+
+	var intermediateId string
+	// The value in this KV pair is called "key"
+	for issuer, key := range mapping {
+		if key == keyId {
+			// Expect to find the key_id we got from Vault when we
+			// generated the intermediate CSR.
+			intermediateId = issuer
+			break
+		}
+	}
+	if intermediateId == "" {
+		return fmt.Errorf("could not find key_id %q in response from vault", keyId)
+	}
+
+	// For Vault 1.11+ it is important to GET then POST to avoid clobbering fields
+	// like `default_follows_latest_issuer`.
+	// https://developer.hashicorp.com/vault/api-docs/secret/pki#default_follows_latest_issuer
+	resp, err := v.readNamespaced(v.config.IntermediatePKINamespace, v.config.IntermediatePKIPath+"config/issuers")
+	if err != nil {
+		return fmt.Errorf("could not read from /config/issuers: %w", err)
+	}
+	issuersConf := resp.Data
+	// Overwrite the default issuer
+	issuersConf["default"] = intermediateId
+
+	_, err = v.writeNamespaced(v.config.IntermediatePKINamespace, v.config.IntermediatePKIPath+"config/issuers", issuersConf)
+	if err != nil {
+		return fmt.Errorf("could not write default issuer to /config/issuers: %w", err)
+	}
+
+	return nil
+}
+
 // Sign calls the configured role in the intermediate PKI backend to issue
 // a new leaf certificate based on the provided CSR, with the issuing
 // intermediate CA cert attached.

agent/connect/ca/provider_vault_test.go

@@ -902,6 +902,28 @@ func TestVaultProvider_ReconfigureIntermediateTTL(t *testing.T) {
 	require.Equal(t, 333*3600, mountConfig.MaxLeaseTTL)
 }
 
+func TestVaultCAProvider_GenerateIntermediate(t *testing.T) {
+
+	SkipIfVaultNotPresent(t)
+
+	provider, _ := testVaultProviderWithConfig(t, true, nil)
+
+	orig, err := provider.ActiveIntermediate()
+	require.NoError(t, err)
+
+	// This test was created to ensure that our calls to Vault
+	// returns a new Intermediate certificate and further calls
+	// to ActiveIntermediate return the same new cert.
+	new, err := provider.GenerateIntermediate()
+	require.NoError(t, err)
+
+	newActive, err := provider.ActiveIntermediate()
+	require.NoError(t, err)
+
+	require.Equal(t, new, newActive)
+	require.NotEqual(t, orig, new)
+}
+
 func getIntermediateCertTTL(t *testing.T, caConf *structs.CAConfiguration) time.Duration {
 	t.Helper()
 

agent/session_endpoint.go

@@ -47,6 +47,10 @@ func (s *HTTPHandlers) SessionCreate(resp http.ResponseWriter, req *http.Request
 
 	fixupEmptySessionChecks(&args.Session)
 
+	if (s.agent.config.Datacenter != args.Datacenter) && (!s.agent.config.ServerMode) {
+		return nil, fmt.Errorf("cross datacenter lock must be created at server agent")
+	}
+
 	// Create the session, get the ID
 	var out string
 	if err := s.agent.RPC("Session.Apply", &args, &out); err != nil {

website/content/api-docs/peering.mdx

@@ -34,8 +34,8 @@ The table below shows this endpoint's support for
 
 ### JSON Request Body Schema
 
-- `PeerName` `(string: <required>)` - The name assigned to the peer cluster.
-  The `PeerName` is used to reference the peer cluster in service discovery queries
+- `Peer` `(string: <required>)` - The name assigned to the peer cluster.
+  The `Peer` is used to reference the peer cluster in service discovery queries
   and configuration entries such as `service-intentions`. This field must be a
   valid DNS hostname label.
 
@@ -54,7 +54,7 @@ You can specify one or more load balancers or external IPs that route external t
 
 ```json
 {
-  "PeerName": "cluster-02",
+  "Peer": "cluster-02",
   "Meta": {
     "env": "production"
   }
@@ -101,8 +101,8 @@ The table below shows this endpoint's support for
 
 ### JSON Request Body Schema
 
-- `PeerName` `(string: <required>)` - The name assigned to the peer cluster.
-  The `PeerName` is used to reference the peer cluster in service discovery queries
+- `Peer` `(string: <required>)` - The name assigned to the peer cluster.
+  The `Peer` is used to reference the peer cluster in service discovery queries
   and configuration entries such as `service-intentions`. This field must be a
   valid DNS hostname label.
 
@@ -121,7 +121,7 @@ The table below shows this endpoint's support for
 
 ```json
 {
-  "PeerName": "cluster-01",
+  "Peer": "cluster-01",
   "PeeringToken": "eyJDQSI6bnVsbCwiU2V...",
   "Meta": {
     "env": "production"

website/content/commands/lock.mdx

@@ -24,6 +24,8 @@ If the lock holder count is more than one, then a semaphore is used instead.
 A semaphore allows more than a single holder, but this is less efficient than
 a simple lock. This follows the [semaphore algorithm](https://learn.hashicorp.com/consul/developer-configuration/semaphore).
 
+To apply a lock to a remote WAN federated datacenter, run the command with the `-datacenter=<name>` flag on a server agent. You cannot use the command with `-datacenter` on client agents because they are unavailable to the remote datacenter.
+
 All locks using the same prefix must agree on the value of `-n`. If conflicting
 values of `-n` are provided, an error will be returned.
 

website/content/docs/agent/config/config-files.mdx

@@ -102,6 +102,7 @@ Valid time units are 'ns', 'us' (or 'µs'), 'ms', 's', 'm', 'h'."
   - `http` - The HTTP API. Defaults to `client_addr`
   - `https` - The HTTPS API. Defaults to `client_addr`
   - `grpc` - The gRPC API. Defaults to `client_addr`
+  - `grpc_tls` - The gRPC API with TLS. Defaults to `client_addr`
 
 - `alt_domain` Equivalent to the [`-alt-domain` command-line flag](/docs/agent/config/cli-flags#_alt_domain)
 

website/content/docs/architecture/index.mdx

@@ -2,7 +2,7 @@
 layout: docs
 page_title: Consul Architecture
 description: >-
-  Consul datacenters consist of clusters of server agents (control plane) and client agents deployed alongside service instances (dataplane). Learn how these components and their different communication methods make Consul possible.
+  Consul datacenters consist of clusters of server agents (control plane) and client agents deployed alongside service instances (data plane). Learn how these components and their different communication methods make Consul possible.
 ---
 
 # Consul Architecture
@@ -47,10 +47,12 @@ Consul servers establish consensus using the Raft algorithm on port `8300`. Refe
 
 ### Client agents
 
-Consul clients report node and service health status to the Consul cluster. In a typical deployment, you must run client agents on every compute node in your datacenter. Clients use remote procedure calls (RPC) to interact with servers. By default, clients send RPC requests to the servers on port `8300`. 
+Consul clients report node and service health status to the Consul cluster. In a typical deployment, you must run client agents on every compute node in your datacenter. Clients use remote procedure calls (RPC) to interact with servers. By default, clients send RPC requests to the servers on port `8300`.
 
 There are no limits to the number of client agents or services you can use with Consul, but production deployments should distribute services across multiple Consul datacenters. Using a multi-datacenter deployment enhances infrastructure resilience and limits control plane issues. We recommend deploying a maximum of 5,000 client agents per datacenter. Some large organizations have deployed tens of thousands of client agents and hundreds of thousands of service instances across a multi-datacenter deployment. Refer to [Cross-datacenter requests](#cross-datacenter-requests) for additional information.
 
+You can also run Consul with an alternate service mesh configuration that deploys Envoy proxies but not client agents. Refer to [Simplified Service Mesh with Consul Dataplanes](/docs/connect/dataplane) for more information.
+
 ## LAN gossip pool
 
 Client and server agents participate in a LAN gossip pool so that they can distribute and perform node [health checks](/docs/discovery/checks). Agents in the pool propagate the health check information across the cluster. Agent gossip communication occurs on port `8301` using UDP. Agent gossip falls back to TCP if UDP is not available. Refer to [Gossip Protocol](/docs/architecture/gossip) for additional information. 

website/content/docs/connect/ca/vault.mdx

@@ -29,6 +29,8 @@ must be met:
   were introduced in Vault 0.10.3. Prior versions of Vault are not
   compatible with Connect.
 
+~> **Note:** Do not use Vault v1.11.0+ as Consul's Connect CA provider — the intermediate CA will become unable to issue the leaf nodes required by service mesh, and by Consul client agents if using auto-encrypt or auto-config and using TLS for agent communication. If you are already using Vault 1.11+ as a Connect CA, refer to this [Knowledge Base article](https://support.hashicorp.com/hc/en-us/articles/11308460105491) for more information about the underlying cause and recommended workaround.
+
 ## Configuration
 
 The Vault CA is enabled by setting the CA provider to `"vault"` and

website/content/docs/connect/cluster-peering/create-manage-peering.mdx

@@ -37,7 +37,7 @@ Every time you generate a peering token, a single-use establishment secret is em
 In `cluster-01`, use the [`/peering/token` endpoint](/api-docs/peering#generate-a-peering-token) to issue a request for a peering token.
 
 ```shell-session
-$ curl --request POST --data '{"PeerName":"cluster-02"}' --url http://localhost:8500/v1/peering/token
+$ curl --request POST --data '{"Peer":"cluster-02"}' --url http://localhost:8500/v1/peering/token
 ```
 
 The CLI outputs the peering token, which is a base64-encoded string containing the token details.
@@ -48,7 +48,7 @@ Create a JSON file that contains the first cluster's name and the peering token.
 
 ```json
 {
-    "PeerName": "cluster-01",
+    "Peer": "cluster-01",
     "PeeringToken": "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImF1ZCI6IlNvbHIifQ.5T7L_L1MPfQ_5FjKGa1fTPqrzwK4bNSM812nW6oyjb8"
 }
 ```

website/content/docs/connect/cluster-peering/k8s.mdx

@@ -7,13 +7,18 @@ description: >-
 
 # Cluster Peering on Kubernetes
 
-To establish a cluster peering connection on Kubernetes, you need to enable the feature in the Helm chart and create custom resource definitions (CRDs) for each side of the peering.
+To establish a cluster peering connection on Kubernetes, you need to enable several pre-requisite values in the Helm chart and create custom resource definitions (CRDs) for each side of the peering.
+
+The following Helm values are mandatory for cluster peering:
+- [`global.tls.enabled = true`](/docs/k8s/helm#v-global-tls-enabled)
+- [`meshGateway.enabled = true`](/docs/k8s/helm#v-meshgateway-enabled)
 
 The following CRDs are used to create and manage a peering connection:
 
 - `PeeringAcceptor`: Generates a peering token and accepts an incoming peering connection.
 - `PeeringDialer`: Uses a peering token to make an outbound peering connection with the cluster that generated the token.
 
+Peering connections, including both data-plane and control-plane traffic, will be routed over mesh gateways. 
 As of Consul v1.14, you can also [implement service failovers and redirects to control traffic](/consul/docs/connect/l7-traffic) between peers.
 
 > To learn how to peer clusters and connect services across peers in AWS Elastic Kubernetes Service (EKS) environments, complete the [Consul Cluster Peering on Kubernetes tutorial](https://learn.hashicorp.com/tutorials/consul/cluster-peering-aws?utm_source=docs).
@@ -52,14 +57,13 @@ Complete the following procedure after you have provisioned a Kubernetes cluster
       image: "hashicorp/consul:1.14.0"
       peering:
         enabled: true
+      tls:
+        enabled: true
     connectInject:
       enabled: true
     dns:
       enabled: true
       enableRedirection: true
-    server:
-      exposeService:
-        enabled: true
     controller:
       enabled: true
     meshGateway:
@@ -68,10 +72,6 @@ Complete the following procedure after you have provisioned a Kubernetes cluster
     ```
 
     </CodeBlockConfig>
-
-    These Helm values configure the servers in each cluster so that they expose ports over a Kubernetes load balancer service. For additional configuration options, refer to [`server.exposeService`](/docs/k8s/helm#v-server-exposeservice).
-
-    When generating a peering token from one of the clusters, Consul includes a load balancer address in the token so that the peering stream goes through the load balancer in front of the servers. For additional configuration options, refer to [`global.peering.tokenGeneration`](/docs/k8s/helm#v-global-peering-tokengeneration).
   
 ### Install Consul on Kubernetes
 
@@ -94,7 +94,7 @@ Install Consul on Kubernetes by using the CLI to apply `values.yaml` to each clu
       ```
 
      ```shell-session
-    $ helm install ${HELM_RELEASE_NAME} hashicorp/consul --create-namespace --namespace consul --version "1.0.0" --values values.yaml --kube-context $CLUSTER2_CONTEXT
+    $ helm install ${HELM_RELEASE_NAME} hashicorp/consul --create-namespace --namespace consul --version "1.0.0" --values values.yaml --set global.datacenter=dc2 --kube-context $CLUSTER2_CONTEXT
      ```
 
 ## Create a peering connection for Consul on Kubernetes
@@ -288,11 +288,15 @@ The examples described in this section demonstrate how to export a service named
 
 1. Apply the intentions to the second cluster.
 
+    <CodeBlockConfig>
+
     ```shell-session
     $ kubectl --context $CLUSTER2_CONTEXT apply --filename intention.yaml
     ```
 
-1. Add the `"consul.hashicorp.com/connect-inject": "true"` annotation to your service's pods before deploying the workload so that the services in `cluster-01` can dial `backend` in `cluster-02`. To dial the upstream service from an application, configure the application so that that requests are sent to the correct DNS name as specified in [Service Virtual IP Lookups](/docs/discovery/dns#service-virtual-ip-lookups). In the following example, the annotation that allows the workload to join the mesh and the configuration provided to the workload that enables the workload to dial the upstream service using the correct DNS name is highlighted. 
+    </CodeBlockConfig>
+
+1. Add the `"consul.hashicorp.com/connect-inject": "true"` annotation to your service's pods before deploying the workload so that the services in `cluster-01` can dial `backend` in `cluster-02`. To dial the upstream service from an application, configure the application so that that requests are sent to the correct DNS name as specified in [Service Virtual IP Lookups](/docs/discovery/dns#service-virtual-ip-lookups). In the following example, the annotation that allows the workload to join the mesh and the configuration provided to the workload that enables the workload to dial the upstream service using the correct DNS name is highlighted. [Service Virtual IP Lookups for Consul Enterprise](/docs/discovery/dns#service-virtual-ip-lookups-for-consul-enterprise) details how you would similarly format a DNS name including partitions and namespaces.
 
     <CodeBlockConfig filename="frontend.yaml" highlight="36,51">
 
@@ -366,7 +370,7 @@ The examples described in this section demonstrate how to export a service named
 
 1. Run the following command in `frontend` and then check the output to confirm that you peered your clusters successfully.
 
-    <CodeBlockConfig filename="frontend.yaml" highlight="31">
+    <CodeBlockConfig highlight="31">
 
     ```shell-session
     $ kubectl --context $CLUSTER1_CONTEXT exec -it $(kubectl --context $CLUSTER1_CONTEXT get pod -l app=frontend -o name) -- curl localhost:9090