Changes to adopt Consul Dataplane architecture for Consul ECS

.github/workflows/terraform-ci.yml

@@ -105,43 +105,31 @@ jobs:
       # HCP is always disabled for tests on PRs.
       matrix:
         name:
-          - acceptance-1.15-FARGATE-HCP
-          - acceptance-1.14-FARGATE-HCP
-          - acceptance-1.13-FARGATE
-          - acceptance-1.15-EC2
-          - acceptance-1.14-EC2
-          - acceptance-1.13-EC2-HCP
+          - acceptance-1.16-FARGATE-HCP
+          - acceptance-1.16-EC2-HCP
+          - acceptance-1.16-FARGATE
+          - acceptance-1.16-EC2
         include:
-          - name: acceptance-1.15-FARGATE-HCP
-            consul-version: '1.15.4'
+          - name: acceptance-1.16-FARGATE-HCP
+            consul-version: '1.16.1'
             enable-hcp: true
             launch-type: FARGATE
 
-          - name: acceptance-1.14-FARGATE-HCP
-            consul-version: '1.14.7'
+          - name: acceptance-1.16-EC2-HCP
+            consul-version: '1.16.1'
             enable-hcp: true
-            launch-type: FARGATE
+            launch-type: EC2
 
-          - name: acceptance-1.13-FARGATE
-            consul-version: '1.13.9'
+          - name: acceptance-1.16-FARGATE
+            consul-version: '1.16.1'
             enable-hcp: false
             launch-type: FARGATE
 
-          - name: acceptance-1.15-EC2
-            consul-version: '1.15.4'
-            enable-hcp: false
-            launch-type: EC2
-
-          - name: acceptance-1.14-EC2
-            consul-version: '1.14.8'
+          - name: acceptance-1.16-EC2
+            consul-version: '1.16.1'
             enable-hcp: false
             launch-type: EC2
 
-          - name: acceptance-1.13-EC2-HCP
-            consul-version: '1.13.8'
-            enable-hcp: true
-            launch-type: EC2
-
     steps:
     - name: Checkout
       uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
@@ -182,6 +170,7 @@ jobs:
         VARS="-var tags={\"build_url\":\"$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID\"}"
         VARS+=' -var launch_type=${{ matrix.launch-type }}'
         VARS+=' -var consul_version=${{ matrix.consul-version }}'
+        VARS+=' -var hcp_project_id=${{ secrets.HCP_PROJECT_ID }}'
         case $GITHUB_REF_NAME in
             main | release/*) VARS+=" -var enable_hcp=${{ matrix.enable-hcp }}";;
             *) VARS+=" -var enable_hcp=false";;
@@ -205,6 +194,7 @@ jobs:
         VARS="-var tags={\"build_url\":\"$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID\"}"
         VARS+=' -var launch_type=${{ matrix.launch-type }}'
         VARS+=' -var consul_version=${{ matrix.consul-version }}'
+        VARS+=' -var hcp_project_id=${{ secrets.HCP_PROJECT_ID }}'
         case $GITHUB_REF_NAME in
             main | release/*) VARS+=" -var enable_hcp=${{ matrix.enable-hcp }}";;
             *) VARS+=" -var enable_hcp=false";;

CHANGELOG.md

@@ -1,3 +1,101 @@
+## Unreleased
+
+BREAKING CHANGES
+* Adopt the architecture described in [Simplified Service Mesh with Consul Dataplane](https://developer.hashicorp.com/consul/docs/connect/dataplane) for ECS.[[GH-199]](https://github.com/hashicorp/terraform-aws-consul-ecs/pull/199)
+* Following changes are made to the `mesh-task` submodule: [[GH-188]](https://github.com/hashicorp/terraform-aws-consul-ecs/pull/188)
+  - Remove `consul-client` container definition from the ECS task definition.
+  - Rename `mesh-init` container to `consul-ecs-control-plane` and the `mesh-init` command to `control-plane`.
+  - Remove the `sidecar-proxy` container and replace it with the `consul-dataplane` container.
+  - Remove the `consul-ecs-health-sync` container definition.
+  - Remove the following input variables
+    - `envoy_image`
+    - `checks`
+    - `retry_join`
+    - `consul_http_addr`
+    - `client_token_auth_method_name`
+    - `gossip_key_secret_arn`
+    - `consul_server_ca_cert_arn`
+    - `consul_agent_configuration`
+    - `enable_acl_token_replication`
+    - `consul_datacenter`
+    - `consul_primary_datacenter`
+  - Add the following input variables
+    - `skip_server_watch`: To prevent the consul-dataplane and consul-ecs-control-plane containers from watching the Consul servers for changes. Useful for situations where Consul servers are behind a load balancer.
+    - `consul_dataplane_image`: Consul Dataplane's Docker image.
+    - `envoy_readiness_port`: Port that is exposed by Envoy which can be hit to determine its readiness.
+    - `consul_server_hosts`: Address of Consul servers. Can be an IP, DNS name or an `exec=` string specifying the script that outputs IP address(es).
+    - `tls_server_name`: The server name to use as the SNI host when connecting via TLS to Consul's HTTP and gRPC interfaces.
+    - `ca_cert_file`: Path of the CA certificate file for Consul's internal HTTP and gRPC interfaces.
+    - `consul_ca_cert_arn`: ARN of the Secrets Manager secret containing the Consul server CA certificate for Consul's internal gRPC and HTTP interfaces.
+    - `consul_grpc_ca_cert_arn`: ARN of the Secrets Manager secret containing the Consul server CA certificate for Consul's internal gRPC communications. Overrides `var.consul_ca_cert_arn`.
+    - `consul_https_ca_cert_arn`: ARN of the Secrets Manager secret containing the CA certificate for Consul server's HTTP interface. Overrides `var.consul_ca_cert_arn`.
+    - `http_config`: Contains HTTP specific TLS settings.
+    - `grpc_config`: Contains gRPC specific TLS settings.
+  - Add IAM policies to fetch `consul_ca_cert_arn`, `consul_grpc_ca_cert_arn` and `consul_https_ca_cert_arn` from Secrets manager.
+  - Add `consulServers` field to `local.config` which gets passed to the `control-plane` container.
+* Rename `acl-controller` submodule to `controller`. Following are the changes made to the same: [[GH-188]](https://github.com/hashicorp/terraform-aws-consul-ecs/pull/188)
+  - Rename `consul-acl-controller` container to `consul-ecs-controller`.
+  - Pass the `CONSUL_ECS_CONFIG_JSON`(which contains the configuration for configuring Consul on ECS) to the `consul-ecs-controller` container similar to how it is being done in the `mesh-task` submodule.
+  - Remove the following CLI flags that were getting passed to the existing command
+    - `-iam-role-path`
+    - `-partitions-enabled`
+    - `-partition`
+  - Remove the following variables
+    - `consul_server_http_addr`
+    - `consul_server_ca_cert_arn`
+  - Add the following variables
+    - `consul_ca_cert_arn`: ARN of the Secrets Manager secret containing the Consul server CA certificate for Consul's internal gRPC and HTTP interfaces.
+    - `consul_grpc_ca_cert_arn`: ARN of the Secrets Manager secret containing the Consul server CA certificate for Consul's internal gRPC communications. Overrides `var.consul_ca_cert_arn`.
+    - `consul_https_ca_cert_arn`: ARN of the Secrets Manager secret containing the CA certificate for Consul server's HTTP interface. Overrides `var.consul_ca_cert_arn`.
+    - `consul_server_hosts`: Address of Consul servers. Can be an IP, DNS name or an `exec=` string specifying the script that outputs IP address(es).
+    - `tls`: Whether to enable TLS for the controller to Consul server traffic.
+    - `tls_server_name`: The server name to use as the SNI host when connecting via TLS to Consul's HTTP and gRPC interfaces.
+    - `http_config`: Contains HTTP specific TLS settings for controller to Control plane traffic.
+    - `grpc_config`: Contains gRPC specific TLS settings for controller to Control plane traffic.
+  - Add IAM policies to fetch `consul_ca_cert_arn`, `consul_grpc_ca_cert_arn` and `consul_https_ca_cert_arn` from Secrets manager.
+* Following changes are made to the `gateway-task` submodule: [[GH-189]](https://github.com/hashicorp/terraform-aws-consul-ecs/pull/189)
+  - Remove `consul-client` container definition from the ECS task definition.
+  - Rename `mesh-init` container to `consul-ecs-control-plane` and the `mesh-init` command to `control-plane`.
+  - Remove the `sidecar-proxy` container and replace it with the `consul-dataplane` container.
+  - Remove the `consul-ecs-health-sync` container definition.
+  - Remove the following input variables
+    - `envoy_image`
+    - `retry_join`
+    - `consul_http_addr`
+    - `client_token_auth_method_name`
+    - `gossip_key_secret_arn`
+    - `consul_server_ca_cert_arn`
+    - `consul_agent_configuration`
+    - `enable_acl_token_replication`
+    - `consul_datacenter`
+    - `consul_primary_datacenter`
+    - `audit_logging`
+  - Add the following input variables
+    - `skip_server_watch`: To prevent the consul-dataplane and consul-ecs-control-plane containers from watching the Consul servers for changes. Useful for situations where Consul servers are behind a load balancer.
+    - `consul-dataplane-image`: Consul Dataplane's Docker image.
+    - `envoy_readiness_port`: Port that is exposed by Envoy which can be hit to determine its readiness.
+    - `consul_server_hosts`: Address of Consul servers. Can be an IP, DNS name or an `exec=` string specifying the script that outputs IP address(es).
+    - `tls_server_name`: The server name to use as the SNI host when connecting via TLS to Consul's HTTP and gRPC interfaces.
+    - `consul_ca_cert_arn`: ARN of the Secrets Manager secret containing the Consul server CA certificate for Consul's internal gRPC and HTTP interfaces.
+    - `consul_grpc_ca_cert_arn`: ARN of the Secrets Manager secret containing the Consul server CA certificate for Consul's internal gRPC communications. Overrides `var.consul_ca_cert_arn`.
+    - `consul_https_ca_cert_arn`: ARN of the Secrets Manager secret containing the CA certificate for Consul server's HTTP interface. Overrides `var.consul_ca_cert_arn`.
+    - `http_config`: Contains HTTP specific TLS settings for the consul-ecs-control-plane to Consul server traffic.
+    - `grpc_config`: Contains gRPC specific TLS settings for the consul-ecs-control-plane to Consul server traffic.
+  - Add IAM policies to fetch `consul_ca_cert_arn`, `consul_grpc_ca_cert_arn` and `consul_https_ca_cert_arn` from Secrets manager.
+  - Add `consulServers` field to `local.config` which gets passed to the `control-plane` container.
+* Following are the changes made to `dev-server` submodule: [[GH-191]](https://github.com/hashicorp/terraform-aws-consul-ecs/pull/191)
+  - Remove the following variables:
+    - `gossip_encryption_enabled`
+    - `generate_gossip_encryption_key`
+    - `gossip_key_secret_arn`
+* Add changes to the `dev-server-ec2` and `dev-server-fargate` examples to adopt the changes made to `mesh-task` submodule. [[GH-191]](https://github.com/hashicorp/terraform-aws-consul-ecs/pull/191)
+* Add changes to the `mesh-gateways` example to adopt the Consul Dataplane based architeture on ECS. [[GH-192]](https://github.com/hashicorp/terraform-aws-consul-ecs/pull/192)
+* Add changes to the `admin-partitions` example to adopt the Consul Dataplane based architeture on ECS. [[GH-193]](https://github.com/hashicorp/terraform-aws-consul-ecs/pull/193)
+
+
+IMPROVEMENTS
+* examples/cluster-peering: Add example terraform to illustrate Consul's cluster peering usecase on ECS. [[GH-194]](https://github.com/hashicorp/terraform-aws-consul-ecs/pull/194)
+
 ## 0.6.1 (Jul 20, 2023)
 
 IMPROVEMENTS

README.md

@@ -5,7 +5,7 @@ AWS ECS (Elastic Container Service).
 
 ## Documentation
 
-See https://www.consul.io/docs/ecs for full documentation.
+See https://developer.hashicorp.com/consul/docs/ecs for full documentation.
 
 ## Architecture
 
@@ -16,19 +16,19 @@ additional containers known as sidecar containers to your task definition.
 
 Specifically, it adds the following containers:
 
-* `consul-ecs-mesh-init` – Runs at startup to set up initial configuration for Consul and Envoy.
-* `consul-client` – Runs for the full lifecycle of the task. This container runs a
-  [Consul client](https://www.consul.io/docs/architecture) that connects with
-  Consul servers and configures the sidecar proxy.
-* `sidecar-proxy` – Runs for the full lifecycle of the task. This container runs
-  [Envoy](https://www.envoyproxy.io/) which is used to proxy and control
-  service mesh traffic. All requests to and from the application run through
-  the sidecar proxy.
-* `health-sync` - Runs for the full lifecycle of the task. This container
-  syncs health check statuses from ECS into Consul.
+* `consul-ecs-control-plane` – Runs for the full lifecycle of the task.
+  * At startup it connects to the available Consul servers and performs a login with the configured IAM Auth method to obtain an ACL token with appropriate privileges.
+  * Using the token, it registers the service and proxy entities to Consul's catalog.
+  * It then bootstraps the configuration JSON required by the Consul dataplane container and writes it to a shared volume.
+  * After this, the container enters into its reconciliation loop where it periodically syncs the health of ECS containers into Consul.
+  * Upon receiving SIGTERM, it marks the corresponding service instance in Consul as unhealthy and waits for the dataplane container to shutdown.
+  * Finally, it deregisters the service and proxy entities from Consul's catalog and performs a Consul logout.
+* `consul-dataplane` – Runs for the full lifecycle of the task. This container runs
+  the [Consul dataplane](https://github.com/hashicorp/consul-dataplane) that configures and starts the Envoy proxy, which controls all the service mesh traffic. All requests to and from the application run through
+  the proxy.
 
-The `acl-controller` module runs a controller that automatically provisions ACL tokens
-for tasks on the mesh.
+The `controller` module runs a controller that automatically provisions ACL tokens
+for tasks on the mesh. It also deregisters service instances from Consul for missing/finished tasks in ECS.
 
 The `dev-server` module runs a development/testing-only Consul server as an
 ECS task.
@@ -47,8 +47,8 @@ See https://www.consul.io/docs/ecs.
 * [dev-server](https://github.com/hashicorp/terraform-aws-consul-ecs/blob/main/modules/dev-server) [**For Development/Testing Only**]: This module deploys a Consul server onto your ECS Cluster
   for development/testing purposes. The server does not have persistent storage and so is not suitable for production deployments.
 
-* [acl-controller](https://github.com/hashicorp/terraform-aws-consul-ecs/blob/main/modules/acl-controller): This modules deploys a controller that automatically provisions ACL tokens
-  for services on the Consul service mesh.
+* [controller](https://github.com/hashicorp/terraform-aws-consul-ecs/blob/main/modules/controller): This modules deploys a controller that automatically provisions ACL tokens
+  for services on the Consul service mesh. It also keeps an eye on the tasks and deregisters the service instances of those tasks that go missing or get finished.
 
 ## Roadmap
 

examples/admin-partitions/README.md

@@ -9,7 +9,7 @@ The [Terraform code](./terraform/) in this example manages the following infrast
 - A client [`mesh-task`](../../modules/mesh-task/) running in an ECS cluster scoped to a Consul Admin Partition and Namespace.
 - A server [`mesh-task`](../../modules/mesh-task/) running in a separate ECS cluster scoped to a different Consul Admin Partition and Namespace.
 
-![Admin Partitions Example](../../_docs/ap-example.png)
+![Admin Partitions Example](https://github.com/hashicorp/terraform-aws-consul-ecs/blob/main/_docs/ap-example.png)
 
 To enable cross-partition communication the following conditions must be met:
 - Both ECS clusters must be in the same region and VPC. This is performed by the Terraform setup.

examples/admin-partitions/terraform/aws_ecs.tf

@@ -25,17 +25,18 @@ resource "random_string" "rand_suffix" {
 
 module "vpc" {
   source  = "terraform-aws-modules/vpc/aws"
-  version = "2.78.0"
+  version = "5.0.0"
 
-  name                 = local.ecs_name
-  cidr                 = "10.0.0.0/16"
-  azs                  = data.aws_availability_zones.available.names
-  private_subnets      = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"]
-  public_subnets       = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"]
-  enable_nat_gateway   = true
-  single_nat_gateway   = true
-  enable_dns_hostnames = true
-  tags                 = var.tags
+  name                          = local.ecs_name
+  cidr                          = "10.0.0.0/16"
+  azs                           = data.aws_availability_zones.available.names
+  private_subnets               = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"]
+  public_subnets                = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"]
+  enable_nat_gateway            = true
+  single_nat_gateway            = true
+  enable_dns_hostnames          = true
+  tags                          = var.tags
+  manage_default_security_group = false
 }
 
 resource "aws_ecs_cluster" "cluster_1" {

examples/admin-partitions/terraform/client.tf

@@ -10,27 +10,35 @@ resource "random_string" "client_suffix" {
   special = false
 }
 
-// Create ACL controller
-module "acl_controller_client" {
-  source = "../../../modules/acl-controller"
+// Create ECS controller
+module "ecs_controller_client" {
+  source = "../../../modules/controller"
   log_configuration = {
     logDriver = "awslogs"
     options = {
       awslogs-group         = aws_cloudwatch_log_group.log_group.name
       awslogs-region        = var.region
-      awslogs-stream-prefix = "consul-acl-controller-${local.client_suffix}"
+      awslogs-stream-prefix = "consul-ecs-controller-${local.client_suffix}"
     }
   }
   launch_type                       = local.launch_type
   consul_bootstrap_token_secret_arn = aws_secretsmanager_secret.bootstrap_token.arn
-  consul_server_http_addr           = hcp_consul_cluster.this.consul_private_endpoint_url
+  consul_server_hosts               = local.server_host
   ecs_cluster_arn                   = aws_ecs_cluster.cluster_1.arn
   region                            = var.region
   subnets                           = module.vpc.private_subnets
   name_prefix                       = local.client_suffix
   consul_ecs_image                  = var.consul_ecs_image
   consul_partitions_enabled         = true
   consul_partition                  = consul_admin_partition.part1.name
+
+  tls = true
+  http_config = {
+    port = 443
+  }
+  grpc_config = {
+    port = 8502
+  }
 }
 
 // Create services.
@@ -66,7 +74,7 @@ module "example_client" {
       initProcessEnabled = true
     }
   }]
-  retry_join = jsondecode(base64decode(hcp_consul_cluster.this.consul_config_file))["retry_join"]
+  consul_server_hosts = local.server_host
   upstreams = [
     {
       destinationName      = "example_server_${local.server_suffix}"
@@ -85,15 +93,19 @@ module "example_client" {
   }
   outbound_only = true
 
-  tls                       = true
-  acls                      = true
-  consul_http_addr          = hcp_consul_cluster.this.consul_private_endpoint_url
-  gossip_key_secret_arn     = aws_secretsmanager_secret.gossip_key.arn
-  consul_server_ca_cert_arn = aws_secretsmanager_secret.consul_ca_cert.arn
-  consul_ecs_image          = var.consul_ecs_image
-  consul_partition          = consul_admin_partition.part1.name
-  consul_namespace          = consul_namespace.ns1.name
-  consul_image              = var.consul_image
+  tls                    = true
+  acls                   = true
+  consul_ecs_image       = var.consul_ecs_image
+  consul_dataplane_image = var.consul_dataplane_image
+  consul_partition       = consul_admin_partition.part1.name
+  consul_namespace       = consul_namespace.ns1.name
 
   additional_task_role_policies = [aws_iam_policy.execute_command.arn]
+
+  http_config = {
+    port = 443
+  }
+  grpc_config = {
+    port = 8502
+  }
 }