Merge pull request #19 from aws-ia/s3-bucket

S3 bucket fix

README.md

@@ -129,13 +129,15 @@ No modules.
 |------|-------------|------|---------|:--------:|
 | <a name="input_airflow_configuration_options"></a> [airflow\_configuration\_options](#input\_airflow\_configuration\_options) | (Optional) The airflow\_configuration\_options parameter specifies airflow override options. | `any` | `null` | no |
 | <a name="input_airflow_version"></a> [airflow\_version](#input\_airflow\_version) | (Optional) Airflow version of your environment, will be set by default to the latest version that MWAA supports. | `string` | `null` | no |
+| <a name="input_create_iam_role"></a> [create\_iam\_role](#input\_create\_iam\_role) | Create IAM role for MWAA | `bool` | `true` | no |
+| <a name="input_create_s3_bucket"></a> [create\_s3\_bucket](#input\_create\_s3\_bucket) | Create new S3 bucket for MWAA. | `string` | `true` | no |
 | <a name="input_create_security_group"></a> [create\_security\_group](#input\_create\_security\_group) | Create security group for MWAA | `bool` | `true` | no |
 | <a name="input_dag_s3_path"></a> [dag\_s3\_path](#input\_dag\_s3\_path) | (Required) The relative path to the DAG folder on your Amazon S3 storage bucket. For example, dags. | `string` | `"dags"` | no |
 | <a name="input_environment_class"></a> [environment\_class](#input\_environment\_class) | (Optional) Environment class for the cluster. Possible options are mw1.small, mw1.medium, mw1.large.<br>Will be set by default to mw1.small. Please check the AWS Pricing for more information about the environment classes. | `string` | `"mw1.small"` | no |
 | <a name="input_execution_role_arn"></a> [execution\_role\_arn](#input\_execution\_role\_arn) | (Required) The Amazon Resource Name (ARN) of the task execution role that the Amazon MWAA and its environment can assume<br>Mandatory if `create_iam_role=false` | `string` | `null` | no |
 | <a name="input_force_detach_policies"></a> [force\_detach\_policies](#input\_force\_detach\_policies) | IAM role Force detach policies | `bool` | `false` | no |
 | <a name="input_iam_role_additional_policies"></a> [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `list(string)` | `[]` | no |
-| <a name="input_iam_role_name"></a> [iam\_role\_name](#input\_iam\_role\_name) | IAM Role Name to be created if execution\_role\_arn is null | `bool` | `null` | no |
+| <a name="input_iam_role_name"></a> [iam\_role\_name](#input\_iam\_role\_name) | IAM Role Name to be created if execution\_role\_arn is null | `string` | `null` | no |
 | <a name="input_iam_role_path"></a> [iam\_role\_path](#input\_iam\_role\_path) | IAM role path | `string` | `"/"` | no |
 | <a name="input_iam_role_permissions_boundary"></a> [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | IAM role Permission boundary | `string` | `null` | no |
 | <a name="input_kms_key"></a> [kms\_key](#input\_kms\_key) | (Optional) The Amazon Resource Name (ARN) of your KMS key that you want to use for encryption.<br>Will be set to the ARN of the managed KMS key aws/airflow by default. | `string` | `null` | no |
@@ -151,7 +153,7 @@ No modules.
 | <a name="input_schedulers"></a> [schedulers](#input\_schedulers) | (Optional) The number of schedulers that you want to run in your environment. | `string` | `null` | no |
 | <a name="input_security_group_ids"></a> [security\_group\_ids](#input\_security\_group\_ids) | Security group IDs for MWAA | `list(string)` | `[]` | no |
 | <a name="input_source_bucket_arn"></a> [source\_bucket\_arn](#input\_source\_bucket\_arn) | (Required) The Amazon Resource Name (ARN) of your Amazon S3 storage bucket. For example, arn:aws:s3:::airflow-mybucketname | `string` | `null` | no |
-| <a name="input_source_bucket_name"></a> [source\_bucket\_name](#input\_source\_bucket\_name) | New bucket will be created with the given name for MWAA | `string` | `null` | no |
+| <a name="input_source_bucket_name"></a> [source\_bucket\_name](#input\_source\_bucket\_name) | New bucket will be created with the given name for MWAA when create\_s3\_bucket=true | `string` | `null` | no |
 | <a name="input_source_cidr"></a> [source\_cidr](#input\_source\_cidr) | (Required) Source CIDR block which will be allowed on MWAA SG to access Airflow UI<br>Used only if `create_security_group=true` | `list(string)` | `[]` | no |
 | <a name="input_tags"></a> [tags](#input\_tags) | (Optional) A map of resource tags to associate with the resource | `map(string)` | `{}` | no |
 | <a name="input_vpc_id"></a> [vpc\_id](#input\_vpc\_id) | (Required) VPC ID to deploy the MWAA Environment.<br>Mandatory if `create_security_group=true` | `string` | `""` | no |

examples/basic/README.md

@@ -71,7 +71,7 @@ This is a very simple workflow that has two tasks that use the BashOperator to e
 
 **Terraform generated S3 bucket**
 
-Copy this file using the AWS Cli using the following command, replacing the `{mwaa_dags_folder}` with the name of your MWAA environment's S3 bucket.
+Copy this file using the AWS cli using the following command, replacing the `{mwaa_dags_folder}` with the name of your MWAA environment's S3 bucket.
 
 ```sh
 cd dags
@@ -117,7 +117,7 @@ You can additionally check the MWAA environment CloudWatch logs to make sure tha
 ## Cleanup
 To clean up your environment, destroy the Terraform module.
 
-NOTE: Empty the S3 bucket created by this module before executing the `terraform destroy`
+*NOTE:* Empty the S3 bucket created by this module before executing the `terraform destroy`
 
 ```sh
 terraform destroy -auto-approve

examples/basic/main.tf

@@ -9,21 +9,86 @@ provider "aws" {
 
 data "aws_availability_zones" "available" {}
 
+data "aws_caller_identity" "current" {}
+
 locals {
-  azs = slice(data.aws_availability_zones.available.names, 0, 2)
+  azs         = slice(data.aws_availability_zones.available.names, 0, 2)
+  bucket_name = format("%s-%s", "aws-ia-mwaa", data.aws_caller_identity.current.account_id)
 }
+
+#-----------------------------------------------------------
+# Create an S3 bucket and upload sample DAG
+#-----------------------------------------------------------
+#tfsec:ignore:AWS017 tfsec:ignore:AWS002 tfsec:ignore:AWS077
+resource "aws_s3_bucket" "this" {
+  bucket = local.bucket_name
+  tags   = var.tags
+}
+
+resource "aws_s3_bucket_acl" "this" {
+  bucket = aws_s3_bucket.this.id
+  acl    = "private"
+}
+
+resource "aws_s3_bucket_versioning" "this" {
+  bucket = aws_s3_bucket.this.id
+  versioning_configuration {
+    status = "Enabled"
+  }
+}
+
+resource "aws_s3_bucket_server_side_encryption_configuration" "this" {
+  bucket = aws_s3_bucket.this.id
+
+  rule {
+    apply_server_side_encryption_by_default {
+      sse_algorithm = "AES256"
+    }
+  }
+}
+
+resource "aws_s3_bucket_public_access_block" "this" {
+  bucket                  = aws_s3_bucket.this.id
+  block_public_acls       = true
+  block_public_policy     = true
+  ignore_public_acls      = true
+  restrict_public_buckets = true
+}
+
+# Upload DAGS
+resource "aws_s3_object" "object1" {
+  for_each = fileset("dags/", "*")
+  bucket   = aws_s3_bucket.this.id
+  key      = "dags/${each.value}"
+  source   = "dags/${each.value}"
+  etag     = filemd5("dags/${each.value}")
+}
+
+# Upload plugins/requirements.txt
+resource "aws_s3_object" "reqs" {
+  for_each = fileset("mwaa/", "*")
+  bucket   = aws_s3_bucket.this.id
+  key      = each.value
+  source   = "mwaa/${each.value}"
+  etag     = filemd5("mwaa/${each.value}")
+}
+
 #-----------------------------------------------------------
 # NOTE: MWAA Airflow environment takes minimum of 20 mins
 #-----------------------------------------------------------
 module "mwaa" {
   source = "../.."
 
-  name                 = "basic-mwaa"
-  airflow_version      = "2.2.2"
-  environment_class    = "mw1.medium"
-  dag_s3_path          = "dags"
-  plugins_s3_path      = "plugins.zip"
-  requirements_s3_path = "requirements.txt"
+  name              = var.name
+  airflow_version   = "2.2.2"
+  environment_class = "mw1.medium"
+  create_s3_bucket  = false
+  source_bucket_arn = aws_s3_bucket.this.arn
+  dag_s3_path       = "dags"
+
+  ## If uploading requirements.txt or plugins, you can enable these via these options
+  #plugins_s3_path      = "plugins.zip"
+  #requirements_s3_path = "requirements.txt"
 
   logging_configuration = {
     dag_processing_logs = {
@@ -52,32 +117,23 @@ module "mwaa" {
     }
   }
 
-  airflow_configuration_options = { # Checkout the suggested Airflow configurations under https://docs.aws.amazon.com/mwaa/latest/userguide/configuring-env-variables.html
-    "core.default_task_retries"            = 3
-    "celery.worker_autoscale"              = "5,5"
-    "core.check_slas"                      = "false"
-    "core.dag_concurrency"                 = 96
-    "core.dag_file_processor_timeout"      = 600
-    "core.dagbag_import_timeout"           = 600
-    "core.max_active_runs_per_dag"         = 32
-    "core.parallelism"                     = 64
-    "scheduler.processor_poll_interval"    = 15
-    "logging.logging_level"                = "INFO"
-    "core.dag_file_processor_timeout"      = 120
-    "web_server.web_server_master_timeout" = 480
-    "web_server.web_server_worker_timeout" = 480
+  airflow_configuration_options = {
+    "core.load_default_connections" = "false"
+    "core.load_examples"            = "false"
+    "webserver.dag_default_view"    = "tree"
+    "webserver.dag_orientation"     = "TB"
   }
+
   min_workers        = 1
-  max_workers        = 25
+  max_workers        = 2
   vpc_id             = module.vpc.vpc_id
   private_subnet_ids = module.vpc.private_subnets
 
   webserver_access_mode = "PUBLIC_ONLY"   # Choose the Private network option(PRIVATE_ONLY) if your Apache Airflow UI is only accessed within a corporate network, and you do not require access to public repositories for web server requirements installation
   source_cidr           = ["10.1.0.0/16"] # Add your IP address to access Airflow UI
-  tags                  = var.tags
-  # create_security_group = true # change to to `false` to bring your sec group using `security_group_ids`
-  # source_bucket_arn = "<ENTER_S3_BUCKET_ARN>" # Module creates a new S3 bucket if `source_bucket_arn` is not specified
-  # execution_role_arn = "<ENTER_YOUR_IAM_ROLE_ARN>" # Module creates a new IAM role if `execution_role_arn` is not specified
+
+  tags = var.tags
+
 }
 
 #---------------------------------------------------------------

examples/basic/variables.tf

@@ -7,7 +7,7 @@ variable "name" {
 variable "region" {
   description = "region"
   type        = string
-  default     = "us-west-2"
+  default     = "eu-west-1"
 }
 
 variable "tags" {

locals.tf

@@ -1,9 +1,9 @@
 locals {
-  execution_role_arn = var.execution_role_arn != null ? var.execution_role_arn : aws_iam_role.mwaa[0].arn
+  execution_role_arn = var.create_iam_role ? aws_iam_role.mwaa[0].arn : var.execution_role_arn
 
-  security_group_ids = length(var.security_group_ids) > 0 ? var.security_group_ids : [aws_security_group.mwaa[0].id]
+  security_group_ids = var.create_security_group ? [aws_security_group.mwaa[0].id] : var.security_group_ids
 
-  source_bucket_arn = var.source_bucket_arn != null ? var.source_bucket_arn : aws_s3_bucket.mwaa[0].arn
+  source_bucket_arn = var.create_s3_bucket ? aws_s3_bucket.mwaa[0].arn : var.source_bucket_arn
 
   default_airflow_configuration_options = {
     "logging.logging_level" = "INFO"

main.tf

@@ -68,7 +68,7 @@ resource "aws_mwaa_environment" "mwaa" {
 # IAM Role
 # ---------------------------------------------------------------------------------------------------------------------
 resource "aws_iam_role" "mwaa" {
-  count = var.execution_role_arn != null ? 0 : 1
+  count = var.create_iam_role ? 1 : 0
 
   name                  = var.iam_role_name != null ? var.iam_role_name : null
   name_prefix           = var.iam_role_name != null ? null : "mwaa-executor"
@@ -82,7 +82,7 @@ resource "aws_iam_role" "mwaa" {
 }
 
 resource "aws_iam_role_policy" "mwaa" {
-  count = var.execution_role_arn != null ? 0 : 1
+  count = var.create_iam_role ? 1 : 0
 
   name_prefix = "mwaa-executor"
   role        = aws_iam_role.mwaa[0].id
@@ -98,25 +98,23 @@ resource "aws_iam_role_policy_attachment" "mwaa" {
 # ---------------------------------------------------------------------------------------------------------------------
 # MWAA S3 Bucket
 # ---------------------------------------------------------------------------------------------------------------------
-#tfsec:ignore:AWS017
-#tfsec:ignore:AWS002
-#tfsec:ignore:AWS077
+#tfsec:ignore:AWS017 tfsec:ignore:AWS002 tfsec:ignore:AWS077
 resource "aws_s3_bucket" "mwaa" {
-  count = var.source_bucket_arn != null ? 0 : 1
+  count = var.create_s3_bucket ? 1 : 0
 
   bucket_prefix = var.source_bucket_name != null ? var.source_bucket_name : format("%s-%s-", "mwaa", data.aws_caller_identity.current.account_id)
   tags          = var.tags
 }
 
 resource "aws_s3_bucket_acl" "mwaa" {
-  count = var.source_bucket_arn != null ? 0 : 1
+  count = var.create_s3_bucket ? 1 : 0
 
   bucket = aws_s3_bucket.mwaa[0].id
   acl    = "private"
 }
 
 resource "aws_s3_bucket_server_side_encryption_configuration" "mwaa" {
-  count = var.source_bucket_arn != null ? 0 : 1
+  count = var.create_s3_bucket ? 1 : 0
 
   bucket = aws_s3_bucket.mwaa[0].id
 
@@ -128,7 +126,7 @@ resource "aws_s3_bucket_server_side_encryption_configuration" "mwaa" {
 }
 
 resource "aws_s3_bucket_versioning" "mwaa" {
-  count = var.source_bucket_arn != null ? 0 : 1
+  count = var.create_s3_bucket ? 1 : 0
 
   bucket = aws_s3_bucket.mwaa[0].id
 
@@ -138,7 +136,7 @@ resource "aws_s3_bucket_versioning" "mwaa" {
 }
 
 resource "aws_s3_bucket_public_access_block" "mwaa" {
-  count = var.source_bucket_arn != null ? 0 : 1
+  count = var.create_s3_bucket ? 1 : 0
 
   bucket = aws_s3_bucket.mwaa[0].id
 
@@ -149,14 +147,14 @@ resource "aws_s3_bucket_public_access_block" "mwaa" {
 }
 
 resource "aws_s3_object" "plugins" {
-  count = var.source_bucket_arn != null ? 0 : 1
+  count = var.create_s3_bucket ? 1 : 0
 
   key    = "plugins.zip"
   bucket = aws_s3_bucket.mwaa[0].id
 }
 
 resource "aws_s3_object" "python_requirements" {
-  count = var.source_bucket_arn != null ? 0 : 1
+  count = var.create_s3_bucket ? 1 : 0
 
   key    = "requirements.txt"
   bucket = aws_s3_bucket.mwaa[0].id
@@ -166,7 +164,7 @@ resource "aws_s3_object" "python_requirements" {
 # MWAA Security Group
 # ---------------------------------------------------------------------------------------------------------------------
 resource "aws_security_group" "mwaa" {
-  count = var.create_security_group == true ? 1 : 0
+  count = var.create_security_group ? 1 : 0
 
   name_prefix = "mwaa-"
   description = "Security group for MWAA environment"
@@ -180,7 +178,7 @@ resource "aws_security_group" "mwaa" {
 }
 
 resource "aws_security_group_rule" "mwaa_sg_inbound" {
-  count = var.create_security_group == true ? 1 : 0
+  count = var.create_security_group ? 1 : 0
 
   type                     = "ingress"
   from_port                = 0
@@ -192,7 +190,7 @@ resource "aws_security_group_rule" "mwaa_sg_inbound" {
 }
 
 resource "aws_security_group_rule" "mwaa_sg_inbound_vpn" {
-  count = var.create_security_group == true && length(var.source_cidr) > 0 ? 1 : 0
+  count = var.create_security_group && length(var.source_cidr) > 0 ? 1 : 0
 
   type              = "ingress"
   from_port         = 443
@@ -205,7 +203,7 @@ resource "aws_security_group_rule" "mwaa_sg_inbound_vpn" {
 
 #tfsec:ignore:AWS007
 resource "aws_security_group_rule" "mwaa_sg_outbound" {
-  count = var.create_security_group == true ? 1 : 0
+  count = var.create_security_group ? 1 : 0
 
   type              = "egress"
   from_port         = 0

variables.tf

@@ -127,9 +127,15 @@ variable "tags" {
 #----------------------------------------------------------------
 # MWAA IAM Role
 #----------------------------------------------------------------
+variable "create_iam_role" {
+  description = "Create IAM role for MWAA"
+  type        = bool
+  default     = true
+}
+
 variable "iam_role_name" {
   description = "IAM Role Name to be created if execution_role_arn is null"
-  type        = bool
+  type        = string
   default     = null
 }
 
@@ -168,35 +174,41 @@ variable "execution_role_arn" {
 #----------------------------------------------------------------
 # MWAA S3 Bucket
 #----------------------------------------------------------------
-variable "source_bucket_arn" {
-  description = "(Required) The Amazon Resource Name (ARN) of your Amazon S3 storage bucket. For example, arn:aws:s3:::airflow-mybucketname"
+variable "create_s3_bucket" {
+  description = "Create new S3 bucket for MWAA. "
   type        = string
-  default     = null
+  default     = true
 }
 
 variable "source_bucket_name" {
   description = <<-EOD
-  New bucket will be created with the given name for MWAA
+  New bucket will be created with the given name for MWAA when create_s3_bucket=true
   EOD
   type        = string
   default     = null
 }
 
+variable "source_bucket_arn" {
+  description = "(Required) The Amazon Resource Name (ARN) of your Amazon S3 storage bucket. For example, arn:aws:s3:::airflow-mybucketname"
+  type        = string
+  default     = null
+}
+
 #----------------------------------------------------------------
 # MWAA Security groups
 #----------------------------------------------------------------
-variable "security_group_ids" {
-  description = "Security group IDs for MWAA"
-  type        = list(string)
-  default     = []
-}
-
 variable "create_security_group" {
   description = "Create security group for MWAA"
   type        = bool
   default     = true
 }
 
+variable "security_group_ids" {
+  description = "Security group IDs for MWAA"
+  type        = list(string)
+  default     = []
+}
+
 variable "vpc_id" {
   description = <<-EOD
   (Required) VPC ID to deploy the MWAA Environment.