From 0c239010b4f181d442ad5fdb103774232a7b3773 Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Mon, 8 Jul 2024 13:51:11 -0500 Subject: [PATCH] Using ARM64 Architecture --- .../ecs_services/airflow_metrics.tf | 26 +++----- .../ecs_services/airflow_scheduler.tf | 32 +++------- infrastructure/ecs_services/airflow_server.tf | 11 +++- .../ecs_services/airflow_standalone_task.tf | 21 +------ infrastructure/ecs_services/airflow_worker.tf | 27 ++------ infrastructure/ecs_services/ecr.tf | 24 ++----- infrastructure/ecs_services/efs.tf | 62 ------------------- infrastructure/ecs_services/locals.tf | 29 +++++++++ 8 files changed, 68 insertions(+), 164 deletions(-) delete mode 100644 infrastructure/ecs_services/efs.tf create mode 100644 infrastructure/ecs_services/locals.tf diff --git a/infrastructure/ecs_services/airflow_metrics.tf b/infrastructure/ecs_services/airflow_metrics.tf index 0752781..d86f586 100644 --- a/infrastructure/ecs_services/airflow_metrics.tf +++ b/infrastructure/ecs_services/airflow_metrics.tf @@ -34,16 +34,6 @@ resource "aws_ecs_task_definition" "airflow_metrics" { requires_compatibilities = ["FARGATE"] volume { name = "efs-${var.prefix}" - efs_volume_configuration { - file_system_id = aws_efs_file_system.efs.id - root_directory = "/mnt/data" - transit_encryption = "ENABLED" - transit_encryption_port = 2999 - authorization_config { - access_point_id = aws_efs_access_point.access.id - iam = "ENABLED" - } - } } container_definitions = jsonencode([ { @@ -55,13 +45,6 @@ resource "aws_ecs_task_definition" "airflow_metrics" { entryPoint = [ "python" ] - mountPoints : [ - { - "containerPath" : "/opt/airflow/dags_efs", - "sourceVolume" : "efs-${var.prefix}" - - } - ] command = [ "scripts/put_airflow_worker_autoscaling_metric_data.py", "--cluster-name", @@ -75,7 +58,14 @@ resource "aws_ecs_task_definition" "airflow_metrics" { "--period", "30" ] - environment = var.airflow_task_common_environment + environment = concat(var.airflow_task_common_environment, + [ + { + name = "SERVICES_HASH" + value = local.services_hashes + } + + ]) user = "50000:0" logConfiguration = { logDriver = "awslogs" diff --git a/infrastructure/ecs_services/airflow_scheduler.tf b/infrastructure/ecs_services/airflow_scheduler.tf index 1a5bb4b..c689391 100644 --- a/infrastructure/ecs_services/airflow_scheduler.tf +++ b/infrastructure/ecs_services/airflow_scheduler.tf @@ -48,19 +48,6 @@ resource "aws_ecs_task_definition" "airflow_scheduler" { cpu_architecture = "ARM64" } requires_compatibilities = ["FARGATE"] -# volume { -# name = "efs-${var.prefix}" -# efs_volume_configuration { -# file_system_id = aws_efs_file_system.efs.id -# root_directory = "/mnt/data" -# transit_encryption = "ENABLED" -# transit_encryption_port = 2999 -# authorization_config { -# access_point_id = aws_efs_access_point.access.id -# iam = "ENABLED" -# } -# } -# } container_definitions = jsonencode([ { @@ -68,13 +55,7 @@ resource "aws_ecs_task_definition" "airflow_scheduler" { image = join(":", [aws_ecr_repository.airflow.repository_url, "latest"]) cpu = var.scheduler_cpu memory = var.scheduler_memory -# mountPoints : [ -# { -# "containerPath" : "/opt/airflow/dags_efs", -# "sourceVolume" : "efs-${var.prefix}" -# -# } -# ] + healthcheck = { command = [ "CMD-SHELL", @@ -92,7 +73,14 @@ resource "aws_ecs_task_definition" "airflow_scheduler" { linuxParameters = { initProcessEnabled = true } - environment = var.airflow_task_common_environment + environment = concat(var.airflow_task_common_environment, + [ + { + name = "SERVICES_HASH" + value = local.services_hashes + } + + ]) user = "50000:0" logConfiguration = { logDriver = "awslogs" @@ -159,7 +147,7 @@ resource "aws_ecs_service" "airflow_scheduler" { platform_version = "1.4.0" scheduling_strategy = "REPLICA" # Update from requirements - force_new_deployment = var.force_new_ecs_service_deployment + #force_new_deployment = var.force_new_ecs_service_deployment } diff --git a/infrastructure/ecs_services/airflow_server.tf b/infrastructure/ecs_services/airflow_server.tf index 1f461d2..884c9ed 100644 --- a/infrastructure/ecs_services/airflow_server.tf +++ b/infrastructure/ecs_services/airflow_server.tf @@ -46,7 +46,14 @@ resource "aws_ecs_task_definition" "airflow_webserver" { } essential = true command = ["webserver"] - environment = var.airflow_task_common_environment + environment = concat(var.airflow_task_common_environment, + [ + { + name = "SERVICES_HASH" + value = [local.config_folder_hash, local.services_build_path_hash] + } + + ]) user = "50000:0" logConfiguration = { logDriver = "awslogs" @@ -109,7 +116,7 @@ resource "aws_ecs_service" "airflow_webserver" { container_port = 8080 } # Update from services folder - force_new_deployment = var.force_new_ecs_service_deployment + #force_new_deployment = var.force_new_ecs_service_deployment # This can be used to update tasks to use a newer container image with same # image/tag combination (e.g., myimage:latest) } diff --git a/infrastructure/ecs_services/airflow_standalone_task.tf b/infrastructure/ecs_services/airflow_standalone_task.tf index 2e0151d..b7a31ff 100644 --- a/infrastructure/ecs_services/airflow_standalone_task.tf +++ b/infrastructure/ecs_services/airflow_standalone_task.tf @@ -34,32 +34,13 @@ resource "aws_ecs_task_definition" "airflow_standalone_task" { cpu_architecture = "ARM64" } requires_compatibilities = ["FARGATE"] -# volume { -# name = "efs-${var.prefix}" -# efs_volume_configuration { -# file_system_id = aws_efs_file_system.efs.id -# root_directory = "/mnt/data" -# transit_encryption = "ENABLED" -# transit_encryption_port = 2999 -# authorization_config { -# access_point_id = aws_efs_access_point.access.id -# iam = "ENABLED" -# } -# } -# } + container_definitions = jsonencode([ { name = "airflow" image = join(":", [aws_ecr_repository.airflow.repository_url, "latest"]) cpu = 256 memory = 512 -# mountPoints : [ -# { -# "containerPath" : "/opt/airflow/dags_efs", -# "sourceVolume" : "efs-${var.prefix}" -# -# } -# ] essential = true command = ["version"] environment = var.airflow_task_common_environment diff --git a/infrastructure/ecs_services/airflow_worker.tf b/infrastructure/ecs_services/airflow_worker.tf index 89d4804..e73e935 100644 --- a/infrastructure/ecs_services/airflow_worker.tf +++ b/infrastructure/ecs_services/airflow_worker.tf @@ -19,20 +19,6 @@ resource "aws_ecs_task_definition" "airflow_worker" { cpu_architecture = "ARM64" } requires_compatibilities = ["FARGATE"] - volume { - name = "efs-${var.prefix}" - efs_volume_configuration { - file_system_id = aws_efs_file_system.efs.id - #root_directory = "/mnt/data" - transit_encryption = "ENABLED" - transit_encryption_port = 2999 - authorization_config { - access_point_id = aws_efs_access_point.access.id - iam = "ENABLED" - } - } - } - container_definitions = jsonencode([ { name = "worker" @@ -41,13 +27,6 @@ resource "aws_ecs_task_definition" "airflow_worker" { memory = var.worker_memory essential = true command = var.worker_cmd != [] ? var.worker_cmd : ["celery", "worker"] - mountPoints : [ - { - "containerPath" : "/opt/airflow/dags_efs", - "sourceVolume" : "efs-${var.prefix}" - - } - ] linuxParameters = { initProcessEnabled = true } @@ -59,6 +38,10 @@ resource "aws_ecs_task_definition" "airflow_worker" { { name = "DUMB_INIT_SETSID" value = "0" + }, + { + name = "WORKER_HASHES" + value = local.workers_hashes } ] ) @@ -116,7 +99,7 @@ resource "aws_ecs_service" "airflow_worker" { weight = 1 } # Update from workers folder - force_new_deployment = var.force_new_ecs_service_deployment + # force_new_deployment = var.force_new_ecs_service_deployment } diff --git a/infrastructure/ecs_services/ecr.tf b/infrastructure/ecs_services/ecr.tf index 0ecf4ef..db6c320 100644 --- a/infrastructure/ecs_services/ecr.tf +++ b/infrastructure/ecs_services/ecr.tf @@ -37,24 +37,12 @@ resource "aws_ecr_lifecycle_policy" "ecr_policy" { }) } -locals { - - services_build_path = "../${path.root}/airflow_services" - dag_folder_path = "../${path.root}/dags" - scripts_path = "../${path.root}/scripts" - config_path = "../${path.root}/configuration" - worker_build_path = "../${path.root}/airflow_worker" -} - - resource "null_resource" "build_ecr_image" { triggers = { - build_path = sha1(join("", [for f in fileset(local.services_build_path, "**") : filesha1("${local.services_build_path}/${f}")])) - scripts_path = sha1(join("", [for f in fileset(local.scripts_path, "**") : filesha1("${local.scripts_path}/${f}")])) - dag_folder_path = sha1(join("", [for f in fileset(local.dag_folder_path, "**") : filesha1("${local.dag_folder_path}/${f}")])) - config_folder_path = sha1(join("", [for f in fileset(local.config_path, "**") : filesha1("${local.config_path}/${f}")])) - - + services_build_path_hash = local.services_build_path_hash + scripts_folder_hash = local.scripts_folder_hash + dag_folder_hash = local.dag_folder_hash + config_folder_hash = local.config_folder_hash } provisioner "local-exec" { @@ -71,8 +59,8 @@ resource "null_resource" "build_ecr_image" { resource "null_resource" "build_worker_ecr_image" { triggers = { - build_path_worker = sha1(join("", [for f in fileset(local.worker_build_path, "**") : filesha1("${local.worker_build_path}/${f}")])) - dag_folder_path = sha1(join("", [for f in fileset(local.dag_folder_path, "**") : filesha1("${local.dag_folder_path}/${f}")])) + worker_folder_hash = local.worker_folder_hash + dag_folder_hash = local.dag_folder_hash } provisioner "local-exec" { diff --git a/infrastructure/ecs_services/efs.tf b/infrastructure/ecs_services/efs.tf deleted file mode 100644 index 7769d07..0000000 --- a/infrastructure/ecs_services/efs.tf +++ /dev/null @@ -1,62 +0,0 @@ -##### -# EFS -##### - -locals { - task_security_group_ids = [ - aws_security_group.airflow_worker_service.id, - aws_security_group.airflow_scheduler_service.id, - aws_security_group.airflow_standalone_task.id, - aws_security_group.airflow_metrics_service.id, - aws_security_group.airflow_webserver_service.id - ] -} -resource "aws_efs_file_system" "efs" { - creation_token = "${var.prefix}-efs" - - tags = { - Name = "${var.prefix}-efs" - } -} - -resource "aws_efs_access_point" "access" { - file_system_id = aws_efs_file_system.efs.id - posix_user { - uid = 50000 - gid = 0 - } - root_directory { - path = "/mnt/data" - creation_info { - owner_uid = 50000 - owner_gid = 0 - permissions = 777 - } - } -} - -resource "aws_security_group" "efs" { - name = "${var.prefix}-efs-sg" - vpc_id = var.vpc_id - - ingress { - protocol = "tcp" - from_port = 2999 - to_port = 2999 - security_groups = local.task_security_group_ids - cidr_blocks = var.efs_allowed_cider_block - } - ingress { - description = "NFS traffic from VPC" - from_port = 2049 - to_port = 2049 - protocol = "tcp" - cidr_blocks = var.efs_allowed_cider_block - } -} -resource "aws_efs_mount_target" "mount" { - count = 2 - file_system_id = aws_efs_file_system.efs.id - subnet_id = var.private_subnet_ids[count.index] - security_groups = [aws_security_group.efs.id] -} \ No newline at end of file diff --git a/infrastructure/ecs_services/locals.tf b/infrastructure/ecs_services/locals.tf new file mode 100644 index 0000000..f2e74e8 --- /dev/null +++ b/infrastructure/ecs_services/locals.tf @@ -0,0 +1,29 @@ +locals { + + services_build_path = "../${path.root}/airflow_services" + dag_folder_path = "../${path.root}/dags" + scripts_path = "../${path.root}/scripts" + config_path = "../${path.root}/configuration" + worker_build_path = "../${path.root}/airflow_worker" +} + + + +locals { + + services_build_path_hash = sha1(join("", [for f in fileset(local.services_build_path, "**") : filesha1("${local.services_build_path}/${f}")])) + dag_folder_hash = sha1(join("", [for f in fileset(local.dag_folder_path, "**") : filesha1("${local.dag_folder_path}/${f}")])) + scripts_folder_hash = sha1(join("", [for f in fileset(local.scripts_path, "**") : filesha1("${local.scripts_path}/${f}")])) + config_folder_hash = sha1(join("", [for f in fileset(local.config_path, "**") : filesha1("${local.config_path}/${f}")])) + worker_folder_hash = sha1(join("", [for f in fileset(local.worker_build_path, "**") : filesha1("${local.worker_build_path}/${f}")])) +} + + +locals { + + services_hashes = [local.scripts_folder_hash, local.dag_folder_hash, local.config_folder_hash, local.services_build_path_hash] + workers_hashes = [local.dag_folder_hash, local.config_folder_hash, local.worker_folder_hash] + +} + +