Skip to content

Commit

Permalink
Fix hash of a dynamic changing config file
Browse files Browse the repository at this point in the history
  • Loading branch information
@amarouane-ABDELHAK
amarouane-ABDELHAK committed Jul 26, 2024
1 parent 1219671 commit 6eeace6
Show file tree
Hide file tree
Showing 9 changed files with 53 additions and 50 deletions.
12 changes: 6 additions & 6 deletions infrastructure/ecs_services/airflow_metrics.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ resource "aws_security_group" "airflow_metrics_service" {

resource "aws_ecs_task_definition" "airflow_metrics" {
family = "${var.prefix}-metrics"
depends_on = [null_resource.build_ecr_image, aws_ecr_repository.airflow]
depends_on = [null_resource.build_ecr_image, aws_ecr_repository.airflow]
cpu = 256
memory = 512
execution_role_arn = aws_iam_role.ecs_task_execution_role.arn
Expand Down Expand Up @@ -58,13 +58,13 @@ resource "aws_ecs_task_definition" "airflow_metrics" {
]
environment = concat(var.airflow_task_common_environment,
[
{
{
name = "SERVICES_HASH"
value = join(",", local.services_hashes)
}
}

])
user = "50000:0"
user = "50000:0"
logConfiguration = {
logDriver = "awslogs"
options = {
Expand Down Expand Up @@ -95,8 +95,8 @@ resource "aws_ecs_service" "airflow_metrics" {
assign_public_ip = false
security_groups = [aws_security_group.airflow_metrics_service.id]
}
platform_version = "1.4.0"
scheduling_strategy = "REPLICA"
platform_version = "1.4.0"
scheduling_strategy = "REPLICA"
# Update from scripts folder
force_new_deployment = var.force_new_ecs_service_deployment
}
12 changes: 6 additions & 6 deletions infrastructure/ecs_services/airflow_scheduler.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ resource "aws_ssm_parameter" "airflow_ecs_cloudwatch_agent_config" {

resource "aws_ecs_task_definition" "airflow_scheduler" {
family = "${var.prefix}-scheduler"
depends_on = [null_resource.build_ecr_image, aws_ecr_repository.airflow]
depends_on = [null_resource.build_ecr_image, aws_ecr_repository.airflow]
cpu = var.scheduler_cpu
memory = var.scheduler_memory
execution_role_arn = aws_iam_role.ecs_task_execution_role.arn
Expand Down Expand Up @@ -77,13 +77,13 @@ resource "aws_ecs_task_definition" "airflow_scheduler" {
}
environment = concat(var.airflow_task_common_environment,
[
{
{
name = "SERVICES_HASH"
value = join(",", local.services_hashes)
}
}

])
user = "50000:0"
user = "50000:0"
logConfiguration = {
logDriver = "awslogs"
options = {
Expand Down Expand Up @@ -146,8 +146,8 @@ resource "aws_ecs_service" "airflow_scheduler" {
assign_public_ip = false
security_groups = [aws_security_group.airflow_scheduler_service.id]
}
platform_version = "1.4.0"
scheduling_strategy = "REPLICA"
platform_version = "1.4.0"
scheduling_strategy = "REPLICA"
# Update from requirements
#force_new_deployment = var.force_new_ecs_service_deployment
}
Expand Down
12 changes: 6 additions & 6 deletions infrastructure/ecs_services/airflow_server.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ resource "aws_cloudwatch_log_group" "airflow_webserver" {

resource "aws_ecs_task_definition" "airflow_webserver" {
family = "${var.prefix}-webserver"
depends_on = [null_resource.build_ecr_image, aws_ecr_repository.airflow]
depends_on = [null_resource.build_ecr_image, aws_ecr_repository.airflow]
cpu = 1024
memory = 2048
execution_role_arn = aws_iam_role.ecs_task_execution_role.arn
Expand Down Expand Up @@ -45,17 +45,17 @@ resource "aws_ecs_task_definition" "airflow_webserver" {
linuxParameters = {
initProcessEnabled = true
}
essential = true
command = ["webserver"]
essential = true
command = ["webserver"]
environment = concat(var.airflow_task_common_environment,
[
{
{
name = "SERVICES_HASH"
value = "${local.config_folder_hash},${local.services_build_path_hash}"
}
}

])
user = "50000:0"
user = "50000:0"
logConfiguration = {
logDriver = "awslogs"
options = {
Expand Down
2 changes: 1 addition & 1 deletion infrastructure/ecs_services/airflow_standalone_task.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ resource "aws_ecs_task_definition" "airflow_standalone_task" {
execution_role_arn = aws_iam_role.ecs_task_execution_role.arn
task_role_arn = aws_iam_role.airflow_task.arn
network_mode = "awsvpc"
runtime_platform {
runtime_platform {
operating_system_family = "LINUX"
cpu_architecture = var.task_cpu_architecture
}
Expand Down
2 changes: 1 addition & 1 deletion infrastructure/ecs_services/airflow_worker.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ resource "aws_cloudwatch_log_group" "airflow_worker" {

resource "aws_ecs_task_definition" "airflow_worker" {
family = "${var.prefix}-worker"
depends_on = [null_resource.build_worker_ecr_image, aws_ecr_repository.airflow]
depends_on = [null_resource.build_worker_ecr_image, aws_ecr_repository.airflow]
cpu = var.worker_cpu # 4096
memory = var.worker_memory # 4096 *2
execution_role_arn = aws_iam_role.ecs_task_execution_role.arn
Expand Down
8 changes: 4 additions & 4 deletions infrastructure/ecs_services/ecr.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,10 @@ resource "aws_ecr_lifecycle_policy" "ecr_policy" {
}
resource "null_resource" "build_ecr_image" {
triggers = {
services_build_path_hash = local.services_build_path_hash
scripts_folder_hash = local.scripts_folder_hash
dag_folder_hash = local.dag_folder_hash
config_folder_hash = local.config_folder_hash
services_build_path_hash = local.services_build_path_hash
scripts_folder_hash = local.scripts_folder_hash
dag_folder_hash = local.dag_folder_hash
config_folder_hash = local.config_folder_hash
}

provisioner "local-exec" {
Expand Down
25 changes: 14 additions & 11 deletions infrastructure/ecs_services/locals.tf
View file
Original file line number Diff line number Diff line change
@@ -1,28 +1,31 @@
locals {

services_build_path = "../${path.root}/airflow_services"
dag_folder_path = "../${path.root}/dags"
scripts_path = "../${path.root}/scripts"
config_path = "../${path.root}/infrastructure/configuration"
worker_build_path = "../${path.root}/airflow_worker"
services_build_path = "../${path.root}/airflow_services"
dag_folder_path = "../${path.root}/dags"
scripts_path = "../${path.root}/scripts"
config_path = "../${path.root}/infrastructure/configuration"
worker_build_path = "../${path.root}/airflow_worker"
config_files = [for f in fileset(local.config_path, "**") : f if f != "airflow.cfg"]


}



locals {

services_build_path_hash = sha1(join("", [for f in fileset(local.services_build_path, "**") : filesha1("${local.services_build_path}/${f}")]))
dag_folder_hash = sha1(join("", [for f in fileset(local.dag_folder_path, "**") : filesha1("${local.dag_folder_path}/${f}")]))
services_build_path_hash = sha1(join("", [for f in fileset(local.services_build_path, "**") : filesha1("${local.services_build_path}/${f}")]))
dag_folder_hash = sha1(join("", [for f in fileset(local.dag_folder_path, "**") : filesha1("${local.dag_folder_path}/${f}")]))
scripts_folder_hash = sha1(join("", [for f in fileset(local.scripts_path, "**") : filesha1("${local.scripts_path}/${f}")]))
config_folder_hash = sha1(join("", [for f in fileset(local.config_path, "**") : filesha1("${local.config_path}/${f}")]))
worker_folder_hash = sha1(join("", [for f in fileset(local.worker_build_path, "**") : filesha1("${local.worker_build_path}/${f}")]))
config_folder_hash = sha1(join("", [for f in local.config_files : filesha1("${local.config_path}/${f}")]))
worker_folder_hash = sha1(join("", [for f in fileset(local.worker_build_path, "**") : filesha1("${local.worker_build_path}/${f}")]))
}


locals {

services_hashes = [local.scripts_folder_hash, local.dag_folder_hash, local.config_folder_hash, local.services_build_path_hash]
workers_hashes = [local.dag_folder_hash, local.config_folder_hash, local.worker_folder_hash]
services_hashes = [local.scripts_folder_hash, local.dag_folder_hash, local.config_folder_hash, local.services_build_path_hash]
workers_hashes = [local.dag_folder_hash, local.config_folder_hash, local.worker_folder_hash]

}

Expand Down
24 changes: 12 additions & 12 deletions infrastructure/main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,18 +29,18 @@ module "database" {


module "secrets" {
source = "./secrets"
db_endpoint = module.database.database_endpoint
db_name = var.airflow_db.db_name
db_password = module.database.database_password
db_port = var.airflow_db.port
db_username = var.airflow_db.username
fernet_key = var.fernet_key
prefix = var.prefix
airflow_admin_username = var.airflow_admin_username
airflow_admin_password = var.airflow_admin_password
webserver_url = module.ecs_services.airflow_url
airflow_custom_variables = merge({db_secret_name = module.secrets.airflow_secrets}, var.airflow_custom_variables)
source = "./secrets"
db_endpoint = module.database.database_endpoint
db_name = var.airflow_db.db_name
db_password = module.database.database_password
db_port = var.airflow_db.port
db_username = var.airflow_db.username
fernet_key = var.fernet_key
prefix = var.prefix
airflow_admin_username = var.airflow_admin_username
airflow_admin_password = var.airflow_admin_password
webserver_url = module.ecs_services.airflow_url
airflow_custom_variables = merge({ db_secret_name = module.secrets.airflow_secrets }, var.airflow_custom_variables)
}


Expand Down
6 changes: 3 additions & 3 deletions infrastructure/varaibles.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ variable "extra_airflow_configuration" {
}

variable "workers_logs_retention_days" {
type = number
type = number
default = 1
}

Expand All @@ -190,10 +190,10 @@ variable "workers_logs_retention_days" {
variable "airflow_custom_variables" {
description = "Airflow custom variables"
type = map(string)
default = {}
default = {}
}

variable "infrastructure_foldername"{
variable "infrastructure_foldername" {
default = "infrastructure"
}

Expand Down

0 comments on commit 6eeace6

Please sign in to comment.