From 8989a29f31e8375dc64ae40c5495100052531f10 Mon Sep 17 00:00:00 2001 From: Pietro Avolio Date: Mon, 26 Jun 2023 15:14:53 +0200 Subject: [PATCH 1/3] Adding hash to pipelines processing step function execution names to prevent collisions --- .../pipeline_management/process_deployment_map.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py index 9fcb432e4..8c03d7a80 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py @@ -11,6 +11,7 @@ import tempfile from typing import Any, TypedDict import yaml +import hashlib from yaml.error import YAMLError import boto3 @@ -173,9 +174,11 @@ def start_executions( # AWS Step Functions supports max 80 characters. # Since the run_id equals 49 characters plus the dash, we have 30 # characters available. To ensure we don't run over, lets use a - # truncated version instead: - truncated_pipeline_name = full_pipeline_name[:30] - sfn_execution_name = f"{truncated_pipeline_name}-{run_id}" + # truncated version concatenated with an hash generated from + # the pipeline name + truncated_pipeline_name = full_pipeline_name[:24] + truncated_pipeline_name_hash = hashlib.md5( bytes(full_pipeline_name, 'utf-8') ).hexdigest()[:5] + sfn_execution_name = f"{truncated_pipeline_name}-{truncated_pipeline_name_hash}-{run_id}" sfn_client.start_execution( stateMachineArn=PIPELINE_MANAGEMENT_STATEMACHINE, name=sfn_execution_name, From af8543853742655e284e2b5eb4a56cce6f676fc2 Mon Sep 17 00:00:00 2001 From: Pietro Avolio Date: Mon, 26 Jun 2023 15:25:09 +0200 Subject: [PATCH 2/3] Adding hash to pipelines processing step function execution names to prevent collisions --- .../pipeline_management/process_deployment_map.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py index 8c03d7a80..2c1b26061 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py @@ -177,8 +177,8 @@ def start_executions( # truncated version concatenated with an hash generated from # the pipeline name truncated_pipeline_name = full_pipeline_name[:24] - truncated_pipeline_name_hash = hashlib.md5( bytes(full_pipeline_name, 'utf-8') ).hexdigest()[:5] - sfn_execution_name = f"{truncated_pipeline_name}-{truncated_pipeline_name_hash}-{run_id}" + execution_unique_hash = hashlib.md5( bytes(full_pipeline_name + run_id, 'utf-8') ).hexdigest()[:5] + sfn_execution_name = f"{truncated_pipeline_name}-{execution_unique_hash}-{run_id}" sfn_client.start_execution( stateMachineArn=PIPELINE_MANAGEMENT_STATEMACHINE, name=sfn_execution_name, From 286e9e1afb5be649349cf10f28d28d6122127702 Mon Sep 17 00:00:00 2001 From: Simon Kok Date: Mon, 24 Jul 2023 13:39:45 +0200 Subject: [PATCH 3/3] Apply linting fixes --- .../pipeline_management/process_deployment_map.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py index 2c1b26061..0f538f113 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py @@ -10,8 +10,8 @@ import json import tempfile from typing import Any, TypedDict -import yaml import hashlib +import yaml from yaml.error import YAMLError import boto3 @@ -177,7 +177,8 @@ def start_executions( # truncated version concatenated with an hash generated from # the pipeline name truncated_pipeline_name = full_pipeline_name[:24] - execution_unique_hash = hashlib.md5( bytes(full_pipeline_name + run_id, 'utf-8') ).hexdigest()[:5] + name_bytes_to_hash = bytes(full_pipeline_name + run_id, 'utf-8') + execution_unique_hash = hashlib.md5(name_bytes_to_hash).hexdigest()[:5] sfn_execution_name = f"{truncated_pipeline_name}-{execution_unique_hash}-{run_id}" sfn_client.start_execution( stateMachineArn=PIPELINE_MANAGEMENT_STATEMACHINE,