From 6275fecc8b57bba85945e1c39b3e9ab672e42411 Mon Sep 17 00:00:00 2001
From: Daniel Sotirhos <dsotirho@ucsc.edu>
Date: Tue, 14 May 2024 11:15:29 -0700
Subject: [PATCH] Fix: Race between layer and Lambda update (#5927)

---
 Makefile                          |  1 +
 scripts/delete_lambda_versions.py | 41 +++++++++++++++++++++++++++++++
 src/azul/terraform.py             |  4 +++
 3 files changed, 46 insertions(+)
 create mode 100644 scripts/delete_lambda_versions.py

diff --git a/Makefile b/Makefile
index 729ca1213c..ea1cf41443 100644
--- a/Makefile
+++ b/Makefile
@@ -99,6 +99,7 @@ $(1)terraform: lambdas
 
 .PHONY: $(1)deploy
 $(1)deploy: check_python $(1)terraform
+	python $(project_root)/scripts/delete_lambda_versions.py
 	python $(project_root)/scripts/post_deploy_tdr.py
 endef
 
diff --git a/scripts/delete_lambda_versions.py b/scripts/delete_lambda_versions.py
new file mode 100644
index 0000000000..d2c5da69ac
--- /dev/null
+++ b/scripts/delete_lambda_versions.py
@@ -0,0 +1,41 @@
+"""
+Delete all published AWS Lambda versions except for "$LATEST" in the current
+deployment.
+"""
+
+import logging
+
+from azul.deployment import (
+    aws,
+)
+from azul.lambdas import (
+    Lambdas,
+)
+from azul.logging import (
+    configure_script_logging,
+)
+
+log = logging.getLogger(__name__)
+
+
+def main():
+    lambdas = Lambdas().list_lambdas()
+    for lambda_ in lambdas:
+        response = aws.lambda_.list_versions_by_function(
+            FunctionName=lambda_.name
+        )
+        for version in response['Versions']:
+            if version['Version'] == '$LATEST':
+                pass
+            else:
+                version_number = version['Version']
+                log.info('Deleting published version %s of %s', version_number, lambda_.name)
+                aws.lambda_.delete_function(
+                    FunctionName=lambda_.name,
+                    Qualifier=version_number
+                )
+
+
+if __name__ == '__main__':
+    configure_script_logging(log)
+    main()
diff --git a/src/azul/terraform.py b/src/azul/terraform.py
index d2e6ca5fcb..26fa339aad 100644
--- a/src/azul/terraform.py
+++ b/src/azul/terraform.py
@@ -708,6 +708,10 @@ def tf_config(self, app_name):
         for resource in resources['aws_lambda_function'].values():
             assert 'layers' not in resource
             resource['layers'] = ['${aws_lambda_layer_version.dependencies.arn}']
+            # Publishing the Lambda function as a new version prevents possible
+            # race conditions when an update to the function's configuration and
+            # code rely on the update of each other in order to work correctly.
+            resource['publish'] = True
             env = config.es_endpoint_env(
                 es_endpoint=(
                     aws.es_endpoint