From 467db3df9f50357280a466d3bbccb2afb9997236 Mon Sep 17 00:00:00 2001 From: ian-whitestone Date: Thu, 31 Dec 2020 10:09:43 -0700 Subject: [PATCH] Make zappa update accept docker image * Raise a NotImplementedError for zappa rollback. Related #2188 --- zappa/cli.py | 44 ++++++++++++++++++++-------------- zappa/core.py | 65 +++++++++++++++++++++++++++++++-------------------- 2 files changed, 66 insertions(+), 43 deletions(-) diff --git a/zappa/cli.py b/zappa/cli.py index ca2ab8d41..6fa118b58 100755 --- a/zappa/cli.py +++ b/zappa/cli.py @@ -430,6 +430,9 @@ def positive_int(s): update_parser.add_argument( '-n', '--no-upload', help="Update configuration where appropriate, but don't upload new code" ) + update_parser.add_argument( + '-d', '--docker-image-uri', help='Update Lambda with a specific docker image hosted in AWS Elastic Container Registry' + ) ## # Debug @@ -573,7 +576,7 @@ def dispatch_command(self, command, stage): json=self.vargs['json'] ) elif command == 'update': # pragma: no cover - self.update(self.vargs['zip'], self.vargs['no_upload']) + self.update(self.vargs['zip'], self.vargs['no_upload'], self.vargs['docker_image_uri']) elif command == 'rollback': # pragma: no cover self.rollback(self.vargs['num_rollback']) elif command == 'invoke': # pragma: no cover @@ -873,7 +876,7 @@ def deploy(self, source_zip=None, docker_image_uri=None): self.zappa.add_api_stage_to_api_key(api_key=self.api_key, api_id=api_id, stage_name=self.api_stage) if self.stage_config.get('touch', True): - self.zappa.wait_until_lambda_function_is_active(function_name=self.lambda_name) + self.zappa.wait_until_lambda_function_is_ready(function_name=self.lambda_name) self.touch_endpoint(endpoint_url) # Finally, delete the local copy our zip package @@ -889,13 +892,12 @@ def deploy(self, source_zip=None, docker_image_uri=None): click.echo(deployment_string) - # TODO: update here... - def update(self, source_zip=None, no_upload=False): + def update(self, source_zip=None, no_upload=False, docker_image_uri=None): """ Repackage and update the function code. """ - if not source_zip: + if not source_zip and not docker_image_uri: # Make sure we're in a venv. self.check_venv() @@ -973,7 +975,11 @@ def update(self, source_zip=None, no_upload=False): num_revisions=self.num_retained_versions, concurrency=self.lambda_concurrency, ) - if source_zip and source_zip.startswith('s3://'): + if docker_image_uri: + kwargs['docker_image_uri'] = docker_image_uri + self.lambda_arn = self.zappa.update_lambda_function(**kwargs) + self.zappa.wait_until_lambda_function_is_ready(function_name=self.lambda_name) + elif source_zip and source_zip.startswith('s3://'): bucket, key_name = parse_s3_url(source_zip) kwargs.update(dict( bucket=bucket, @@ -991,7 +997,7 @@ def update(self, source_zip=None, no_upload=False): self.lambda_arn = self.zappa.update_lambda_function(**kwargs) # Remove the uploaded zip from S3, because it is now registered.. - if not source_zip and not no_upload: + if not source_zip and not no_upload and not docker_image_uri: self.remove_uploaded_zip() # Update the configuration, in case there are changes. @@ -1010,7 +1016,7 @@ def update(self, source_zip=None, no_upload=False): ) # Finally, delete the local copy our zip package - if not source_zip and not no_upload: + if not source_zip and not no_upload and not docker_image_uri: if self.stage_config.get('delete_local_zip', True): self.remove_local_zip() @@ -1086,6 +1092,7 @@ def update(self, source_zip=None, no_upload=False): deployed_string = deployed_string + " (" + api_url + ")" if self.stage_config.get('touch', True): + self.zappa.wait_until_lambda_function_is_ready(function_name=self.lambda_name) if api_url: self.touch_endpoint(api_url) elif endpoint_url: @@ -1452,13 +1459,15 @@ def tabular_print(title, value): status_dict["Lambda Name"] = self.lambda_name status_dict["Lambda ARN"] = self.lambda_arn status_dict["Lambda Role ARN"] = conf['Role'] - status_dict["Lambda Handler"] = conf['Handler'] status_dict["Lambda Code Size"] = conf['CodeSize'] status_dict["Lambda Version"] = conf['Version'] status_dict["Lambda Last Modified"] = conf['LastModified'] status_dict["Lambda Memory Size"] = conf['MemorySize'] status_dict["Lambda Timeout"] = conf['Timeout'] - status_dict["Lambda Runtime"] = conf['Runtime'] + # Handler & Runtime won't be present for lambda Docker deployments + # https://github.com/Miserlou/Zappa/issues/2188 + status_dict["Lambda Handler"] = conf.get('Handler', '') + status_dict["Lambda Runtime"] = conf.get('Runtime', '') if 'VpcConfig' in conf.keys(): status_dict["Lambda VPC ID"] = conf.get('VpcConfig', {}).get('VpcId', 'Not assigned') else: @@ -2310,6 +2319,13 @@ def create_package(self, output=None): settings_s = self.get_zappa_settings_string() + # Copy our Django app into root of our package. + # It doesn't work otherwise. + if self.django_settings: + base = __file__.rsplit(os.sep, 1)[0] + django_py = ''.join(os.path.join(base, 'ext', 'django_zappa.py')) + lambda_zip.write(django_py, 'django_zappa_app.py') + # Lambda requires a specific chmod temp_settings = tempfile.NamedTemporaryFile(delete=False) os.chmod(temp_settings.name, 0o644) @@ -2459,13 +2475,6 @@ def get_zappa_settings_string(self): if authorizer_function: settings_s += "AUTHORIZER_FUNCTION='{0!s}'\n".format(authorizer_function) - # Copy our Django app into root of our package. - # It doesn't work otherwise. - if self.django_settings: - base = __file__.rsplit(os.sep, 1)[0] - django_py = ''.join(os.path.join(base, 'ext', 'django_zappa.py')) - lambda_zip.write(django_py, 'django_zappa_app.py') - # async response async_response_table = self.stage_config.get('async_response_table', '') settings_s += "ASYNC_RESPONSE_TABLE='{0!s}'\n".format(async_response_table) @@ -2750,7 +2759,6 @@ def touch_endpoint(self, endpoint_url): return touch_path = self.stage_config.get('touch_path', '/') - req = requests.get(endpoint_url + touch_path) # Sometimes on really large packages, it can take 60-90 secs to be diff --git a/zappa/core.py b/zappa/core.py index d4be0b72e..9e3bc7465 100644 --- a/zappa/core.py +++ b/zappa/core.py @@ -1107,7 +1107,7 @@ def create_lambda_function( self, return resource_arn - def update_lambda_function(self, bucket, function_name, s3_key=None, publish=True, local_zip=None, num_revisions=None, concurrency=None): + def update_lambda_function(self, bucket, function_name, s3_key=None, publish=True, local_zip=None, num_revisions=None, concurrency=None, docker_image_uri=None): """ Given a bucket and key (or a local path) of a valid Lambda-zip, a function name and a handler, update that Lambda function's code. Optionally, delete previous versions if they exceed the optional limit. @@ -1118,7 +1118,9 @@ def update_lambda_function(self, bucket, function_name, s3_key=None, publish=Tru FunctionName=function_name, Publish=publish ) - if local_zip: + if docker_image_uri: + kwargs['ImageUri'] = docker_image_uri + elif local_zip: kwargs['ZipFile'] = local_zip else: kwargs['S3Bucket'] = bucket @@ -1213,6 +1215,7 @@ def update_lambda_configuration( self, # Check if there are any remote aws lambda env vars so they don't get trashed. # https://github.com/Miserlou/Zappa/issues/987, Related: https://github.com/Miserlou/Zappa/issues/765 lambda_aws_config = self.lambda_client.get_function_configuration(FunctionName=function_name) + is_docker_deployment = lambda_aws_config['PackageType'] == 'Image' if "Environment" in lambda_aws_config: lambda_aws_environment_variables = lambda_aws_config["Environment"].get("Variables", {}) # Append keys that are remote but not in settings file @@ -1220,22 +1223,26 @@ def update_lambda_configuration( self, if key not in aws_environment_variables: aws_environment_variables[key] = value - response = self.lambda_client.update_function_configuration( - FunctionName=function_name, - Runtime=runtime, - Role=self.credentials_arn, - Handler=handler, - Description=description, - Timeout=timeout, - MemorySize=memory_size, - VpcConfig=vpc_config, - Environment={'Variables': aws_environment_variables}, - KMSKeyArn=aws_kms_key_arn, - TracingConfig={ + kwargs = { + 'FunctionName': function_name, + 'Role': self.credentials_arn, + 'Description': description, + 'Timeout': timeout, + 'MemorySize': memory_size, + 'VpcConfig':vpc_config, + 'Environment':{'Variables': aws_environment_variables}, + 'KMSKeyArn': aws_kms_key_arn, + 'TracingConfig': { 'Mode': 'Active' if self.xray_tracing else 'PassThrough' }, - Layers=layers - ) + } + + if not is_docker_deployment: + kwargs['Handler'] = handler + kwargs['Runtime'] = runtime + kwargs['Layers'] = layers + + response = self.lambda_client.update_function_configuration(**kwargs) resource_arn = response['FunctionArn'] @@ -1271,6 +1278,12 @@ def rollback_lambda_function_version(self, function_name, versions_back=1, publi """ response = self.lambda_client.list_versions_by_function(FunctionName=function_name) + # https://github.com/Miserlou/Zappa/pull/2192 + if response['Versions'][-1]["PackageType"] == "Image": + raise NotImplementedError( + "Zappa's rollback functionality is not available for Docker based deployments" + ) + # Take into account $LATEST if len(response['Versions']) < versions_back + 1: print("We do not have {} revisions. Aborting".format(str(versions_back))) @@ -1290,30 +1303,32 @@ def rollback_lambda_function_version(self, function_name, versions_back=1, publi return response['FunctionArn'] - def is_lambda_function_active(self, function_name): + def is_lambda_function_ready(self, function_name): """ - Checks if a lambda function is active, given a name. + Checks if a lambda function is active and no updates are in progress. """ - response = self.lambda_client.get_function( - FunctionName=function_name) - return response['Configuration']['State'] == 'Active' + response = self.lambda_client.get_function(FunctionName=function_name) + return ( + response['Configuration']['State'] == 'Active' + and response['Configuration']['LastUpdateStatus'] != 'InProgress' + ) - def wait_until_lambda_function_is_active(self, function_name): + def wait_until_lambda_function_is_ready(self, function_name): """ Continuously check if a lambda function is active. For functions deployed with a docker image instead of a ZIP package, the function can take a few seconds longer - to be created, so we must wait before running any status + to be created or update, so we must wait before running any status checks against the function. """ show_waiting_message = True while True: - if self.is_lambda_function_active(function_name): + if self.is_lambda_function_ready(function_name): break if show_waiting_message: - print("Waiting until lambda function is active.") + print("Waiting until lambda function is ready.") show_waiting_message = False time.sleep(1)