From 72e64dbf30b94185998c937e5ed31087a51f0013 Mon Sep 17 00:00:00 2001 From: Tatiana Litvinova Date: Wed, 4 Jan 2023 18:24:43 +0100 Subject: [PATCH] Pre-import cdk libraries at the beginning of cluster create and update Import of cdk libraries at lambda cold start takes 12 to 16 seconds, sometimes causing lambda timeout. Importing them in a separate thread, started at the beginning of create and update operations and executed concurrently with stack validation allows to reduce the time to complete the operation. --- .../pcluster/api/controllers/import_cdk.py | 27 +++++++++++++++++++ cli/src/pcluster/models/cluster.py | 3 +++ 2 files changed, 30 insertions(+) create mode 100644 cli/src/pcluster/api/controllers/import_cdk.py diff --git a/cli/src/pcluster/api/controllers/import_cdk.py b/cli/src/pcluster/api/controllers/import_cdk.py new file mode 100644 index 0000000000..4bd503849c --- /dev/null +++ b/cli/src/pcluster/api/controllers/import_cdk.py @@ -0,0 +1,27 @@ +import logging +import time +from threading import Thread + +LOGGER = logging.getLogger(__name__) + + +def import_cdk(): + """Import cdk libraries.""" + LOGGER.info("Start importing") + begin = time.time() + from aws_cdk.core import App # noqa: F401 pylint: disable=import-outside-toplevel + + from pcluster.templates.cluster_stack import ClusterCdkStack # noqa: F401 pylint: disable=import-outside-toplevel + + LOGGER.info("Import complete in %i seconds", time.time() - begin) + + +def start(): + """ + Import cdk libraries in a separate thread. + + :return: thread importing cdk libraries + """ + thread = Thread(target=import_cdk) + thread.start() + return thread diff --git a/cli/src/pcluster/models/cluster.py b/cli/src/pcluster/models/cluster.py index 3c92169757..423c2c664c 100644 --- a/cli/src/pcluster/models/cluster.py +++ b/cli/src/pcluster/models/cluster.py @@ -29,6 +29,7 @@ from jinja2.sandbox import SandboxedEnvironment from marshmallow import ValidationError +from pcluster.api.controllers.import_cdk import start as start_cdk_import from pcluster.api.models import Metadata from pcluster.aws.aws_api import AWSApi from pcluster.aws.common import AWSClientError, BadRequestError, LimitExceededError, StackNotFoundError, get_region @@ -352,6 +353,7 @@ def create( raises ClusterActionError: in case of generic error raises ConfigValidationError: if configuration is invalid """ + start_cdk_import() creation_result = None artifact_dir_generated = False try: @@ -887,6 +889,7 @@ def update( raises ConfigValidationError: if configuration is invalid raises ClusterUpdateError: if update is not allowed """ + start_cdk_import() try: target_config, changes, ignored_validation_failures = self.validate_update_request( target_source_config, validator_suppressors, validation_failure_level, force