Skip to content

Commit

Permalink
Pre-import cdk libraries at the beginning of cluster create and update
Browse files Browse the repository at this point in the history
Import of cdk libraries at lambda cold start takes 12 to 16 seconds, sometimes causing lambda timeout.
Importing them in a separate thread, started at the beginning of create and update operations
and executed concurrently with stack validation allows to reduce the time to complete the operation.
  • Loading branch information
awslitvit committed Jan 5, 2023
1 parent 550806e commit e5800c9
Show file tree
Hide file tree
Showing 2 changed files with 30 additions and 0 deletions.
3 changes: 3 additions & 0 deletions cli/src/pcluster/models/cluster.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@
from pcluster.models.s3_bucket import S3Bucket, S3BucketFactory, S3FileFormat, create_s3_presigned_url, parse_bucket_url
from pcluster.schemas.cluster_schema import ClusterSchema
from pcluster.templates.cdk_builder import CDKTemplateBuilder
from pcluster.templates.import_cdk import start as start_cdk_import
from pcluster.utils import (
datetime_to_epoch,
generate_random_name_with_prefix,
Expand Down Expand Up @@ -352,6 +353,7 @@ def create(
raises ClusterActionError: in case of generic error
raises ConfigValidationError: if configuration is invalid
"""
start_cdk_import()
creation_result = None
artifact_dir_generated = False
try:
Expand Down Expand Up @@ -887,6 +889,7 @@ def update(
raises ConfigValidationError: if configuration is invalid
raises ClusterUpdateError: if update is not allowed
"""
start_cdk_import()
try:
target_config, changes, ignored_validation_failures = self.validate_update_request(
target_source_config, validator_suppressors, validation_failure_level, force
Expand Down
27 changes: 27 additions & 0 deletions cli/src/pcluster/templates/import_cdk.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
import logging
import time
from threading import Thread

LOGGER = logging.getLogger(__name__)


def import_cdk():
"""Import cdk libraries."""
LOGGER.info("Start importing")
begin = time.time()
from aws_cdk.core import App # noqa: F401 pylint: disable=import-outside-toplevel

from pcluster.templates.cluster_stack import ClusterCdkStack # noqa: F401 pylint: disable=import-outside-toplevel

LOGGER.info("Import complete in %i seconds", time.time() - begin)


def start():
"""
Import cdk libraries in a separate thread.
:return: thread importing cdk libraries
"""
thread = Thread(target=import_cdk)
thread.start()
return thread

0 comments on commit e5800c9

Please sign in to comment.