From 4ba4ed89e2ecf67c1042f8ea01cce2878282def6 Mon Sep 17 00:00:00 2001 From: Carolyn Nguyen Date: Thu, 2 Sep 2021 11:16:32 -0700 Subject: [PATCH 1/5] Make hyperparameters compatible with placeholders --- src/stepfunctions/steps/sagemaker.py | 12 ++++-------- tests/unit/test_sagemaker_steps.py | 10 +++------- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/src/stepfunctions/steps/sagemaker.py b/src/stepfunctions/steps/sagemaker.py index 9530478..5f34b7b 100644 --- a/src/stepfunctions/steps/sagemaker.py +++ b/src/stepfunctions/steps/sagemaker.py @@ -127,8 +127,9 @@ def __init__(self, state_id, estimator, job_name, data=None, hyperparameters=Non parameters['InputDataConfig'][0]['DataSource']['S3DataSource']['S3Uri.$'] = data_uri if hyperparameters is not None: - if estimator.hyperparameters() is not None: - hyperparameters = self.__merge_hyperparameters(hyperparameters, estimator.hyperparameters()) + if not isinstance(hyperparameters, Placeholder): + if estimator.hyperparameters() is not None: + hyperparameters = self.__merge_hyperparameters(hyperparameters, estimator.hyperparameters()) parameters['HyperParameters'] = hyperparameters if experiment_config is not None: @@ -173,12 +174,7 @@ def __merge_hyperparameters(self, training_step_hyperparameters, estimator_hyper estimator_hyperparameters (dict): Hyperparameters specified in the estimator """ merged_hyperparameters = estimator_hyperparameters.copy() - for key, value in training_step_hyperparameters.items(): - if key in merged_hyperparameters: - logger.info( - f"hyperparameter property: <{key}> with value: <{merged_hyperparameters[key]}> provided in the" - f" estimator will be overwritten with value provided in constructor: <{value}>") - merged_hyperparameters[key] = value + merge_dicts(merged_hyperparameters, training_step_hyperparameters) return merged_hyperparameters class TransformStep(Task): diff --git a/tests/unit/test_sagemaker_steps.py b/tests/unit/test_sagemaker_steps.py index 664a498..631a342 100644 --- a/tests/unit/test_sagemaker_steps.py +++ b/tests/unit/test_sagemaker_steps.py @@ -275,6 +275,7 @@ def test_training_step_creation_with_placeholders(pca_estimator): execution_input = ExecutionInput(schema={ 'Data': str, 'OutputPath': str, + 'HyperParameters': str }) step_input = StepInput(schema={ @@ -292,6 +293,7 @@ def test_training_step_creation_with_placeholders(pca_estimator): 'TrialComponentDisplayName': 'Training' }, tags=DEFAULT_TAGS, + hyperparameters=execution_input['HyperParameters'] ) assert step.to_dict() == { 'Type': 'Task', @@ -312,13 +314,7 @@ def test_training_step_creation_with_placeholders(pca_estimator): 'VolumeSizeInGB': 30 }, 'RoleArn': EXECUTION_ROLE, - 'HyperParameters': { - 'feature_dim': '50000', - 'num_components': '10', - 'subtract_mean': 'True', - 'algorithm_mode': 'randomized', - 'mini_batch_size': '200' - }, + 'HyperParameters.$': "$$.Execution.Input['HyperParameters']", 'InputDataConfig': [ { 'ChannelName': 'training', From eb9a08a5a2a761812cbe7aaccd6a377ac8ad6f7e Mon Sep 17 00:00:00 2001 From: Carolyn Nguyen Date: Wed, 8 Sep 2021 18:26:33 -0700 Subject: [PATCH 2/5] feat: Support placeholders with hyperparameters passed to TrainingStep --- src/stepfunctions/steps/sagemaker.py | 7 +++---- tests/unit/test_sagemaker_steps.py | 20 +++++++++++++++++--- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/stepfunctions/steps/sagemaker.py b/src/stepfunctions/steps/sagemaker.py index 5f34b7b..0818a4c 100644 --- a/src/stepfunctions/steps/sagemaker.py +++ b/src/stepfunctions/steps/sagemaker.py @@ -69,7 +69,7 @@ def __init__(self, state_id, estimator, job_name, data=None, hyperparameters=Non * (list[sagemaker.amazon.amazon_estimator.RecordSet]) - A list of :class:`sagemaker.amazon.amazon_estimator.RecordSet` objects, where each instance is a different channel of training data. - hyperparameters (dict, optional): Parameters used for training. + hyperparameters (dict[str, str] or dict[str, Placeholder], optional): Parameters used for training. Hyperparameters supplied will be merged with the Hyperparameters specified in the estimator. If there are duplicate entries, the value provided through this property will be used. (Default: Hyperparameters specified in the estimator.) mini_batch_size (int): Specify this argument only when estimator is a built-in estimator of an Amazon algorithm. For other estimators, batch size should be specified in the estimator. @@ -127,9 +127,8 @@ def __init__(self, state_id, estimator, job_name, data=None, hyperparameters=Non parameters['InputDataConfig'][0]['DataSource']['S3DataSource']['S3Uri.$'] = data_uri if hyperparameters is not None: - if not isinstance(hyperparameters, Placeholder): - if estimator.hyperparameters() is not None: - hyperparameters = self.__merge_hyperparameters(hyperparameters, estimator.hyperparameters()) + if estimator.hyperparameters() is not None: + hyperparameters = self.__merge_hyperparameters(hyperparameters, estimator.hyperparameters()) parameters['HyperParameters'] = hyperparameters if experiment_config is not None: diff --git a/tests/unit/test_sagemaker_steps.py b/tests/unit/test_sagemaker_steps.py index 631a342..40c1964 100644 --- a/tests/unit/test_sagemaker_steps.py +++ b/tests/unit/test_sagemaker_steps.py @@ -275,7 +275,9 @@ def test_training_step_creation_with_placeholders(pca_estimator): execution_input = ExecutionInput(schema={ 'Data': str, 'OutputPath': str, - 'HyperParameters': str + 'num_components': str, + 'HyperParamA': str, + 'HyperParamB': str, }) step_input = StepInput(schema={ @@ -293,7 +295,11 @@ def test_training_step_creation_with_placeholders(pca_estimator): 'TrialComponentDisplayName': 'Training' }, tags=DEFAULT_TAGS, - hyperparameters=execution_input['HyperParameters'] + hyperparameters={ + 'num_components': execution_input['num_components'], + 'HyperParamA': execution_input['HyperParamA'], + 'HyperParamB': execution_input['HyperParamB'] + } ) assert step.to_dict() == { 'Type': 'Task', @@ -314,7 +320,15 @@ def test_training_step_creation_with_placeholders(pca_estimator): 'VolumeSizeInGB': 30 }, 'RoleArn': EXECUTION_ROLE, - 'HyperParameters.$': "$$.Execution.Input['HyperParameters']", + 'HyperParameters': { + 'HyperParamA.$': "$$.Execution.Input['HyperParamA']", + 'HyperParamB.$': "$$.Execution.Input['HyperParamB']", + 'algorithm_mode': 'randomized', + 'feature_dim': 50000, + 'mini_batch_size': 200, + 'num_components.$': "$$.Execution.Input['num_components']", + 'subtract_mean': True + }, 'InputDataConfig': [ { 'ChannelName': 'training', From a6cbd8176cc09db3438acf3d34730c454e4d49c2 Mon Sep 17 00:00:00 2001 From: Carolyn Nguyen Date: Thu, 9 Sep 2021 18:16:19 -0700 Subject: [PATCH 3/5] Placeholder hyperparameters overwrite the estimator hyperparameters --- src/stepfunctions/steps/sagemaker.py | 17 +++++-- tests/unit/test_sagemaker_steps.py | 72 +++++++++++++++++++++++++++- 2 files changed, 83 insertions(+), 6 deletions(-) diff --git a/src/stepfunctions/steps/sagemaker.py b/src/stepfunctions/steps/sagemaker.py index 0818a4c..b47b7d6 100644 --- a/src/stepfunctions/steps/sagemaker.py +++ b/src/stepfunctions/steps/sagemaker.py @@ -69,9 +69,10 @@ def __init__(self, state_id, estimator, job_name, data=None, hyperparameters=Non * (list[sagemaker.amazon.amazon_estimator.RecordSet]) - A list of :class:`sagemaker.amazon.amazon_estimator.RecordSet` objects, where each instance is a different channel of training data. - hyperparameters (dict[str, str] or dict[str, Placeholder], optional): Parameters used for training. - Hyperparameters supplied will be merged with the Hyperparameters specified in the estimator. + hyperparameters: Parameters used for training. + * (dict[str, str], optional) - Hyperparameters supplied will be merged with the Hyperparameters specified in the estimator. If there are duplicate entries, the value provided through this property will be used. (Default: Hyperparameters specified in the estimator.) + * (Placeholder, optional) - Hyperparameters supplied will overwrite the Hyperparameters specified in the estimator. mini_batch_size (int): Specify this argument only when estimator is a built-in estimator of an Amazon algorithm. For other estimators, batch size should be specified in the estimator. experiment_config (dict, optional): Specify the experiment config for the training. (Default: None) wait_for_completion (bool, optional): Boolean value set to `True` if the Task state should wait for the training job to complete before proceeding to the next step in the workflow. Set to `False` if the Task state should submit the training job and proceed to the next step. (default: True) @@ -127,8 +128,9 @@ def __init__(self, state_id, estimator, job_name, data=None, hyperparameters=Non parameters['InputDataConfig'][0]['DataSource']['S3DataSource']['S3Uri.$'] = data_uri if hyperparameters is not None: - if estimator.hyperparameters() is not None: - hyperparameters = self.__merge_hyperparameters(hyperparameters, estimator.hyperparameters()) + if not isinstance(hyperparameters, Placeholder): + if estimator.hyperparameters() is not None: + hyperparameters = self.__merge_hyperparameters(hyperparameters, estimator.hyperparameters()) parameters['HyperParameters'] = hyperparameters if experiment_config is not None: @@ -173,7 +175,12 @@ def __merge_hyperparameters(self, training_step_hyperparameters, estimator_hyper estimator_hyperparameters (dict): Hyperparameters specified in the estimator """ merged_hyperparameters = estimator_hyperparameters.copy() - merge_dicts(merged_hyperparameters, training_step_hyperparameters) + for key, value in training_step_hyperparameters.items(): + if key in merged_hyperparameters: + logger.info( + f"hyperparameter property: <{key}> with value: <{merged_hyperparameters[key]}> provided in the" + f" estimator will be overwritten with value provided in constructor: <{value}>") + merged_hyperparameters[key] = value return merged_hyperparameters class TransformStep(Task): diff --git a/tests/unit/test_sagemaker_steps.py b/tests/unit/test_sagemaker_steps.py index 40c1964..a0b2b72 100644 --- a/tests/unit/test_sagemaker_steps.py +++ b/tests/unit/test_sagemaker_steps.py @@ -272,6 +272,77 @@ def test_training_step_creation(pca_estimator): @patch('botocore.client.BaseClient._make_api_call', new=mock_boto_api_call) @patch.object(boto3.session.Session, 'region_name', 'us-east-1') def test_training_step_creation_with_placeholders(pca_estimator): + execution_input = ExecutionInput(schema={ + 'Data': str, + 'OutputPath': str, + 'HyperParameters': str + }) + + step_input = StepInput(schema={ + 'JobName': str, + }) + + step = TrainingStep('Training', + estimator=pca_estimator, + job_name=step_input['JobName'], + data=execution_input['Data'], + output_data_config_path=execution_input['OutputPath'], + experiment_config={ + 'ExperimentName': 'pca_experiment', + 'TrialName': 'pca_trial', + 'TrialComponentDisplayName': 'Training' + }, + tags=DEFAULT_TAGS, + hyperparameters=execution_input['HyperParameters'] + ) + assert step.to_dict() == { + 'Type': 'Task', + 'Parameters': { + 'AlgorithmSpecification': { + 'TrainingImage': PCA_IMAGE, + 'TrainingInputMode': 'File' + }, + 'OutputDataConfig': { + 'S3OutputPath.$': "$$.Execution.Input['OutputPath']" + }, + 'StoppingCondition': { + 'MaxRuntimeInSeconds': 86400 + }, + 'ResourceConfig': { + 'InstanceCount': 1, + 'InstanceType': 'ml.c4.xlarge', + 'VolumeSizeInGB': 30 + }, + 'RoleArn': EXECUTION_ROLE, + 'HyperParameters.$': "$$.Execution.Input['HyperParameters']", + 'InputDataConfig': [ + { + 'ChannelName': 'training', + 'DataSource': { + 'S3DataSource': { + 'S3DataDistributionType': 'FullyReplicated', + 'S3DataType': 'S3Prefix', + 'S3Uri.$': "$$.Execution.Input['Data']" + } + } + } + ], + 'ExperimentConfig': { + 'ExperimentName': 'pca_experiment', + 'TrialName': 'pca_trial', + 'TrialComponentDisplayName': 'Training' + }, + 'TrainingJobName.$': "$['JobName']", + 'Tags': DEFAULT_TAGS_LIST + }, + 'Resource': 'arn:aws:states:::sagemaker:createTrainingJob.sync', + 'End': True + } + + +@patch('botocore.client.BaseClient._make_api_call', new=mock_boto_api_call) +@patch.object(boto3.session.Session, 'region_name', 'us-east-1') +def test_training_step_creation_with_hyperparameters_containing_placeholders(pca_estimator): execution_input = ExecutionInput(schema={ 'Data': str, 'OutputPath': str, @@ -353,7 +424,6 @@ def test_training_step_creation_with_placeholders(pca_estimator): 'End': True } - @patch('botocore.client.BaseClient._make_api_call', new=mock_boto_api_call) @patch.object(boto3.session.Session, 'region_name', 'us-east-1') def test_training_step_creation_with_debug_hook(pca_estimator_with_debug_hook): From effe984e102e01777f576f8243a3e1eb44c6eb90 Mon Sep 17 00:00:00 2001 From: Carolyn Nguyen <83104894+ca-nguyen@users.noreply.github.com> Date: Thu, 9 Sep 2021 22:18:40 -0700 Subject: [PATCH 4/5] Update Hyperparameters description Co-authored-by: Adam Wong <55506708+wong-a@users.noreply.github.com> --- src/stepfunctions/steps/sagemaker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stepfunctions/steps/sagemaker.py b/src/stepfunctions/steps/sagemaker.py index b47b7d6..a87ce60 100644 --- a/src/stepfunctions/steps/sagemaker.py +++ b/src/stepfunctions/steps/sagemaker.py @@ -72,7 +72,7 @@ def __init__(self, state_id, estimator, job_name, data=None, hyperparameters=Non hyperparameters: Parameters used for training. * (dict[str, str], optional) - Hyperparameters supplied will be merged with the Hyperparameters specified in the estimator. If there are duplicate entries, the value provided through this property will be used. (Default: Hyperparameters specified in the estimator.) - * (Placeholder, optional) - Hyperparameters supplied will overwrite the Hyperparameters specified in the estimator. + * (Placeholder, optional) - The TrainingStep will use the hyperparameters specified by the Placeholder's value instead of the hyperparameters specified in the estimator. mini_batch_size (int): Specify this argument only when estimator is a built-in estimator of an Amazon algorithm. For other estimators, batch size should be specified in the estimator. experiment_config (dict, optional): Specify the experiment config for the training. (Default: None) wait_for_completion (bool, optional): Boolean value set to `True` if the Task state should wait for the training job to complete before proceeding to the next step in the workflow. Set to `False` if the Task state should submit the training job and proceed to the next step. (default: True) From e55c3bc83165cf66a8c1b8624a222da64a6fbc25 Mon Sep 17 00:00:00 2001 From: Carolyn Nguyen Date: Thu, 9 Sep 2021 22:46:08 -0700 Subject: [PATCH 5/5] Updated test assert scope to focus on hyperparameters --- src/stepfunctions/steps/sagemaker.py | 2 +- tests/unit/test_sagemaker_steps.py | 61 +++++----------------------- 2 files changed, 11 insertions(+), 52 deletions(-) diff --git a/src/stepfunctions/steps/sagemaker.py b/src/stepfunctions/steps/sagemaker.py index a87ce60..a3ddd47 100644 --- a/src/stepfunctions/steps/sagemaker.py +++ b/src/stepfunctions/steps/sagemaker.py @@ -70,7 +70,7 @@ def __init__(self, state_id, estimator, job_name, data=None, hyperparameters=Non :class:`sagemaker.amazon.amazon_estimator.RecordSet` objects, where each instance is a different channel of training data. hyperparameters: Parameters used for training. - * (dict[str, str], optional) - Hyperparameters supplied will be merged with the Hyperparameters specified in the estimator. + * (dict, optional) - Hyperparameters supplied will be merged with the Hyperparameters specified in the estimator. If there are duplicate entries, the value provided through this property will be used. (Default: Hyperparameters specified in the estimator.) * (Placeholder, optional) - The TrainingStep will use the hyperparameters specified by the Placeholder's value instead of the hyperparameters specified in the estimator. mini_batch_size (int): Specify this argument only when estimator is a built-in estimator of an Amazon algorithm. For other estimators, batch size should be specified in the estimator. diff --git a/tests/unit/test_sagemaker_steps.py b/tests/unit/test_sagemaker_steps.py index a0b2b72..02c6083 100644 --- a/tests/unit/test_sagemaker_steps.py +++ b/tests/unit/test_sagemaker_steps.py @@ -367,63 +367,22 @@ def test_training_step_creation_with_hyperparameters_containing_placeholders(pca }, tags=DEFAULT_TAGS, hyperparameters={ - 'num_components': execution_input['num_components'], + 'num_components': execution_input['num_components'], # This will overwrite the value that was defined in the pca_estimator 'HyperParamA': execution_input['HyperParamA'], 'HyperParamB': execution_input['HyperParamB'] } ) - assert step.to_dict() == { - 'Type': 'Task', - 'Parameters': { - 'AlgorithmSpecification': { - 'TrainingImage': PCA_IMAGE, - 'TrainingInputMode': 'File' - }, - 'OutputDataConfig': { - 'S3OutputPath.$': "$$.Execution.Input['OutputPath']" - }, - 'StoppingCondition': { - 'MaxRuntimeInSeconds': 86400 - }, - 'ResourceConfig': { - 'InstanceCount': 1, - 'InstanceType': 'ml.c4.xlarge', - 'VolumeSizeInGB': 30 - }, - 'RoleArn': EXECUTION_ROLE, - 'HyperParameters': { - 'HyperParamA.$': "$$.Execution.Input['HyperParamA']", - 'HyperParamB.$': "$$.Execution.Input['HyperParamB']", - 'algorithm_mode': 'randomized', - 'feature_dim': 50000, - 'mini_batch_size': 200, - 'num_components.$': "$$.Execution.Input['num_components']", - 'subtract_mean': True - }, - 'InputDataConfig': [ - { - 'ChannelName': 'training', - 'DataSource': { - 'S3DataSource': { - 'S3DataDistributionType': 'FullyReplicated', - 'S3DataType': 'S3Prefix', - 'S3Uri.$': "$$.Execution.Input['Data']" - } - } - } - ], - 'ExperimentConfig': { - 'ExperimentName': 'pca_experiment', - 'TrialName': 'pca_trial', - 'TrialComponentDisplayName': 'Training' - }, - 'TrainingJobName.$': "$['JobName']", - 'Tags': DEFAULT_TAGS_LIST - }, - 'Resource': 'arn:aws:states:::sagemaker:createTrainingJob.sync', - 'End': True + assert step.to_dict()['Parameters']['HyperParameters'] == { + 'HyperParamA.$': "$$.Execution.Input['HyperParamA']", + 'HyperParamB.$': "$$.Execution.Input['HyperParamB']", + 'algorithm_mode': 'randomized', + 'feature_dim': 50000, + 'mini_batch_size': 200, + 'num_components.$': "$$.Execution.Input['num_components']", + 'subtract_mean': True } + @patch('botocore.client.BaseClient._make_api_call', new=mock_boto_api_call) @patch.object(boto3.session.Session, 'region_name', 'us-east-1') def test_training_step_creation_with_debug_hook(pca_estimator_with_debug_hook):