diff --git a/components/aws/sagemaker/Changelog.md b/components/aws/sagemaker/Changelog.md new file mode 100644 index 00000000000..bb1d8cca0e3 --- /dev/null +++ b/components/aws/sagemaker/Changelog.md @@ -0,0 +1,40 @@ +# Change log for AWS SageMaker Components + +The version of the AWS SageMaker Components is determined by the docker image tag used in YAML spec +Repository: https://hub.docker.com/repository/docker/amazon/aws-sagemaker-kfp-components + +--------------------------------------------- + +**Change log for version 0.3.0** +- Remove data_location parameters from all components + (Use "channes" parameter instead) + +> Pull requests : [#3518](https://github.com/kubeflow/pipelines/pull/3518) + + +**Change log for version 2.0 (Apr 14, 2020)** +- Fix bug in Ground Truth component +- Add user agent header to boto3 client + +> Pull requests: [#3474](https://github.com/kubeflow/pipelines/pull/3474), [#3487](https://github.com/kubeflow/pipelines/pull/3487) + + +--------------------------------------------- + +## Old + +These are the old images which were in https://hub.docker.com/r/redbackthomson/aws-kubeflow-sagemaker/tags + +**Change log 20200402** +- Fix for vpc issue +- Add license files +- Use AmazonLinux instead of Ubuntu +- Pin the pip packages + + +> Pull requests: [#3374](https://github.com/kubeflow/pipelines/pull/3374), [#3397](https://github.com/kubeflow/pipelines/pull/3397) + +No change log available for older images +Please check git log + + diff --git a/components/aws/sagemaker/THIRD-PARTY-LICENSES.txt b/components/aws/sagemaker/THIRD-PARTY-LICENSES.txt index 441f56a2437..179d9b3ec3f 100644 --- a/components/aws/sagemaker/THIRD-PARTY-LICENSES.txt +++ b/components/aws/sagemaker/THIRD-PARTY-LICENSES.txt @@ -1,4 +1,4 @@ -** Amazon SageMaker Components for Kubeflow Pipelines; version 0.2.0 -- +** Amazon SageMaker Components for Kubeflow Pipelines; version 0.3.0 -- https://github.com/kubeflow/pipelines/tree/master/components/aws/sagemaker Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. ** boto3; version 1.12.33 -- https://github.com/boto/boto3/ diff --git a/components/aws/sagemaker/batch_transform/component.yaml b/components/aws/sagemaker/batch_transform/component.yaml index 795629e8d36..6e0ad637ac8 100644 --- a/components/aws/sagemaker/batch_transform/component.yaml +++ b/components/aws/sagemaker/batch_transform/component.yaml @@ -74,7 +74,7 @@ outputs: - {name: output_location, description: 'S3 URI of the transform job results.'} implementation: container: - image: amazon/aws-sagemaker-kfp-components:0.2 + image: amazon/aws-sagemaker-kfp-components:0.3.0 command: ['python'] args: [ batch_transform.py, diff --git a/components/aws/sagemaker/common/_utils.py b/components/aws/sagemaker/common/_utils.py index 5501a6d9838..0ae3ff73f20 100644 --- a/components/aws/sagemaker/common/_utils.py +++ b/components/aws/sagemaker/common/_utils.py @@ -126,11 +126,6 @@ def create_training_job_request(args): ### Update input channels, must have at least one specified if len(args['channels']) > 0: request['InputDataConfig'] = args['channels'] - # Max number of input channels/data locations is 20, but currently only 8 data location parameters are exposed separately. - # Source: Input data configuration description in the SageMaker create training job form - for i in range(1, len(args['channels']) + 1): - if args['data_location_' + str(i)]: - request['InputDataConfig'][i-1]['DataSource']['S3DataSource']['S3Uri'] = args['data_location_' + str(i)] else: logging.error("Must specify at least one input channel.") raise Exception('Could not create job request') @@ -517,11 +512,6 @@ def create_hyperparameter_tuning_job_request(args): ### Update input channels, must have at least one specified if len(args['channels']) > 0: request['TrainingJobDefinition']['InputDataConfig'] = args['channels'] - # Max number of input channels/data locations is 20, but currently only 8 data location parameters are exposed separately. - # Source: Input data configuration description in the SageMaker create hyperparameter tuning job form - for i in range(1, len(args['channels']) + 1): - if args['data_location_' + str(i)]: - request['TrainingJobDefinition']['InputDataConfig'][i-1]['DataSource']['S3DataSource']['S3Uri'] = args['data_location_' + str(i)] else: logging.error("Must specify at least one input channel.") raise Exception('Could not make job request') diff --git a/components/aws/sagemaker/deploy/component.yaml b/components/aws/sagemaker/deploy/component.yaml index c43803d8d1d..15ae51073f2 100644 --- a/components/aws/sagemaker/deploy/component.yaml +++ b/components/aws/sagemaker/deploy/component.yaml @@ -79,7 +79,7 @@ outputs: - {name: endpoint_name, description: 'Endpoint name'} implementation: container: - image: amazon/aws-sagemaker-kfp-components:0.2 + image: amazon/aws-sagemaker-kfp-components:0.3.0 command: ['python'] args: [ deploy.py, diff --git a/components/aws/sagemaker/ground_truth/component.yaml b/components/aws/sagemaker/ground_truth/component.yaml index 259a04ca03b..06823416e14 100644 --- a/components/aws/sagemaker/ground_truth/component.yaml +++ b/components/aws/sagemaker/ground_truth/component.yaml @@ -88,7 +88,7 @@ outputs: - {name: active_learning_model_arn, description: 'The ARN for the most recent Amazon SageMaker model trained as part of automated data labeling.'} implementation: container: - image: amazon/aws-sagemaker-kfp-components:0.2 + image: amazon/aws-sagemaker-kfp-components:0.3.0 command: ['python'] args: [ ground_truth.py, diff --git a/components/aws/sagemaker/hyperparameter_tuning/README.md b/components/aws/sagemaker/hyperparameter_tuning/README.md index 843b908aac7..1437702f628 100644 --- a/components/aws/sagemaker/hyperparameter_tuning/README.md +++ b/components/aws/sagemaker/hyperparameter_tuning/README.md @@ -26,7 +26,6 @@ integer_parameters | The array of IntegerParameterRange objects that specify ran continuous_parameters | The array of ContinuousParameterRange objects that specify ranges of continuous hyperparameters that you want to search | Yes | Yes | List of Dicts | | [] | categorical_parameters | The array of CategoricalParameterRange objects that specify ranges of categorical hyperparameters that you want to search | Yes | Yes | List of Dicts | | [] | channels | A list of dicts specifying the input channels (at least one); refer to [documentation](https://github.com/awsdocs/amazon-sagemaker-developer-guide/blob/master/doc_source/API_Channel.md) for parameters | No | No | List of Dicts | | | -data_location_[1, 8] | The S3 URI of the input data source for channel [1, 8] | Yes | Yes | | | output_location | The Amazon S3 path where you want Amazon SageMaker to store the results of the transform job | No | No | String | | | output_encryption_key | The AWS KMS key that Amazon SageMaker uses to encrypt the model artifacts | Yes | Yes | String | | | instance_type | The ML compute instance type | Yes | No | String | ml.m4.xlarge, ml.m4.2xlarge, ml.m4.4xlarge, ml.m4.10xlarge, ml.m4.16xlarge, ml.m5.large, ml.m5.xlarge, ml.m5.2xlarge, ml.m5.4xlarge, ml.m5.12xlarge, ml.m5.24xlarge, ml.c4.xlarge, ml.c4.2xlarge, ml.c4.4xlarge, ml.c4.8xlarge, ml.p2.xlarge, ml.p2.8xlarge, ml.p2.16xlarge, ml.p3.2xlarge, ml.p3.8xlarge, ml.p3.16xlarge, ml.c5.xlarge, ml.c5.2xlarge, ml.c5.4xlarge, ml.c5.9xlarge, ml.c5.18xlarge | ml.m4.xlarge | @@ -52,7 +51,6 @@ Notes: * Specify training image OR algorithm name. Use the image parameter for Bring Your Own Container (BYOC) algorithms, and algorithm name for Amazon built-in algorithms, custom algorithm resources in SageMaker, and algorithms subscribed to from the AWS Marketplace. * Specify VPC security group IDs AND VPC subnets to specify the VPC that you want the training jobs to connect to. * Specify warm start type AND 1 to 5 parent HPO jobs to launch the hyperparameter tuning job with previous jobs as a starting point. -* The parameters, data_location_1 through 8, is intended to be used for inputting the S3 URI outputs from previous steps in the pipeline, for example, from a Ground Truth labeling job. Otherwise, the S3 data location can be specified directly in the channels parameter. ## Outputs Name | Description diff --git a/components/aws/sagemaker/hyperparameter_tuning/component.yaml b/components/aws/sagemaker/hyperparameter_tuning/component.yaml index 31c419bdcb1..1e5b315e258 100644 --- a/components/aws/sagemaker/hyperparameter_tuning/component.yaml +++ b/components/aws/sagemaker/hyperparameter_tuning/component.yaml @@ -45,30 +45,6 @@ inputs: default: '[]' - name: channels description: 'A list of dicts specifying the input channels. Must have at least one.' - - name: data_location_1 - description: 'The S3 URI of the input data source for channel 1.' - default: '' - - name: data_location_2 - description: 'The S3 URI of the input data source for channel 2.' - default: '' - - name: data_location_3 - description: 'The S3 URI of the input data source for channel 3.' - default: '' - - name: data_location_4 - description: 'The S3 URI of the input data source for channel 4.' - default: '' - - name: data_location_5 - description: 'The S3 URI of the input data source for channel 5.' - default: '' - - name: data_location_6 - description: 'The S3 URI of the input data source for channel 6.' - default: '' - - name: data_location_7 - description: 'The S3 URI of the input data source for channel 7.' - default: '' - - name: data_location_8 - description: 'The S3 URI of the input data source for channel 8.' - default: '' - name: output_location description: 'The Amazon S3 path where you want Amazon SageMaker to store the model artifacts is from the best training job.' - name: output_encryption_key @@ -139,7 +115,7 @@ outputs: description: 'The registry path of the Docker image that contains the training algorithm' implementation: container: - image: amazon/aws-sagemaker-kfp-components:0.2 + image: amazon/aws-sagemaker-kfp-components:0.3.0 command: ['python'] args: [ hyperparameter_tuning.py, @@ -160,14 +136,6 @@ implementation: --continuous_parameters, {inputValue: continuous_parameters}, --categorical_parameters, {inputValue: categorical_parameters}, --channels, {inputValue: channels}, - --data_location_1, {inputValue: data_location_1}, - --data_location_2, {inputValue: data_location_2}, - --data_location_3, {inputValue: data_location_3}, - --data_location_4, {inputValue: data_location_4}, - --data_location_5, {inputValue: data_location_5}, - --data_location_6, {inputValue: data_location_6}, - --data_location_7, {inputValue: data_location_7}, - --data_location_8, {inputValue: data_location_8}, --output_location, {inputValue: output_location}, --output_encryption_key, {inputValue: output_encryption_key}, --instance_type, {inputValue: instance_type}, diff --git a/components/aws/sagemaker/hyperparameter_tuning/src/hyperparameter_tuning.py b/components/aws/sagemaker/hyperparameter_tuning/src/hyperparameter_tuning.py index 132b8a2b9af..68fe2fefa58 100644 --- a/components/aws/sagemaker/hyperparameter_tuning/src/hyperparameter_tuning.py +++ b/components/aws/sagemaker/hyperparameter_tuning/src/hyperparameter_tuning.py @@ -35,14 +35,6 @@ def create_parser(): parser.add_argument('--continuous_parameters', type=_utils.str_to_json_list, required=False, help='The array of ContinuousParameterRange objects that specify ranges of continuous hyperparameters that you want to search.', default='[]') parser.add_argument('--categorical_parameters', type=_utils.str_to_json_list, required=False, help='The array of CategoricalParameterRange objects that specify ranges of categorical hyperparameters that you want to search.', default='[]') parser.add_argument('--channels', type=_utils.str_to_json_list, required=True, help='A list of dicts specifying the input channels. Must have at least one.') - parser.add_argument('--data_location_1', type=str.strip, required=False, help='The S3 URI of the input data source for channel 1.', default='') - parser.add_argument('--data_location_2', type=str.strip, required=False, help='The S3 URI of the input data source for channel 2.', default='') - parser.add_argument('--data_location_3', type=str.strip, required=False, help='The S3 URI of the input data source for channel 3.', default='') - parser.add_argument('--data_location_4', type=str.strip, required=False, help='The S3 URI of the input data source for channel 4.', default='') - parser.add_argument('--data_location_5', type=str.strip, required=False, help='The S3 URI of the input data source for channel 5.', default='') - parser.add_argument('--data_location_6', type=str.strip, required=False, help='The S3 URI of the input data source for channel 6.', default='') - parser.add_argument('--data_location_7', type=str.strip, required=False, help='The S3 URI of the input data source for channel 7.', default='') - parser.add_argument('--data_location_8', type=str.strip, required=False, help='The S3 URI of the input data source for channel 8.', default='') parser.add_argument('--output_location', type=str.strip, required=True, help='The Amazon S3 path where you want Amazon SageMaker to store the results of the transform job.') parser.add_argument('--output_encryption_key', type=str.strip, required=False, help='The AWS KMS key that Amazon SageMaker uses to encrypt the model artifacts.', default='') parser.add_argument('--instance_type', choices=['ml.m4.xlarge', 'ml.m4.2xlarge', 'ml.m4.4xlarge', 'ml.m4.10xlarge', 'ml.m4.16xlarge', 'ml.m5.large', 'ml.m5.xlarge', 'ml.m5.2xlarge', 'ml.m5.4xlarge', diff --git a/components/aws/sagemaker/model/component.yaml b/components/aws/sagemaker/model/component.yaml index 0b9b4298895..52ddcc2d663 100644 --- a/components/aws/sagemaker/model/component.yaml +++ b/components/aws/sagemaker/model/component.yaml @@ -45,7 +45,7 @@ outputs: - {name: model_name, description: 'The model name Sagemaker created'} implementation: container: - image: amazon/aws-sagemaker-kfp-components:0.2 + image: amazon/aws-sagemaker-kfp-components:0.3.0 command: ['python'] args: [ create_model.py, diff --git a/components/aws/sagemaker/train/README.md b/components/aws/sagemaker/train/README.md index c98f2c52533..14cdaf22db6 100644 --- a/components/aws/sagemaker/train/README.md +++ b/components/aws/sagemaker/train/README.md @@ -37,9 +37,6 @@ max_wait_time | The maximum time in seconds you are willing to wait for a manage checkpoint_config | Dictionary of information about the output location for managed spot training checkpoint data | Yes | Dict | | {} | tags | Key-value pairs to categorize AWS resources | Yes | Dict | | {} | -Notes : -* The parameters, data_location_1 through 8, is intended to be used for inputting the S3 URI outputs from previous steps in the pipeline, for example, from a Ground Truth labeling job. Otherwise, the S3 data location can be specified directly in the channels parameter. - ## Output Stores the Model in the s3 bucket you specified diff --git a/components/aws/sagemaker/train/component.yaml b/components/aws/sagemaker/train/component.yaml index cc0735b38f9..b326a22965b 100644 --- a/components/aws/sagemaker/train/component.yaml +++ b/components/aws/sagemaker/train/component.yaml @@ -26,30 +26,6 @@ inputs: default: '{}' - name: channels description: 'A list of dicts specifying the input channels. Must have at least one.' - - name: data_location_1 - description: 'The S3 URI of the input data source for channel 1.' - default: '' - - name: data_location_2 - description: 'The S3 URI of the input data source for channel 2.' - default: '' - - name: data_location_3 - description: 'The S3 URI of the input data source for channel 3.' - default: '' - - name: data_location_4 - description: 'The S3 URI of the input data source for channel 4.' - default: '' - - name: data_location_5 - description: 'The S3 URI of the input data source for channel 5.' - default: '' - - name: data_location_6 - description: 'The S3 URI of the input data source for channel 6.' - default: '' - - name: data_location_7 - description: 'The S3 URI of the input data source for channel 7.' - default: '' - - name: data_location_8 - description: 'The S3 URI of the input data source for channel 8.' - default: '' - name: instance_type description: 'The ML compute instance type.' default: 'ml.m4.xlarge' @@ -103,7 +79,7 @@ outputs: - {name: training_image, description: 'The registry path of the Docker image that contains the training algorithm'} implementation: container: - image: amazon/aws-sagemaker-kfp-components:0.2 + image: amazon/aws-sagemaker-kfp-components:0.3.0 command: ['python'] args: [ train.py, @@ -117,14 +93,6 @@ implementation: --training_input_mode, {inputValue: training_input_mode}, --hyperparameters, {inputValue: hyperparameters}, --channels, {inputValue: channels}, - --data_location_1, {inputValue: data_location_1}, - --data_location_2, {inputValue: data_location_2}, - --data_location_3, {inputValue: data_location_3}, - --data_location_4, {inputValue: data_location_4}, - --data_location_5, {inputValue: data_location_5}, - --data_location_6, {inputValue: data_location_6}, - --data_location_7, {inputValue: data_location_7}, - --data_location_8, {inputValue: data_location_8}, --instance_type, {inputValue: instance_type}, --instance_count, {inputValue: instance_count}, --volume_size, {inputValue: volume_size}, diff --git a/components/aws/sagemaker/train/src/train.py b/components/aws/sagemaker/train/src/train.py index 07d1b13cf36..448e86341b3 100644 --- a/components/aws/sagemaker/train/src/train.py +++ b/components/aws/sagemaker/train/src/train.py @@ -27,14 +27,6 @@ def create_parser(): parser.add_argument('--training_input_mode', choices=['File', 'Pipe'], type=str.strip, help='The input mode that the algorithm supports. File or Pipe.', default='File') parser.add_argument('--hyperparameters', type=_utils.str_to_json_dict, help='Dictionary of hyperparameters for the the algorithm.', default='{}') parser.add_argument('--channels', type=_utils.str_to_json_list, required=True, help='A list of dicts specifying the input channels. Must have at least one.') - parser.add_argument('--data_location_1', type=str.strip, required=False, help='The S3 URI of the input data source for channel 1.', default='') - parser.add_argument('--data_location_2', type=str.strip, required=False, help='The S3 URI of the input data source for channel 2.', default='') - parser.add_argument('--data_location_3', type=str.strip, required=False, help='The S3 URI of the input data source for channel 3.', default='') - parser.add_argument('--data_location_4', type=str.strip, required=False, help='The S3 URI of the input data source for channel 4.', default='') - parser.add_argument('--data_location_5', type=str.strip, required=False, help='The S3 URI of the input data source for channel 5.', default='') - parser.add_argument('--data_location_6', type=str.strip, required=False, help='The S3 URI of the input data source for channel 6.', default='') - parser.add_argument('--data_location_7', type=str.strip, required=False, help='The S3 URI of the input data source for channel 7.', default='') - parser.add_argument('--data_location_8', type=str.strip, required=False, help='The S3 URI of the input data source for channel 8.', default='') parser.add_argument('--instance_type', required=True, choices=['ml.m4.xlarge', 'ml.m4.2xlarge', 'ml.m4.4xlarge', 'ml.m4.10xlarge', 'ml.m4.16xlarge', 'ml.m5.large', 'ml.m5.xlarge', 'ml.m5.2xlarge', 'ml.m5.4xlarge', 'ml.m5.12xlarge', 'ml.m5.24xlarge', 'ml.c4.xlarge', 'ml.c4.2xlarge', 'ml.c4.4xlarge', 'ml.c4.8xlarge', 'ml.p2.xlarge', 'ml.p2.8xlarge', 'ml.p2.16xlarge', 'ml.p3.2xlarge', 'ml.p3.8xlarge', 'ml.p3.16xlarge', 'ml.c5.xlarge', 'ml.c5.2xlarge', 'ml.c5.4xlarge', 'ml.c5.9xlarge', 'ml.c5.18xlarge'], type=str.strip, help='The ML compute instance type.', default='ml.m4.xlarge') diff --git a/components/aws/sagemaker/workteam/component.yaml b/components/aws/sagemaker/workteam/component.yaml index 33352ae8803..af37ae61483 100644 --- a/components/aws/sagemaker/workteam/component.yaml +++ b/components/aws/sagemaker/workteam/component.yaml @@ -27,7 +27,7 @@ outputs: - {name: workteam_arn, description: 'The ARN of the workteam.'} implementation: container: - image: amazon/aws-sagemaker-kfp-components:0.2 + image: amazon/aws-sagemaker-kfp-components:0.3.0 command: ['python'] args: [ workteam.py, diff --git a/samples/contrib/aws-samples/ground_truth_pipeline_demo/mini-image-classification-pipeline.py b/samples/contrib/aws-samples/ground_truth_pipeline_demo/mini-image-classification-pipeline.py index 988a125ed3c..efabfd7e637 100644 --- a/samples/contrib/aws-samples/ground_truth_pipeline_demo/mini-image-classification-pipeline.py +++ b/samples/contrib/aws-samples/ground_truth_pipeline_demo/mini-image-classification-pipeline.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 import kfp +import json +import copy from kfp import components from kfp import dsl from kfp.aws import use_aws_secret @@ -9,6 +11,24 @@ sagemaker_gt_op = components.load_component_from_file('../../../../components/aws/sagemaker/ground_truth/component.yaml') sagemaker_train_op = components.load_component_from_file('../../../../components/aws/sagemaker/train/component.yaml') +channelObjList = [] + +channelObj = { + 'ChannelName': '', + 'DataSource': { + 'S3DataSource': { + 'S3Uri': '', + 'S3DataType': 'AugmentedManifestFile', + 'S3DataDistributionType': 'FullyReplicated', + 'AttributeNames': ['source-ref', 'category'] + } + }, + 'ContentType': 'application/x-recordio', + 'CompressionType': 'None', + 'RecordWrapperType': 'RecordIO' +} + + @dsl.pipeline( name='Ground Truth image classification test pipeline', description='SageMaker Ground Truth job test' @@ -38,30 +58,6 @@ def ground_truth_test(region='us-west-2', training_algorithm_name='image classification', training_input_mode='Pipe', training_hyperparameters='{"num_classes": "2", "num_training_samples": "14", "mini_batch_size": "2"}', - training_channels='[{"ChannelName": "train", \ - "DataSource": { \ - "S3DataSource": { \ - "S3Uri": "", \ - "S3DataType": "AugmentedManifestFile", \ - "S3DataDistributionType": "FullyReplicated", \ - "AttributeNames": ["source-ref", "category"] \ - } \ - }, \ - "ContentType": "application/x-recordio", \ - "CompressionType": "None", \ - "RecordWrapperType": "RecordIO"}, \ - {"ChannelName": "validation", \ - "DataSource": { \ - "S3DataSource": { \ - "S3Uri": "", \ - "S3DataType": "AugmentedManifestFile", \ - "S3DataDistributionType": "FullyReplicated", \ - "AttributeNames": ["source-ref", "category"] \ - } \ - }, \ - "ContentType": "application/x-recordio", \ - "CompressionType": "None", \ - "RecordWrapperType": "RecordIO"}]', training_output_location='s3://your-bucket-name/mini-image-classification/training-output', training_instance_type='ml.p2.xlarge', training_instance_count='1', @@ -119,14 +115,19 @@ def ground_truth_test(region='us-west-2', max_concurrent_tasks=ground_truth_max_concurrent_tasks ).apply(use_aws_secret('aws-secret', 'AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY')) + channelObj['ChannelName'] = 'train' + channelObj['DataSource']['S3DataSource']['S3Uri'] = str(ground_truth_train.outputs['output_manifest_location']) + channelObjList.append(copy.deepcopy(channelObj)) + channelObj['ChannelName'] = 'validation' + channelObj['DataSource']['S3DataSource']['S3Uri'] = str(ground_truth_validation.outputs['output_manifest_location']) + channelObjList.append(copy.deepcopy(channelObj)) + training = sagemaker_train_op( region=region, algorithm_name=training_algorithm_name, training_input_mode=training_input_mode, hyperparameters=training_hyperparameters, - channels=training_channels, - data_location_1=ground_truth_train.outputs['output_manifest_location'], - data_location_2=ground_truth_validation.outputs['output_manifest_location'], + channels=json.dumps(channelObjList), instance_type=training_instance_type, instance_count=training_instance_count, volume_size=training_volume_size, @@ -136,4 +137,4 @@ def ground_truth_test(region='us-west-2', ).apply(use_aws_secret('aws-secret', 'AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY')) if __name__ == '__main__': - kfp.compiler.Compiler().compile(hpo_test, __file__ + '.zip') # noqa: F821 TODO + kfp.compiler.Compiler().compile(ground_truth_test, __file__ + '.zip') diff --git a/samples/contrib/aws-samples/mnist-kmeans-sagemaker/kmeans-hpo-pipeline.py b/samples/contrib/aws-samples/mnist-kmeans-sagemaker/kmeans-hpo-pipeline.py index d6ebf5f2cb6..c3cf49f14d9 100644 --- a/samples/contrib/aws-samples/mnist-kmeans-sagemaker/kmeans-hpo-pipeline.py +++ b/samples/contrib/aws-samples/mnist-kmeans-sagemaker/kmeans-hpo-pipeline.py @@ -1,12 +1,40 @@ #!/usr/bin/env python3 import kfp +import json +import copy from kfp import components from kfp import dsl from kfp.aws import use_aws_secret sagemaker_hpo_op = components.load_component_from_file('../../../../components/aws/sagemaker/hyperparameter_tuning/component.yaml') + +channelObjList = [] + +channelObj = { + 'ChannelName': '', + 'DataSource': { + 'S3DataSource': { + 'S3Uri': '', + 'S3DataType': 'S3Prefix', + 'S3DataDistributionType': 'FullyReplicated' + } + }, + 'ContentType': '', + 'CompressionType': 'None', + 'RecordWrapperType': 'None', + 'InputMode': 'File' +} + +channelObj['ChannelName'] = 'train' +channelObj['DataSource']['S3DataSource']['S3Uri'] = 's3://kubeflow-pipeline-data/mnist_kmeans_example/train_data' +channelObjList.append(copy.deepcopy(channelObj)) +channelObj['ChannelName'] = 'test' +channelObj['DataSource']['S3DataSource']['S3Uri'] = 's3://kubeflow-pipeline-data/mnist_kmeans_example/test_data' +channelObjList.append(copy.deepcopy(channelObj)) + + @dsl.pipeline( name='MNIST HPO test pipeline', description='SageMaker hyperparameter tuning job test' @@ -26,30 +54,7 @@ def hpo_test(region='us-west-2', {"Name": "extra_center_factor", "MinValue": "10", "MaxValue": "20"}]', continuous_parameters='[]', categorical_parameters='[{"Name": "init_method", "Values": ["random", "kmeans++"]}]', - channels='[{"ChannelName": "train", \ - "DataSource": { \ - "S3DataSource": { \ - "S3Uri": "s3://kubeflow-pipeline-data/mnist_kmeans_example/data", \ - "S3DataType": "S3Prefix", \ - "S3DataDistributionType": "FullyReplicated" \ - } \ - }, \ - "ContentType": "", \ - "CompressionType": "None", \ - "RecordWrapperType": "None", \ - "InputMode": "File"}, \ - {"ChannelName": "test", \ - "DataSource": { \ - "S3DataSource": { \ - "S3Uri": "s3://kubeflow-pipeline-data/mnist_kmeans_example/data", \ - "S3DataType": "S3Prefix", \ - "S3DataDistributionType": "FullyReplicated" \ - } \ - }, \ - "ContentType": "", \ - "CompressionType": "None", \ - "RecordWrapperType": "None", \ - "InputMode": "File"}]', + channels=json.dumps(channelObjList), output_location='s3://kubeflow-pipeline-data/mnist_kmeans_example/output', output_encryption_key='', instance_type='ml.p2.16xlarge', diff --git a/samples/contrib/aws-samples/mnist-kmeans-sagemaker/mnist-classification-pipeline.py b/samples/contrib/aws-samples/mnist-kmeans-sagemaker/mnist-classification-pipeline.py index fa8471d934f..f02d47b817d 100644 --- a/samples/contrib/aws-samples/mnist-kmeans-sagemaker/mnist-classification-pipeline.py +++ b/samples/contrib/aws-samples/mnist-kmeans-sagemaker/mnist-classification-pipeline.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 import kfp +import json +import copy from kfp import components from kfp import dsl from kfp.aws import use_aws_secret @@ -11,6 +13,34 @@ sagemaker_deploy_op = components.load_component_from_file('../../../../components/aws/sagemaker/deploy/component.yaml') sagemaker_batch_transform_op = components.load_component_from_file('../../../../components/aws/sagemaker/batch_transform/component.yaml') + +hpoChannels = [] +trainChannels = [] + +channelObj = { + 'ChannelName': '', + 'DataSource': { + 'S3DataSource': { + 'S3Uri': '', + 'S3DataType': 'S3Prefix', + 'S3DataDistributionType': 'FullyReplicated' + } + }, + 'ContentType': '', + 'CompressionType': 'None', + 'RecordWrapperType': 'None', + 'InputMode': 'File' +} + +channelObj['ChannelName'] = 'train' +channelObj['DataSource']['S3DataSource']['S3Uri'] = 's3://kubeflow-pipeline-data/mnist_kmeans_example/train_data' +hpoChannels.append(copy.deepcopy(channelObj)) +trainChannels.append(copy.deepcopy(channelObj)) +channelObj['ChannelName'] = 'test' +channelObj['DataSource']['S3DataSource']['S3Uri'] = 's3://kubeflow-pipeline-data/mnist_kmeans_example/test_data' +hpoChannels.append(copy.deepcopy(channelObj)) + + @dsl.pipeline( name='MNIST Classification pipeline', description='MNIST Classification using KMEANS in SageMaker' @@ -26,30 +56,7 @@ def mnist_classification(region='us-west-2', hpo_integer_parameters='[{"Name": "mini_batch_size", "MinValue": "500", "MaxValue": "600"}, {"Name": "extra_center_factor", "MinValue": "10", "MaxValue": "20"}]', hpo_continuous_parameters='[]', hpo_categorical_parameters='[{"Name": "init_method", "Values": ["random", "kmeans++"]}]', - hpo_channels='[{"ChannelName": "train", \ - "DataSource": { \ - "S3DataSource": { \ - "S3Uri": "s3://kubeflow-pipeline-data/mnist_kmeans_example/train_data", \ - "S3DataType": "S3Prefix", \ - "S3DataDistributionType": "FullyReplicated" \ - } \ - }, \ - "ContentType": "", \ - "CompressionType": "None", \ - "RecordWrapperType": "None", \ - "InputMode": "File"}, \ - {"ChannelName": "test", \ - "DataSource": { \ - "S3DataSource": { \ - "S3Uri": "s3://kubeflow-pipeline-data/mnist_kmeans_example/test_data", \ - "S3DataType": "S3Prefix", \ - "S3DataDistributionType": "FullyReplicated" \ - } \ - }, \ - "ContentType": "", \ - "CompressionType": "None", \ - "RecordWrapperType": "None", \ - "InputMode": "File"}]', + hpo_channels=json.dumps(hpoChannels), hpo_spot_instance='False', hpo_max_wait_time='3600', hpo_checkpoint_config='{}', @@ -64,18 +71,7 @@ def mnist_classification(region='us-west-2', endpoint_url='', network_isolation='True', traffic_encryption='False', - train_channels='[{"ChannelName": "train", \ - "DataSource": { \ - "S3DataSource": { \ - "S3Uri": "s3://kubeflow-pipeline-data/mnist_kmeans_example/train_data", \ - "S3DataType": "S3Prefix", \ - "S3DataDistributionType": "FullyReplicated" \ - } \ - }, \ - "ContentType": "", \ - "CompressionType": "None", \ - "RecordWrapperType": "None", \ - "InputMode": "File"}]', + train_channels=json.dumps(trainChannels), train_spot_instance='False', train_max_wait_time='3600', train_checkpoint_config='{}', diff --git a/samples/contrib/aws-samples/simple_train_pipeline/README.md b/samples/contrib/aws-samples/simple_train_pipeline/README.md index 8f5c42d503f..812bfbbb483 100644 --- a/samples/contrib/aws-samples/simple_train_pipeline/README.md +++ b/samples/contrib/aws-samples/simple_train_pipeline/README.md @@ -7,20 +7,13 @@ An example pipeline with only [train component](https://github.com/kubeflow/pipe 2. Get and store data in S3 buckets. You can get sample data using this code. Create a new file `s3_sample_data_creator.py` with following content : ```buildoutcfg + import io + import boto3 import pickle, gzip, numpy, urllib.request, json from urllib.parse import urlparse - - # Load the dataset - urllib.request.urlretrieve("http://deeplearning.net/data/mnist/mnist.pkl.gz", "mnist.pkl.gz") - with gzip.open('mnist.pkl.gz', 'rb') as f: - train_set, valid_set, test_set = pickle.load(f, encoding='latin1') - - - # Upload dataset to S3 from sagemaker.amazon.common import write_numpy_to_dense_tensor - import io - import boto3 + ########################################################################################### # This is the only thing that you need to change in this code # Give the name of your S3 bucket @@ -28,7 +21,14 @@ An example pipeline with only [train component](https://github.com/kubeflow/pipe bucket = '' ########################################################################################### - + + # Load the dataset + urllib.request.urlretrieve("http://deeplearning.net/data/mnist/mnist.pkl.gz", "mnist.pkl.gz") + with gzip.open('mnist.pkl.gz', 'rb') as f: + train_set, valid_set, test_set = pickle.load(f, encoding='latin1') + + + # Upload dataset to S3 data_key = 'mnist_kmeans_example/data' data_location = 's3://{}/{}'.format(bucket, data_key) print('Data will be uploaded to: {}'.format(data_location)) @@ -41,8 +41,34 @@ An example pipeline with only [train component](https://github.com/kubeflow/pipe boto3.resource('s3').Bucket(bucket).Object(data_key).upload_fileobj(buf) ``` Run this file `python s3_sample_data_creator.py` -3. Prepare an IAM role with permissions to run SageMaker jobs and access to S3 buckets. https://docs.aws.amazon.com/sagemaker/latest/dg/sagemaker-roles.html - Go to you AWS account -> IAM -> Roles -> Create Role -> Select SageMaker Service -> Next -> Next -> Next -> Give "SageMakerExecutorKFP" as Role Name -> Create Role -> Click on the role that you created -> Attach policy -> AmazonS3FullAccess -> Attach Policy -> Note down the role ARN +3. Prepare an IAM role with permissions to run SageMaker jobs and access to S3 buckets. + + create a new file "trust.json" with following content + ```buildoutcfg + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "", + "Effect": "Allow", + "Principal": { + "Service": "sagemaker.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] + } + ``` + ```buildoutcfg + + # run these commands to create a role named "SageMakerExecutorKFP" with SageMaker and S3 access + aws iam create-role --role-name SageMakerExecutorKFP --assume-role-policy-document file://trust.json + aws iam attach-role-policy --policy-arn arn:aws:iam::aws:policy/AmazonSageMakerFullAccess --role-name SageMakerExecutorKFP + aws iam attach-role-policy --policy-arn arn:aws:iam::aws:policy/AmazonS3FullAccess --role-name SageMakerExecutorKFP + + # Note down the role ARN + aws iam get-role --role-name SageMakerExecutorKFP # | jq .Role.Arn + ``` 4. Add 'aws-secret' to your Kubeflow namespace. ``` # 1. get aws key and secret in base64 format: diff --git a/samples/contrib/aws-samples/simple_train_pipeline/training-pipeline.py b/samples/contrib/aws-samples/simple_train_pipeline/training-pipeline.py index 90de0eb861e..d21320fdf4a 100644 --- a/samples/contrib/aws-samples/simple_train_pipeline/training-pipeline.py +++ b/samples/contrib/aws-samples/simple_train_pipeline/training-pipeline.py @@ -1,12 +1,36 @@ #!/usr/bin/env python3 import kfp +import json +import copy from kfp import components from kfp import dsl from kfp.aws import use_aws_secret sagemaker_train_op = components.load_component_from_file('../../../../components/aws/sagemaker/train/component.yaml') +channelObjList = [] + +channelObj = { + 'ChannelName': '', + 'DataSource': { + 'S3DataSource': { + 'S3Uri': '', + 'S3DataType': 'S3Prefix', + 'S3DataDistributionType': 'FullyReplicated' + } + }, + 'ContentType': '', + 'CompressionType': 'None', + 'RecordWrapperType': 'None', + 'InputMode': 'File' +} + +channelObj['ChannelName'] = 'train' +channelObj['DataSource']['S3DataSource']['S3Uri'] = 's3://kubeflow-pipeline-data/mnist_kmeans_example/data' +channelObjList.append(copy.deepcopy(channelObj)) + + @dsl.pipeline( name='Training pipeline', description='SageMaker training job test' @@ -17,22 +41,7 @@ def training( image='382416733822.dkr.ecr.us-east-1.amazonaws.com/kmeans:1', training_input_mode='File', hyperparameters='{"k": "10", "feature_dim": "784"}', - channels='[ \ - { \ - "ChannelName": "train", \ - "DataSource": { \ - "S3DataSource": { \ - "S3Uri": "s3://kubeflow-pipeline-data/mnist_kmeans_example/data", \ - "S3DataType": "S3Prefix", \ - "S3DataDistributionType": "FullyReplicated" \ - } \ - }, \ - "ContentType": "", \ - "CompressionType": "None", \ - "RecordWrapperType": "None", \ - "InputMode": "File" \ - } \ - ]', + channels=json.dumps(channelObjList), instance_type='ml.p2.xlarge', instance_count='1', volume_size='50',