re:Invent 2020 features

.dictionary

@@ -0,0 +1,23 @@
+str
+Args
+datetime
+smexperiments
+json
+ARN
+boolean
+boto
+DSL
+enum
+enums
+config
+Jupyter
+hyperparameter
+hyperparameters
+subexpression
+kwargs
+API
+UI
+runtime
+AutoML
+subfolders
+sagemaker

.header_template.txt

@@ -0,0 +1,12 @@
+Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"). You
+may not use this file except in compliance with the License. A copy of
+the License is located at
+
+http://aws.amazon.com/apache2.0/
+
+or in the "license" file accompanying this file. This file is
+distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+ANY KIND, either express or implied. See the License for the specific
+language governing permissions and limitations under the License.

.pylintrc

@@ -3,7 +3,7 @@
 # * https://github.com/google/seq2seq/blob/master/pylintrc
 # * https://github.com/google/pygtrie/blob/master/.pylintrc
 
-[MASTER]
+[MAIN]
 
 # Specify a configuration file.
 #rcfile=
@@ -289,7 +289,7 @@ spelling-dict=
 spelling-ignore-words=
 
 # A path to a file that contains private dictionary; one word per line.
-spelling-private-dict-file=
+spelling-private-dict-file=.dictionary
 
 # Tells whether to store unknown words to indicated private dictionary in
 # --spelling-private-dict-file option instead of raising a message.

pytest.ini

@@ -0,0 +1,5 @@
+[pytest]
+addopts = --strict
+markers =
+    docker: tests that require docker
+    slow: marks tests as slow

setup.py

@@ -1,10 +1,10 @@
-# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"). You
 # may not use this file except in compliance with the License. A copy of
 # the License is located at
 #
-#     http://aws.amazon.com/apache2.0/
+#   http://aws.amazon.com/apache2.0/
 #
 # or in the "license" file accompanying this file. This file is
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
@@ -24,7 +24,7 @@ def read(fname):
 
 
 # Declare minimal set for installation
-required_packages = ["boto3>=1.12.8"]
+required_packages = ["boto3>=1.16.27"]
 
 # Open readme with original (i.e. LF) newlines
 # to prevent the all too common "`long_description_content_type` missing"
@@ -68,6 +68,8 @@ def read(fname):
             "pytest-rerunfailures",
             "pytest-xdist",
             "docker",
+            "pandas",
+            "sklearn",
         ]
     },
 )

src/__init__.py

@@ -0,0 +1,13 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""Placeholder docstring."""

src/smexperiments/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"). You
 # may not use this file except in compliance with the License. A copy of

src/smexperiments/_base_types.py

@@ -1,4 +1,4 @@
-# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"). You
 # may not use this file except in compliance with the License. A copy of

src/smexperiments/_boto_functions.py

@@ -1,4 +1,4 @@
-# Copyright 019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"). You
 # may not use this file except in compliance with the License. A copy of

src/smexperiments/_environment.py

@@ -1,3 +1,15 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
 import enum
 import json
 import os
@@ -21,7 +33,7 @@ class TrialComponentEnvironment(object):
 
     Attributes:
         environment_type (EnvironmentType): The environment type.
-        source_arn (str): The arn of the current job.
+        source_arn (str): The ARN of the current job.
     """
 
     environment_type = None
@@ -36,7 +48,7 @@ def load(cls, training_job_arn_env=TRAINING_JOB_ARN_ENV, processing_job_config_p
         """Loads source arn of current job from environment.
 
         Args:
-            training_job_arn_env (str): The environment key for training job arn.
+            training_job_arn_env (str): The environment key for training job ARN.
             processing_job_config_path (str): The processing job config path.
 
         Returns:

src/smexperiments/_utils.py

@@ -1,4 +1,4 @@
-# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"). You
 # may not use this file except in compliance with the License. A copy of
@@ -17,6 +17,7 @@
 import boto3
 import botocore
 import logging
+from importlib import import_module
 
 
 def sagemaker_client():
@@ -90,3 +91,15 @@ def get_or_create_default_bucket(boto_session, default_bucket_prefix="sagemaker"
         else:
             raise
     return default_bucket
+
+
+def get_module(module_name):
+    """Imports an module.
+
+    Args:
+        module_name (str): Name of the module to importt.
+
+    Returns:
+        [obj]: The imported module
+    """
+    return import_module(module_name)

src/smexperiments/api_types.py

@@ -1,4 +1,4 @@
-# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"). You
 # may not use this file except in compliance with the License. A copy of
@@ -10,7 +10,7 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
-"""Contains api objects for SageMaker experiments."""
+"""Contains API objects for SageMaker experiments."""
 import numbers
 
 from smexperiments import _base_types
@@ -20,7 +20,7 @@ class ExperimentSummary(_base_types.ApiObject):
     """Summary model of an experiment.
 
     Attributes:
-        experiment_arn (str): Arn of the experiment.
+        experiment_arn (str): ARN of the experiment.
         experiment_name (str): Name of the experiment.
         creation_time (datetime): Date experiment was created.
         last_modified_time (datetime): Date experiment was last modified.
@@ -72,7 +72,7 @@ class TrialSummary(_base_types.ApiObject):
     """Summary model of a trial.
 
     Attributes:
-        trial_arn (str): The arn of the trial.
+        trial_arn (str): The ARN of the trial.
         trial_name (str): The name of the trial.
         creation_time (datetime):  When the trial was created.
         last_modified_time (datetime): When the trial was last modified.
@@ -185,9 +185,9 @@ class TrialComponentSummary(_base_types.ApiObject):
 
     Attributes:
         trial_component_name (str): Name of trial component.
-        trial_component_arn (str): Arn of the trial component.
+        trial_component_arn (str): ARN of the trial component.
         display_name (str): Friendly display name.
-        source_arn (str): Arn of the trial component source.
+        source_arn (str): ARN of the trial component source.
         status (str): Status.
         start_time (datetime): Start time.
         end_time (datetime): End time.
@@ -220,7 +220,7 @@ class ExperimentSource(_base_types.ApiObject):
     """ExperimentSource
 
     Attributes:
-        source_arn (str): The arn of the source.
+        source_arn (str): The ARN of the source.
     """
 
     source_arn = None
@@ -233,7 +233,7 @@ class TrialSource(_base_types.ApiObject):
     """TrialSource
 
     Attributes:
-        source_arn (str): The arn of the source.
+        source_arn (str): The ARN of the source.
     """
 
     source_arn = None
@@ -246,7 +246,7 @@ class TrialComponentSource(_base_types.ApiObject):
     """TrialComponentsource
 
     Attributes:
-        source_arn (str): The arn of the source.
+        source_arn (str): The ARN of the source.
     """
 
     source_arn = None
@@ -356,7 +356,7 @@ class ExperimentSearchResult(_base_types.ApiObject):
     """Summary model of an Experiment search result.
 
     Attributes:
-        experiment_arn (str): Arn of the experiment.
+        experiment_arn (str): ARN of the experiment.
         experiment_name (str): Name of the experiment.
         display_name (str): Display name of the experiment.
         source (dict): The source of the experiment
@@ -384,7 +384,7 @@ class TrialSearchResult(_base_types.ApiObject):
     """Summary model of an Trial search result.
 
     Attributes:
-        trial_arn (str): Arn of the trial.
+        trial_arn (str): ARN of the trial.
         trial_name (str): Name of the trial.
         display_name (str): Display name of the trial.
         source (dict): The source of the trial.
@@ -424,7 +424,7 @@ class TrialComponentSearchResult(_base_types.ApiObject):
     """Summary model of an Trial Component search result.
 
     Attributes:
-        trial_component_arn (str): Arn of the trial component.
+        trial_component_arn (str): ARN of the trial component.
         trial_component_name (str): Name of the trial component.
         display_name (str): Display name of the trial component.
         source (dict): The source of the trial component.

src/smexperiments/experiment.py

@@ -1,4 +1,4 @@
-# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"). You
 # may not use this file except in compliance with the License. A copy of
@@ -10,7 +10,6 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
-
 """Contains the SageMaker Experiment class."""
 from smexperiments import _base_types, api_types, trial, _utils, trial_component
 import time
@@ -77,7 +76,7 @@ def delete(self):
         Returns:
             dict: Delete experiment API response.
         """
-        self._invoke_api(self._boto_delete_method, self._boto_delete_members)
+        return self._invoke_api(self._boto_delete_method, self._boto_delete_members)
 
     @classmethod
     def load(cls, experiment_name, sagemaker_boto_client=None):

src/smexperiments/metrics.py

@@ -1,4 +1,4 @@
-# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"). You
 # may not use this file except in compliance with the License. A copy of
@@ -45,6 +45,7 @@ def log_metric(self, metric_name, value, timestamp=None, iteration_number=None):
 
         Raises:
             SageMakerMetricsWriterException: If the metrics file is closed.
+            AttributeError: If file has been initialized and the writer hasn't been closed.
         """
         raw_metric_data = _RawMetricData(
             metric_name=metric_name, value=value, timestamp=timestamp, iteration_number=iteration_number

src/smexperiments/search_expression.py

@@ -1,4 +1,4 @@
-# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"). You
 # may not use this file except in compliance with the License. A copy of
@@ -18,7 +18,7 @@
 @unique
 class Operator(Enum):
     """Search operators"""
-    
+
     EQUALS = "Equals"
     NOT_EQUALS = "NotEquals"
     GREATER_THAN = "GreaterThan"
@@ -33,14 +33,14 @@ class Operator(Enum):
 @unique
 class BooleanOperator(Enum):
     """Boolean search operation enum"""
-    
+
     AND = "And"
     OR = "Or"
 
 
 class SearchObject(ApiObject):
     """SearchObject"""
-    
+
     def to_boto(self):
         """Convert a search object to boto"""
         return ApiObject.to_boto(self)
@@ -101,7 +101,7 @@ def __init__(self, filters=None, nested_filters=None, sub_expressions=None, bool
             filters (list): list of Filter objects
             nested_filters (list): list of Nested Filters objects
             sub_expressions (list): list of Search Expresssion objects
-            operator (dict): one of the boolean operator enums
+            boolean_operator (dict): one of the boolean operator enums
         """
         if filters is None and nested_filters is None and sub_expressions is None:
             raise ValueError("You must specify at least one subexpression, filter, or nested filter")