From 0d62f307bbed4df9c05403b46923337af9098890 Mon Sep 17 00:00:00 2001 From: Rahul Huilgol Date: Thu, 5 Dec 2019 14:17:16 -0800 Subject: [PATCH 1/2] Remove unused classes --- smdebug/core/actions/__init__.py | 3 -- smdebug/core/actions/action_base.py | 10 ------ smdebug/core/actions/terminate_smjob.py | 24 ------------- smdebug/core/sagemaker_utils.py | 42 ---------------------- smdebug/trials/trial_catalog.py | 46 ------------------------- tests/analysis/trials/test_local.py | 6 ---- 6 files changed, 131 deletions(-) delete mode 100644 smdebug/core/actions/__init__.py delete mode 100644 smdebug/core/actions/action_base.py delete mode 100644 smdebug/core/actions/terminate_smjob.py delete mode 100644 smdebug/trials/trial_catalog.py diff --git a/smdebug/core/actions/__init__.py b/smdebug/core/actions/__init__.py deleted file mode 100644 index c604dd987..000000000 --- a/smdebug/core/actions/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# Local -from .action_base import Action -from .terminate_smjob import TerminateSagemakerJob diff --git a/smdebug/core/actions/action_base.py b/smdebug/core/actions/action_base.py deleted file mode 100644 index 0c7d14ce0..000000000 --- a/smdebug/core/actions/action_base.py +++ /dev/null @@ -1,10 +0,0 @@ -# First Party -from smdebug.core.logger import get_logger - - -class Action: - def __init__(self): - self.logger = get_logger() - - def run(self, rule_name, **kwargs): - pass diff --git a/smdebug/core/actions/terminate_smjob.py b/smdebug/core/actions/terminate_smjob.py deleted file mode 100644 index 3d8600a5f..000000000 --- a/smdebug/core/actions/terminate_smjob.py +++ /dev/null @@ -1,24 +0,0 @@ -# First Party -from smdebug.core.sagemaker_utils import SageMakerUtils - -# Local -from .action_base import Action - - -class TerminateSagemakerJob(Action): - def __init__(self, sm_job_name): - super().__init__() - self.job_name = sm_job_name - - def run(self, rule_name, **kwargs): - try: - # todo fix hardcoding of arn inside this function - SageMakerUtils.terminate_sagemaker_job(self.job_name) - # tags = [{'Key':"TerminatedBy", 'Value': rule_name} , - # {'Key':'TerminationTime', 'Value': str(time.time())} ] - # SageMakerUtils.add_tags(self.job_name, tags) - except Exception as e: - self.logger.warning( - "Caught exception when running TerminateSagemakerJob " - "action for smjob:{} Exception:{}".format(self.sm_job_name, e) - ) diff --git a/smdebug/core/sagemaker_utils.py b/smdebug/core/sagemaker_utils.py index 9c6987e6a..786f2fa9e 100644 --- a/smdebug/core/sagemaker_utils.py +++ b/smdebug/core/sagemaker_utils.py @@ -1,12 +1,6 @@ # Standard Library import os -# Third Party -import boto3 - -# First Party -from smdebug.core.config_constants import DEFAULT_SAGEMAKER_OUTDIR - def is_sagemaker_job(): """ @@ -17,39 +11,3 @@ def is_sagemaker_job(): :return: True or False """ return "TRAINING_JOB_NAME" in os.environ - - -def get_sagemaker_out_dir(): - return DEFAULT_SAGEMAKER_OUTDIR - - -class SageMakerUtils: - @staticmethod - def is_sagemaker_job_finished(jobname, returnMock=None): - if returnMock is not None: - return returnMock - client = boto3.client("sagemaker") - response = client.describe_training_job(TrainingJobName=jobname) - status = response["TrainingJobStatus"] - if status in ["InProgress", "Stopping"]: - return False - elif status in ["Completed", "Failed", "Stopped"]: - return True # return 1 if the job is finished - - @staticmethod - def terminate_sagemaker_job(jobname): - client = boto3.client("sagemaker") - try: - client.stop_training_job(TrainingJobName=jobname) - except Exception as e: - print(e) - - @staticmethod - def add_tags(sm_job_name, tags): - client = boto3.client("sagemaker") - # TODO create resource arn here - resource_arn = "arn:aws:sagemaker:us-east-1:072677473360:training-job/" + sm_job_name - try: - client.add_tags(ResourceArn=resource_arn, Tags=tags) - except Exception as e: - print(e) diff --git a/smdebug/trials/trial_catalog.py b/smdebug/trials/trial_catalog.py deleted file mode 100644 index 032c6c907..000000000 --- a/smdebug/trials/trial_catalog.py +++ /dev/null @@ -1,46 +0,0 @@ -# Standard Library -import os - - -class TrialCatalog(object): - def __init__(self): - self.trials = {} - - def add_trial(self, trial_name, trial_object): - self.trials[trial_name] = trial_object - - def get_trial(self, trial_name): - return self.trials[trial_name] - - def get_trials(self): - return self.trials.keys() - - -class LocalTrialCatalog(TrialCatalog): - def __init__(self, localdir): - super().__init__() - self.localdir = localdir - - def list_candidates(self): - files_and_folders = os.listdir(self.localdir) - folders = [x for x in files_and_folders if os.path.isdir(os.path.join(self.localdir, x))] - return folders - - -""" -class SageMakerTrialCatalog(TrialCatalog): - def __init__(self,endpoint,port): - super().__init__() - self.endpoint = endpoint - self.port = port - self.client = InfluxDBClient(host=self.endpoint, port=self.port) - self.client.switch_database('deb') - - - def list_candidates(self): - points = self.client.query(f"select distinct(expid) from execdata") - res = [] - for p in poinsmd.get_points(): - res.append(p['distinct']) - return res -""" diff --git a/tests/analysis/trials/test_local.py b/tests/analysis/trials/test_local.py index 5f343dfd5..da7ccbb0b 100644 --- a/tests/analysis/trials/test_local.py +++ b/tests/analysis/trials/test_local.py @@ -7,17 +7,11 @@ # First Party from smdebug.trials import LocalTrial -from smdebug.trials.trial_catalog import LocalTrialCatalog def check_local(localdir, trial_name, num_steps, num_tensors): - tc = LocalTrialCatalog(localdir=localdir) - assert trial_name in tc.list_candidates() path = os.path.join(localdir, trial_name) trial_obj = LocalTrial(name=trial_name, dirname=path) - tc.add_trial(trial_name, trial_obj) - trial_obj2 = tc.get_trial(trial_name) - assert trial_obj == trial_obj2 check_trial(trial_obj, num_tensors=num_tensors, num_steps=num_steps) From e997f56e48c1c5370502847779ca5f0f3002c104 Mon Sep 17 00:00:00 2001 From: Rahul Huilgol Date: Thu, 5 Dec 2019 14:20:51 -0800 Subject: [PATCH 2/2] Remove actions from rule --- smdebug/rules/rule.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/smdebug/rules/rule.py b/smdebug/rules/rule.py index 8ed2b582c..062fb492d 100644 --- a/smdebug/rules/rule.py +++ b/smdebug/rules/rule.py @@ -22,7 +22,6 @@ def __init__(self, base_trial, other_trials=None): self.req_tensors = RequiredTensors(self.base_trial, self.other_trials) - self.actions = None self.logger = get_logger() self.rule_name = self.__class__.__name__ @@ -56,13 +55,4 @@ def invoke(self, step): val = self.invoke_at_step(step) if val: - self.run_actions() raise RuleEvaluationConditionMet(self.rule_name, step) - - def register_action(self, actions): - self.actions = actions - - def run_actions(self): - if self.actions is not None: - for action in self.actions: - action.run(rule_name=self.__class__.__name__)