PR 49 CI tests

config/tests.sh

@@ -23,23 +23,23 @@ export SMDEBUG_LOG_LEVEL=info
 
 export OUT_DIR=upload/$CURRENT_COMMIT_PATH
 export REPORT_DIR=$OUT_DIR/pytest_reports
-python -m pytest -W=ignore --html=$REPORT_DIR/report_analysis.html --self-contained-html tests/analysis
-python -m pytest -W=ignore --html=$REPORT_DIR/report_core.html --self-contained-html tests/core
-
-if [ "$run_pytest_xgboost" = "enable" ] ; then
-    run_for_framework xgboost
-fi
-
-if [ "$run_pytest_tensorflow" = "enable" ] ; then
-    run_for_framework tensorflow
-fi
-
-if [ "$run_pytest_mxnet" = "enable" ] ; then
-    run_for_framework mxnet
-fi
-
-if [ "$run_pytest_pytorch" = "enable" ] ; then
-    run_for_framework pytorch
-fi
+#python -m pytest -W=ignore --html=$REPORT_DIR/report_analysis.html --self-contained-html tests/analysis
+#python -m pytest -W=ignore --html=$REPORT_DIR/report_core.html --self-contained-html tests/core
+#
+#if [ "$run_pytest_xgboost" = "enable" ] ; then
+#    run_for_framework xgboost
+#fi
+#
+#if [ "$run_pytest_tensorflow" = "enable" ] ; then
+#    run_for_framework tensorflow
+#fi
+#
+#if [ "$run_pytest_mxnet" = "enable" ] ; then
+#    run_for_framework mxnet
+#fi
+#
+#if [ "$run_pytest_pytorch" = "enable" ] ; then
+#    run_for_framework pytorch
+#fi
 
 check_logs $REPORT_DIR/*

examples/mxnet/notebooks/MNISTSimpleInteractiveAnalysis.ipynb

@@ -324,7 +324,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "good_trial.tensors()"
+    "good_trial.tensor_names()"
    ]
   },
   {
@@ -419,7 +419,7 @@
    "outputs": [],
    "source": [
     "def plot_gradients( lt ):\n",
-    "    for tname in lt.tensors():\n",
+    "    for tname in lt.tensor_names():\n",
     "        if not 'gradient' in tname: continue\n",
     "        steps, data = get_data(lt, tname)\n",
     "        plt.plot( steps, data, label=tname)\n",

examples/mxnet/notebooks/mxnet-realtime-analysis.ipynb

@@ -355,7 +355,7 @@
    "outputs": [],
    "source": [
     "# feel free to inspect all tensors logged by uncommenting below line\n",
-    "# trial.tensors()"
+    "# trial.tensor_names()"
    ]
   },
   {

examples/mxnet/notebooks/tensor_plot.py

@@ -63,15 +63,15 @@ def load_tensors(self):
 
             # input image into the neural network
             if self.label is not None:
-                for tname in self.trial.tensors(regex=self.label):
+                for tname in self.trial.tensor_names(regex=self.label):
                     tensor = self.trial.tensor(tname).value(step)
                     if self.color_channel == 1:
                         self.input[step] = tensor[0, 0, :, :]
                     elif self.color_channel == 3:
                         self.input[step] = tensor[0, :, :, 3]
 
             # iterate over tensors that match the regex
-            for tname in self.trial.tensors(regex=self.regex):
+            for tname in self.trial.tensor_names(regex=self.regex):
                 tensor = self.trial.tensor(tname).value(step)
                 # get max value of tensors to set axis dimension accordingly
                 for dim in tensor.shape:
@@ -119,7 +119,7 @@ def load_tensors(self):
 
             # model output
             if self.prediction is not None:
-                for tname in self.trial.tensors(regex=self.prediction):
+                for tname in self.trial.tensor_names(regex=self.prediction):
                     tensor = self.trial.tensor(tname).value(step)
                     # predicted class (batch size, propabilities per clas)
                     if len(tensor.shape) == 2:

examples/pytorch/notebooks/PyTorch-SimpleInteractiveAnalysis.ipynb

@@ -736,7 +736,7 @@
     }
    ],
    "source": [
-    "good_trial.tensors()"
+    "good_trial.tensor_names()"
    ]
   },
   {
@@ -874,7 +874,7 @@
    "outputs": [],
    "source": [
     "def plot_gradients( lt ):\n",
-    "    for tname in lt.tensors():\n",
+    "    for tname in lt.tensor_names():\n",
     "        if not 'gradient' in tname: continue\n",
     "        steps, data = get_data(lt, tname)\n",
     "        plt.plot( steps, data, label=tname)\n",

examples/rules/scripts/my_custom_rule.py

@@ -8,7 +8,7 @@ def __init__(self, base_trial, threshold=10.0):
         self.threshold = float(threshold)
 
     def set_required_tensors(self, step):
-        for tname in self.base_trial.tensors(collection="gradients"):
+        for tname in self.base_trial.tensor_names(collection="gradients"):
             self.req_tensors.add(tname, steps=[step])
 
     def invoke_at_step(self, step):

smdebug/rules/req_tensors.py

@@ -57,7 +57,7 @@ def _add_steps_for_tensor(self, name, steps):
 
     def add(self, name, steps, should_match_regex=False):
         if should_match_regex:
-            names = self.trial.tensors(regex=[name])
+            names = self.trial.tensor_names(regex=[name])
             for name in names:
                 self._add_steps_for_tensor(name, steps)
         else:

smdebug/trials/trial.py

@@ -121,7 +121,7 @@ def __repr__(self):
             f"    path={self.path},\n"
             f"    steps={self.steps()},\n"
             f"    collections={list(self.collections().keys())},\n"
-            f"    tensors={self.tensors()},\n"
+            f"    tensor_names={self.tensor_names()},\n"
             f")"
         )
 
@@ -314,21 +314,25 @@ def _tensors_matching_regex(self, regex_list) -> set:
                 matched_tensornames.add(tensorname)
         return matched_tensornames
 
-    def _tensors_in_collection(self, collection) -> set:
+    @staticmethod
+    def _parse_collection_name(collection):
         if isinstance(collection, Collection):
             coll_name = collection.name
         elif type(collection) is str:
             coll_name = collection
         else:
             raise TypeError(f"Invalid argument type for collection {collection.__class__}")
+        return coll_name
 
+    def _tensors_in_collection(self, collection) -> set:
+        coll_name = self._parse_collection_name(collection)
         rval = set(self.collection(coll_name).tensor_names)
         regex = self.collection(coll_name).include_regex
         if regex:
             rval.update(self._tensors_matching_regex(regex))
         return rval
 
-    def tensors(self, *, step=None, mode=ModeKeys.GLOBAL, regex=None, collection=None) -> list:
+    def tensor_names(self, *, step=None, mode=ModeKeys.GLOBAL, regex=None, collection=None) -> list:
         self.maybe_refresh()
         ts = set()
         if step is None and mode == ModeKeys.GLOBAL:
@@ -340,17 +344,21 @@ def tensors(self, *, step=None, mode=ModeKeys.GLOBAL, regex=None, collection=Non
 
         if regex is None and collection is None:
             return sorted(list(ts))
+        elif regex is not None and collection is not None:
+            raise ValueError("Only one of `regex` or `collection` can be passed to this method")
         else:
-            xs = set()
-            if regex is not None:
-                xs.update(self._tensors_matching_regex(regex))
             if collection is not None:
-                collection_tensors_saved = set(self._tensors.keys()).intersection(
-                    self._tensors_in_collection(collection)
-                )
-                xs.update(collection_tensors_saved)
-
-        return sorted(list(ts.intersection(xs)))
+                xs = set(self._tensors.keys()).intersection(self._tensors_in_collection(collection))
+                matching_tensors_saved = ts.intersection(xs)
+                if len(matching_tensors_saved) == 0:
+                    coll_name = self._parse_collection_name(collection)
+                    self.logger.warning(f"No tensors from the collection {coll_name} were saved")
+            else:
+                xs = self._tensors_matching_regex(regex)
+                matching_tensors_saved = ts.intersection(xs)
+                if len(matching_tensors_saved) == 0:
+                    self.logger.warning(f"No tensors matching the regex pattern given were saved")
+            return sorted(list(matching_tensors_saved))
 
     def _tensors_for_step(self, step, mode=ModeKeys.GLOBAL) -> list:
         step = self._mode_to_global[mode][step] if mode != ModeKeys.GLOBAL else step

tests/analysis/rules/test_rule_no_refresh.py

@@ -17,7 +17,7 @@ def __init__(self, base_trial):
             super().__init__(base_trial=base_trial)
 
         def set_required_tensors(self, step):
-            for t in self.base_trial.tensors():
+            for t in self.base_trial.tensor_names():
                 self.req_tensors.add(t, steps=[step])
 
         def invoke_at_step(self, step):

tests/analysis/trials/test_has_passed_step_scenarios.py

@@ -480,14 +480,14 @@ def test_override_if_too_many_steps_skipped():
         trial.last_index_token
         == "has_step_scenarios/too-many-steps-skipped/index/000000000/000000000009_worker_2.json"
     )
-    trial.tensors()
+    trial.tensor_names()
     assert trial.last_complete_step == 9
     assert (
         trial.last_index_token
         == "has_step_scenarios/too-many-steps-skipped/index/000000000/000000000009_worker_2.json"
     )
-    trial.tensors()
-    trial.tensors()
+    trial.tensor_names()
+    trial.tensor_names()
     assert trial.last_complete_step == 9
     assert (
         trial.last_index_token

tests/analysis/trials/test_modes.py

@@ -57,7 +57,7 @@ def test_mode_data():
         assert tr.has_passed_step(s + 1) == StepState.NOT_YET_AVAILABLE
         assert tr.has_passed_step(s + 1, mode=modes.TRAIN) == StepState.NOT_YET_AVAILABLE
 
-    assert len(tr.tensors()) == 1
+    assert len(tr.tensor_names()) == 1
     assert len(tr.steps()) == 10
     assert len(tr.steps(mode=modes.TRAIN)) == 5
     assert len(tr.steps(mode=modes.EVAL)) == 5

tests/analysis/trials/test_refresh.py

@@ -56,8 +56,8 @@ def help_test_refresh(path):
         )
     tr = create_trial(path + trial_name)
 
-    assert "foo_" + str(num_tensors + 1) not in tr.tensors()
-    assert "foo_1" in tr.tensors()
+    assert "foo_" + str(num_tensors + 1) not in tr.tensor_names()
+    assert "foo_1" in tr.tensor_names()
     assert len(tr.steps()) == num_steps
     assert len(tr.tensor("foo_1").steps()) == num_steps
 
@@ -117,8 +117,8 @@ def help_test_no_refresh(path):
         )
     tr = create_trial(path + trial_name)
 
-    assert "foo_" + str(num_tensors + 1) not in tr.tensors()
-    assert "foo_1" in tr.tensors()
+    assert "foo_" + str(num_tensors + 1) not in tr.tensor_names()
+    assert "foo_1" in tr.tensor_names()
     assert len(tr.steps()) == num_steps
     assert len(tr.tensor("foo_1").steps()) == num_steps
 

tests/analysis/trials/test_tensors_api.py

@@ -4,6 +4,7 @@
 
 # Third Party
 import numpy as np
+import pytest
 from tests.analysis.utils import generate_data
 
 # First Party
@@ -42,11 +43,14 @@ def test_tensors(out_dir):
     tr.collection("test").add_tensor_name("boo_5")
     tr.collection("test").add_tensor_name("boo_6")
     tr.collection("test").add_tensor_name("boo_17")  # missing tensor
-    print(tr.tensors())
-    assert len(tr.tensors()) == num_tensors * 2
-    assert len(tr.tensors(regex="foo")) == num_tensors
-    assert len(tr.tensors(collection="test")) == num_tensors + 2
-    assert len(tr.tensors(collection=tr.collection("test"))) == num_tensors + 2
+    print(tr.tensor_names())
+    assert len(tr.tensor_names()) == num_tensors * 2
+    assert len(tr.tensor_names(regex="foo")) == num_tensors
+    assert len(tr.tensor_names(collection="test")) == num_tensors + 2
+    assert len(tr.tensor_names(collection=tr.collection("test"))) == num_tensors + 2
+
+    with pytest.raises(ValueError):
+        tr.tensor_names(collection=tr.collection("test"), regex="a")
 
 
 def test_mode_data():
@@ -78,11 +82,11 @@ def test_mode_data():
             )
         fw.close()
 
-    assert trial.tensors() == ["arr_1", "arr_2"]
-    assert trial.tensors(step=0) == ["arr_1"]
-    assert trial.tensors(step=1) == ["arr_2"]
-    assert trial.tensors(step=0, mode=modes.TRAIN) == ["arr_1"]
-    assert trial.tensors(step=0, mode=modes.EVAL) == ["arr_2"]
+    assert trial.tensor_names() == ["arr_1", "arr_2"]
+    assert trial.tensor_names(step=0) == ["arr_1"]
+    assert trial.tensor_names(step=1) == ["arr_2"]
+    assert trial.tensor_names(step=0, mode=modes.TRAIN) == ["arr_1"]
+    assert trial.tensor_names(step=0, mode=modes.EVAL) == ["arr_2"]
 
-    assert trial.tensors(mode=modes.TRAIN) == ["arr_1"]
-    assert trial.tensors(mode=modes.EVAL) == ["arr_2"]
+    assert trial.tensor_names(mode=modes.TRAIN) == ["arr_1"]
+    assert trial.tensor_names(mode=modes.EVAL) == ["arr_2"]

tests/analysis/utils.py

@@ -43,8 +43,8 @@ def generate_data(
 
 
 def check_trial(trial_obj, num_steps, num_tensors):
-    assert len(trial_obj.tensors()) == num_tensors
-    for t in trial_obj.tensors():
+    assert len(trial_obj.tensor_names()) == num_tensors
+    for t in trial_obj.tensor_names():
         assert len(trial_obj.tensor(t).steps()) == num_steps
         for s in trial_obj.tensor(t).steps():
             v = trial_obj.tensor(t).value(s)

tests/core/test_handler.py

@@ -29,7 +29,7 @@ def __init__(self):
                 )
             ]
 
-    # input to get_index_for_tensors is a dict {path:{tensornames:[step_nums]}}
+    # input to get_index_for_tensors is a dict {path:{tensor_names:[step_nums]}}
     # output of that fn is dict {path:{tname:[(step_num, TensorLocation)]}}
     def get_index_for_tensors(self, t_dict):
         dict_to_return = dict()

tests/core/test_hook_save_scalar.py

@@ -179,11 +179,11 @@ def check_trials(out_dir, save_steps, coll_name, saved_scalars=None):
     """
     trial = create_trial(path=out_dir, name="test output")
     assert trial
-    tensor_list = set(trial.tensors()) & set(trial.tensors(collection=coll_name))
+    tensor_list = set(trial.tensor_names()) & set(trial.tensor_names(collection=coll_name))
     for tname in tensor_list:
         if tname not in saved_scalars:
             assert len(trial.tensor(tname).steps()) == len(save_steps)
-    scalar_list = trial.tensors(regex="^scalar")
+    scalar_list = trial.tensor_names(regex="^scalar")
     if scalar_list:
         assert len(set(saved_scalars) & set(scalar_list)) == len(saved_scalars)
 

tests/mxnet/test_hook_all_zero.py

@@ -35,9 +35,9 @@ def test_hook_all_zero(hook=None, out_dir=None):
     assert tr
     assert len(tr.steps()) == 4
 
-    tnames = tr.tensors(regex="conv._input")
+    tnames = tr.tensor_names(regex="conv._input")
     print(tnames)
-    tname = tr.tensors(regex="conv._input")[0]
+    tname = tr.tensor_names(regex="conv._input")[0]
     print(tname)
     print(tr.tensor(tname).steps())
     conv_tensor_value = tr.tensor(tname).value(step_num=0)

tests/mxnet/test_hook_loss_collection.py

@@ -27,8 +27,8 @@ def test_loss_collection_default():
     assert tr
     assert len(tr.steps()) == 4
 
-    print(tr.tensors())
-    tname = tr.tensors(regex=".*loss")[0]
+    print(tr.tensor_names())
+    tname = tr.tensor_names(regex=".*loss")[0]
     loss_tensor = tr.tensor(tname)
     loss_val = loss_tensor.value(step_num=1)
     assert len(loss_val) > 0
@@ -51,8 +51,8 @@ def test_loss_collection_with_no_other_collections():
     assert tr
     assert len(tr.steps()) == 4
 
-    print(tr.tensors())
-    tname = tr.tensors(regex=".*loss")[0]
+    print(tr.tensor_names())
+    tname = tr.tensor_names(regex=".*loss")[0]
     loss_tensor = tr.tensor(tname)
     loss_val = loss_tensor.value(step_num=1)
     assert len(loss_val) > 0