Fix exported_collections variable for TF

Author: rahul003

smdebug/tensorflow/base_hook.py

@@ -90,7 +90,7 @@ def __init__(
         # This will be None if the var wasn't set, i.e. not param server
         self.tf_config_json = load_tf_config_json(os.getenv("TF_CONFIG"))
         self._hook_supported = None
-
+        self._exported_collections = False
         self._distribution_strategy = {
             ModeKeys.TRAIN: None,
             ModeKeys.EVAL: None,

smdebug/tensorflow/keras.py

@@ -514,15 +514,15 @@ def _on_any_batch_end(self, batch, mode, logs=None):
         self._save_tensors_post_step(batch, logs)
 
         if self._prepared_tensors[mode]:
-            if self.exported_collections is False:
+            if self._exported_collections is False:
                 # in keras, these collections change when mode changes
                 # but rest of the project isn't yet capable of handling this
                 # this means that collections like outputs, or other collections with intermediate tensors
                 # will only have tensor names from first mode
 
                 # this means sometimes collections will be exported after 1 step
                 self.export_collections()
-                self.exported_collections = True
+                self._exported_collections = True
 
             if self._exported_model[self.mode] is False:
                 # confirmed that keras has same graph for all modes

smdebug/tensorflow/session.py

@@ -292,9 +292,9 @@ def begin(self):
             self._export_model()
             self._exported_model[self.mode] = True
 
-        if self.exported_collections is False:
+        if self._exported_collections is False:
             self.export_collections()
-            self.exported_collections = True
+            self._exported_collections = True
 
     def _get_tensors_to_save_this_step(self) -> set:
         tensors_to_save = set()

tests/zero_code_change/README.md

@@ -0,0 +1,9 @@
+You can run the ZCC tests on DLAMI or DLC for supported containers as follows from the main folder of the repository:
+
+```
+PYTHONPATH=. python tests/zero_code_change/tensorflow_integration_tests.py
+```
+
+```
+pytest tests/zero_code_change/horovod/
+```