From 65086bf0730e68e17b2ee053bf4c775a3890cc93 Mon Sep 17 00:00:00 2001 From: NihalHarish Date: Wed, 16 Sep 2020 23:40:12 -0700 Subject: [PATCH 1/6] init --- smdebug/tensorflow/keras.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/smdebug/tensorflow/keras.py b/smdebug/tensorflow/keras.py index 67d325d87..0239f7dd3 100644 --- a/smdebug/tensorflow/keras.py +++ b/smdebug/tensorflow/keras.py @@ -73,6 +73,7 @@ def __init__( ) # stores tensors custom tensors saved by users every step self.saved_layers = dict() self.has_registered_model = False + # supports_tf_logs property was introduced in TF 2.3.0 # it indicates to the framework that the callback is not # limited to reading only numpy logs @@ -757,6 +758,13 @@ def _save_layer_values(self, logs): for layer_name, layer_input, layer_output in logs: # Cast layer_name to str since it can also be of type bytes # when run with mirrored strategy + if isinstance(layer_name, tf.Tensor): + # Tensor.name is meaningless with eager execution + layer_name = str(layer_name.numpy(), "utf-8") + elif isinstance(layer_name, tf.Variable): + layer_name = layer_name.name + elif isinstance(layer_name, bytes): + layer_name = str(layer_name, "utf-8") if len(layer_input) == 1: # Layer Inputs are flattened and passed as a list into # the next layer. Unpacking it speeds up the _make_numpy fn. From f6634d867589deb1bd731182f0edbdcf158e5c53 Mon Sep 17 00:00:00 2001 From: NihalHarish Date: Sat, 19 Sep 2020 22:41:59 -0700 Subject: [PATCH 2/6] rebase --- smdebug/tensorflow/keras.py | 1 - 1 file changed, 1 deletion(-) diff --git a/smdebug/tensorflow/keras.py b/smdebug/tensorflow/keras.py index 0239f7dd3..0ce302512 100644 --- a/smdebug/tensorflow/keras.py +++ b/smdebug/tensorflow/keras.py @@ -73,7 +73,6 @@ def __init__( ) # stores tensors custom tensors saved by users every step self.saved_layers = dict() self.has_registered_model = False - # supports_tf_logs property was introduced in TF 2.3.0 # it indicates to the framework that the callback is not # limited to reading only numpy logs From e88e54fb4b926715c1224cc934fac58e9201a4e0 Mon Sep 17 00:00:00 2001 From: NihalHarish Date: Sun, 20 Sep 2020 01:20:13 -0700 Subject: [PATCH 3/6] test --- tests/tensorflow2/test_keras.py | 58 +++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/tests/tensorflow2/test_keras.py b/tests/tensorflow2/test_keras.py index 0c9da2d89..9e75fb1ed 100644 --- a/tests/tensorflow2/test_keras.py +++ b/tests/tensorflow2/test_keras.py @@ -7,6 +7,7 @@ `python tests/tensorflow2/test_keras.py` from the main directory. """ # Standard Library +import re import time # Third Party @@ -29,6 +30,19 @@ from smdebug.tensorflow import ReductionConfig, SaveConfig +def get_model(): + model = tf.keras.models.Sequential( + [ + # WA for TF issue https://github.com/tensorflow/tensorflow/issues/36279 + tf.keras.layers.Flatten(input_shape=(28, 28, 1)), + tf.keras.layers.Dense(128, activation="relu"), + tf.keras.layers.Dropout(0.2), + tf.keras.layers.Dense(10, activation="softmax"), + ] + ) + return model + + def helper_keras_fit( trial_dir, save_all=False, @@ -48,15 +62,7 @@ def helper_keras_fit( (x_train, y_train), (x_test, y_test) = mnist.load_data() x_train, x_test = x_train / 255, x_test / 255 - model = tf.keras.models.Sequential( - [ - tf.keras.layers.Flatten(input_shape=(28, 28)), - tf.keras.layers.Dense(128, activation="relu"), - tf.keras.layers.Dropout(0.2), - tf.keras.layers.Dense(10, activation="softmax"), - ] - ) - + model = get_model() if hook is None: if save_config is None: save_config = SaveConfig(save_interval=3) @@ -124,17 +130,7 @@ def helper_keras_gradtape( (tf.cast(x_train[..., tf.newaxis] / 255, tf.float32), tf.cast(y_train, tf.int64)) ) dataset = dataset.shuffle(1000).batch(batch_size) - - model = tf.keras.models.Sequential( - [ - # WA for TF issue https://github.com/tensorflow/tensorflow/issues/36279 - tf.keras.layers.Flatten(input_shape=(28, 28, 1)), - tf.keras.layers.Dense(128, activation="relu"), - tf.keras.layers.Dropout(0.2), - tf.keras.layers.Dense(10, activation="softmax"), - ] - ) - + model = get_model() if hook is None: if save_config is None: save_config = SaveConfig(save_interval=3) @@ -549,6 +545,28 @@ def test_include_regex(out_dir, tf_eager_mode): assert tr.tensor(tname).value(0) is not None +@pytest.mark.slow +def test_layer_names(out_dir, tf_eager_mode): + hook = smd.KerasHook( + out_dir, + save_config=SaveConfig(save_interval=9), + include_collections=[CollectionKeys.LAYERS], + ) + helper_keras_fit( + out_dir, + hook=hook, + save_config=SaveConfig(save_interval=9), + steps=["train"], + run_eagerly=tf_eager_mode, + ) + + tr = create_trial_fast_refresh(out_dir) + tnames = tr.tensor_names(collection=CollectionKeys.LAYERS) + pattern = r"^(flatten|dense|dropout)(_\d+)?\/(inputs|outputs)" + for tname in tnames: + assert re.match(pattern=pattern, string=tname) is not None + + @pytest.mark.skip_if_non_eager @pytest.mark.slow def test_clash_with_tb_callback(out_dir): From 71676846dd3b15b4b60875dce309144b40ae3d84 Mon Sep 17 00:00:00 2001 From: NihalHarish Date: Sun, 20 Sep 2020 15:35:17 -0700 Subject: [PATCH 4/6] retrigger CI From afe2f957efa8019bfaa0b19d5d85cd94a6ad43ce Mon Sep 17 00:00:00 2001 From: NihalHarish Date: Fri, 25 Sep 2020 02:12:29 -0700 Subject: [PATCH 5/6] gradtape test --- tests/tensorflow2/test_keras.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/tensorflow2/test_keras.py b/tests/tensorflow2/test_keras.py index 9e75fb1ed..df3acc474 100644 --- a/tests/tensorflow2/test_keras.py +++ b/tests/tensorflow2/test_keras.py @@ -567,6 +567,22 @@ def test_layer_names(out_dir, tf_eager_mode): assert re.match(pattern=pattern, string=tname) is not None +@pytest.mark.slow +def test_layer_names_gradient_tape(out_dir): + hook = smd.KerasHook( + out_dir, + save_config=SaveConfig(save_interval=9), + include_collections=[CollectionKeys.LAYERS], + ) + helper_keras_gradtape(out_dir, hook=hook, save_config=SaveConfig(save_interval=9)) + + tr = create_trial_fast_refresh(out_dir) + tnames = tr.tensor_names(collection=CollectionKeys.LAYERS) + pattern = r"^(flatten|dense|dropout)(_\d+)?\/(inputs|outputs)" + for tname in tnames: + assert re.match(pattern=pattern, string=tname) is not None + + @pytest.mark.skip_if_non_eager @pytest.mark.slow def test_clash_with_tb_callback(out_dir): From 591c0373d609f2057111e5dd2d02fce7856d7bad Mon Sep 17 00:00:00 2001 From: NihalHarish Date: Fri, 25 Sep 2020 15:50:18 -0700 Subject: [PATCH 6/6] refactor --- tests/tensorflow2/test_keras.py | 34 +++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/tests/tensorflow2/test_keras.py b/tests/tensorflow2/test_keras.py index df3acc474..7228b7ec9 100644 --- a/tests/tensorflow2/test_keras.py +++ b/tests/tensorflow2/test_keras.py @@ -182,6 +182,24 @@ def helper_keras_gradtape( hook.close() +@pytest.mark.skip_if_non_eager +@pytest.mark.slow +def test_layer_names_gradient_tape(out_dir): + hook = smd.KerasHook( + out_dir, + save_config=SaveConfig(save_interval=9), + include_collections=[CollectionKeys.LAYERS], + ) + helper_keras_gradtape(out_dir, hook=hook, save_config=SaveConfig(save_interval=9)) + + tr = create_trial_fast_refresh(out_dir) + tnames = tr.tensor_names(collection=CollectionKeys.LAYERS) + pattern = r"^(flatten|dense|dropout)(_\d+)?\/(inputs|outputs)" + for tname in tnames: + assert re.match(pattern=pattern, string=tname) is not None + + +@pytest.mark.skip_if_non_eager def test_keras_gradtape_shapes(out_dir): hook = smd.KerasHook( out_dir=out_dir, @@ -567,22 +585,6 @@ def test_layer_names(out_dir, tf_eager_mode): assert re.match(pattern=pattern, string=tname) is not None -@pytest.mark.slow -def test_layer_names_gradient_tape(out_dir): - hook = smd.KerasHook( - out_dir, - save_config=SaveConfig(save_interval=9), - include_collections=[CollectionKeys.LAYERS], - ) - helper_keras_gradtape(out_dir, hook=hook, save_config=SaveConfig(save_interval=9)) - - tr = create_trial_fast_refresh(out_dir) - tnames = tr.tensor_names(collection=CollectionKeys.LAYERS) - pattern = r"^(flatten|dense|dropout)(_\d+)?\/(inputs|outputs)" - for tname in tnames: - assert re.match(pattern=pattern, string=tname) is not None - - @pytest.mark.skip_if_non_eager @pytest.mark.slow def test_clash_with_tb_callback(out_dir):