From 9f4ce1e6c900f28c0ee1f5dc96c6be0ac119c137 Mon Sep 17 00:00:00 2001 From: NihalHarish Date: Thu, 30 Jul 2020 00:58:04 -0700 Subject: [PATCH 1/6] load from s3 path --- tests/tensorflow/test_keras_to_estimator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/tensorflow/test_keras_to_estimator.py b/tests/tensorflow/test_keras_to_estimator.py index d171990bc..bd9d4ebf6 100644 --- a/tests/tensorflow/test_keras_to_estimator.py +++ b/tests/tensorflow/test_keras_to_estimator.py @@ -17,7 +17,9 @@ def test_keras_to_estimator(out_dir): def input_fn(): split = tfds.Split.TRAIN - dataset = tfds.load("iris", split=split, as_supervised=True) + dataset = tfds.load( + "iris", data_dir="s3://smdebug-testing/datasets/", split=split, as_supervised=True + ) dataset = dataset.map(lambda features, labels: ({"dense_input": features}, labels)) dataset = dataset.batch(32).repeat() return dataset From 80b27aa23d7f65db3256c685e60ffc7bb5aaf24e Mon Sep 17 00:00:00 2001 From: NihalHarish Date: Thu, 30 Jul 2020 01:02:24 -0700 Subject: [PATCH 2/6] load from s3 path --- tests/tensorflow2/test_keras.py | 4 +++- tests/zero_code_change/test_tensorflow_integration.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/tensorflow2/test_keras.py b/tests/tensorflow2/test_keras.py index 3359826fb..67299d937 100644 --- a/tests/tensorflow2/test_keras.py +++ b/tests/tensorflow2/test_keras.py @@ -749,7 +749,9 @@ def test_keras_to_estimator(out_dir, tf_eager_mode): def input_fn(): split = tfds.Split.TRAIN - dataset = tfds.load("iris", split=split, as_supervised=True) + dataset = tfds.load( + "iris", data_dir="s3://smdebug-testing/datasets/", split=split, as_supervised=True + ) dataset = dataset.map(lambda features, labels: ({"dense_input": features}, labels)) dataset = dataset.batch(32).repeat() return dataset diff --git a/tests/zero_code_change/test_tensorflow_integration.py b/tests/zero_code_change/test_tensorflow_integration.py index d002a94e7..f25ad3450 100644 --- a/tests/zero_code_change/test_tensorflow_integration.py +++ b/tests/zero_code_change/test_tensorflow_integration.py @@ -421,7 +421,9 @@ def test_keras_to_estimator(script_mode): def input_fn(): split = tfds.Split.TRAIN - dataset = tfds.load("iris", split=split, as_supervised=True) + dataset = tfds.load( + "iris", data_dir="s3://smdebug-testing/datasets/", split=split, as_supervised=True + ) dataset = dataset.map(lambda features, labels: ({"dense_input": features}, labels)) dataset = dataset.batch(32).repeat() return dataset From 09b914cf6121522d43d45e638f801da075d64974 Mon Sep 17 00:00:00 2001 From: NihalHarish Date: Thu, 30 Jul 2020 01:53:58 -0700 Subject: [PATCH 3/6] constants file --- tests/constants.py | 1 + tests/tensorflow/test_keras_to_estimator.py | 3 ++- tests/tensorflow2/test_keras.py | 3 ++- tests/zero_code_change/test_tensorflow_integration.py | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 tests/constants.py diff --git a/tests/constants.py b/tests/constants.py new file mode 100644 index 000000000..df660933b --- /dev/null +++ b/tests/constants.py @@ -0,0 +1 @@ +TF_IRIS_DATASET_S3_PATH = "s3://smdebug-testing/datasets/" diff --git a/tests/tensorflow/test_keras_to_estimator.py b/tests/tensorflow/test_keras_to_estimator.py index bd9d4ebf6..e5031da75 100644 --- a/tests/tensorflow/test_keras_to_estimator.py +++ b/tests/tensorflow/test_keras_to_estimator.py @@ -1,6 +1,7 @@ # Third Party import tensorflow as tf import tensorflow_datasets as tfds +from tests.constants import TF_IRIS_DATASET_S3_PATH # First Party from smdebug.tensorflow import EstimatorHook, modes @@ -18,7 +19,7 @@ def test_keras_to_estimator(out_dir): def input_fn(): split = tfds.Split.TRAIN dataset = tfds.load( - "iris", data_dir="s3://smdebug-testing/datasets/", split=split, as_supervised=True + "iris", data_dir=TF_IRIS_DATASET_S3_PATH, split=split, as_supervised=True ) dataset = dataset.map(lambda features, labels: ({"dense_input": features}, labels)) dataset = dataset.batch(32).repeat() diff --git a/tests/tensorflow2/test_keras.py b/tests/tensorflow2/test_keras.py index 67299d937..6d87639ca 100644 --- a/tests/tensorflow2/test_keras.py +++ b/tests/tensorflow2/test_keras.py @@ -13,6 +13,7 @@ import pytest import tensorflow.compat.v2 as tf import tensorflow_datasets as tfds +from tests.constants import TF_IRIS_DATASET_S3_PATH from tests.tensorflow2.utils import is_tf_2_2 from tests.tensorflow.utils import create_trial_fast_refresh @@ -750,7 +751,7 @@ def test_keras_to_estimator(out_dir, tf_eager_mode): def input_fn(): split = tfds.Split.TRAIN dataset = tfds.load( - "iris", data_dir="s3://smdebug-testing/datasets/", split=split, as_supervised=True + "iris", data_dir=TF_IRIS_DATASET_S3_PATH, split=split, as_supervised=True ) dataset = dataset.map(lambda features, labels: ({"dense_input": features}, labels)) dataset = dataset.batch(32).repeat() diff --git a/tests/zero_code_change/test_tensorflow_integration.py b/tests/zero_code_change/test_tensorflow_integration.py index f25ad3450..4688db676 100644 --- a/tests/zero_code_change/test_tensorflow_integration.py +++ b/tests/zero_code_change/test_tensorflow_integration.py @@ -422,7 +422,7 @@ def test_keras_to_estimator(script_mode): def input_fn(): split = tfds.Split.TRAIN dataset = tfds.load( - "iris", data_dir="s3://smdebug-testing/datasets/", split=split, as_supervised=True + "iris", data_dir=TF_IRIS_DATASET_S3_PATH, split=split, as_supervised=True ) dataset = dataset.map(lambda features, labels: ({"dense_input": features}, labels)) dataset = dataset.batch(32).repeat() From acf4990be42b73cf9ac7ce68cac03e42e6c14fa5 Mon Sep 17 00:00:00 2001 From: NihalHarish Date: Thu, 30 Jul 2020 02:52:35 -0700 Subject: [PATCH 4/6] import --- tests/zero_code_change/test_tensorflow_integration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/zero_code_change/test_tensorflow_integration.py b/tests/zero_code_change/test_tensorflow_integration.py index 4688db676..7c9c25c9b 100644 --- a/tests/zero_code_change/test_tensorflow_integration.py +++ b/tests/zero_code_change/test_tensorflow_integration.py @@ -21,6 +21,7 @@ import pytest import tensorflow.compat.v1 as tf import tensorflow_datasets as tfds +from tests.constants import TF_IRIS_DATASET_S3_PATH from tests.tensorflow.hooks.test_mirrored_strategy import test_basic from tests.tensorflow.keras.test_keras_mirrored import test_tf_keras from tests.zero_code_change.tf_utils import ( From 4049e8635c117a30f9b28448daafbefc50a2df79 Mon Sep 17 00:00:00 2001 From: NihalHarish Date: Thu, 30 Jul 2020 06:28:30 -0700 Subject: [PATCH 5/6] retrigger CI From 7d8130e2edead289c346b80d617bc36a993cb7fb Mon Sep 17 00:00:00 2001 From: NihalHarish Date: Thu, 30 Jul 2020 11:04:04 -0700 Subject: [PATCH 6/6] rename variable --- tests/constants.py | 2 +- tests/tensorflow/test_keras_to_estimator.py | 6 ++---- tests/tensorflow2/test_keras.py | 6 ++---- tests/zero_code_change/test_tensorflow_integration.py | 4 ++-- 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/tests/constants.py b/tests/constants.py index df660933b..b7b95f3bf 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -1 +1 @@ -TF_IRIS_DATASET_S3_PATH = "s3://smdebug-testing/datasets/" +TEST_DATASET_S3_PATH = "s3://smdebug-testing/datasets/" diff --git a/tests/tensorflow/test_keras_to_estimator.py b/tests/tensorflow/test_keras_to_estimator.py index e5031da75..0122b34b9 100644 --- a/tests/tensorflow/test_keras_to_estimator.py +++ b/tests/tensorflow/test_keras_to_estimator.py @@ -1,7 +1,7 @@ # Third Party import tensorflow as tf import tensorflow_datasets as tfds -from tests.constants import TF_IRIS_DATASET_S3_PATH +from tests.constants import TEST_DATASET_S3_PATH # First Party from smdebug.tensorflow import EstimatorHook, modes @@ -18,9 +18,7 @@ def test_keras_to_estimator(out_dir): def input_fn(): split = tfds.Split.TRAIN - dataset = tfds.load( - "iris", data_dir=TF_IRIS_DATASET_S3_PATH, split=split, as_supervised=True - ) + dataset = tfds.load("iris", data_dir=TEST_DATASET_S3_PATH, split=split, as_supervised=True) dataset = dataset.map(lambda features, labels: ({"dense_input": features}, labels)) dataset = dataset.batch(32).repeat() return dataset diff --git a/tests/tensorflow2/test_keras.py b/tests/tensorflow2/test_keras.py index 6d87639ca..d7fc42759 100644 --- a/tests/tensorflow2/test_keras.py +++ b/tests/tensorflow2/test_keras.py @@ -13,7 +13,7 @@ import pytest import tensorflow.compat.v2 as tf import tensorflow_datasets as tfds -from tests.constants import TF_IRIS_DATASET_S3_PATH +from tests.constants import TEST_DATASET_S3_PATH from tests.tensorflow2.utils import is_tf_2_2 from tests.tensorflow.utils import create_trial_fast_refresh @@ -750,9 +750,7 @@ def test_keras_to_estimator(out_dir, tf_eager_mode): def input_fn(): split = tfds.Split.TRAIN - dataset = tfds.load( - "iris", data_dir=TF_IRIS_DATASET_S3_PATH, split=split, as_supervised=True - ) + dataset = tfds.load("iris", data_dir=TEST_DATASET_S3_PATH, split=split, as_supervised=True) dataset = dataset.map(lambda features, labels: ({"dense_input": features}, labels)) dataset = dataset.batch(32).repeat() return dataset diff --git a/tests/zero_code_change/test_tensorflow_integration.py b/tests/zero_code_change/test_tensorflow_integration.py index 7c9c25c9b..95a1df5f9 100644 --- a/tests/zero_code_change/test_tensorflow_integration.py +++ b/tests/zero_code_change/test_tensorflow_integration.py @@ -21,7 +21,7 @@ import pytest import tensorflow.compat.v1 as tf import tensorflow_datasets as tfds -from tests.constants import TF_IRIS_DATASET_S3_PATH +from tests.constants import TEST_DATASET_S3_PATH from tests.tensorflow.hooks.test_mirrored_strategy import test_basic from tests.tensorflow.keras.test_keras_mirrored import test_tf_keras from tests.zero_code_change.tf_utils import ( @@ -423,7 +423,7 @@ def test_keras_to_estimator(script_mode): def input_fn(): split = tfds.Split.TRAIN dataset = tfds.load( - "iris", data_dir=TF_IRIS_DATASET_S3_PATH, split=split, as_supervised=True + "iris", data_dir=TEST_DATASET_S3_PATH, split=split, as_supervised=True ) dataset = dataset.map(lambda features, labels: ({"dense_input": features}, labels)) dataset = dataset.batch(32).repeat()