From bc97548577e77a4d7cd3a941807d36b89baa819e Mon Sep 17 00:00:00 2001 From: sampathweb <1437573+sampathweb@users.noreply.github.com> Date: Thu, 22 Feb 2024 11:57:18 -0600 Subject: [PATCH 01/11] CI - Add kaggle creds to pull model --- .kokoro/github/ubuntu/gpu/build.sh | 3 +++ .kokoro/github/ubuntu/gpu/jax/continuous.cfg | 18 ++++++++++++++++++ .kokoro/github/ubuntu/gpu/jax/presubmit.cfg | 18 ++++++++++++++++++ .../github/ubuntu/gpu/keras2/continuous.cfg | 18 ++++++++++++++++++ .kokoro/github/ubuntu/gpu/keras2/presubmit.cfg | 18 ++++++++++++++++++ .../ubuntu/gpu/tensorflow/continuous.cfg | 18 ++++++++++++++++++ .../github/ubuntu/gpu/tensorflow/presubmit.cfg | 18 ++++++++++++++++++ .kokoro/github/ubuntu/gpu/torch/continuous.cfg | 18 ++++++++++++++++++ .kokoro/github/ubuntu/gpu/torch/presubmit.cfg | 18 ++++++++++++++++++ 9 files changed, 147 insertions(+) diff --git a/.kokoro/github/ubuntu/gpu/build.sh b/.kokoro/github/ubuntu/gpu/build.sh index 2017b77c8..d3ed48cb6 100644 --- a/.kokoro/github/ubuntu/gpu/build.sh +++ b/.kokoro/github/ubuntu/gpu/build.sh @@ -21,6 +21,9 @@ nvcc --version cd "src/github/keras-nlp" pip install -U pip setuptools psutil +export KAGGLE_USERNAME="${KOKORO_KEYSTORE_DIR}/73361_keras_kaggle_username" +export KAGGLE_KEY="${KOKORO_KEYSTORE_DIR}/73361_keras_kaggle_secret_key" + if [ "${KERAS2:-0}" == "1" ] then echo "Keras2 detected." diff --git a/.kokoro/github/ubuntu/gpu/jax/continuous.cfg b/.kokoro/github/ubuntu/gpu/jax/continuous.cfg index 1b9ffb605..63351a9f4 100644 --- a/.kokoro/github/ubuntu/gpu/jax/continuous.cfg +++ b/.kokoro/github/ubuntu/gpu/jax/continuous.cfg @@ -12,5 +12,23 @@ env_vars: { value: "jax" } +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73361 + keyname: "keras_kaggle_username" + } + } +} + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73361 + keyname: "keras_kaggle_secret_key" + } + } +} + # Set timeout to 60 mins from default 180 mins timeout_mins: 60 \ No newline at end of file diff --git a/.kokoro/github/ubuntu/gpu/jax/presubmit.cfg b/.kokoro/github/ubuntu/gpu/jax/presubmit.cfg index 1b9ffb605..63351a9f4 100644 --- a/.kokoro/github/ubuntu/gpu/jax/presubmit.cfg +++ b/.kokoro/github/ubuntu/gpu/jax/presubmit.cfg @@ -12,5 +12,23 @@ env_vars: { value: "jax" } +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73361 + keyname: "keras_kaggle_username" + } + } +} + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73361 + keyname: "keras_kaggle_secret_key" + } + } +} + # Set timeout to 60 mins from default 180 mins timeout_mins: 60 \ No newline at end of file diff --git a/.kokoro/github/ubuntu/gpu/keras2/continuous.cfg b/.kokoro/github/ubuntu/gpu/keras2/continuous.cfg index 7e971ac96..03fa92222 100644 --- a/.kokoro/github/ubuntu/gpu/keras2/continuous.cfg +++ b/.kokoro/github/ubuntu/gpu/keras2/continuous.cfg @@ -12,5 +12,23 @@ env_vars: { value: "1" } +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73361 + keyname: "keras_kaggle_username" + } + } +} + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73361 + keyname: "keras_kaggle_secret_key" + } + } +} + # Set timeout to 60 mins from default 180 mins timeout_mins: 60 \ No newline at end of file diff --git a/.kokoro/github/ubuntu/gpu/keras2/presubmit.cfg b/.kokoro/github/ubuntu/gpu/keras2/presubmit.cfg index 7e971ac96..03fa92222 100644 --- a/.kokoro/github/ubuntu/gpu/keras2/presubmit.cfg +++ b/.kokoro/github/ubuntu/gpu/keras2/presubmit.cfg @@ -12,5 +12,23 @@ env_vars: { value: "1" } +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73361 + keyname: "keras_kaggle_username" + } + } +} + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73361 + keyname: "keras_kaggle_secret_key" + } + } +} + # Set timeout to 60 mins from default 180 mins timeout_mins: 60 \ No newline at end of file diff --git a/.kokoro/github/ubuntu/gpu/tensorflow/continuous.cfg b/.kokoro/github/ubuntu/gpu/tensorflow/continuous.cfg index b85ee6f4e..c707593fc 100644 --- a/.kokoro/github/ubuntu/gpu/tensorflow/continuous.cfg +++ b/.kokoro/github/ubuntu/gpu/tensorflow/continuous.cfg @@ -12,5 +12,23 @@ env_vars: { value: "tensorflow" } +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73361 + keyname: "keras_kaggle_username" + } + } +} + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73361 + keyname: "keras_kaggle_secret_key" + } + } +} + # Set timeout to 60 mins from default 180 mins timeout_mins: 60 \ No newline at end of file diff --git a/.kokoro/github/ubuntu/gpu/tensorflow/presubmit.cfg b/.kokoro/github/ubuntu/gpu/tensorflow/presubmit.cfg index b85ee6f4e..c707593fc 100644 --- a/.kokoro/github/ubuntu/gpu/tensorflow/presubmit.cfg +++ b/.kokoro/github/ubuntu/gpu/tensorflow/presubmit.cfg @@ -12,5 +12,23 @@ env_vars: { value: "tensorflow" } +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73361 + keyname: "keras_kaggle_username" + } + } +} + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73361 + keyname: "keras_kaggle_secret_key" + } + } +} + # Set timeout to 60 mins from default 180 mins timeout_mins: 60 \ No newline at end of file diff --git a/.kokoro/github/ubuntu/gpu/torch/continuous.cfg b/.kokoro/github/ubuntu/gpu/torch/continuous.cfg index 5d25106b3..345159e80 100644 --- a/.kokoro/github/ubuntu/gpu/torch/continuous.cfg +++ b/.kokoro/github/ubuntu/gpu/torch/continuous.cfg @@ -12,5 +12,23 @@ env_vars: { value: "torch" } +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73361 + keyname: "keras_kaggle_username" + } + } +} + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73361 + keyname: "keras_kaggle_secret_key" + } + } +} + # Set timeout to 60 mins from default 180 mins timeout_mins: 60 \ No newline at end of file diff --git a/.kokoro/github/ubuntu/gpu/torch/presubmit.cfg b/.kokoro/github/ubuntu/gpu/torch/presubmit.cfg index 5d25106b3..345159e80 100644 --- a/.kokoro/github/ubuntu/gpu/torch/presubmit.cfg +++ b/.kokoro/github/ubuntu/gpu/torch/presubmit.cfg @@ -12,5 +12,23 @@ env_vars: { value: "torch" } +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73361 + keyname: "keras_kaggle_username" + } + } +} + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73361 + keyname: "keras_kaggle_secret_key" + } + } +} + # Set timeout to 60 mins from default 180 mins timeout_mins: 60 \ No newline at end of file From 452502f211205b63858a3fdac1181720fc6c7d38 Mon Sep 17 00:00:00 2001 From: sampathweb <1437573+sampathweb@users.noreply.github.com> Date: Fri, 23 Feb 2024 14:28:40 -0800 Subject: [PATCH 02/11] add kaggle env variables --- .kokoro/github/ubuntu/gpu/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.kokoro/github/ubuntu/gpu/build.sh b/.kokoro/github/ubuntu/gpu/build.sh index d3ed48cb6..3e3f92d46 100644 --- a/.kokoro/github/ubuntu/gpu/build.sh +++ b/.kokoro/github/ubuntu/gpu/build.sh @@ -21,8 +21,8 @@ nvcc --version cd "src/github/keras-nlp" pip install -U pip setuptools psutil -export KAGGLE_USERNAME="${KOKORO_KEYSTORE_DIR}/73361_keras_kaggle_username" -export KAGGLE_KEY="${KOKORO_KEYSTORE_DIR}/73361_keras_kaggle_secret_key" +export KAGGLE_USERNAME="$(cat ${KOKORO_KEYSTORE_DIR}/73361_keras_kaggle_username)" +export KAGGLE_KEY="$(cat ${KOKORO_KEYSTORE_DIR}/73361_keras_kaggle_secret_key)" if [ "${KERAS2:-0}" == "1" ] then From 252ed7fe7d21897522fe5e55f88aa17a2794b946 Mon Sep 17 00:00:00 2001 From: sampathweb <1437573+sampathweb@users.noreply.github.com> Date: Fri, 23 Feb 2024 14:33:29 -0800 Subject: [PATCH 03/11] Kaggle env: --- .kokoro/github/ubuntu/gpu/build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.kokoro/github/ubuntu/gpu/build.sh b/.kokoro/github/ubuntu/gpu/build.sh index 3e3f92d46..733f97203 100644 --- a/.kokoro/github/ubuntu/gpu/build.sh +++ b/.kokoro/github/ubuntu/gpu/build.sh @@ -1,3 +1,6 @@ +export KAGGLE_USERNAME="$(cat ${KOKORO_KEYSTORE_DIR}/73361_keras_kaggle_username)" +export KAGGLE_KEY="$(cat ${KOKORO_KEYSTORE_DIR}/73361_keras_kaggle_secret_key)" + set -e set -x @@ -21,9 +24,6 @@ nvcc --version cd "src/github/keras-nlp" pip install -U pip setuptools psutil -export KAGGLE_USERNAME="$(cat ${KOKORO_KEYSTORE_DIR}/73361_keras_kaggle_username)" -export KAGGLE_KEY="$(cat ${KOKORO_KEYSTORE_DIR}/73361_keras_kaggle_secret_key)" - if [ "${KERAS2:-0}" == "1" ] then echo "Keras2 detected." From 8c4e5a55e13c0436bec7732c58769dfc980c4eb6 Mon Sep 17 00:00:00 2001 From: sampathweb <1437573+sampathweb@users.noreply.github.com> Date: Fri, 23 Feb 2024 14:35:32 -0800 Subject: [PATCH 04/11] Kaggle env: --- .kokoro/github/ubuntu/gpu/build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.kokoro/github/ubuntu/gpu/build.sh b/.kokoro/github/ubuntu/gpu/build.sh index 733f97203..29b4a88db 100644 --- a/.kokoro/github/ubuntu/gpu/build.sh +++ b/.kokoro/github/ubuntu/gpu/build.sh @@ -1,7 +1,8 @@ +set -e + export KAGGLE_USERNAME="$(cat ${KOKORO_KEYSTORE_DIR}/73361_keras_kaggle_username)" export KAGGLE_KEY="$(cat ${KOKORO_KEYSTORE_DIR}/73361_keras_kaggle_secret_key)" -set -e set -x cd "${KOKORO_ROOT}/" From 507574578214e733ae5aa70fc70a8af6999208ad Mon Sep 17 00:00:00 2001 From: sampathweb <1437573+sampathweb@users.noreply.github.com> Date: Fri, 23 Feb 2024 15:07:19 -0800 Subject: [PATCH 05/11] Kaggle env: --- .kokoro/github/ubuntu/gpu/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kokoro/github/ubuntu/gpu/build.sh b/.kokoro/github/ubuntu/gpu/build.sh index 29b4a88db..3d3c55e46 100644 --- a/.kokoro/github/ubuntu/gpu/build.sh +++ b/.kokoro/github/ubuntu/gpu/build.sh @@ -1,9 +1,9 @@ set -e -export KAGGLE_USERNAME="$(cat ${KOKORO_KEYSTORE_DIR}/73361_keras_kaggle_username)" export KAGGLE_KEY="$(cat ${KOKORO_KEYSTORE_DIR}/73361_keras_kaggle_secret_key)" set -x +export KAGGLE_USERNAME="$(cat ${KOKORO_KEYSTORE_DIR}/73361_keras_kaggle_username)" cd "${KOKORO_ROOT}/" From 3b734e8f00fec0a7a7f0fdb8afe408ac1402ba44 Mon Sep 17 00:00:00 2001 From: sampathweb <1437573+sampathweb@users.noreply.github.com> Date: Fri, 23 Feb 2024 15:14:13 -0800 Subject: [PATCH 06/11] Kaggle env: --- .kokoro/github/ubuntu/gpu/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kokoro/github/ubuntu/gpu/build.sh b/.kokoro/github/ubuntu/gpu/build.sh index 3d3c55e46..ca1249402 100644 --- a/.kokoro/github/ubuntu/gpu/build.sh +++ b/.kokoro/github/ubuntu/gpu/build.sh @@ -1,9 +1,9 @@ set -e export KAGGLE_KEY="$(cat ${KOKORO_KEYSTORE_DIR}/73361_keras_kaggle_secret_key)" +export KAGGLE_USERNAME="$(cat ${KOKORO_KEYSTORE_DIR}/73361_keras_kaggle_username)" set -x -export KAGGLE_USERNAME="$(cat ${KOKORO_KEYSTORE_DIR}/73361_keras_kaggle_username)" cd "${KOKORO_ROOT}/" From 1e773802e3f135419b438dc1abaab9fb54ac1fea Mon Sep 17 00:00:00 2001 From: sampathweb <1437573+sampathweb@users.noreply.github.com> Date: Mon, 26 Feb 2024 11:18:41 -0800 Subject: [PATCH 07/11] Update Build script for Kokoro --- .kokoro/github/ubuntu/gpu/build.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.kokoro/github/ubuntu/gpu/build.sh b/.kokoro/github/ubuntu/gpu/build.sh index ca1249402..8837a87c3 100644 --- a/.kokoro/github/ubuntu/gpu/build.sh +++ b/.kokoro/github/ubuntu/gpu/build.sh @@ -3,8 +3,16 @@ set -e export KAGGLE_KEY="$(cat ${KOKORO_KEYSTORE_DIR}/73361_keras_kaggle_secret_key)" export KAGGLE_USERNAME="$(cat ${KOKORO_KEYSTORE_DIR}/73361_keras_kaggle_username)" +if [[ -z "${KAGGLE_KEY}" ]]; then + echo "KAGGLE_KEY is NOT set" +fi + set -x +if [[ -z "${KAGGLE_USERNAME}" ]]; then + echo "KAGGLE_USERNAME is NOT set" +fi + cd "${KOKORO_ROOT}/" sudo update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 1 From d49329a22d16967c0c005368d75ee174711f90c8 Mon Sep 17 00:00:00 2001 From: sampathweb <1437573+sampathweb@users.noreply.github.com> Date: Mon, 26 Feb 2024 13:54:40 -0800 Subject: [PATCH 08/11] Add Kaggle env var --- .kokoro/github/ubuntu/gpu/build.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.kokoro/github/ubuntu/gpu/build.sh b/.kokoro/github/ubuntu/gpu/build.sh index 8837a87c3..87cd20649 100644 --- a/.kokoro/github/ubuntu/gpu/build.sh +++ b/.kokoro/github/ubuntu/gpu/build.sh @@ -5,14 +5,16 @@ export KAGGLE_USERNAME="$(cat ${KOKORO_KEYSTORE_DIR}/73361_keras_kaggle_username if [[ -z "${KAGGLE_KEY}" ]]; then echo "KAGGLE_KEY is NOT set" + exit 1 fi -set -x - if [[ -z "${KAGGLE_USERNAME}" ]]; then echo "KAGGLE_USERNAME is NOT set" + exit 1 fi +set -x + cd "${KOKORO_ROOT}/" sudo update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 1 From 71a2e7545d4a544f01b68efecc53812d6c002610 Mon Sep 17 00:00:00 2001 From: sampathweb <1437573+sampathweb@users.noreply.github.com> Date: Fri, 1 Mar 2024 11:22:21 -0800 Subject: [PATCH 09/11] set gemma preset to extra_large --- keras_nlp/models/gemma/gemma_backbone_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/keras_nlp/models/gemma/gemma_backbone_test.py b/keras_nlp/models/gemma/gemma_backbone_test.py index add2ac199..855d49658 100644 --- a/keras_nlp/models/gemma/gemma_backbone_test.py +++ b/keras_nlp/models/gemma/gemma_backbone_test.py @@ -54,8 +54,9 @@ def test_saved_model(self): ) @pytest.mark.kaggle_key_required - @pytest.mark.large + @pytest.mark.extra_large def test_smallest_preset(self): + # TODO: Fails with OOM on current GPU CI self.run_preset_test( cls=GemmaBackbone, preset="gemma_2b_en", From 2f426ed97d3e74679ea38ab13123a32a450e7763 Mon Sep 17 00:00:00 2001 From: sampathweb <1437573+sampathweb@users.noreply.github.com> Date: Mon, 4 Mar 2024 13:36:32 -0600 Subject: [PATCH 10/11] Change Gemma small preset to bfloat16 --- keras_nlp/models/gemma/gemma_backbone_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/keras_nlp/models/gemma/gemma_backbone_test.py b/keras_nlp/models/gemma/gemma_backbone_test.py index 855d49658..78f04a25e 100644 --- a/keras_nlp/models/gemma/gemma_backbone_test.py +++ b/keras_nlp/models/gemma/gemma_backbone_test.py @@ -54,9 +54,9 @@ def test_saved_model(self): ) @pytest.mark.kaggle_key_required - @pytest.mark.extra_large + @pytest.mark.large def test_smallest_preset(self): - # TODO: Fails with OOM on current GPU CI + keras.config.set_floatx("float16") self.run_preset_test( cls=GemmaBackbone, preset="gemma_2b_en", From 0842f7014d7de070d75295e1b46abcba8ce429cd Mon Sep 17 00:00:00 2001 From: sampathweb <1437573+sampathweb@users.noreply.github.com> Date: Mon, 4 Mar 2024 13:59:06 -0600 Subject: [PATCH 11/11] Change Gemma small preset to xlarge --- keras_nlp/models/gemma/gemma_backbone_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/keras_nlp/models/gemma/gemma_backbone_test.py b/keras_nlp/models/gemma/gemma_backbone_test.py index 78f04a25e..855d49658 100644 --- a/keras_nlp/models/gemma/gemma_backbone_test.py +++ b/keras_nlp/models/gemma/gemma_backbone_test.py @@ -54,9 +54,9 @@ def test_saved_model(self): ) @pytest.mark.kaggle_key_required - @pytest.mark.large + @pytest.mark.extra_large def test_smallest_preset(self): - keras.config.set_floatx("float16") + # TODO: Fails with OOM on current GPU CI self.run_preset_test( cls=GemmaBackbone, preset="gemma_2b_en",