From 482cf726420c7f65d75ff3fab9c56a8b13b721d4 Mon Sep 17 00:00:00 2001 From: ericharper Date: Sun, 18 Jun 2023 16:28:57 -0600 Subject: [PATCH 01/13] update container Signed-off-by: ericharper --- .github/workflows/import-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/import-test.yml b/.github/workflows/import-test.yml index 5fc34347710d..6a465de69bf8 100644 --- a/.github/workflows/import-test.yml +++ b/.github/workflows/import-test.yml @@ -12,7 +12,7 @@ jobs: # Check https://hub.docker.com/r/pytorch/pytorch/tags for latest tags container: - image: pytorch/pytorch:1.11.0-cuda11.3-cudnn8-runtime + image: pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime steps: - uses: actions/checkout@v2 From f65539aee4affdba8ba3052be2020990c2a2849f Mon Sep 17 00:00:00 2001 From: ericharper Date: Sun, 18 Jun 2023 16:43:07 -0600 Subject: [PATCH 02/13] run import tests in parallel, isntall Cython Signed-off-by: ericharper --- .github/workflows/import-test.yml | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/.github/workflows/import-test.yml b/.github/workflows/import-test.yml index 6a465de69bf8..1bf9ca8de0ff 100644 --- a/.github/workflows/import-test.yml +++ b/.github/workflows/import-test.yml @@ -7,7 +7,7 @@ on: - "**" jobs: - ci-import-check: + build-nemo: runs-on: ubuntu-latest # Check https://hub.docker.com/r/pytorch/pytorch/tags for latest tags @@ -25,6 +25,8 @@ jobs: - name: Install nemo dependencies id: nemo-wheel run: | + # install Cython + pip install Cython # install test requirements pip install -r requirements/requirements_test.txt # Build nemo as a wheel @@ -34,6 +36,17 @@ jobs: DIST_FILE=$(find ./dist -name "*.whl" | head -n 1) echo "::set-output name=DIST_FILE::${DIST_FILE}" + outputs: + dist_file: ${{ steps.nemo-wheel.outputs.DIST_FILE }} + + test-asr-imports: + needs: build-nemo + runs-on: ubuntu-latest + container: + image: pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime + + steps: + - uses: actions/checkout@v2 - name: Test ASR Domain Imports run: | # Install NeMo Domain @@ -43,6 +56,14 @@ jobs: # Uninstall NeMo pip uninstall -y nemo_toolkit + test-tts-imports: + needs: build-nemo + runs-on: ubuntu-latest + container: + image: pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime + + steps: + - uses: actions/checkout@v2 - name: Test TTS Domain Imports run: | # Install NeMo Domain @@ -52,6 +73,12 @@ jobs: # Uninstall NeMo pip uninstall -y nemo_toolkit + test-nlp-imports: + needs: build-nemo + runs-on: ubuntu-latest + container: + image: pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime + - name: Test NLP Domain Imports run: | # Install NeMo Domain From 77c9ee7f996e7c0f5ce3d81070d9a36ba5a504b3 Mon Sep 17 00:00:00 2001 From: ericharper Date: Sun, 18 Jun 2023 16:49:49 -0600 Subject: [PATCH 03/13] fix typo Signed-off-by: ericharper --- .github/workflows/import-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/import-test.yml b/.github/workflows/import-test.yml index 1bf9ca8de0ff..e4ca728aebe1 100644 --- a/.github/workflows/import-test.yml +++ b/.github/workflows/import-test.yml @@ -79,6 +79,8 @@ jobs: container: image: pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime + steps: + - uses: actions/checkout@v2 - name: Test NLP Domain Imports run: | # Install NeMo Domain From 99af3619243e6b661eb1fe6ef66842ca51a66144 Mon Sep 17 00:00:00 2001 From: ericharper Date: Sun, 18 Jun 2023 16:50:43 -0600 Subject: [PATCH 04/13] remove redundant comment Signed-off-by: ericharper --- .github/workflows/import-test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/import-test.yml b/.github/workflows/import-test.yml index e4ca728aebe1..7fa325dfce5a 100644 --- a/.github/workflows/import-test.yml +++ b/.github/workflows/import-test.yml @@ -25,7 +25,6 @@ jobs: - name: Install nemo dependencies id: nemo-wheel run: | - # install Cython pip install Cython # install test requirements pip install -r requirements/requirements_test.txt From 5cf689934ea4f49111a888c45d4d790d2dda7628 Mon Sep 17 00:00:00 2001 From: ericharper Date: Sun, 18 Jun 2023 17:03:21 -0600 Subject: [PATCH 05/13] fix more typos Signed-off-by: ericharper --- .github/workflows/import-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/import-test.yml b/.github/workflows/import-test.yml index 7fa325dfce5a..d63d12c12d75 100644 --- a/.github/workflows/import-test.yml +++ b/.github/workflows/import-test.yml @@ -49,7 +49,7 @@ jobs: - name: Test ASR Domain Imports run: | # Install NeMo Domain - pip install "${{ steps.nemo-wheel.outputs.DIST_FILE }}[asr]" + pip install "${{ needs.build-nemo.outputs.dist_file }}[asr]" # Run import checks python tests/core_ptl/check_imports.py --domain "asr" # Uninstall NeMo @@ -66,7 +66,7 @@ jobs: - name: Test TTS Domain Imports run: | # Install NeMo Domain - pip install "${{ steps.nemo-wheel.outputs.DIST_FILE }}[tts]" + pip install "${{ needs.build-nemo.outputs.dist_file }}[tts]" # Run import checks python tests/core_ptl/check_imports.py --domain "tts" # Uninstall NeMo @@ -83,7 +83,7 @@ jobs: - name: Test NLP Domain Imports run: | # Install NeMo Domain - pip install "${{ steps.nemo-wheel.outputs.DIST_FILE }}[nlp]" + pip install "${{ needs.build-nemo.outputs.dist_file }}[nlp]" # Run import checks python tests/core_ptl/check_imports.py --domain "nlp" # Uninstall NeMo From 8af4ff1d58ff6496ba901c9c2e6f08f2aaabe398 Mon Sep 17 00:00:00 2001 From: ericharper Date: Sun, 18 Jun 2023 17:22:05 -0600 Subject: [PATCH 06/13] upload and download wheel Signed-off-by: ericharper --- .github/workflows/import-test.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/.github/workflows/import-test.yml b/.github/workflows/import-test.yml index d63d12c12d75..c27a03782eb9 100644 --- a/.github/workflows/import-test.yml +++ b/.github/workflows/import-test.yml @@ -35,14 +35,27 @@ jobs: DIST_FILE=$(find ./dist -name "*.whl" | head -n 1) echo "::set-output name=DIST_FILE::${DIST_FILE}" + - name: Upload wheel as artifact + uses: actions/upload-artifact@v2 + with: + name: nemo-wheel-file + path: ${{ steps.nemo-wheel.outputs.DIST_FILE }} + outputs: dist_file: ${{ steps.nemo-wheel.outputs.DIST_FILE }} + test-asr-imports: needs: build-nemo runs-on: ubuntu-latest container: image: pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime + + - name: Download wheel file + uses: actions/download-artifact@v2 + with: + name: nemo-wheel-file + path: ./dist steps: - uses: actions/checkout@v2 @@ -61,6 +74,12 @@ jobs: container: image: pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime + - name: Download wheel file + uses: actions/download-artifact@v2 + with: + name: nemo-wheel-file + path: ./dist + steps: - uses: actions/checkout@v2 - name: Test TTS Domain Imports @@ -78,6 +97,12 @@ jobs: container: image: pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime + - name: Download wheel file + uses: actions/download-artifact@v2 + with: + name: nemo-wheel-file + path: ./dist + steps: - uses: actions/checkout@v2 - name: Test NLP Domain Imports From 441d1d0a6a8c739b091e889cd25260afd3b360e7 Mon Sep 17 00:00:00 2001 From: ericharper Date: Sun, 18 Jun 2023 17:25:11 -0600 Subject: [PATCH 07/13] fix typos Signed-off-by: ericharper --- .github/workflows/import-test.yml | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/.github/workflows/import-test.yml b/.github/workflows/import-test.yml index c27a03782eb9..f26ae11262bd 100644 --- a/.github/workflows/import-test.yml +++ b/.github/workflows/import-test.yml @@ -59,6 +59,11 @@ jobs: steps: - uses: actions/checkout@v2 + - name: Download wheel file + uses: actions/download-artifact@v2 + with: + name: nemo-wheel-file + path: ./dist - name: Test ASR Domain Imports run: | # Install NeMo Domain @@ -74,14 +79,14 @@ jobs: container: image: pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime - - name: Download wheel file - uses: actions/download-artifact@v2 - with: - name: nemo-wheel-file - path: ./dist steps: - uses: actions/checkout@v2 + - name: Download wheel file + uses: actions/download-artifact@v2 + with: + name: nemo-wheel-file + path: ./dist - name: Test TTS Domain Imports run: | # Install NeMo Domain @@ -97,14 +102,14 @@ jobs: container: image: pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime - - name: Download wheel file - uses: actions/download-artifact@v2 - with: - name: nemo-wheel-file - path: ./dist steps: - uses: actions/checkout@v2 + - name: Download wheel file + uses: actions/download-artifact@v2 + with: + name: nemo-wheel-file + path: ./dist - name: Test NLP Domain Imports run: | # Install NeMo Domain From c19d15eadcb38aae8391fbc8c01706ddda5c3f1a Mon Sep 17 00:00:00 2001 From: ericharper Date: Sun, 18 Jun 2023 17:28:25 -0600 Subject: [PATCH 08/13] fix typos Signed-off-by: ericharper --- .github/workflows/import-test.yml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/.github/workflows/import-test.yml b/.github/workflows/import-test.yml index f26ae11262bd..4cc8a2cdff11 100644 --- a/.github/workflows/import-test.yml +++ b/.github/workflows/import-test.yml @@ -16,12 +16,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Update base dependencies run: | apt-get update && apt-get install -y build-essential apt-get install -y libsndfile1 make - - name: Install nemo dependencies id: nemo-wheel run: | @@ -34,13 +32,11 @@ jobs: # Preserve wheel location DIST_FILE=$(find ./dist -name "*.whl" | head -n 1) echo "::set-output name=DIST_FILE::${DIST_FILE}" - - name: Upload wheel as artifact uses: actions/upload-artifact@v2 with: name: nemo-wheel-file path: ${{ steps.nemo-wheel.outputs.DIST_FILE }} - outputs: dist_file: ${{ steps.nemo-wheel.outputs.DIST_FILE }} @@ -50,13 +46,6 @@ jobs: runs-on: ubuntu-latest container: image: pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime - - - name: Download wheel file - uses: actions/download-artifact@v2 - with: - name: nemo-wheel-file - path: ./dist - steps: - uses: actions/checkout@v2 - name: Download wheel file @@ -78,8 +67,6 @@ jobs: runs-on: ubuntu-latest container: image: pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime - - steps: - uses: actions/checkout@v2 - name: Download wheel file @@ -101,8 +88,6 @@ jobs: runs-on: ubuntu-latest container: image: pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime - - steps: - uses: actions/checkout@v2 - name: Download wheel file From 05a6cf06107ba3d715c14efe8f36a04d15acc10e Mon Sep 17 00:00:00 2001 From: ericharper Date: Sun, 18 Jun 2023 17:31:38 -0600 Subject: [PATCH 09/13] test order Signed-off-by: ericharper --- .github/workflows/import-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/import-test.yml b/.github/workflows/import-test.yml index 4cc8a2cdff11..d10b20191e07 100644 --- a/.github/workflows/import-test.yml +++ b/.github/workflows/import-test.yml @@ -47,12 +47,12 @@ jobs: container: image: pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime steps: - - uses: actions/checkout@v2 - name: Download wheel file uses: actions/download-artifact@v2 with: name: nemo-wheel-file path: ./dist + - uses: actions/checkout@v2 - name: Test ASR Domain Imports run: | # Install NeMo Domain @@ -68,12 +68,12 @@ jobs: container: image: pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime steps: - - uses: actions/checkout@v2 - name: Download wheel file uses: actions/download-artifact@v2 with: name: nemo-wheel-file path: ./dist + - uses: actions/checkout@v2 - name: Test TTS Domain Imports run: | # Install NeMo Domain @@ -89,12 +89,12 @@ jobs: container: image: pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime steps: - - uses: actions/checkout@v2 - name: Download wheel file uses: actions/download-artifact@v2 with: name: nemo-wheel-file path: ./dist + - uses: actions/checkout@v2 - name: Test NLP Domain Imports run: | # Install NeMo Domain From ab6ada2bb5e13af1a892fb7dcddc10f09b309cf0 Mon Sep 17 00:00:00 2001 From: ericharper Date: Sun, 18 Jun 2023 17:38:33 -0600 Subject: [PATCH 10/13] remove name Signed-off-by: ericharper --- .github/workflows/import-test.yml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/import-test.yml b/.github/workflows/import-test.yml index d10b20191e07..376d7a1bec10 100644 --- a/.github/workflows/import-test.yml +++ b/.github/workflows/import-test.yml @@ -47,12 +47,11 @@ jobs: container: image: pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime steps: - - name: Download wheel file - uses: actions/download-artifact@v2 + - uses: actions/checkout@v2 + - uses: actions/download-artifact@v2 with: name: nemo-wheel-file path: ./dist - - uses: actions/checkout@v2 - name: Test ASR Domain Imports run: | # Install NeMo Domain @@ -68,12 +67,11 @@ jobs: container: image: pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime steps: - - name: Download wheel file - uses: actions/download-artifact@v2 + - uses: actions/checkout@v2 + - uses: actions/download-artifact@v2 with: name: nemo-wheel-file path: ./dist - - uses: actions/checkout@v2 - name: Test TTS Domain Imports run: | # Install NeMo Domain @@ -89,12 +87,11 @@ jobs: container: image: pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime steps: - - name: Download wheel file - uses: actions/download-artifact@v2 + - uses: actions/checkout@v2 + - uses: actions/download-artifact@v2 with: name: nemo-wheel-file path: ./dist - - uses: actions/checkout@v2 - name: Test NLP Domain Imports run: | # Install NeMo Domain From 4118b881479a5ca8b2ea89d840713c7d75b8484b Mon Sep 17 00:00:00 2001 From: ericharper Date: Sun, 18 Jun 2023 17:39:26 -0600 Subject: [PATCH 11/13] fix indent Signed-off-by: ericharper --- .github/workflows/import-test.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/import-test.yml b/.github/workflows/import-test.yml index 376d7a1bec10..b00a3af821a6 100644 --- a/.github/workflows/import-test.yml +++ b/.github/workflows/import-test.yml @@ -49,9 +49,9 @@ jobs: steps: - uses: actions/checkout@v2 - uses: actions/download-artifact@v2 - with: - name: nemo-wheel-file - path: ./dist + with: + name: nemo-wheel-file + path: ./dist - name: Test ASR Domain Imports run: | # Install NeMo Domain @@ -69,9 +69,9 @@ jobs: steps: - uses: actions/checkout@v2 - uses: actions/download-artifact@v2 - with: - name: nemo-wheel-file - path: ./dist + with: + name: nemo-wheel-file + path: ./dist - name: Test TTS Domain Imports run: | # Install NeMo Domain @@ -89,9 +89,9 @@ jobs: steps: - uses: actions/checkout@v2 - uses: actions/download-artifact@v2 - with: - name: nemo-wheel-file - path: ./dist + with: + name: nemo-wheel-file + path: ./dist - name: Test NLP Domain Imports run: | # Install NeMo Domain From 51d0810b2c43732f9b704484c003cb487eb0608d Mon Sep 17 00:00:00 2001 From: ericharper Date: Sun, 18 Jun 2023 17:42:38 -0600 Subject: [PATCH 12/13] add names back Signed-off-by: ericharper --- .github/workflows/import-test.yml | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/workflows/import-test.yml b/.github/workflows/import-test.yml index b00a3af821a6..00fd97662917 100644 --- a/.github/workflows/import-test.yml +++ b/.github/workflows/import-test.yml @@ -15,7 +15,8 @@ jobs: image: pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime steps: - - uses: actions/checkout@v2 + - name: Checkout repo + uses: actions/checkout@v2 - name: Update base dependencies run: | apt-get update && apt-get install -y build-essential @@ -47,8 +48,10 @@ jobs: container: image: pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime steps: - - uses: actions/checkout@v2 - - uses: actions/download-artifact@v2 + - name: Checkout repo + uses: actions/checkout@v2 + - name: Download wheel as artifact + uses: actions/download-artifact@v2 with: name: nemo-wheel-file path: ./dist @@ -67,8 +70,10 @@ jobs: container: image: pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime steps: - - uses: actions/checkout@v2 - - uses: actions/download-artifact@v2 + - name: Checkout repo + uses: actions/checkout@v2 + - name: Download wheel as artifact + uses: actions/download-artifact@v2 with: name: nemo-wheel-file path: ./dist @@ -87,8 +92,10 @@ jobs: container: image: pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime steps: - - uses: actions/checkout@v2 - - uses: actions/download-artifact@v2 + - name: Checkout repo + uses: actions/checkout@v2 + - name: Download wheel as artifact + uses: actions/download-artifact@v2 with: name: nemo-wheel-file path: ./dist From dffa36f746387aba1bddcbe4e186b70a982efa2f Mon Sep 17 00:00:00 2001 From: ericharper Date: Sun, 18 Jun 2023 17:54:08 -0600 Subject: [PATCH 13/13] don't upload or download just build in parallel Signed-off-by: ericharper --- .github/workflows/import-test.yml | 79 +++++++++++++++---------------- 1 file changed, 38 insertions(+), 41 deletions(-) diff --git a/.github/workflows/import-test.yml b/.github/workflows/import-test.yml index 00fd97662917..e9b10e1e34af 100644 --- a/.github/workflows/import-test.yml +++ b/.github/workflows/import-test.yml @@ -6,14 +6,13 @@ on: paths: - "**" +# Check https://hub.docker.com/r/pytorch/pytorch/tags for latest tags jobs: - build-nemo: - runs-on: ubuntu-latest - # Check https://hub.docker.com/r/pytorch/pytorch/tags for latest tags + test-asr-imports: + runs-on: ubuntu-latest container: image: pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime - steps: - name: Checkout repo uses: actions/checkout@v2 @@ -33,76 +32,74 @@ jobs: # Preserve wheel location DIST_FILE=$(find ./dist -name "*.whl" | head -n 1) echo "::set-output name=DIST_FILE::${DIST_FILE}" - - name: Upload wheel as artifact - uses: actions/upload-artifact@v2 - with: - name: nemo-wheel-file - path: ${{ steps.nemo-wheel.outputs.DIST_FILE }} - outputs: - dist_file: ${{ steps.nemo-wheel.outputs.DIST_FILE }} - - - test-asr-imports: - needs: build-nemo - runs-on: ubuntu-latest - container: - image: pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime - steps: - - name: Checkout repo - uses: actions/checkout@v2 - - name: Download wheel as artifact - uses: actions/download-artifact@v2 - with: - name: nemo-wheel-file - path: ./dist - name: Test ASR Domain Imports run: | # Install NeMo Domain - pip install "${{ needs.build-nemo.outputs.dist_file }}[asr]" + pip install "${{ steps.nemo-wheel.outputs.DIST_FILE }}[asr]" # Run import checks python tests/core_ptl/check_imports.py --domain "asr" # Uninstall NeMo pip uninstall -y nemo_toolkit test-tts-imports: - needs: build-nemo runs-on: ubuntu-latest container: image: pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime steps: - name: Checkout repo uses: actions/checkout@v2 - - name: Download wheel as artifact - uses: actions/download-artifact@v2 - with: - name: nemo-wheel-file - path: ./dist + - name: Update base dependencies + run: | + apt-get update && apt-get install -y build-essential + apt-get install -y libsndfile1 make + - name: Install nemo dependencies + id: nemo-wheel + run: | + pip install Cython + # install test requirements + pip install -r requirements/requirements_test.txt + # Build nemo as a wheel + pip install build + python -m build --no-isolation --wheel + # Preserve wheel location + DIST_FILE=$(find ./dist -name "*.whl" | head -n 1) + echo "::set-output name=DIST_FILE::${DIST_FILE}" - name: Test TTS Domain Imports run: | # Install NeMo Domain - pip install "${{ needs.build-nemo.outputs.dist_file }}[tts]" + pip install "${{ steps.nemo-wheel.outputs.DIST_FILE }}[tts]" # Run import checks python tests/core_ptl/check_imports.py --domain "tts" # Uninstall NeMo pip uninstall -y nemo_toolkit test-nlp-imports: - needs: build-nemo runs-on: ubuntu-latest container: image: pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime steps: - name: Checkout repo uses: actions/checkout@v2 - - name: Download wheel as artifact - uses: actions/download-artifact@v2 - with: - name: nemo-wheel-file - path: ./dist + - name: Update base dependencies + run: | + apt-get update && apt-get install -y build-essential + apt-get install -y libsndfile1 make + - name: Install nemo dependencies + id: nemo-wheel + run: | + pip install Cython + # install test requirements + pip install -r requirements/requirements_test.txt + # Build nemo as a wheel + pip install build + python -m build --no-isolation --wheel + # Preserve wheel location + DIST_FILE=$(find ./dist -name "*.whl" | head -n 1) + echo "::set-output name=DIST_FILE::${DIST_FILE}" - name: Test NLP Domain Imports run: | # Install NeMo Domain - pip install "${{ needs.build-nemo.outputs.dist_file }}[nlp]" + pip install "${{ steps.nemo-wheel.outputs.DIST_FILE }}[nlp]" # Run import checks python tests/core_ptl/check_imports.py --domain "nlp" # Uninstall NeMo