Clean up our testing (#537)

.github/workflows/build_docs.yaml

@@ -39,7 +39,7 @@ jobs:
         run: python -m pip install --upgrade pip
       - name: Install dependencies
         run: |
-          python -m pip install -r requirements.txt
+          python -m pip install torch
           python -m pip install -e .
           cd docs
           python -m pip install -r requirements.txt

.github/workflows/conversion_script_test.yaml

@@ -57,14 +57,13 @@ jobs:
             sudo chmod 600 /swapfile
             sudo mkswap /swapfile
             sudo swapon /swapfile
-            python3 -m pip install awscli==1.32.6
+            python -m pip install awscli==1.32.6
             mkdir -p /tmp/test-artifacts
             aws s3 cp s3://pytorch-multimodal/llama2-7b/consolidated.00.pth /tmp/test-artifacts/llama2-7b-fair
       - name: Install dependencies
         run: |
-          python -m pip install -r requirements.txt
-          python -m pip install -r dev-requirements.txt
-          python -m pip install -e .
+          python -m pip install torch
+          python -m pip install -e ".[dev]"
       - name: Run conversion test
         run: |
           pytest tests/torchtune/_cli/test_convert_checkpoint.py --large-scale True

.github/workflows/recipe_test.yaml

@@ -1,4 +1,4 @@
-name: E2E Recipe Tests
+name: Recipe Tests
 
 on:
   push:
@@ -43,26 +43,14 @@ jobs:
         with:
           role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_torchtune_pytorch-multimodal
           aws-region: us-east-1
-      - name: Download files from S3
-        uses: nick-fields/retry@v2
-        with:
-          max_attempts: 3
-          retry_on: error
-          timeout_seconds: 14400
-          command: |
-            python3 -m pip install awscli==1.32.6
-            mkdir -p /tmp/test-artifacts
-            aws s3 cp s3://pytorch-multimodal/llama2-7b/tokenizer.model /tmp/test-artifacts
-            aws s3 cp s3://pytorch-multimodal/small-ckpt-01242024 /tmp/test-artifacts
-            aws s3 cp s3://pytorch-multimodal/small-ckpt-hf-03082024.pt /tmp/test-artifacts
-            aws s3 cp s3://pytorch-multimodal/small-ckpt-tune-03082024.pt /tmp/test-artifacts
-            aws s3 cp s3://pytorch-multimodal/small-ckpt-meta-03082024.pt /tmp/test-artifacts
+      - name: Install S3 CLI
+        run: |
+          python -m pip install awscli==1.32.6
       - name: Install dependencies
         run: |
-          python -m pip install -r requirements.txt
-          python -m pip install -r dev-requirements.txt
-          python -m pip install -e .
-      - name: Run unit tests with coverage
-        run: pytest tests/recipes --cov=. --cov-report=xml --durations=20 -vv
+          python -m pip install torch
+          python -m pip install -e ".[dev]"
+      - name: Run recipe tests with coverage
+        run: pytest tests -m integration_test --silence-s3-logs --cov=. --cov-report=xml --durations=20 -vv
       - name: Upload Coverage to Codecov
         uses: codecov/codecov-action@v3

.github/workflows/eval_test.yaml → .github/workflows/recipe_test_multi_gpu.yaml

@@ -1,12 +1,13 @@
-name: E2E Eval Tests
+name: Multi-GPU Recipe Tests
 
 on:
   push:
     branches: [ main ]
   pull_request:
+  workflow_dispatch:
 
 concurrency:
-  group: eval-test-${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }}
+  group: recipe-test-multi-gpu-${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }}
   cancel-in-progress: true
 
 permissions:
@@ -18,8 +19,8 @@ defaults:
     shell: bash -l -eo pipefail {0}
 
 jobs:
-  eval_test:
-    runs-on: ubuntu-latest
+  recipe_test_multi_gpu:
+    runs-on: linux.8xlarge.nvidia.gpu
     strategy:
       matrix:
         python-version: ['3.8', '3.9', '3.10', '3.11']
@@ -41,24 +42,14 @@ jobs:
         with:
           role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_torchtune_pytorch-multimodal
           aws-region: us-east-1
-      - name: Download files from S3
-        uses: nick-fields/retry@v2
-        with:
-          max_attempts: 3
-          retry_on: error
-          timeout_seconds: 14400
-          command: |
-            python3 -m pip install awscli==1.32.6
-            mkdir -p /tmp/test-artifacts
-            aws s3 cp s3://pytorch-multimodal/llama2-7b/tokenizer.model /tmp/test-artifacts
-            aws s3 cp s3://pytorch-multimodal/small-ckpt-01242024 /tmp/test-artifacts
-            aws s3 cp s3://pytorch-multimodal/small-ckpt-tune-03082024.pt /tmp/test-artifacts
+      - name: Install S3 CLI
+        run: |
+          python -m pip install awscli==1.32.6
       - name: Install dependencies
         run: |
-          python -m pip install -r requirements.txt
-          python -m pip install -r dev-requirements.txt
-          python -m pip install -e .
-      - name: Run unit tests with coverage
-        run: pytest tests/evals --cov=. --cov-report=xml --durations=20 -vv
+          python -m pip install torch
+          python -m pip install -e ".[dev]"
+      - name: Run recipe tests with coverage
+        run: pytest tests -m integration_test --silence-s3-logs --cov=. --cov-report=xml --durations=20 -vv
       - name: Upload Coverage to Codecov
         uses: codecov/codecov-action@v3

.github/workflows/unit_test.yaml

@@ -33,10 +33,9 @@ jobs:
         run: python -m pip install --upgrade pip
       - name: Install dependencies
         run: |
-          python -m pip install -r requirements.txt
-          python -m pip install -r dev-requirements.txt
-          python -m pip install -e .
+          python -m pip install torch
+          python -m pip install -e ".[dev]"
       - name: Run unit tests with coverage
-        run: pytest tests/torchtune --cov=. --cov-report=xml --durations=20 -vv
+        run: pytest tests --without-integration --without-slow-integration --cov=. --cov-report=xml --durations=20 -vv
       - name: Upload Coverage to Codecov
         uses: codecov/codecov-action@v3

dev-requirements.txt

@@ -1,6 +1,7 @@
 pytest
 pytest-cov
 pytest-mock
+pytest-integration
 pre-commit
 tensorboard
 wandb