Validation

.github/workflows/code-quality.yaml

@@ -20,24 +20,23 @@ defaults:
 jobs:
   code-quality:
     runs-on: ubuntu-20.04
-    timeout-minutes: 20
+    timeout-minutes: 30
     strategy:
       matrix:
         python_version:
-        - '3.9'
-        - '3.10'
+        - "3.9"
+        - "3.10"
         pip_deps:
-        - '[dev]'
+        - "[dev]"
     steps:
     - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
+    - name: Get composite run steps repository
+      uses: actions/checkout@v3
       with:
-        python-version: ${{ matrix.python_version }}
-    - name: Setup
-      run: |
-        set -ex
-        python -m pip install --upgrade 'pip<23' wheel
-        python -m pip install --upgrade .${{ matrix.pip_deps }}
-    - name: Run checks
-      run: |
-        pre-commit run --all-files
+        repository: mosaicml/ci-testing
+        ref: v0.0.3
+        path: ./ci-testing
+    - uses: ./ci-testing/.github/actions/code-quality
+      with:
+        python_version: ${{ matrix.python_version }}
+        pip_deps: ${{ matrix.pip_deps }}

.github/workflows/codeql-analysis.yml

@@ -9,7 +9,7 @@
 # the `language` matrix defined below to confirm you have the correct set of
 # supported CodeQL languages.
 #
-name: 'CodeQL'
+name: "CodeQL"
 
 on:
   push:
@@ -18,7 +18,7 @@ on:
     # The branches below must be a subset of the branches above
     branches: [main]
   schedule:
-  - cron: '0 9 * * 1'  # Every Monday at 09:00 (9:00 AM)
+  - cron: "0 9 * * 1"  # Every Monday at 09:00 (9:00 AM)
 
 jobs:
   analyze:
@@ -32,40 +32,19 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        language: ['python']
+        language: ["python"]
         # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
         # Learn more about CodeQL language support at https://git.io/codeql-language-support
 
     steps:
     - name: Checkout repository
       uses: actions/checkout@v2
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+    - name: Get composite run steps repository
+      uses: actions/checkout@v3
       with:
-        languages: ${{ matrix.language }}
-        setup-python-dependencies: false
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-        # queries: ./path/to/local/query, your-org/your-repo/queries@main
-
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
-
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://git.io/JvXDl
-
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
-
-    # - run: |
-    #   make bootstrap
-    #   make release
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+        repository: mosaicml/ci-testing
+        ref: v0.0.3
+        path: ./ci-testing
+    - uses: ./ci-testing/.github/actions/codeql-analysis
+      with:
+        language: ${{ matrix.language }}

.github/workflows/coverage.yaml

@@ -12,21 +12,12 @@ jobs:
     steps:
     - name: Checkout Repo
       uses: actions/checkout@v3
-    - name: Setup
-      run: |
-        set -ex
-        python -m pip install --upgrade 'pip<23' wheel
-        pip install coverage[toml]==6.5.0
-    - name: Download artifacts
-      uses: actions/download-artifact@v3
+    - name: Get composite run steps repository
+      uses: actions/checkout@v3
+      with:
+        repository: mosaicml/ci-testing
+        ref: v0.0.3
+        path: ./ci-testing
+    - uses: ./ci-testing/.github/actions/coverage
       with:
-        path: ${{ inputs.download-path }}
-    - name: Generate coverage report
-      run: |
-        set -ex
-
-        # Flatten the coverage files
-        ls ${{ inputs.download-path }} | while read x; do mv ${{ inputs.download-path }}/$x/.coverage .coverage.$x; done
-
-        python -m coverage combine
-        python -m coverage report
+        download-path: ${{ inputs.download-path }}

.github/workflows/docker.yaml

@@ -7,28 +7,22 @@ on:
     branches:
     - main
     paths:
-    - ./Dockerfile
+    - Dockerfile
     - .github/workflows/docker.yaml
   workflow_dispatch: {}
 jobs:
   docker-build:
-    runs-on: ubuntu-latest
+    runs-on: mosaic-4wide
     if: github.repository_owner == 'mosaicml'
     strategy:
       matrix:
         include:
-        - name: '2.1.0_cu121'
-          base_image: mosaicml/pytorch:2.1.0_cu121-python3.10-ubuntu20.04
-          dep_groups: '[gpu]'
-        - name: '2.1.0_cu121_flash2'
-          base_image: mosaicml/pytorch:2.1.0_cu121-python3.10-ubuntu20.04
-          dep_groups: '[gpu-flash2]'
-        - name: '2.1.0_cu121_aws'
-          base_image: mosaicml/pytorch:2.1.0_cu121-python3.10-ubuntu20.04-aws
-          dep_groups: '[gpu]'
-        - name: '2.1.0_cu121_flash2_aws'
-          base_image: mosaicml/pytorch:2.1.0_cu121-python3.10-ubuntu20.04-aws
-          dep_groups: '[gpu-flash2]'
+        - name: "2.2.1_cu121_flash2"
+          base_image: mosaicml/pytorch:2.2.1_cu121-python3.11-ubuntu20.04
+          dep_groups: "[gpu-flash2]"
+        - name: "2.2.1_cu121_flash2_aws"
+          base_image: mosaicml/pytorch:2.2.1_cu121-python3.11-ubuntu20.04-aws
+          dep_groups: "[gpu-flash2]"
     steps:
     - name: Maximize Build Space on Worker
       uses: easimon/maximize-build-space@v4
@@ -88,5 +82,6 @@ jobs:
         cache-from: type=registry,ref=${{ env.IMAGE_CACHE }}
         cache-to: type=registry,ref=${{ env.IMAGE_CACHE }},mode=max
         build-args: |
+          BRANCH_NAME=${{ github.head_ref || github.ref_name }}
           BASE_IMAGE=${{ matrix.base_image }}
           DEP_GROUPS=${{ matrix.dep_groups }}

.github/workflows/pr-cpu.yaml

@@ -15,21 +15,23 @@ concurrency:
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
 jobs:
   pytest-cpu:
-    uses: ./.github/workflows/pytest-cpu.yaml
+    uses: mosaicml/ci-testing/.github/workflows/pytest-cpu.yaml@v0.0.3
     strategy:
       matrix:
         include:
-        - name: 'cpu-2.1.0'
-          container: mosaicml/pytorch:2.1.0_cpu-python3.10-ubuntu20.04
-          markers: 'not gpu'
-          pytest_command: 'coverage run -m pytest'
+        - name: "cpu-2.2.1"
+          container: mosaicml/pytorch:2.2.1_cpu-python3.11-ubuntu20.04
+          markers: "not gpu"
+          pytest_command: "coverage run -m pytest"
     name: ${{ matrix.name }}
     if: github.repository_owner == 'mosaicml'
     with:
       container: ${{ matrix.container }}
       name: ${{ matrix.name }}
+      pip_deps: "[all-cpu]"
       pytest-command: ${{ matrix.pytest_command }}
       pytest-markers: ${{ matrix.markers }}
+      safe_directory: llm-foundry
   coverage:
     uses: ./.github/workflows/coverage.yaml
     name: Coverage Results