diff --git a/.github/mcp/mcp_pytest.py b/.github/mcp/mcp_pytest.py
deleted file mode 100644
index e7d4270272..0000000000
--- a/.github/mcp/mcp_pytest.py
+++ /dev/null
@@ -1,148 +0,0 @@
-# Copyright 2022 MosaicML LLM Foundry authors
-# SPDX-License-Identifier: Apache-2.0
-
-"""Run pytest using MCP."""
-
-import argparse
-import time
-
-from mcli import (RunConfig, RunStatus, create_run, follow_run_logs,
-                  wait_for_run_status)
-
-if __name__ == '__main__':
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--name',
-                        type=str,
-                        default='mcp-pytest',
-                        help='Base name of run')
-    parser.add_argument('--cluster',
-                        type=str,
-                        default='r1z4',
-                        help='Cluster to use')
-    parser.add_argument('--gpu_type',
-                        type=str,
-                        default='a100_40gb',
-                        help='Type of GPU to use')
-    parser.add_argument('--gpu_num',
-                        type=int,
-                        default=2,
-                        help='Number of the GPU to use')
-    parser.add_argument('--image',
-                        type=str,
-                        default='mosaicml/pytorch:latest',
-                        help='Docker image to use')
-    parser.add_argument('--git_branch',
-                        type=str,
-                        help='Git branch to check out')
-    parser.add_argument(
-        '--git_commit',
-        type=str,
-        help='Git commit to check out. Overrides git_branch if specified')
-    parser.add_argument(
-        '--pr_number',
-        type=int,
-        help=
-        'PR number to check out. Overrides git_branch/git_commit if specified')
-    parser.add_argument('--pytest_markers',
-                        type=str,
-                        help='Markers to pass to pytest')
-    parser.add_argument('--pytest_command',
-                        type=str,
-                        help='Command to run pytest')
-    parser.add_argument('--timeout',
-                        type=int,
-                        default=1800,
-                        help='Timeout for run (in seconds)')
-    parser.add_argument('--deps_group',
-                        type=str,
-                        help='Dependency group to install')
-    args = parser.parse_args()
-
-    name = args.name
-    git_integration = {
-        'integration_type': 'git_repo',
-        'git_repo': 'mosaicml/llm-foundry',
-        'ssh_clone': 'False',
-    }
-    if args.git_branch is not None and args.git_commit is None:
-        name += f'-branch-{args.git_branch}'
-        git_integration['git_branch'] = args.git_branch
-    if args.git_commit is not None:
-        name += f'-commit-{args.git_commit}'
-        git_integration['git_commit'] = args.git_commit
-
-    command = 'cd llm-foundry'
-
-    # Checkout a specific PR if specified
-    if args.pr_number is not None:
-        name += f'-pr-{args.pr_number}'
-        command += f'''
-
-        git fetch origin pull/{args.pr_number}/head:pr_branch
-
-        git checkout pr_branch
-
-        '''
-
-    # Shorten name if too long
-    if len(name) > 56:
-        name = name[:56]
-
-    clear_tmp_path_flag = '-o tmp_path_retention_policy=none'
-    command += f'''
-
-    pip install --upgrade --user .[{args.deps_group}]
-
-    export COMMON_ARGS="-v --durations=20 -m '{args.pytest_markers}' {clear_tmp_path_flag}"
-
-    make test-dist PYTEST='{args.pytest_command}' EXTRA_ARGS="$COMMON_ARGS" WORLD_SIZE=2
-
-    make test PYTEST='{args.pytest_command}' EXTRA_ARGS="$COMMON_ARGS --codeblocks"
-
-    python -m coverage combine
-
-    python -m coverage report
-    '''
-
-    config = RunConfig(
-        name=name,
-        compute={
-            'cluster': args.cluster,
-            'gpu_type': args.gpu_type,
-            'gpus': args.gpu_num
-        },
-        image=args.image,
-        integrations=[git_integration],
-        command=command,
-        scheduling={'max_duration': args.timeout / 60 / 60},
-        env_variables=[
-            {
-                'key': 'MOSAICML_PLATFORM',
-                'value': 'False',
-            },
-            {
-                'key': 'PYTHONUNBUFFERED',
-                'value': '1',
-            },
-        ],
-    )
-
-    # Create run
-    run = create_run(config)
-    print(f'[GHA] Run created: {run.name}')
-
-    # Wait until run starts before fetching logs
-    run = wait_for_run_status(run, status='running')
-    start_time = time.time()
-    print('[GHA] Run started. Following logs...')
-
-    # Print logs
-    for line in follow_run_logs(run):
-        print(line, end='')
-
-    print('[GHA] Run completed. Waiting for run to finish...')
-    run = wait_for_run_status(run, status=RunStatus.COMPLETED)
-
-    # Fail if command exited with non-zero exit code or timed out (didn't reach COMPLETED)
-    assert run.status == RunStatus.COMPLETED, f'Run did not complete: {run.status} ({run.reason})'
diff --git a/.github/workflows/code-quality.yaml b/.github/workflows/code-quality.yaml
index 6a014b068d..cb2e4f9a3f 100644
--- a/.github/workflows/code-quality.yaml
+++ b/.github/workflows/code-quality.yaml
@@ -8,7 +8,6 @@ on:
     branches:
     - main
     - release/**
-  workflow_call:
   workflow_dispatch:
 # Cancel old runs when a new commit is pushed to the same branch if not on main or dev
 concurrency:
@@ -30,14 +29,13 @@ jobs:
         - "[dev]"
     steps:
     - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
+    - name: Get composite run steps repository
+      uses: actions/checkout@v3
       with:
-        python-version: ${{ matrix.python_version }}
-    - name: Setup
-      run: |
-        set -ex
-        python -m pip install --upgrade 'pip<23' wheel
-        python -m pip install --upgrade .${{ matrix.pip_deps }}
-    - name: Run checks
-      run: |
-        pre-commit run --all-files
+        repository: mosaicml/ci-testing
+        ref: v0.0.2
+        path: ./ci-testing
+    - uses: ./ci-testing/.github/actions/code-quality
+      with:
+        python_version: ${{ matrix.python_version }}
+        pip_deps: ${{ matrix.pip_deps }}
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index 0058b2c425..d0da96aa64 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -39,33 +39,12 @@ jobs:
     steps:
     - name: Checkout repository
       uses: actions/checkout@v2
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+    - name: Get composite run steps repository
+      uses: actions/checkout@v3
       with:
-        languages: ${{ matrix.language }}
-        setup-python-dependencies: false
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-        # queries: ./path/to/local/query, your-org/your-repo/queries@main
-
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
-
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://git.io/JvXDl
-
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
-
-    # - run: |
-    #   make bootstrap
-    #   make release
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+        repository: mosaicml/ci-testing
+        ref: v0.0.2
+        path: ./ci-testing
+    - uses: ./ci-testing/.github/actions/codeql-analysis
+      with:
+        language: ${{ matrix.language }}
diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml
index f89d67ec39..0cb96ca03d 100644
--- a/.github/workflows/coverage.yaml
+++ b/.github/workflows/coverage.yaml
@@ -12,21 +12,12 @@ jobs:
     steps:
     - name: Checkout Repo
       uses: actions/checkout@v3
-    - name: Setup
-      run: |
-        set -ex
-        python -m pip install --upgrade 'pip<23' wheel
-        pip install coverage[toml]==6.5.0
-    - name: Download artifacts
-      uses: actions/download-artifact@v3
+    - name: Get composite run steps repository
+      uses: actions/checkout@v3
+      with:
+        repository: mosaicml/ci-testing
+        ref: v0.0.2
+        path: ./ci-testing
+    - uses: ./ci-testing/.github/actions/coverage
       with:
-        path: ${{ inputs.download-path }}
-    - name: Generate coverage report
-      run: |
-        set -ex
-
-        # Flatten the coverage files
-        ls ${{ inputs.download-path }} | while read x; do mv ${{ inputs.download-path }}/$x/.coverage .coverage.$x; done
-
-        python -m coverage combine
-        python -m coverage report
+        download-path: ${{ inputs.download-path }}
diff --git a/.github/workflows/pr-cpu.yaml b/.github/workflows/pr-cpu.yaml
index 0bba0fadb9..eb8610dad9 100644
--- a/.github/workflows/pr-cpu.yaml
+++ b/.github/workflows/pr-cpu.yaml
@@ -15,7 +15,7 @@ concurrency:
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
 jobs:
   pytest-cpu:
-    uses: ./.github/workflows/pytest-cpu.yaml
+    uses: mosaicml/ci-testing/.github/workflows/pytest-cpu.yaml@v0.0.2
     strategy:
       matrix:
         include:
@@ -28,8 +28,10 @@ jobs:
     with:
       container: ${{ matrix.container }}
       name: ${{ matrix.name }}
+      pip_deps: "[all-cpu]"
       pytest-command: ${{ matrix.pytest_command }}
       pytest-markers: ${{ matrix.markers }}
+      safe_directory: llm-foundry
   coverage:
     uses: ./.github/workflows/coverage.yaml
     name: Coverage Results
diff --git a/.github/workflows/pr-gpu.yaml b/.github/workflows/pr-gpu.yaml
index 05ba590342..afd01425e4 100644
--- a/.github/workflows/pr-gpu.yaml
+++ b/.github/workflows/pr-gpu.yaml
@@ -15,29 +15,30 @@ concurrency:
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
 jobs:
   pytest-gpu:
-    uses: ./.github/workflows/pytest-gpu.yaml
+    uses: mosaicml/ci-testing/.github/workflows/pytest-gpu.yaml@v0.0.2
     strategy:
       matrix:
         include:
         - name: "gpu-2.2.0"
           container: mosaicml/pytorch:2.2.0_cu121-python3.11-ubuntu20.04
           markers: "gpu"
+          pip_deps: "[all]"
           pytest_command: "coverage run -m pytest"
-          deps_group: "all"
         - name: "gpu-2.2.0-flash2"
           container: mosaicml/llm-foundry:2.2.0_cu121_flash2-latest
           markers: "gpu"
+          pip_deps: "[all-flash2]"
           pytest_command: "coverage run -m pytest"
-          deps_group: "all-flash2"
     name: ${{ matrix.name }}
     if: github.repository_owner == 'mosaicml'
     with:
       container: ${{ matrix.container }}
+      git_repo: mosaicml/llm-foundry
       mcloud-timeout: 1800
       name: ${{ matrix.name }}
+      pip_deps: ${{ matrix.pip_deps }}
       pytest-command: ${{ matrix.pytest_command }}
       pytest-markers: ${{ matrix.markers }}
       python-version: 3.9
-      deps-group: ${{ matrix.deps_group }}
     secrets:
       mcloud-api-key: ${{ secrets.MCLOUD_API_KEY }}
diff --git a/.github/workflows/pytest-cpu.yaml b/.github/workflows/pytest-cpu.yaml
deleted file mode 100644
index 3bb3697b91..0000000000
--- a/.github/workflows/pytest-cpu.yaml
+++ /dev/null
@@ -1,48 +0,0 @@
-name: Pytest CPU
-on:
-  workflow_call:
-    inputs:
-      container:
-        required: true
-        type: string
-      name:
-        required: true
-        type: string
-      pytest-command:
-        required: true
-        type: string
-      pytest-markers:
-        required: true
-        type: string
-jobs:
-  pytest-cpu:
-    timeout-minutes: 30
-    runs-on: ubuntu-latest
-    container: ${{ inputs.container }}
-    steps:
-    - name: Checkout Repo
-      uses: actions/checkout@v3
-    - name: Setup
-      run: |
-        set -ex
-        export PATH=/composer-python:$PATH
-        python -m pip install --upgrade 'pip<23' wheel
-        python -m pip install --upgrade .[all-cpu]
-    - name: Run Tests
-      id: tests
-      run: |
-        set -ex
-        export PATH=/composer-python:$PATH
-        export COMMON_ARGS="-v --durations=20 -m '${{ inputs.pytest-markers }}' -o tmp_path_retention_policy=none"
-
-        # Necessary to run git diff for doctests
-        git config --global --add safe.directory /__w/llm-foundry/llm-foundry
-
-        make test PYTEST='${{ inputs.pytest-command }}' EXTRA_ARGS="$COMMON_ARGS --codeblocks"
-        # make test-dist PYTEST='${{ inputs.pytest-command }}' EXTRA_ARGS="$COMMON_ARGS" WORLD_SIZE=2
-
-        python -m coverage combine
-    - uses: actions/upload-artifact@v3
-      with:
-        name: coverage-${{ github.sha }}-${{ inputs.name }}
-        path: .coverage
diff --git a/.github/workflows/pytest-gpu.yaml b/.github/workflows/pytest-gpu.yaml
deleted file mode 100644
index df7cd09c99..0000000000
--- a/.github/workflows/pytest-gpu.yaml
+++ /dev/null
@@ -1,83 +0,0 @@
-name: Pytest GPU
-on:
-  workflow_call:
-    inputs:
-      container:
-        required: true
-        type: string
-      mcloud-timeout:
-        required: false
-        type: number
-        default: 1800
-      name:
-        required: true
-        type: string
-      pytest-command:
-        required: true
-        type: string
-      pytest-markers:
-        required: true
-        type: string
-      python-version:
-        required: false
-        type: string
-        default: 3.9
-      deps-group:
-        required: true
-        type: string
-    secrets:
-      mcloud-api-key:
-        required: true
-jobs:
-  pytest-gpu:
-    timeout-minutes: 60  # ${{ inputs.gha-timeout }} for some reason not able to turn this into an input
-    runs-on: ubuntu-latest
-    env:
-      MOSAICML_API_KEY: ${{ secrets.mcloud-api-key }}
-    steps:
-    - name: Checkout Repo
-      uses: actions/checkout@v3
-    - name: Setup Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ inputs.python-version }}
-    - name: Cache pip
-      uses: actions/cache@v3
-      with:
-        # This path is specific to Ubuntu
-        path: ~/.cache/pip
-        # Look to see if there is a cache hit for the corresponding requirements file
-        key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-
-          ${{ runner.os }}-
-    - name: Setup MCLI
-      run: |
-        set -ex
-        python -m pip install mosaicml-cli
-        mcli version
-    - name: Submit Run
-      id: tests
-      run: |
-        set -ex
-
-        PR_NUMBER="$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH")"
-        REF_ARGS=""
-
-        # Use the PR number if it exists, commit SHA for protected branches and the branch name otherwise
-        if [ -z "$PR_NUMBER" ] || [ "$PR_NUMBER" = "null" ]; then
-          if [[ "$GITHUB_REF" =~ "refs/heads/main" || "$GITHUB_REF" =~ "refs/heads/release" ]]; then
-            REF_ARGS="--git_commit $GITHUB_SHA"
-          else
-            REF_ARGS="--git_branch $GITHUB_REF_NAME"
-          fi
-        else
-          REF_ARGS="--pr_number $PR_NUMBER"
-        fi
-
-        python .github/mcp/mcp_pytest.py \
-              --image '${{ inputs.container }}' \
-              --pytest_markers '${{ inputs.pytest-markers }}' \
-              --pytest_command '${{ inputs.pytest-command }}' \
-              --timeout ${{ inputs.mcloud-timeout }} ${REF_ARGS} \
-              --deps_group ${{ inputs.deps-group }}
diff --git a/.github/workflows/smoketest.yaml b/.github/workflows/smoketest.yaml
index 428de32795..24688cb413 100644
--- a/.github/workflows/smoketest.yaml
+++ b/.github/workflows/smoketest.yaml
@@ -26,16 +26,14 @@ jobs:
         - "3.9"
         - "3.10"
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Get composite run steps repository
+      uses: actions/checkout@v3
       with:
-        python-version: ${{ matrix.python_version }}
-    - name: Setup
-      run: |
-        set -ex
-        python -m pip install --upgrade 'pip<23' wheel
-        python -m pip install --upgrade .
-        python -m pip install pytest==7.2.1 pytest_codeblocks==0.16.1
-    - name: Run checks
-      run: |
-        pytest tests/test_smoketest.py
+        repository: mosaicml/ci-testing
+        ref: v0.0.2
+        path: ./ci-testing
+    - uses: ./ci-testing/.github/actions/smoketest
+      with:
+        python_version: ${{ matrix.python_version }}