diff --git a/.github/workflows/unit_tests_gpu.yml b/.github/workflows/unit_tests_gpu.yml
new file mode 100644
index 000000000..e3789693e
--- /dev/null
+++ b/.github/workflows/unit_tests_gpu.yml
@@ -0,0 +1,48 @@
+name: Unit tests GPU
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+
+jobs:
+  unit-tests:
+
+    strategy:
+      matrix:
+        os: [gpu-runner]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        model: ["gpt2gpu", "phi2gpu"]
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Show GPUs
+        run: |
+          nvidia-smi
+      - name: Install dependencies
+        shell: bash
+        run: |
+          python -m pip install --upgrade pip
+          pip install pytest
+          pip install -e .[test]
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+      - name: Install accelerate
+        run: |
+          pip install accelerate
+      - name: Check GPU available
+        run: |
+          python -c "import torch; assert torch.cuda.is_available()"
+      - name: Run tests (except server)
+        run: |
+          pytest --cov=guidance --cov-report=xml --cov-report=term-missing --selected_model ${{ matrix.model }} -m "not server" ./tests/
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
\ No newline at end of file
diff --git a/tests/conftest.py b/tests/conftest.py
index 0917f9ea7..56c7beee9 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -9,6 +9,11 @@
     "phi2cpu": dict(
         name="transformers:microsoft/phi-2", kwargs={"trust_remote_code": True}
     ),
+    "gpt2gpu": dict(name="transformers:gpt2", kwargs={"device_map": "cuda:0"}),
+    "phi2gpu": dict(
+        name="transformers:microsoft/phi-2",
+        kwargs={"trust_remote_code": True, "device_map": "cuda:0"},
+    ),
 }