Add UI integration tests with playwright

- Add a new 'integration-tests' directory that does white box UI testing - Install dockerspawner in dev-requirements, as the playwright integration tests now need it - Move loading and about page tests to use playwright - Add some tests for the home page - Use a single instance of local-binder-local-hub for doing thes integration tests - Upload playwright traces (https://playwright.dev/python/docs/trace-viewer) on failure so we can debug things better
yuvipanda · Dec 7, 2024 · 8a5e565 · 8a5e565
1 parent 386c11a
commit 8a5e565
Show file tree

Hide file tree

Showing 8 changed files with 312 additions and 106 deletions.
diff --git a/.github/workflows/playwright.yaml b/.github/workflows/playwright.yaml
@@ -0,0 +1,97 @@
+name: Playwright Tests
+
+on:
+  pull_request:
+    paths-ignore:
+      - "**.md"
+      - "**.rst"
+      - "docs/**"
+      - "examples/**"
+      - ".github/workflows/**"
+      - "!.github/workflows/playwright.yaml"
+  push:
+    paths-ignore:
+      - "**.md"
+      - "**.rst"
+      - "docs/**"
+      - "examples/**"
+      - ".github/workflows/**"
+      - "!.github/workflows/playwright.yaml"
+    branches-ignore:
+      - "dependabot/**"
+      - "pre-commit-ci-update-config"
+      - "update-*"
+  workflow_dispatch:
+
+jobs:
+  tests:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 10
+
+    permissions:
+      contents: read
+    env:
+      GITHUB_ACCESS_TOKEN: "${{ secrets.github_token }}"
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup OS level dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install --yes \
+            build-essential \
+            curl \
+            libcurl4-openssl-dev \
+            libssl-dev
+
+      - uses: actions/setup-node@v4
+        id: setup-node
+        with:
+          node-version: "22"
+
+      - name: Cache npm
+        uses: actions/cache@v4
+        with:
+          path: ~/.npm
+          key: node-${{ steps.setup-node.outputs.node-version }}-${{ hashFiles('**/package.json') }}-${{ github.job }}
+
+      - name: Run webpack to build static assets
+        run: |
+          npm install
+          npm run webpack
+
+      - uses: actions/setup-python@v5
+        id: setup-python
+        with:
+          python-version: "3.12"
+
+      - name: Cache pip
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: python-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/*requirements.txt') }}-${{ github.job }}
+
+      - name: Setup test dependencies
+        run: |
+          npm i -g configurable-http-proxy
+
+          pip install --no-binary pycurl -r dev-requirements.txt
+          pip install -e .
+
+      - name: Install playwright browser
+        run: |
+          playwright install firefox
+
+      - name: Run playwright tests
+        run: |
+          py.test --cov=binderhub -s integration-tests/
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: playwright-traces
+          path: test-results/
+
+      # Upload test coverage info to codecov
+      - uses: codecov/codecov-action@v5
diff --git a/binderhub/tests/conftest.py b/binderhub/tests/conftest.py
@@ -42,8 +42,12 @@
 
 
 def pytest_configure(config):
-    """This function has meaning to pytest, for more information, see:
-    https://docs.pytest.org/en/stable/reference.html#pytest.hookspec.pytest_configure
+    """
+    Configure plugins and custom markers
+
+    This function is called by pytest after command line arguments have
+    been parsed. See https://docs.pytest.org/en/stable/reference/reference.html#pytest.hookspec.pytest_configure
+    for more information.
     """
     # register our custom markers
     config.addinivalue_line(

diff --git a/binderhub/tests/test_main.py b/binderhub/tests/test_main.py
@@ -60,29 +60,6 @@ def _resolve_url(page_url, url):
     return f"{parsed.scheme}://{parsed.netloc}{path}{url}"
 
 
-@pytest.mark.remote
-async def test_main_page(app):
-    """Check the main page and any links on it"""
-    r = await async_requests.get(app.url)
-    assert r.status_code == 200
-    soup = BeautifulSoup(r.text, "html5lib")
-
-    # check src links (style, images)
-    for el in soup.find_all(src=True):
-        url = _resolve_url(app.url, el["src"])
-        r = await async_requests.get(url)
-        assert r.status_code == 200, f"{r.status_code} {url}"
-
-    # check hrefs
-    for el in soup.find_all(href=True):
-        href = el["href"]
-        if href.startswith("#"):
-            continue
-        url = _resolve_url(app.url, href)
-        r = await async_requests.get(url)
-        assert r.status_code == 200, f"{r.status_code} {url}"
-
-
 @pytest.mark.remote
 @pytest.mark.helm
 async def test_custom_template(app):
@@ -92,15 +69,6 @@ async def test_custom_template(app):
     assert "test-template" in r.text
 
 
-@pytest.mark.remote
-async def test_about_handler(app):
-    # Check that the about page loads
-    r = await async_requests.get(app.url + "/about")
-    assert r.status_code == 200
-    assert "This website is powered by" in r.text
-    assert binder_version.split("+")[0] in r.text
-
-
 @pytest.mark.remote
 async def test_versions_handler(app):
     # Check that the about page loads
@@ -121,65 +89,6 @@ async def test_versions_handler(app):
     assert data["binderhub"].split("+")[0] == binder_version.split("+")[0]
 
 
-@pytest.mark.parametrize(
-    "provider_prefix,repo,ref,path,path_type,status_code",
-    [
-        ("gh", "binderhub-ci-repos/requirements", "master", "", "", 200),
-        ("gh", "binderhub-ci-repos%2Frequirements", "master", "", "", 400),
-        ("gh", "binderhub-ci-repos/requirements", "master/", "", "", 200),
-        (
-            "gh",
-            "binderhub-ci-repos/requirements",
-            "20c4fe55a9b2c5011d228545e821b1c7b1723652",
-            "index.ipynb",
-            "file",
-            200,
-        ),
-        (
-            "gh",
-            "binderhub-ci-repos/requirements",
-            "20c4fe55a9b2c5011d228545e821b1c7b1723652",
-            "%2Fnotebooks%2Findex.ipynb",
-            "url",
-            200,
-        ),
-        ("gh", "binderhub-ci-repos/requirements", "master", "has%20space", "file", 200),
-        (
-            "gh",
-            "binderhub-ci-repos/requirements",
-            "master/",
-            "%2Fhas%20space%2F",
-            "file",
-            200,
-        ),
-        (
-            "gh",
-            "binderhub-ci-repos/requirements",
-            "master",
-            "%2Fhas%20space%2F%C3%BCnicode.ipynb",
-            "file",
-            200,
-        ),
-    ],
-)
-async def test_loading_page(
-    app, provider_prefix, repo, ref, path, path_type, status_code
-):
-    # repo = f'{org}/{repo_name}'
-    spec = f"{repo}/{ref}"
-    provider_spec = f"{provider_prefix}/{spec}"
-    query = f"{path_type}path={path}" if path else ""
-    uri = f"/v2/{provider_spec}?{query}"
-    r = await async_requests.get(app.url + uri)
-    assert r.status_code == status_code, f"{r.status_code} {uri}"
-    if status_code == 200:
-        soup = BeautifulSoup(r.text, "html5lib")
-        assert soup.find(id="log-container")
-        nbviewer_url = soup.find(id="nbviewer-preview").find("iframe").attrs["src"]
-        r = await async_requests.get(nbviewer_url)
-        assert r.status_code == 200, f"{r.status_code} {nbviewer_url}"
-
-
 @pytest.mark.parametrize(
     "origin,host,expected_origin",
     [

diff --git a/binderhub/tests/utils.py b/binderhub/tests/utils.py
@@ -2,6 +2,7 @@
 
 import asyncio
 import io
+import socket
 from concurrent.futures import ThreadPoolExecutor
 from urllib.parse import urlparse
 
@@ -149,3 +150,12 @@ def _next():
 
 # async_requests.get = requests.get returning a Future, etc.
 async_requests = _AsyncRequests()
+
+
+def random_port() -> int:
+    """Get a single random port."""
+    sock = socket.socket()
+    sock.bind(("", 0))
+    port = sock.getsockname()[1]
+    sock.close()
+    return port
diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -2,13 +2,16 @@ beautifulsoup4[html5lib]
 build
 chartpress>=2.1
 click
+dockerspawner
 jsonschema
 jupyter-repo2docker>=2021.08.0
 jupyter_packaging>=0.10.4,<2
 jupyterhub
+nest-asyncio
 pytest
 pytest-asyncio
 pytest-cov
 pytest-timeout
+pytest_playwright
 requests
 ruamel.yaml>=0.17.30
diff --git a/integration-tests/conftest.py b/integration-tests/conftest.py
@@ -0,0 +1,6 @@
+import nest_asyncio
+
+
+def pytest_configure(config):
+    # Required for playwright to be run from within pytest
+    nest_asyncio.apply()