From e2f5d5b3b970ad38d1809e99352f1a77d5091c99 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Mon, 14 Feb 2022 17:29:24 +0300 Subject: [PATCH] Run tests in parallel --- Makefile | 4 ++-- base-notebook/test/test_container_options.py | 10 +++++----- base-notebook/test/test_start_container.py | 6 +++--- conftest.py | 18 ++++++++++-------- requirements-dev.txt | 1 + test/test_notebook.py | 6 +++--- 6 files changed, 24 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index 21d004cb14..4d6688ccac 100644 --- a/Makefile +++ b/Makefile @@ -207,7 +207,7 @@ run-sudo-shell/%: ## run a bash in interactive mode as root in a stack test/%: ## run tests against a stack (only common tests or common tests + specific tests) @echo "::group::test/$(OWNER)/$(notdir $@)" - @if [ ! -d "$(notdir $@)/test" ]; then TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest -m "not info" test; \ - else TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest -m "not info" test $(notdir $@)/test; fi + @if [ ! -d "$(notdir $@)/test" ]; then TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest -n auto -m "not info" test; \ + else TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest -n auto -m "not info" test $(notdir $@)/test; fi @echo "::endgroup::" test-all: $(foreach I, $(ALL_IMAGES), test/$(I)) ## test all stacks diff --git a/base-notebook/test/test_container_options.py b/base-notebook/test/test_container_options.py index 65e85d24c1..de1261b3bb 100644 --- a/base-notebook/test/test_container_options.py +++ b/base-notebook/test/test_container_options.py @@ -7,7 +7,7 @@ import pytest # type: ignore import requests -from conftest import TrackedContainer +from conftest import TrackedContainer, find_free_port LOGGER = logging.getLogger(__name__) @@ -15,11 +15,11 @@ def test_cli_args(container: TrackedContainer, http_client: requests.Session) -> None: """Container should respect notebook server command line args (e.g., disabling token security)""" + host_port = find_free_port() running_container = container.run_detached( command=["start-notebook.sh", "--NotebookApp.token=''"], - ports={"8888/tcp": None}, + ports={"8888/tcp": host_port}, ) - host_port = container.get_host_port("8888/tcp") resp = http_client.get(f"http://localhost:{host_port}") resp.raise_for_status() logs = running_container.logs().decode("utf-8") @@ -37,11 +37,11 @@ def test_unsigned_ssl( """Container should generate a self-signed SSL certificate and notebook server should use it to enable HTTPS. """ + host_port = find_free_port() running_container = container.run_detached( environment=["GEN_CERT=yes"], - ports={"8888/tcp": None}, + ports={"8888/tcp": host_port}, ) - host_port = container.get_host_port("8888/tcp") # NOTE: The requests.Session backing the http_client fixture does not retry # properly while the server is booting up. An SSL handshake error seems to # abort the retry logic. Forcing a long sleep for the moment until I have diff --git a/base-notebook/test/test_start_container.py b/base-notebook/test/test_start_container.py index 7d9b30f9e2..0322db8272 100644 --- a/base-notebook/test/test_start_container.py +++ b/base-notebook/test/test_start_container.py @@ -7,7 +7,7 @@ import requests import time -from conftest import TrackedContainer +from conftest import TrackedContainer, find_free_port LOGGER = logging.getLogger(__name__) @@ -47,11 +47,12 @@ def test_start_notebook( LOGGER.info( f"Test that the start-notebook launches the {expected_command} server from the env {env} ..." ) + host_port = find_free_port() running_container = container.run_detached( tty=True, environment=env, command=["start-notebook.sh"], - ports={"8888/tcp": None}, + ports={"8888/tcp": host_port}, ) # sleeping some time to let the server start time.sleep(3) @@ -69,7 +70,6 @@ def test_start_notebook( assert len(expected_warnings) == len(warnings) # checking if the server is listening if expected_start: - host_port = container.get_host_port("8888/tcp") resp = http_client.get(f"http://localhost:{host_port}") assert resp.status_code == 200, "Server is not listening" diff --git a/conftest.py b/conftest.py index a002626566..adbada58e0 100644 --- a/conftest.py +++ b/conftest.py @@ -1,7 +1,9 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. +from contextlib import closing import os import logging +import socket from typing import Any, Optional import docker @@ -16,6 +18,14 @@ LOGGER = logging.getLogger(__name__) +def find_free_port() -> str: + """Returns the available host port. Can be called in multiple threads/processes.""" + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: + s.bind(("", 0)) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + return s.getsockname()[1] + + @pytest.fixture(scope="session") def http_client() -> requests.Session: """Requests session with retries and backoff.""" @@ -108,14 +118,6 @@ def run_and_wait( assert rv == 0 or rv["StatusCode"] == 0 return logs - def get_host_port(self, container_port: str) -> str: - """Returns the host port associated with the tracked container's port.""" - assert isinstance(self.container, Container) - self.container.reload() - return self.container.attrs["NetworkSettings"]["Ports"][container_port][0][ - "HostPort" - ] - @staticmethod def get_errors(logs: str) -> list[str]: return TrackedContainer._lines_starting_with(logs, "ERROR") diff --git a/requirements-dev.txt b/requirements-dev.txt index 23cb6bcc58..193cad0c69 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,5 +3,6 @@ packaging plumbum pre-commit pytest +pytest-xdist requests tabulate diff --git a/test/test_notebook.py b/test/test_notebook.py index 163e1ecd26..3dd0cf6678 100644 --- a/test/test_notebook.py +++ b/test/test_notebook.py @@ -3,15 +3,15 @@ import requests -from conftest import TrackedContainer +from conftest import TrackedContainer, find_free_port def test_secured_server( container: TrackedContainer, http_client: requests.Session ) -> None: """Notebook server should eventually request user login.""" - container.run_detached(ports={"8888/tcp": None}) - host_port = container.get_host_port("8888/tcp") + host_port = find_free_port() + container.run_detached(ports={"8888/tcp": host_port}) resp = http_client.get(f"http://localhost:{host_port}") resp.raise_for_status() assert "login_submit" in resp.text, "User login not requested"