Merge pull request #1634 from mathbunnyru/asalikhov/test_all

Run tests on all children images

.github/workflows/docker-amd64.yml

@@ -18,10 +18,8 @@ on:
  - "tensorflow-notebook/**"
 
  - "tagging/**"
- - "test/**"
- - "conftest.py"
+ - "tests/**"
  - "Makefile"
- - "pytest.ini"
  - "requirements-dev.txt"
  push:
  branches:
@@ -40,10 +38,8 @@ on:
  - "tensorflow-notebook/**"
 
  - "tagging/**"
- - "test/**"
- - "conftest.py"
+ - "tests/**"
  - "Makefile"
- - "pytest.ini"
  - "requirements-dev.txt"
  workflow_dispatch:
 

.github/workflows/docker.yml

@@ -18,10 +18,8 @@ on:
  - "tensorflow-notebook/**"
 
  - "tagging/**"
- - "test/**"
- - "conftest.py"
+ - "tests/**"
  - "Makefile"
- - "pytest.ini"
  - "requirements-dev.txt"
  push:
  branches:
@@ -40,10 +38,8 @@ on:
  - "tensorflow-notebook/**"
 
  - "tagging/**"
- - "test/**"
- - "conftest.py"
+ - "tests/**"
  - "Makefile"
- - "pytest.ini"
  - "requirements-dev.txt"
  workflow_dispatch:
 

Makefile

@@ -113,7 +113,7 @@ build-all-multi: $(foreach I, $(MULTI_IMAGES), build-multi/$(I)) $(foreach I, $(
 
 
 check-outdated/%: ## check the outdated mamba/conda packages in a stack and produce a report (experimental)
- @TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest test/test_outdated.py
+ @TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest tests/base-notebook/test_outdated.py
 check-outdated-all: $(foreach I, $(ALL_IMAGES), check-outdated/$(I)) ## check all the stacks for outdated packages
 
 
@@ -205,9 +205,8 @@ run-sudo-shell/%: ## run a bash in interactive mode as root in a stack
 
 
 
-test/%: ## run tests against a stack (only common tests or common tests + specific tests)
+test/%: ## run tests against a stack
  @echo "::group::test/$(OWNER)/$(notdir $@)"
- @if [ ! -d "$(notdir $@)/test" ]; then TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest --numprocesses=auto -m "not info" test; \
- else TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest --numprocesses=auto -m "not info" test $(notdir $@)/test; fi
+ tests/run_tests.py --short-image-name "$(notdir $@)" --owner "$(OWNER)"
  @echo "::endgroup::"
 test-all: $(foreach I, $(ALL_IMAGES), test/$(I)) ## test all stacks

all-spark-notebook/.dockerignore

@@ -1,3 +1,2 @@
 # Documentation
 README.md
-test

base-notebook/.dockerignore

@@ -1,3 +1,2 @@
 # Documentation
 README.md
-test

datascience-notebook/.dockerignore

@@ -1,3 +1,2 @@
 # Documentation
 README.md
-test

docs/contributing/tests.md

@@ -1,42 +1,50 @@
 # Image Tests
 
-We greatly appreciate pull requests that extend the automated tests that vet the basic functionality
-of the Docker images.
+We greatly appreciate pull requests that extend the automated tests that vet the basic functionality of the Docker images.
 
 ## How the Tests Work
 
-GitHub Action executes `make build-test-all` against pull requests submitted to the `jupyter/docker-stacks` repository.
-This `make` command builds every docker image.
-After building each image, the `make` command executes `pytest` to run both image-specific tests like those in
-[base-notebook/test/](https://github.com/jupyter/docker-stacks/tree/master/base-notebook/test) and
-common tests defined in [test/](https://github.com/jupyter/docker-stacks/tree/master/test).
-Both kinds of tests make use of global [pytest fixtures](https://docs.pytest.org/en/latest/reference/fixtures.html)
-defined in the [conftest.py](https://github.com/jupyter/docker-stacks/blob/master/conftest.py) file at the root of the projects.
+A [GitHub Action workflow](https://github.com/jupyter/docker-stacks/blob/master/.github/workflows/docker.yml)
+runs the following commands against pull requests submitted to the `jupyter/docker-stacks` repository:
+
+1. `make -C main build-all-multi` - which builds all the Docker images
+2. `make -C main test-all` - which tests the newly created Docker images
+ This `make` command builds and then tests every docker image.
+
+We use `pytest` module to run tests on the image.
+`conftest.py` and `pytest.ini` in the `tests` folder define the environment in which tests are run.
+More info on `pytest` can be found [here](https://docs.pytest.org/en/latest/contents.html).
+
+The actual image-specific test files are located in folders like `tests/<somestack>-notebook/`.
+
+```{note}
+If your test is located in `tests/<somestack>-notebook/`, it will be run against `jupyter/<somestack>-notebook` image and against all the [images inherited from this image](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#image-relationships.
+```
+
+Many tests make use of global [pytest fixtures](https://docs.pytest.org/en/latest/reference/fixtures.html)
+defined in the [conftest.py](https://github.com/jupyter/docker-stacks/blob/master/tests/conftest.py) file.
 
 ## Unit tests
 
 If you want to run a python script in one of our images, you could add a unit test.
-You can do this by creating a `<somestack>-notebook/test/units/` directory, if it doesn't already exist and put your file there.
-File in this folder will run automatically when tests are run.
-You could see an example for tensorflow package [here](https://github.com/jupyter/docker-stacks/blob/HEAD/tensorflow-notebook/test/units/unit_tensorflow.py).
+You can do this by creating a `tests/<somestack>-notebook/units/` directory, if it doesn't already exist and put your file there.
+Files in this folder will be executed in the container when tests are run.
+You could see an [example for the TensorFlow package here](https://github.com/jupyter/docker-stacks/blob/HEAD/tests/tensorflow-notebook/units/unit_tensorflow.py).
 
 ## Contributing New Tests
 
 Please follow the process below to add new tests:
 
-1. If the test should run against every image built, add your test code to one of the modules in
- [test/](https://github.com/jupyter/docker-stacks/tree/master/test) or create a new module.
-2. If your test should run against a single image, add your test code to one of the modules in
- `some-notebook/test/` or create a new module.
-3. Build one or more images you intend to test and run the tests locally.
+1. Add your test code to one of the modules in `tests/<somestack>-notebook/` directory or create a new module.
+2. Build one or more images you intend to test and run the tests locally.
  If you use `make`, call:
 
  ```bash
  make build/somestack-notebook
  make test/somestack-notebook
  ```
 
-4. [Submit a pull request](https://github.com/PointCloudLibrary/pcl/wiki/A-step-by-step-guide-on-preparing-and-submitting-a-pull-request)
+3. [Submit a pull request](https://github.com/PointCloudLibrary/pcl/wiki/A-step-by-step-guide-on-preparing-and-submitting-a-pull-request)
  (PR) with your changes.
-5. Watch for GitHub to report a build success or failure for your PR on GitHub.
-6. Discuss changes with the maintainers and address any issues running the tests on GitHub.
+4. Watch for GitHub to report a build success or failure for your PR on GitHub.
+5. Discuss changes with the maintainers and address any issues running the tests on GitHub.

minimal-notebook/.dockerignore

@@ -1,3 +1,2 @@
 # Documentation
 README.md
-test

pyspark-notebook/.dockerignore

@@ -1,3 +1,2 @@
 # Documentation
 README.md
-test

r-notebook/.dockerignore

@@ -1,3 +1,2 @@
 # Documentation
 README.md
-test

scipy-notebook/.dockerignore

@@ -1,3 +1,2 @@
 # Documentation
 README.md
-test

tensorflow-notebook/.dockerignore

@@ -1,3 +1,2 @@
 # Documentation
 README.md
-test

tests/README.md

@@ -0,0 +1,3 @@
+# Docker stacks testing
+
+Please, refer to the [testing section of documentation](https://jupyter-docker-stacks.readthedocs.io/en/latest/contributing/tests.html) to see how the tests are run.

test/test_packages.py → tests/base-notebook/test_packages.py

@@ -18,19 +18,18 @@
 
 Example:
 
- $ make test/datascience-notebook
+ $ make test/base-notebook
 
  # [...]
  # test/test_packages.py::test_python_packages
- # --------------------------------------------------------------------------------------------- live log setup ----------------------------------------------------------------------------------------------
- # 2020-03-08 09:56:04 [ INFO] Starting container jupyter/datascience-notebook ... (package_helper.py:51)
- # 2020-03-08 09:56:04 [ INFO] Running jupyter/datascience-notebook with args {'detach': True, 'ports': {'8888/tcp': 8888}, 'tty': True, 'command': ['start.sh', 'bash', '-c', 'sleep infinity']} ... (conftest.py:78)
- # 2020-03-08 09:56:04 [ INFO] Grabing the list of manually requested packages ... (package_helper.py:76)
- # ---------------------------------------------------------------------------------------------- live log call ----------------------------------------------------------------------------------------------
- # 2020-03-08 09:56:07 [ INFO] Testing the import of packages ... (test_packages.py:125)
- # 2020-03-08 09:56:07 [ INFO] Trying to import conda (test_packages.py:127)
- # 2020-03-08 09:56:07 [ INFO] Trying to import notebook (test_packages.py:127)
- # 2020-03-08 09:56:08 [ INFO] Trying to import jupyterhub (test_packages.py:127)
+ # tests/base-notebook/test_packages.py::test_python_packages
+ # ---------------------------------------------------------------------------------------------- live log setup ----------------------------------------------------------------------------------------------
+ # 2022-02-17 16:44:36 [ INFO] Starting container jupyter/base-notebook ... (package_helper.py:55)
+ # 2022-02-17 16:44:36 [ INFO] Running jupyter/base-notebook with args {'detach': True, 'tty': True, 'command': ['start.sh', 'bash', '-c', 'sleep infinity']} ... (conftest.py:95)
+ # 2022-02-17 16:44:37 [ INFO] Grabing the list of manually requested packages ... (package_helper.py:83)
+ # ---------------------------------------------------------------------------------------------- live log call -----------------------------------------------------------------------------------------------
+ # 2022-02-17 16:44:38 [ INFO] Testing the import of packages ... (test_packages.py:144)
+ # 2022-02-17 16:44:38 [ INFO] Trying to import mamba (test_packages.py:146)
  # [...]
 
 """

tests/base-notebook/test_units.py

@@ -0,0 +1,39 @@
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+import logging
+
+from conftest import TrackedContainer
+from images_hierarchy import get_test_dirs
+
+LOGGER = logging.getLogger(__name__)
+
+
+def test_units(container: TrackedContainer) -> None:
+ """Various units tests
+ Add a py file in the `tests/{somestack}-notebook/units` dir and it will be automatically tested
+ """
+ short_image_name = container.image_name[container.image_name.rfind("/") + 1 :]
+ LOGGER.info(f"Running unit tests for: {short_image_name}")
+
+ test_dirs = get_test_dirs(short_image_name)
+
+ for test_dir in test_dirs:
+ host_data_dir = test_dir / "units"
+ LOGGER.info(f"Searching for units tests in {host_data_dir}")
+ cont_data_dir = "/home/jovyan/data"
+
+ if not host_data_dir.exists():
+ LOGGER.info(f"Not found unit tests for image: {container.image_name}")
+ continue
+
+ for test_file in host_data_dir.iterdir():
+ test_file_name = test_file.name
+ LOGGER.info(f"Running unit test: {test_file_name}")
+
+ container.run_and_wait(
+ timeout=30,
+ volumes={str(host_data_dir): {"bind": cont_data_dir, "mode": "ro"}},
+ tty=True,
+ command=["start.sh", "python", f"{cont_data_dir}/{test_file_name}"],
+ )

tests/images_hierarchy.py

@@ -0,0 +1,31 @@
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+from pathlib import Path
+from typing import Optional
+
+THIS_DIR = Path(__file__).parent.resolve()
+
+# Please, take a look at the hierarchy of the images here:
+# https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#image-relationships
+ALL_IMAGES = {
+ "base-notebook": None,
+ "minimal-notebook": "base-notebook",
+ "scipy-notebook": "minimal-notebook",
+ "r-notebook": "minimal-notebook",
+ "tensorflow-notebook": "scipy-notebook",
+ "datascience-notebook": "scipy-notebook",
+ "pyspark-notebook": "scipy-notebook",
+ "all-spark-notebook": "pyspark-notebook",
+}
+
+
+def get_test_dirs(
+ short_image_name: Optional[str],
+) -> list[Path]:
+ if short_image_name is None:
+ return [] # type: ignore
+
+ test_dirs = get_test_dirs(ALL_IMAGES[short_image_name])
+ if (current_image_tests_dir := THIS_DIR / short_image_name).exists():
+ test_dirs.append(current_image_tests_dir)
+ return test_dirs

tests/run_tests.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import argparse
+import logging
+
+import plumbum
+from plumbum.cmd import pytest
+
+from images_hierarchy import get_test_dirs
+
+
+LOGGER = logging.getLogger(__name__)
+
+
+def test_image(short_image_name: str, owner: str) -> None:
+ LOGGER.info(f"Testing image: {short_image_name}")
+ test_dirs = get_test_dirs(short_image_name)
+ LOGGER.info(f"Test dirs to be run: {test_dirs}")
+ with plumbum.local.env(TEST_IMAGE=f"{owner}/{short_image_name}"):
+ (
+ pytest[
+ "--numprocesses",
+ "auto",
+ "-m",
+ "not info",
+ test_dirs,
+ ]
+ & plumbum.FG
+ )
+
+
+if __name__ == "__main__":
+ logging.basicConfig(level=logging.INFO)
+
+ arg_parser = argparse.ArgumentParser()
+ arg_parser.add_argument(
+ "--short-image-name",
+ required=True,
+ help="Short image name to run test on",
+ )
+ arg_parser.add_argument("--owner", required=True, help="Owner of the image")
+
+ args = arg_parser.parse_args()
+
+ test_image(args.short_image_name, args.owner)