From 7ed70ac9a135d3c06bdb818d5e4aad8c20878858 Mon Sep 17 00:00:00 2001 From: Simon Shillaker Date: Tue, 8 Feb 2022 15:39:57 +0000 Subject: [PATCH 01/15] Skeleton for docs --- .github/workflows/tests.yml | 99 ++++++++-------------------------- docker/faabric-base.dockerfile | 4 ++ docs/.gitignore | 4 ++ docs/CMakeLists.txt | 28 ++++++++++ docs/Doxyfile | 11 ++++ docs/Makefile | 20 +++++++ docs/conf.py | 21 ++++++++ docs/index.rst | 20 +++++++ docs/make.bat | 35 ++++++++++++ requirements.txt | 15 +++--- tasks/__init__.py | 2 + tasks/docs.py | 34 ++++++++++++ tasks/util/env.py | 1 - 13 files changed, 210 insertions(+), 84 deletions(-) create mode 100644 docs/.gitignore create mode 100644 docs/CMakeLists.txt create mode 100644 docs/Doxyfile create mode 100644 docs/Makefile create mode 100644 docs/conf.py create mode 100644 docs/index.rst create mode 100644 docs/make.bat create mode 100644 tasks/docs.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 23504bc19..4b3e31dcd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,46 +15,33 @@ jobs: conan-cache: if: github.event.pull_request.draft == false runs-on: ubuntu-latest - env: - HOST_TYPE: ci - REDIS_QUEUE_HOST: redis - REDIS_STATE_HOST: redis container: image: faasm/faabric:0.2.4 defaults: run: working-directory: /code/faabric - services: - redis: - image: redis steps: - # --- Code update --- - - name: "Fetch ref" - run: git fetch origin ${GITHUB_REF}:ci-branch - - name: "Check out branch" - run: git checkout --force ci-branch - # --- Set-up --- - - name: "Ping redis" - run: redis-cli -h redis ping - # --- Cache based on Conan version and ExternalProjects cmake source - - name: Get Conan version - id: get-conan-version - run: | - echo "::set-output name=conan-version::$(conan --version | tr -d '[:alpha:] ')" - - name: Hash ExternalProjects - id: hash-external - run: | - echo "::set-output name=hash-external::$(sha256sum cmake/ExternalProjects.cmake | cut '-d ' -f 1)" - - uses: actions/cache@v2 - with: - path: '~/.conan' - key: ${{ runner.os }}-${{ steps.get-conan-version.outputs.conan-version }}-${{ steps.hash-external.outputs.hash-external }} - restore-keys: | - ${{ runner.os }}-${{ steps.get-conan-version.outputs.conan-version }}- - ${{ runner.os }}- - # --- Build dependencies - - name: "Build dependencies to be shared by all sanitiser runs" + - uses: faasm/conan-cache-action@v1 + - name: "Build dependencies to be shared by all runs" + run: inv dev.cmake -b Debug + + docs: + if: github.event.pull_request.draft == false + needs: [conan-cache] + runs-on: ubuntu-latest + container: + image: faasm/faabric:0.2.4 + defaults: + run: + working-directory: /code/faabric + steps: + - uses: faasm/conan-cache-action@v1 + - name: "Build dependencies to be shared by all runs" run: inv dev.cmake -b Debug + - name: "Build doxygen docs" + run: inv dev.cc docs-doxy + - name: "Build sphinx docs" + run: inv dev.cc docs-sphinx formatting: if: github.event.pull_request.draft == false @@ -109,30 +96,10 @@ jobs: redis: image: redis steps: - # --- Code update --- - - name: "Fetch ref" - run: git fetch origin ${GITHUB_REF}:ci-branch - - name: "Check out branch" - run: git checkout --force ci-branch + - uses: faasm/conan-cache-action@v1 # --- Set-up --- - name: "Ping redis" run: redis-cli -h redis ping - # --- Cache based on Conan version and ExternalProjects cmake source - - name: Get Conan version - id: get-conan-version - run: | - echo "::set-output name=conan-version::$(conan --version | tr -d '[:alpha:] ')" - - name: "Hash ExternalProjects" - id: hash-external - run: | - echo "::set-output name=hash-external::$(sha256sum cmake/ExternalProjects.cmake | cut '-d ' -f 1)" - - uses: actions/cache@v2 - with: - path: '~/.conan' - key: ${{ runner.os }}-${{ steps.get-conan-version.outputs.conan-version }}-${{ steps.hash-external.outputs.hash-external }} - restore-keys: | - ${{ runner.os }}-${{ steps.get-conan-version.outputs.conan-version }}- - ${{ runner.os }}- # --- Tests build - need a debug build for actually running tests --- - name: "Run cmake for tests" run: inv dev.cmake --build=Debug --sanitiser ${{ matrix.sanitiser }} @@ -198,29 +165,9 @@ jobs: redis: image: redis steps: - # --- Code update --- - - name: "Fetch ref" - run: git fetch origin ${GITHUB_REF}:ci-branch - - name: "Check out branch" - run: git checkout --force ci-branch - # --- Cache based on Conan version and ExternalProjects cmake source - - name: "Get Conan version" - id: get-conan-version - run: | - echo "::set-output name=conan-version::$(conan --version | tr -d '[:alpha:] ')" - - name: "Hash ExternalProjects" - id: hash-external - run: | - echo "::set-output name=hash-external::$(sha256sum cmake/ExternalProjects.cmake | cut '-d ' -f 1)" - - uses: actions/cache@v2 + - uses: faasm/conan-cache-action@v1 with: - path: '~/.conan' - key: release-${{ runner.os }}-${{ steps.get-conan-version.outputs.conan-version }}-${{ steps.hash-external.outputs.hash-external }} - restore-keys: | - release-${{ runner.os }}-${{ steps.get-conan-version.outputs.conan-version }}-${{ steps.hash-external.outputs.hash-external }} - release-${{ runner.os }}-${{ steps.get-conan-version.outputs.conan-version }}- - release-${{ runner.os }}- - # --- Examples --- + build-type: "release" - name: "Run cmake shared" run: inv dev.cmake --shared --build=Release - name: "Build Faabric shared library" diff --git a/docker/faabric-base.dockerfile b/docker/faabric-base.dockerfile index 016c368de..10802f718 100644 --- a/docker/faabric-base.dockerfile +++ b/docker/faabric-base.dockerfile @@ -22,6 +22,7 @@ RUN apt install -y \ clang-tidy-13 \ clang-tools-13 \ cmake \ + doxygen \ g++-11 \ git \ kitware-archive-keyring \ @@ -50,6 +51,9 @@ RUN apt install -y \ RUN curl -s -L -o /tmp/conan-latest.deb https://github.com/conan-io/conan/releases/download/1.43.0/conan-ubuntu-64.deb && sudo dpkg -i /tmp/conan-latest.deb && rm -f /tmp/conan-latest.deb +# Update pip +RUN pip install -U pip + # Tidy up RUN apt-get clean autoclean RUN apt-get autoremove diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 000000000..bb1988a6a --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1,4 @@ +doxygen/ +html/ +sphinx/ +_build/ diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt new file mode 100644 index 000000000..2a346eadb --- /dev/null +++ b/docs/CMakeLists.txt @@ -0,0 +1,28 @@ +find_package(Doxygen REQUIRED) +find_package(Sphinx REQUIRED) + +# Find all the public headers +file(GLOB_RECURSE HEADERS ../include/*.h) + +set(DOXYGEN_INDEX_FILE ${CMAKE_CURRENT_SOURCE_DIR}/html/index.html) + +add_custom_command(OUTPUT ${DOXYGEN_INDEX_FILE} + DEPENDS ${HEADERS} + COMMAND ${DOXYGEN_EXECUTABLE} Doxyfile + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + MAIN_DEPENDENCY Doxyfile + COMMENT "Generating docs") + +add_custom_target(docs-doxy ALL DEPENDS ${DOXYGEN_INDEX_FILE}) + +set(SPHINX_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}) +set(SPHINX_BUILD ${CMAKE_CURRENT_BINARY_DIR}/docs/sphinx) + +add_custom_target(docs-sphinx ALL + COMMAND + ${SPHINX_EXECUTABLE} -b html + -Dbreathe_projects.Faabric=${DOXYGEN_OUTPUT_DIR} + ${SPHINX_SOURCE} ${SPHINX_BUILD} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Generating documentation with Sphinx") + diff --git a/docs/Doxyfile b/docs/Doxyfile new file mode 100644 index 000000000..d13f28219 --- /dev/null +++ b/docs/Doxyfile @@ -0,0 +1,11 @@ +DOXYFILE_ENCODING = UTF-8 +PROJECT_NAME = "Faabric" +OUTPUT_DIRECTORY = doxygen +OUTPUT_LANGUAGE = English +FULL_PATH_NAMES = YES +MARKDOWN_SUPPORT = YES +GENERATE_LATEX = NO +GENERATE_XML = YES +GENERATE_HTML = NO +INPUT = ../include/ ../src/ +FILE_PATTERNS = *.h *.cpp diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 000000000..d4bb2cbb9 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 000000000..5c0317ef5 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,21 @@ +from os.path import dirname, realpath, join, expanduser + +DOCS_ROOT = dirname(realpath(__file__)) +DOXYGEN_OUT = join(DOCS_ROOT, "doxygen", "xml") + +project = "Faabric" +copyright = "2022, Simon Shillaker" +author = "Simon Shillaker" + +extensions = ["breathe"] + +templates_path = ["templates"] + +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + +html_theme = "alabaster" + +html_static_path = ["static"] + +breathe_projects = {"Faabric": DOCS_ROOT} +breathe_default_project = "Faabric" diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 000000000..3d8ae89f6 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,20 @@ +.. Faabric documentation master file, created by + sphinx-quickstart on Tue Feb 8 14:45:52 2022. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to Faabric's documentation! +=================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 000000000..153be5e2f --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/requirements.txt b/requirements.txt index e6c0d85d2..eb857f173 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,8 @@ -ansible -black -docker -flake8 -invoke -numpy -redis +black==21.12b0 +breathe==4.32.0 +docker==5.0.3 +flake8==4.0.1 +invoke==1.6.0 +numpy==1.22.0 +redis==4.1.0 +sphinx-rtd-theme==1.0.0 diff --git a/tasks/__init__.py b/tasks/__init__.py index 5bc3f368f..77664b2c6 100644 --- a/tasks/__init__.py +++ b/tasks/__init__.py @@ -3,6 +3,7 @@ from . import call from . import container from . import dev +from . import docs from . import examples from . import git from . import mpi_native @@ -11,6 +12,7 @@ call, container, dev, + docs, examples, git, mpi_native, diff --git a/tasks/docs.py b/tasks/docs.py new file mode 100644 index 000000000..c9e358c7d --- /dev/null +++ b/tasks/docs.py @@ -0,0 +1,34 @@ +from os import makedirs +from shutil import rmtree +from os.path import exists, join +from subprocess import run + +from tasks.util.env import ( + PROJ_ROOT, + FAABRIC_SHARED_BUILD_DIR, + FAABRIC_STATIC_BUILD_DIR, + FAABRIC_INSTALL_PREFIX, +) + +from invoke import task + +DOCS_DIR = join(PROJ_ROOT, "docs") +DOXY_OUT_DIR = join(DOCS_DIR, "doxygen", "xml") +SPHINX_OUT_DIR = join(DOCS_DIR, "sphinx") + + +@task(default=True) +def doxy(ctx): + run("doxygen", cwd=DOCS_DIR, check=True, shell=True) + + +@task +def sphinx(ctx): + makedirs(SPHINX_OUT_DIR, exist_ok=True) + + run( + "sphinx-build -b html {} {}".format(DOCS_DIR, SPHINX_OUT_DIR), + cwd=DOCS_DIR, + check=True, + shell=True, + ) diff --git a/tasks/util/env.py b/tasks/util/env.py index 44a080591..34f96c33d 100644 --- a/tasks/util/env.py +++ b/tasks/util/env.py @@ -3,7 +3,6 @@ HOME_DIR = expanduser("~") PROJ_ROOT = dirname(dirname(dirname(realpath(__file__)))) -ANSIBLE_ROOT = join(PROJ_ROOT, "ansible") _FAABRIC_BUILD_DIR = environ.get("FAABRIC_BUILD_DIR", "/build/faabric") From 2b3208dead1ce2aad33e82de32ad246844b37236 Mon Sep 17 00:00:00 2001 From: Simon Shillaker Date: Tue, 8 Feb 2022 15:40:48 +0000 Subject: [PATCH 02/15] Remove autogen makefile --- docs/make.bat | 35 ----------------------------------- 1 file changed, 35 deletions(-) delete mode 100644 docs/make.bat diff --git a/docs/make.bat b/docs/make.bat deleted file mode 100644 index 153be5e2f..000000000 --- a/docs/make.bat +++ /dev/null @@ -1,35 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=. -set BUILDDIR=_build - -if "%1" == "" goto help - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.https://www.sphinx-doc.org/ - exit /b 1 -) - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% - -:end -popd From 3a487e76e6c3017470b98e4bb83e66cd68411c0c Mon Sep 17 00:00:00 2001 From: Simon Shillaker Date: Tue, 8 Feb 2022 15:45:50 +0000 Subject: [PATCH 03/15] Tidy-up --- .github/workflows/tests.yml | 4 ++-- docs/CMakeLists.txt | 28 ---------------------------- docs/Makefile | 20 -------------------- docs/conf.py | 4 ++-- tasks/docs.py | 10 ++-------- 5 files changed, 6 insertions(+), 60 deletions(-) delete mode 100644 docs/CMakeLists.txt delete mode 100644 docs/Makefile diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4b3e31dcd..087b1e056 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -39,9 +39,9 @@ jobs: - name: "Build dependencies to be shared by all runs" run: inv dev.cmake -b Debug - name: "Build doxygen docs" - run: inv dev.cc docs-doxy + run: inv docs.doxy - name: "Build sphinx docs" - run: inv dev.cc docs-sphinx + run: inv docs.sphinx formatting: if: github.event.pull_request.draft == false diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt deleted file mode 100644 index 2a346eadb..000000000 --- a/docs/CMakeLists.txt +++ /dev/null @@ -1,28 +0,0 @@ -find_package(Doxygen REQUIRED) -find_package(Sphinx REQUIRED) - -# Find all the public headers -file(GLOB_RECURSE HEADERS ../include/*.h) - -set(DOXYGEN_INDEX_FILE ${CMAKE_CURRENT_SOURCE_DIR}/html/index.html) - -add_custom_command(OUTPUT ${DOXYGEN_INDEX_FILE} - DEPENDS ${HEADERS} - COMMAND ${DOXYGEN_EXECUTABLE} Doxyfile - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - MAIN_DEPENDENCY Doxyfile - COMMENT "Generating docs") - -add_custom_target(docs-doxy ALL DEPENDS ${DOXYGEN_INDEX_FILE}) - -set(SPHINX_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}) -set(SPHINX_BUILD ${CMAKE_CURRENT_BINARY_DIR}/docs/sphinx) - -add_custom_target(docs-sphinx ALL - COMMAND - ${SPHINX_EXECUTABLE} -b html - -Dbreathe_projects.Faabric=${DOXYGEN_OUTPUT_DIR} - ${SPHINX_SOURCE} ${SPHINX_BUILD} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - COMMENT "Generating documentation with Sphinx") - diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index d4bb2cbb9..000000000 --- a/docs/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = . -BUILDDIR = _build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/conf.py b/docs/conf.py index 5c0317ef5..f918fc6c1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,4 +1,4 @@ -from os.path import dirname, realpath, join, expanduser +from os.path import dirname, realpath, join DOCS_ROOT = dirname(realpath(__file__)) DOXYGEN_OUT = join(DOCS_ROOT, "doxygen", "xml") @@ -17,5 +17,5 @@ html_static_path = ["static"] -breathe_projects = {"Faabric": DOCS_ROOT} +breathe_projects = {"Faabric": DOXYGEN_OUT} breathe_default_project = "Faabric" diff --git a/tasks/docs.py b/tasks/docs.py index c9e358c7d..7238cc2dc 100644 --- a/tasks/docs.py +++ b/tasks/docs.py @@ -1,14 +1,8 @@ from os import makedirs -from shutil import rmtree -from os.path import exists, join +from os.path import join from subprocess import run -from tasks.util.env import ( - PROJ_ROOT, - FAABRIC_SHARED_BUILD_DIR, - FAABRIC_STATIC_BUILD_DIR, - FAABRIC_INSTALL_PREFIX, -) +from tasks.util.env import PROJ_ROOT from invoke import task From c0922ce733ce0d6a96bcb9e2db21e422771cd871 Mon Sep 17 00:00:00 2001 From: Simon Shillaker Date: Tue, 8 Feb 2022 16:06:23 +0000 Subject: [PATCH 04/15] Add directory to cache actioN --- .github/workflows/tests.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 087b1e056..129d5cefe 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -22,6 +22,8 @@ jobs: working-directory: /code/faabric steps: - uses: faasm/conan-cache-action@v1 + with: + directory: /code/faabric - name: "Build dependencies to be shared by all runs" run: inv dev.cmake -b Debug @@ -36,6 +38,8 @@ jobs: working-directory: /code/faabric steps: - uses: faasm/conan-cache-action@v1 + with: + directory: /code/faabric - name: "Build dependencies to be shared by all runs" run: inv dev.cmake -b Debug - name: "Build doxygen docs" @@ -97,6 +101,8 @@ jobs: image: redis steps: - uses: faasm/conan-cache-action@v1 + with: + directory: /code/faabric # --- Set-up --- - name: "Ping redis" run: redis-cli -h redis ping @@ -168,6 +174,7 @@ jobs: - uses: faasm/conan-cache-action@v1 with: build-type: "release" + directory: /code/faabric - name: "Run cmake shared" run: inv dev.cmake --shared --build=Release - name: "Build Faabric shared library" From 89821889df9c28139ba392e9785fa68a7c5d2e42 Mon Sep 17 00:00:00 2001 From: Simon Shillaker Date: Tue, 8 Feb 2022 16:38:30 +0000 Subject: [PATCH 05/15] RTD config file --- .readthedocs.yaml | 13 +++++++++++++ docs/Doxyfile | 7 ++++++- docs/conf.py | 14 ++++++++++---- docs/{ => source}/development.md | 0 docs/{ => source}/mpi_native.md | 0 requirements.txt | 3 ++- tasks/docs.py | 4 +++- 7 files changed, 34 insertions(+), 7 deletions(-) create mode 100644 .readthedocs.yaml rename docs/{ => source}/development.md (100%) rename docs/{ => source}/mpi_native.md (100%) diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 000000000..49eee0353 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,13 @@ +version: 2 + +build: + os: "ubuntu-20.04" + tools: + python: "3.9" + +sphinx: + configuration: docs/conf.py + +python: + install: + - requirements: requirements.txt diff --git a/docs/Doxyfile b/docs/Doxyfile index d13f28219..69509dfce 100644 --- a/docs/Doxyfile +++ b/docs/Doxyfile @@ -2,10 +2,15 @@ DOXYFILE_ENCODING = UTF-8 PROJECT_NAME = "Faabric" OUTPUT_DIRECTORY = doxygen OUTPUT_LANGUAGE = English + FULL_PATH_NAMES = YES MARKDOWN_SUPPORT = YES +RECURSIVE = YES +QUIET = YES + GENERATE_LATEX = NO GENERATE_XML = YES GENERATE_HTML = NO -INPUT = ../include/ ../src/ + +INPUT = ../include ../src FILE_PATTERNS = *.h *.cpp diff --git a/docs/conf.py b/docs/conf.py index f918fc6c1..3eacdd963 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,21 +1,27 @@ from os.path import dirname, realpath, join +from subprocess import run +from os import environ DOCS_ROOT = dirname(realpath(__file__)) DOXYGEN_OUT = join(DOCS_ROOT, "doxygen", "xml") +# Generate doxygen files in RTD +IS_READ_THE_DOCS = environ.get("READTHEDOCS", None) == "True" +if IS_READ_THE_DOCS: + run("doxygen", cwd=DOCS_ROOT, check=True, shell=True) + project = "Faabric" copyright = "2022, Simon Shillaker" author = "Simon Shillaker" -extensions = ["breathe"] +extensions = ["breathe", "myst_parser"] -templates_path = ["templates"] +templates_path = ["source/templates"] +html_static_path = ["source/static"] exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] html_theme = "alabaster" -html_static_path = ["static"] - breathe_projects = {"Faabric": DOXYGEN_OUT} breathe_default_project = "Faabric" diff --git a/docs/development.md b/docs/source/development.md similarity index 100% rename from docs/development.md rename to docs/source/development.md diff --git a/docs/mpi_native.md b/docs/source/mpi_native.md similarity index 100% rename from docs/mpi_native.md rename to docs/source/mpi_native.md diff --git a/requirements.txt b/requirements.txt index eb857f173..9f520d22a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ breathe==4.32.0 docker==5.0.3 flake8==4.0.1 invoke==1.6.0 -numpy==1.22.0 +myst-parser==0.16.1 +numpy==1.21.5 redis==4.1.0 sphinx-rtd-theme==1.0.0 diff --git a/tasks/docs.py b/tasks/docs.py index 7238cc2dc..684d79e2f 100644 --- a/tasks/docs.py +++ b/tasks/docs.py @@ -1,4 +1,4 @@ -from os import makedirs +from os import makedirs, environ from os.path import join from subprocess import run @@ -10,6 +10,8 @@ DOXY_OUT_DIR = join(DOCS_DIR, "doxygen", "xml") SPHINX_OUT_DIR = join(DOCS_DIR, "sphinx") +IS_READ_THE_DOCS = environ.get("READTHEDOCS", None) == "True" + @task(default=True) def doxy(ctx): From c8398bef72d79505ad884c83cbdb8754a6f61be4 Mon Sep 17 00:00:00 2001 From: Simon Shillaker Date: Tue, 8 Feb 2022 16:42:24 +0000 Subject: [PATCH 06/15] Hack to install deps --- .github/workflows/tests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 129d5cefe..504e780d7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -40,6 +40,10 @@ jobs: - uses: faasm/conan-cache-action@v1 with: directory: /code/faabric + - name: "TEMP - install doxygen" + run: apt install -y doxygen + - name: "TEMP - install python deps" + run: pip install -r requirements.txt - name: "Build dependencies to be shared by all runs" run: inv dev.cmake -b Debug - name: "Build doxygen docs" From c944266c6ed4b284fcb73424d7371bffe70a1c95 Mon Sep 17 00:00:00 2001 From: Simon Shillaker Date: Tue, 8 Feb 2022 18:59:32 +0000 Subject: [PATCH 07/15] Added breathe-apidoc --- docs/.gitignore | 3 +-- docs/conf.py | 8 ++++++++ docs/index.rst | 12 ++++++------ tasks/docs.py | 13 +++++++++++++ 4 files changed, 28 insertions(+), 8 deletions(-) diff --git a/docs/.gitignore b/docs/.gitignore index bb1988a6a..b9244c26a 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -1,4 +1,3 @@ doxygen/ -html/ +apidoc/ sphinx/ -_build/ diff --git a/docs/conf.py b/docs/conf.py index 3eacdd963..3d2b44b37 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -4,12 +4,20 @@ DOCS_ROOT = dirname(realpath(__file__)) DOXYGEN_OUT = join(DOCS_ROOT, "doxygen", "xml") +APIDOC_OUT_DIR = join(DOCS_ROOT, "apidoc") # Generate doxygen files in RTD IS_READ_THE_DOCS = environ.get("READTHEDOCS", None) == "True" if IS_READ_THE_DOCS: run("doxygen", cwd=DOCS_ROOT, check=True, shell=True) + run( + "breathe-apidoc {} -o {}".format(DOXYGEN_OUT, APIDOC_OUT_DIR), + cwd=DOCS_ROOT, + check=True, + shell=True, + ) + project = "Faabric" copyright = "2022, Simon Shillaker" author = "Simon Shillaker" diff --git a/docs/index.rst b/docs/index.rst index 3d8ae89f6..d8d73aa62 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -10,11 +10,11 @@ Welcome to Faabric's documentation! :maxdepth: 2 :caption: Contents: + source/development + source/mpi_native + apidoc/namespacelist + apidoc/classlist +Most of this documentation is auto-generated from the code, but includes a few +extra pieces of stand-alone content referenced in the sidebar. -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/tasks/docs.py b/tasks/docs.py index 684d79e2f..0a4830d6d 100644 --- a/tasks/docs.py +++ b/tasks/docs.py @@ -10,6 +10,8 @@ DOXY_OUT_DIR = join(DOCS_DIR, "doxygen", "xml") SPHINX_OUT_DIR = join(DOCS_DIR, "sphinx") +APIDOC_OUT_DIR = join(DOCS_DIR, "apidoc") + IS_READ_THE_DOCS = environ.get("READTHEDOCS", None) == "True" @@ -18,6 +20,17 @@ def doxy(ctx): run("doxygen", cwd=DOCS_DIR, check=True, shell=True) +@task +def apidoc(ctx): + makedirs(APIDOC_OUT_DIR, exist_ok=True) + run( + "breathe-apidoc {} -o {}".format(DOXY_OUT_DIR, APIDOC_OUT_DIR), + cwd=DOCS_DIR, + check=True, + shell=True, + ) + + @task def sphinx(ctx): makedirs(SPHINX_OUT_DIR, exist_ok=True) From 6562c645de1728871923a74cc5be9eb82795504b Mon Sep 17 00:00:00 2001 From: Simon Shillaker Date: Tue, 8 Feb 2022 19:04:54 +0000 Subject: [PATCH 08/15] Simplify sphinx build --- docs/conf.py | 23 +++++++++++------------ tasks/docs.py | 25 ++----------------------- 2 files changed, 13 insertions(+), 35 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 3d2b44b37..de863ab39 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,22 +1,21 @@ from os.path import dirname, realpath, join from subprocess import run -from os import environ +from os import makedirs DOCS_ROOT = dirname(realpath(__file__)) DOXYGEN_OUT = join(DOCS_ROOT, "doxygen", "xml") APIDOC_OUT_DIR = join(DOCS_ROOT, "apidoc") -# Generate doxygen files in RTD -IS_READ_THE_DOCS = environ.get("READTHEDOCS", None) == "True" -if IS_READ_THE_DOCS: - run("doxygen", cwd=DOCS_ROOT, check=True, shell=True) - - run( - "breathe-apidoc {} -o {}".format(DOXYGEN_OUT, APIDOC_OUT_DIR), - cwd=DOCS_ROOT, - check=True, - shell=True, - ) +makedirs(APIDOC_OUT_DIR, exist_ok=True) + +run("doxygen", cwd=DOCS_ROOT, check=True, shell=True) + +run( + "breathe-apidoc {} -o {}".format(DOXYGEN_OUT, APIDOC_OUT_DIR), + cwd=DOCS_ROOT, + check=True, + shell=True, +) project = "Faabric" copyright = "2022, Simon Shillaker" diff --git a/tasks/docs.py b/tasks/docs.py index 0a4830d6d..5754d6488 100644 --- a/tasks/docs.py +++ b/tasks/docs.py @@ -1,4 +1,4 @@ -from os import makedirs, environ +from os import makedirs from os.path import join from subprocess import run @@ -7,32 +7,11 @@ from invoke import task DOCS_DIR = join(PROJ_ROOT, "docs") -DOXY_OUT_DIR = join(DOCS_DIR, "doxygen", "xml") SPHINX_OUT_DIR = join(DOCS_DIR, "sphinx") -APIDOC_OUT_DIR = join(DOCS_DIR, "apidoc") - -IS_READ_THE_DOCS = environ.get("READTHEDOCS", None) == "True" - @task(default=True) -def doxy(ctx): - run("doxygen", cwd=DOCS_DIR, check=True, shell=True) - - -@task -def apidoc(ctx): - makedirs(APIDOC_OUT_DIR, exist_ok=True) - run( - "breathe-apidoc {} -o {}".format(DOXY_OUT_DIR, APIDOC_OUT_DIR), - cwd=DOCS_DIR, - check=True, - shell=True, - ) - - -@task -def sphinx(ctx): +def generate(ctx): makedirs(SPHINX_OUT_DIR, exist_ok=True) run( From 0bc7f9c3be516897bf61888087b74f3361276a3d Mon Sep 17 00:00:00 2001 From: Simon Shillaker Date: Tue, 8 Feb 2022 19:23:23 +0000 Subject: [PATCH 09/15] Revert unnecessary changes to GHA --- .github/workflows/tests.yml | 95 ++++++++++++++++++++++++++++++------- 1 file changed, 77 insertions(+), 18 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 504e780d7..c785014b2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,16 +15,45 @@ jobs: conan-cache: if: github.event.pull_request.draft == false runs-on: ubuntu-latest + env: + HOST_TYPE: ci + REDIS_QUEUE_HOST: redis + REDIS_STATE_HOST: redis container: image: faasm/faabric:0.2.4 defaults: run: working-directory: /code/faabric + services: + redis: + image: redis steps: - - uses: faasm/conan-cache-action@v1 + # --- Code update --- + - name: "Fetch ref" + run: git fetch origin ${GITHUB_REF}:ci-branch + - name: "Check out branch" + run: git checkout --force ci-branch + # --- Set-up --- + - name: "Ping redis" + run: redis-cli -h redis ping + # --- Cache based on Conan version and ExternalProjects cmake source + - name: Get Conan version + id: get-conan-version + run: | + echo "::set-output name=conan-version::$(conan --version | tr -d '[:alpha:] ')" + - name: Hash ExternalProjects + id: hash-external + run: | + echo "::set-output name=hash-external::$(sha256sum cmake/ExternalProjects.cmake | cut '-d ' -f 1)" + - uses: actions/cache@v2 with: - directory: /code/faabric - - name: "Build dependencies to be shared by all runs" + path: '~/.conan' + key: ${{ runner.os }}-${{ steps.get-conan-version.outputs.conan-version }}-${{ steps.hash-external.outputs.hash-external }} + restore-keys: | + ${{ runner.os }}-${{ steps.get-conan-version.outputs.conan-version }}- + ${{ runner.os }}- + # --- Build dependencies + - name: "Build dependencies to be shared by all sanitiser runs" run: inv dev.cmake -b Debug docs: @@ -37,19 +66,12 @@ jobs: run: working-directory: /code/faabric steps: - - uses: faasm/conan-cache-action@v1 - with: - directory: /code/faabric - name: "TEMP - install doxygen" run: apt install -y doxygen - name: "TEMP - install python deps" run: pip install -r requirements.txt - - name: "Build dependencies to be shared by all runs" - run: inv dev.cmake -b Debug - - name: "Build doxygen docs" - run: inv docs.doxy - - name: "Build sphinx docs" - run: inv docs.sphinx + - name: "Build docs" + run: inv docs formatting: if: github.event.pull_request.draft == false @@ -104,12 +126,30 @@ jobs: redis: image: redis steps: - - uses: faasm/conan-cache-action@v1 - with: - directory: /code/faabric + # --- Code update --- + - name: "Fetch ref" + run: git fetch origin ${GITHUB_REF}:ci-branch + - name: "Check out branch" + run: git checkout --force ci-branch # --- Set-up --- - name: "Ping redis" run: redis-cli -h redis ping + # --- Cache based on Conan version and ExternalProjects cmake source + - name: Get Conan version + id: get-conan-version + run: | + echo "::set-output name=conan-version::$(conan --version | tr -d '[:alpha:] ')" + - name: "Hash ExternalProjects" + id: hash-external + run: | + echo "::set-output name=hash-external::$(sha256sum cmake/ExternalProjects.cmake | cut '-d ' -f 1)" + - uses: actions/cache@v2 + with: + path: '~/.conan' + key: ${{ runner.os }}-${{ steps.get-conan-version.outputs.conan-version }}-${{ steps.hash-external.outputs.hash-external }} + restore-keys: | + ${{ runner.os }}-${{ steps.get-conan-version.outputs.conan-version }}- + ${{ runner.os }}- # --- Tests build - need a debug build for actually running tests --- - name: "Run cmake for tests" run: inv dev.cmake --build=Debug --sanitiser ${{ matrix.sanitiser }} @@ -175,10 +215,29 @@ jobs: redis: image: redis steps: - - uses: faasm/conan-cache-action@v1 + # --- Code update --- + - name: "Fetch ref" + run: git fetch origin ${GITHUB_REF}:ci-branch + - name: "Check out branch" + run: git checkout --force ci-branch + # --- Cache based on Conan version and ExternalProjects cmake source + - name: "Get Conan version" + id: get-conan-version + run: | + echo "::set-output name=conan-version::$(conan --version | tr -d '[:alpha:] ')" + - name: "Hash ExternalProjects" + id: hash-external + run: | + echo "::set-output name=hash-external::$(sha256sum cmake/ExternalProjects.cmake | cut '-d ' -f 1)" + - uses: actions/cache@v2 with: - build-type: "release" - directory: /code/faabric + path: '~/.conan' + key: release-${{ runner.os }}-${{ steps.get-conan-version.outputs.conan-version }}-${{ steps.hash-external.outputs.hash-external }} + restore-keys: | + release-${{ runner.os }}-${{ steps.get-conan-version.outputs.conan-version }}-${{ steps.hash-external.outputs.hash-external }} + release-${{ runner.os }}-${{ steps.get-conan-version.outputs.conan-version }}- + release-${{ runner.os }}- + # --- Examples --- - name: "Run cmake shared" run: inv dev.cmake --shared --build=Release - name: "Build Faabric shared library" From b05267c50159feda5b4bbcc7b21c2adf38f30a30 Mon Sep 17 00:00:00 2001 From: Simon Shillaker Date: Fri, 11 Feb 2022 10:41:43 +0100 Subject: [PATCH 10/15] Populate classes with members, rtd theme --- docs/Doxyfile | 2 +- docs/conf.py | 15 ++++++++++++--- docs/index.rst | 16 ++++++++++------ include/faabric/endpoint/Endpoint.h | 4 ++++ 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/docs/Doxyfile b/docs/Doxyfile index 69509dfce..e34fbae45 100644 --- a/docs/Doxyfile +++ b/docs/Doxyfile @@ -3,7 +3,7 @@ PROJECT_NAME = "Faabric" OUTPUT_DIRECTORY = doxygen OUTPUT_LANGUAGE = English -FULL_PATH_NAMES = YES +FULL_PATH_NAMES = NO MARKDOWN_SUPPORT = YES RECURSIVE = YES QUIET = YES diff --git a/docs/conf.py b/docs/conf.py index de863ab39..134d1d4a2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -2,6 +2,11 @@ from subprocess import run from os import makedirs +# ---------------------------------------- +# This is the Sphinx configuration file, which wraps +# the calls to Doxygen and Breathe. +# ---------------------------------------- + DOCS_ROOT = dirname(realpath(__file__)) DOXYGEN_OUT = join(DOCS_ROOT, "doxygen", "xml") APIDOC_OUT_DIR = join(DOCS_ROOT, "apidoc") @@ -11,7 +16,9 @@ run("doxygen", cwd=DOCS_ROOT, check=True, shell=True) run( - "breathe-apidoc {} -o {}".format(DOXYGEN_OUT, APIDOC_OUT_DIR), + "breathe-apidoc {} -o {} -f -m".format( + DOXYGEN_OUT, APIDOC_OUT_DIR + ), cwd=DOCS_ROOT, check=True, shell=True, @@ -21,14 +28,16 @@ copyright = "2022, Simon Shillaker" author = "Simon Shillaker" -extensions = ["breathe", "myst_parser"] +extensions = ["breathe", "myst_parser", "sphinx_rtd_theme"] templates_path = ["source/templates"] html_static_path = ["source/static"] exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] -html_theme = "alabaster" +html_theme = "sphinx_rtd_theme" breathe_projects = {"Faabric": DOXYGEN_OUT} breathe_default_project = "Faabric" +breathe_default_members = ("members", "undoc-members") + diff --git a/docs/index.rst b/docs/index.rst index d8d73aa62..5b4e22b57 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,18 +3,22 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -Welcome to Faabric's documentation! +Faabric Documentation =================================== +These docs include some pages on specific topics, as well as the autogenerated +docs from the code. + .. toctree:: :maxdepth: 2 - :caption: Contents: + :caption: Topics source/development source/mpi_native - apidoc/namespacelist - apidoc/classlist -Most of this documentation is auto-generated from the code, but includes a few -extra pieces of stand-alone content referenced in the sidebar. +.. toctree:: + :maxdepth: 1 + :caption: Code + apidoc/namespacelist + apidoc/classlist diff --git a/include/faabric/endpoint/Endpoint.h b/include/faabric/endpoint/Endpoint.h index d335b66ea..c77b4ad8a 100644 --- a/include/faabric/endpoint/Endpoint.h +++ b/include/faabric/endpoint/Endpoint.h @@ -6,6 +6,10 @@ #include namespace faabric::endpoint { + +/** + * Superclass for HTTP endpoints. + */ class Endpoint { public: From 3f3870020e2286d0fb263aae65b2ca2dbc30bdfd Mon Sep 17 00:00:00 2001 From: Simon Shillaker Date: Fri, 11 Feb 2022 11:19:25 +0100 Subject: [PATCH 11/15] Remove comment --- docs/index.rst | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 5b4e22b57..91f475c55 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,8 +1,3 @@ -.. Faabric documentation master file, created by - sphinx-quickstart on Tue Feb 8 14:45:52 2022. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - Faabric Documentation =================================== From 60eb7f57ae6af31aaff55bd39b233d3e2ef4ee5d Mon Sep 17 00:00:00 2001 From: Simon Shillaker Date: Fri, 11 Feb 2022 11:22:02 +0100 Subject: [PATCH 12/15] Update README --- README.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index d75df5b2b..0bc6fe7c5 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,10 @@ distributed serverless runtimes. It began life as part of [Faasm](https://github.com/faasm/faasm), but is now a stand-alone library that can be used as a base for other distributed runtimes. +See the [documentation](https://faabric.readthedocs.io/en/latest/) for more +details. + ## Usage You can see a simple example in the [examples](examples) directory. -## Development - -See the [development docs](docs/development.md) for more information on -developing Faabric. - From 5c02816ac732651c5e953a51fcf0da8d94576719 Mon Sep 17 00:00:00 2001 From: Simon Shillaker Date: Fri, 11 Feb 2022 11:34:02 +0100 Subject: [PATCH 13/15] Formatting --- docs/conf.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 134d1d4a2..dddf2a98d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,9 +16,7 @@ run("doxygen", cwd=DOCS_ROOT, check=True, shell=True) run( - "breathe-apidoc {} -o {} -f -m".format( - DOXYGEN_OUT, APIDOC_OUT_DIR - ), + "breathe-apidoc {} -o {} -f -m".format(DOXYGEN_OUT, APIDOC_OUT_DIR), cwd=DOCS_ROOT, check=True, shell=True, @@ -40,4 +38,3 @@ breathe_projects = {"Faabric": DOXYGEN_OUT} breathe_default_project = "Faabric" breathe_default_members = ("members", "undoc-members") - From 3b9247bee6b4836f471a67a5f44f88a3258c75cc Mon Sep 17 00:00:00 2001 From: Simon Shillaker Date: Fri, 11 Feb 2022 11:35:47 +0100 Subject: [PATCH 14/15] Check out code in docs build --- .github/workflows/tests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4f9c498f0..0ccc0b589 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -37,6 +37,10 @@ jobs: run: working-directory: /code/faabric steps: + - name: "Fetch ref" + run: git fetch origin ${GITHUB_REF}:ci-branch + - name: "Check out branch" + run: git checkout --force ci-branch - name: "TEMP - install doxygen" run: apt install -y doxygen - name: "TEMP - install python deps" From 229aa54d1ac297ae8a3e07240e0eeafad92d0eaa Mon Sep 17 00:00:00 2001 From: Simon Shillaker Date: Fri, 11 Feb 2022 11:40:16 +0100 Subject: [PATCH 15/15] Remove conan-cache dep from docs build --- .github/workflows/tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0ccc0b589..7a648c5d4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -29,7 +29,6 @@ jobs: docs: if: github.event.pull_request.draft == false - needs: [conan-cache] runs-on: ubuntu-latest container: image: faasm/faabric:0.2.4