Merge pull request #119 from EthicalML/114_further_tests_on_ci

Added further tests to CI and provide Dockerimage with builds to swiftshader

.dockerignore

@@ -0,0 +1,5 @@
+build/*
+examples/*
+docker-builders/
+swiftshader/
+

.github/workflows/cpp_tests.yml

@@ -0,0 +1,34 @@
+name: C++ Tests
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  cpp-tests:
+
+    runs-on: ubuntu-18.04
+    container: axsauze/kompute-builder:0.2
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: recursive
+    - name: configure-cpp
+      run: |
+        cmake -Bbuild/ \
+                  -DKOMPUTE_OPT_INSTALL=0                    \
+                  -DKOMPUTE_OPT_REPO_SUBMODULE_BUILD=1       \
+                  -DKOMPUTE_OPT_BUILD_TESTS=1                \
+                  -DKOMPUTE_OPT_ENABLE_SPDLOG=1              \
+                  -DSPDLOG_INSTALL=1
+    - name: build-cpp
+      run: |
+        make mk_build_tests
+    - name: test-cpp
+      run: |
+        export VK_ICD_FILENAMES=/swiftshader/vk_swiftshader_icd.json
+        make mk_run_tests_cpu_only
+

.github/workflows/python_tests.yml

@@ -0,0 +1,32 @@
+name: Python Tests
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  python-tests:
+
+    runs-on: ubuntu-18.04
+    container: axsauze/kompute-builder:0.2
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: recursive
+    - name: configure-python
+      run: |
+        pip3 install -r python/test/requirements-dev.txt
+    - name: build-python
+      run: |
+        pip3 install .
+    - name: test-python
+      run: |
+        export VK_ICD_FILENAMES=/swiftshader/vk_swiftshader_icd.json
+        pytest -v python/test/test_array_multiplication.py
+        pytest -v python/test/test_kompute.py -k "test_opmult"
+        pytest -v python/test/test_logistic_regression.py
+
+

.gitignore

@@ -183,4 +183,7 @@ tmp/
 build/
 release/
 
+# Kompute
+swiftshader/
+
 

Makefile

@@ -56,37 +56,56 @@ mk_cmake:
 	cmake \
 		-Bbuild \
 		$(MK_CMAKE_EXTRA_FLAGS) \
-		-DCMAKE_TOOLCHAIN_FILE=$(VCPKG_UNIX_PATH) \
+		-DKOMPUTE_EXTRA_CXX_FLAGS=$(MK_KOMPUTE_EXTRA_CXX_FLAGS) \
 		-DCMAKE_BUILD_TYPE=$(MK_BUILD_TYPE) \
 		-DCMAKE_INSTALL_PREFIX=$(MK_INSTALL_PATH) \
-		-DKOMPUTE_EXTRA_CXX_FLAGS=$(MK_KOMPUTE_EXTRA_CXX_FLAGS) \
 		-DKOMPUTE_OPT_INSTALL=1 \
-		-DKOMPUTE_OPT_REPO_SUBMODULE_BUILD=0 \
+		-DKOMPUTE_OPT_REPO_SUBMODULE_BUILD=1 \
 		-DKOMPUTE_OPT_BUILD_TESTS=1 \
 		-DKOMPUTE_OPT_BUILD_DOCS=1 \
 		-DKOMPUTE_OPT_BUILD_SHADERS=1 \
 		-DKOMPUTE_OPT_BUILD_SINGLE_HEADER=1 \
 		-DKOMPUTE_OPT_ENABLE_SPDLOG=1 \
+		-DSPDLOG_INSTALL=1 \
 		-G "Unix Makefiles"
 
 mk_build_all:
-	make -C build/
+	cmake --build build/. --parallel
 
 mk_build_docs:
-	make -C build/ docs
+	cmake --build build/. --target docs --parallel
 
 mk_build_kompute:
-	make -C build/ kompute
+	cmake --build build/. --target kompute --parallel
 
 mk_build_tests:
-	make -C build/ test_kompute
+	cmake --build build/ --target test_kompute --parallel
 
 mk_run_docs: mk_build_docs
 	(cd build/docs/sphinx && python2.7 -m SimpleHTTPServer)
 
 mk_run_tests: mk_build_tests
 	./build/test/test_kompute $(FILTER_TESTS)
 
+mk_build_swiftshader_library:
+	git clone https://github.com/google/swiftshader || echo "Assuming already cloned"
+	# GCC 8 or above is required otherwise error on "filesystem" lib will appear
+	CC="/usr/bin/gcc-8" CXX="/usr/bin/g++-8" cmake swiftshader/. -Bswiftshader/build/
+	cmake --build swiftshader/build/. --parallel
+
+mk_run_tests_cpu: export VK_ICD_FILENAMES=$(PWD)/swiftshader/build/vk_swiftshader_icd.json
+mk_run_tests_cpu: mk_build_swiftshader_library mk_build_tests mk_run_tests_cpu_only
+
+mk_run_tests_cpu_only:
+	./build/test/test_kompute --gtest_filter="TestLogisticRegressionAlgorithm.*"
+	./build/test/test_kompute --gtest_filter="TestManager.*"
+	./build/test/test_kompute --gtest_filter="TestOpAlgoBase.ShaderCompiledDataFromConstructor"
+	./build/test/test_kompute --gtest_filter="TestOpTensorCopy.*"
+	./build/test/test_kompute --gtest_filter="TestOpTensorCreate.*"
+	./build/test/test_kompute --gtest_filter="TestOpTensorSync.*"
+	./build/test/test_kompute --gtest_filter="TestSequence.*"
+	./build/test/test_kompute --gtest_filter="TestTensor.*"
+
 
 ####### Visual studio build shortcut commands #######
 
@@ -110,6 +129,7 @@ vs_cmake:
 		-DKOMPUTE_OPT_BUILD_SHADERS=1 \
 		-DKOMPUTE_OPT_BUILD_SINGLE_HEADER=1 \
 		-DKOMPUTE_OPT_ENABLE_SPDLOG=1 \
+		-DSPDLOG_INSTALL=1 \
 		-G "Visual Studio 16 2019"
 
 vs_build_all:
@@ -136,16 +156,29 @@ vs_run_tests: vs_build_tests
 ####### Create release ######
 
 update_builder_image:
+	docker build . -f docker-builders/KomputeBuilder.Dockerfile \
+		-t axsauze/kompute-builder:0.2
+	docker push axsauze/kompute-builder:0.2
+
+update_vulkan_sdk:
 	docker build -f builders/Dockerfile.linux . \
-		-t axsauze/kompute-builder:0.1
-	docker push axsauze/kompute-builder:0.1
+		-t axsauze/vulkan-sdk:0.1
+	docker push axsauze/vulkan-sdk:0.1
 
 create_linux_release:
 	docker run -it \
 		-v $(pwd):/workspace \
 		axsauze/kompute-builder:0.1 \
 		/workspace/scripts/build_release_linux.sh
 
+
+####### Run CI Commands #######
+
+# This command uses act to replicate github action
+# https://github.com/nektos/act
+run_ci:
+	act
+
 ####### General project commands #######
 
 install_python_reqs:

README.md

@@ -46,7 +46,7 @@
 
 Kompute is provided as a single header file [`Kompute.hpp`](#setup). See [build-system section](#build-overview) for configurations available.
 
-### Trying the Colab Notebook
+### Interactive Notebook & Hands on Videos
 
 You are able to try out the interactive Colab Notebooks which allow you to use a free GPU. The available examples are the Python and C++ examples below:
 
@@ -80,6 +80,40 @@ You are able to try out the interactive Colab Notebooks which allow you to use a
 </table>
 
 
+You can also check out the two following getting started videos presented at the FOSDEM 2021 conference. 
+
+Both talks have annotated sections - the intro for both is almost the same so you can skip to the more specific content.
+
+<table>
+<tr>
+
+<td width="50%">
+<h5>For <a href="https://www.youtube.com/watch?v=Xz4fiQNmGSA">C++ & Vulkan SDK</a> Enthusiasts</h5>
+</td>
+
+<td>
+<h5>For <a href="https://www.youtube.com/watch?v=AJRyZ09IUdg">Python & Machine Learning</a> Enthusiasts</h5>
+</td>
+
+</tr>
+<tr>
+
+<td width="50%">
+<a href="https://www.youtube.com/watch?v=Xz4fiQNmGSA">
+<img src="https://raw.githubusercontent.com/EthicalML/vulkan-kompute/master/docs/images/kompute-cpp-video.png">
+</a>
+</td>
+
+<td>
+<a href="https://www.youtube.com/watch?v=AJRyZ09IUdg">
+<img src="https://raw.githubusercontent.com/EthicalML/vulkan-kompute/master/docs/images/kompute-python-video.png">
+</a>
+</td>
+
+</tr>
+</table>
+
+
 ### Your First Kompute
 
 Below you can find both the C++ and Python version of a simple GPU multiplication snippet with Kompute.
@@ -336,21 +370,27 @@ To update the documentation you will need to:
 
 ##### Running tests
 
-To run tests you can use the helper top level Makefile
+Running the unit tests has been significantly simplified for contributors.
 
-For visual studio you can run
+The tests run on CPU, and can be triggered using the ACT command line interface (https://github.com/nektos/act) - once you install the command line (And start the Docker daemon) you just have to type:
 
 ```
-make vs_cmake
-make vs_run_tests VS_BUILD_TYPE="Release"
+$ act
+
+[Python Tests/python-tests] 🚀  Start image=axsauze/kompute-builder:0.2
+[C++ Tests/cpp-tests      ] 🚀  Start image=axsauze/kompute-builder:0.2
+[C++ Tests/cpp-tests      ]   🐳  docker run image=axsauze/kompute-builder:0.2 entrypoint=["/usr/bin/tail" "-f" "/dev/null"] cmd=[]
+[Python Tests/python-tests]   🐳  docker run image=axsauze/kompute-builder:0.2 entrypoint=["/usr/bin/tail" "-f" "/dev/null"] cmd=[]
+...
 ```
 
-For unix you can run
+The repository contains unit tests for the C++ and Python code, and can be found under the `test/` and `python/test` folder.
 
-```
-make mk_cmake MK_BUILD_TYPE="Release"
-make mk_run_tests
-```
+The tests are currently run through the CI using Github Actions. It uses the images found in `docker-builders/`.
+
+In order to minimise hardware requirements the tests can run without a GPU, directly in the CPU using [Swiftshader](https://github.com/google/swiftshader).
+
+For more information on how the CI and tests are setup, you can go to the [CI, Docker and Tests Section](https://kompute.cc/overview/ci-tests.html) in the documentation.
 
 ## Motivations
 

docker-builders/KomputeBuilder.Dockerfile

@@ -0,0 +1,35 @@
+FROM ubuntu:18.04
+
+# Base packages from default ppa
+RUN apt-get update -y
+RUN apt-get install -y wget
+RUN apt-get install -y gnupg
+RUN apt-get install -y ca-certificates
+RUN apt-get install -y software-properties-common
+
+# Repository to latest cmake
+RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add -
+RUN apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main'
+
+# Repository for latest git (needed for gh actions)
+RUN add-apt-repository -y ppa:git-core/ppa
+
+# Refresh repositories
+RUN apt update -y
+
+RUN apt install -y git
+RUN apt-get install -y cmake g++
+RUN apt-get install -y libvulkan-dev
+# Swiftshader dependencies
+RUN apt-get install -y libx11-dev zlib1g-dev
+RUN apt-get install -y libxext-dev
+
+COPY --from=axsauze/swiftshader:0.1 /swiftshader/ /swiftshader/
+
+# Setup Python
+RUN apt-get install -y python3-pip
+
+RUN mkdir builder
+WORKDIR /builder
+
+

docker-builders/Makefile

@@ -0,0 +1,19 @@
+
+build_kompute_builder:
+	docker build .. -f KomputeBuilder.Dockerfile -t axsauze/kompute-builder:0.2
+
+push_kompute_builder: build_kompute_builder
+	docker push axsauze/kompute-builder:0.2
+
+build_swiftshader:
+	docker build .. -f Swiftshader.Dockerfile -t axsauze/swiftshader:0.1
+
+push_swiftshader: build_swiftshader
+	docker push axsauze/swiftshader:0.1
+
+build_vulkan_sdk:
+	docker build .. -f VulkanSDK.Dockerfile -t axsauze/vulkan-sdk:0.1
+
+push_vulkan_sdk: build_vulkan_sdk
+	docker push axsauze/vulkan-sdk:0.1
+