Removing GLSLang as core dependency

.ccls

@@ -24,4 +24,5 @@
 -I./single_include/
 -I./vk_ndk_wrapper_include/
 -I./test/compiled_shaders_include/
+-I./test/utils/
 

.github/workflows/cpp_tests.yml

@@ -10,7 +10,7 @@ jobs:
   cpp-tests:
 
     runs-on: ubuntu-18.04
-    container: axsauze/kompute-builder:0.2
+    container: axsauze/kompute-builder:0.3
 
     steps:
     - uses: actions/checkout@v2

.github/workflows/python_tests.yml

@@ -10,7 +10,7 @@ jobs:
   python-tests:
 
     runs-on: ubuntu-18.04
-    container: axsauze/kompute-builder:0.2
+    container: axsauze/kompute-builder:0.3
 
     steps:
     - uses: actions/checkout@v2

.gitignore

@@ -186,5 +186,5 @@ release/
 # Kompute
 swiftshader/
 vk_swiftshader_icd.json
-
+tmp_kp_shader.comp.spv
 

.gitmodules

@@ -14,10 +14,6 @@
 	path = python/pybind11
 	url = https://github.com/pybind/pybind11
     branch = v2.6.1
-[submodule "external/glslang"]
-	path = external/glslang
-	url = https://github.com/KhronosGroup/glslang/
-    branch = 11.1.0
 [submodule "external/fmt"]
 	path = external/fmt
 	url = https://github.com/fmtlib/fmt

CHANGELOG.md

@@ -34,7 +34,7 @@
 **Closed issues:**
 
 - Update memory barriers to align with tensor staging/primary memory revamp [\#181](https://github.com/EthicalML/vulkan-kompute/issues/181)
-- Move shader defaultResource inside kp::Shader class [\#175](https://github.com/EthicalML/vulkan-kompute/issues/175)
+- Move shader defaultResource inside kp_test_utils::Shader class [\#175](https://github.com/EthicalML/vulkan-kompute/issues/175)
 - Reach at least 90% code coverage on tests [\#170](https://github.com/EthicalML/vulkan-kompute/issues/170)
 - Add functionality to re-record sequence as now it's possible to update the underlying algorithm [\#169](https://github.com/EthicalML/vulkan-kompute/issues/169)
 - Use numpy arrays as default return value [\#166](https://github.com/EthicalML/vulkan-kompute/issues/166)

CMakeLists.txt

@@ -19,7 +19,6 @@ option(KOMPUTE_OPT_ENABLE_SPDLOG "Extra compile flags for Kompute, see docs for
 option(KOMPUTE_OPT_REPO_SUBMODULE_BUILD "Use the submodule repos instead of external package manager" 0)
 option(KOMPUTE_OPT_ANDROID_BUILD "Enable android compilation flags required" 0)
 option(KOMPUTE_OPT_DISABLE_VK_DEBUG_LAYERS "Explicitly disable debug layers even on debug" 0)
-option(KOMPUTE_OPT_DISABLE_SHADER_UTILS "Remove shader util code and dependencies including glslang" 0)
 option(KOMPUTE_OPT_DEPENDENCIES_SHARED_LIBS "Whether to use shared libraries for dependencies for install" 0)
 option(KOMPUTE_OPT_BUILD_AS_SHARED_LIB "Whether to build kompute as shared library" 0)
 # Build flags
@@ -55,19 +54,15 @@ if(KOMPUTE_OPT_DISABLE_VK_DEBUG_LAYERS)
     set(KOMPUTE_EXTRA_CXX_FLAGS "${KOMPUTE_EXTRA_CXX_FLAGS} -DKOMPUTE_DISABLE_VK_DEBUG_LAYERS=1")
 endif()
 
-if(NOT KOMPUTE_OPT_DISABLE_SHADER_UTILS)
-    if(KOMPUTE_OPT_INSTALL)
-        # Enable install parameters for glslang (overrides parameters passed)
-        # When install is enabled the glslang libraries become shared
-        set(ENABLE_GLSLANG_INSTALL ON CACHE BOOL "Enables install of glslang" FORCE)
+if(KOMPUTE_OPT_INSTALL)
+    # Enable install parameters for glslang (overrides parameters passed)
+    # When install is enabled the glslang libraries become shared
+    set(ENABLE_GLSLANG_INSTALL ON CACHE BOOL "Enables install of glslang" FORCE)
 
-        # By default we enable shared library based installation
-        if(KOMPUTE_OPT_DEPENDENCIES_SHARED_LIBS)
-            set(BUILD_SHARED_LIBS ON CACHE BOOL "Enables build of shared libraries" FORCE)
-        endif()
+    # By default we enable shared library based installation
+    if(KOMPUTE_OPT_DEPENDENCIES_SHARED_LIBS)
+        set(BUILD_SHARED_LIBS ON CACHE BOOL "Enables build of shared libraries" FORCE)
     endif()
-else()
-    set(KOMPUTE_EXTRA_CXX_FLAGS "${KOMPUTE_EXTRA_CXX_FLAGS} -DKOMPUTE_DISABLE_SHADER_UTILS=1")
 endif()
 
 set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DDEBUG=1 ${KOMPUTE_EXTRA_CXX_FLAGS} -DUSE_DEBUG_EXTENTIONS")

Makefile

@@ -194,7 +194,7 @@ build_single_header:
 		"single_include/kompute/Kompute.hpp"
 
 win_build_xxd:
-	cd external/bin/ && gcc -o xxd.exe xxd.c -DCYGWIN
+	cd external/bin/ && gcc.exe -o xxd.exe xxd.c -DCYGWIN
 
 format:
 	$(CLANG_FORMAT_BIN) -i -style="{BasedOnStyle: mozilla, IndentWidth: 4}" src/*.cpp src/include/kompute/*.hpp test/*cpp

README.md

@@ -74,7 +74,8 @@ void kompute(const std::string& shader) {
     kp::Constants pushConstsB({ 3.0 });
 
     auto algorithm = mgr.algorithm(params,
-                                   kp::Shader::compileSource(shader),
+                                   // See documentation shader section for compileSource
+                                   compileSource(shader),
                                    workgroup,
                                    specConsts,
                                    pushConstsA);
@@ -165,6 +166,7 @@ def kompute(shader):
     push_consts_a = [2]
     push_consts_b = [3]
 
+    # See documentation shader section for compile_source
     spirv = kp.Shader.compile_source(shader)
 
     algo = mgr.algorithm(params, spirv, workgroup, spec_consts, push_consts_a)
@@ -372,7 +374,7 @@ You can also access the <a href="https://github.com/EthicalML/vulkan-kompute/tre
 
 ### Simple examples
 
-* [Pass shader as raw string](https://kompute.cc/overview/advanced-examples.html#simple-shader-example)
+* [Simple multiplication example](https://kompute.cc/overview/advanced-examples.html#simple-shader-example)
 * [Record batch commands with a Kompute Sequence](https://kompute.cc/overview/advanced-examples.html#record-batch-commands)
 * [Run Asynchronous Operations](https://kompute.cc/overview/advanced-examples.html#asynchronous-operations)
 * [Run Parallel Operations Across Multiple GPU Queues](https://kompute.cc/overview/advanced-examples.html#parallel-operations)

docker-builders/KomputeBuilder.Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
+FROM ubuntu:20.04
 
 # Base packages from default ppa
 RUN apt-get update -y
@@ -26,9 +26,14 @@ RUN apt-get install -y libxext-dev
 
 COPY --from=axsauze/swiftshader:0.1 /swiftshader/ /swiftshader/
 
+# GLSLANG tools for tests
+RUN apt-get install -y glslang-tools
+
 # Setup Python
 RUN apt-get install -y python3-pip
 
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10
+
 RUN mkdir builder
 WORKDIR /builder
 

docker-builders/Makefile

@@ -1,9 +1,9 @@
 
 build_kompute_builder:
-	docker build .. -f KomputeBuilder.Dockerfile -t axsauze/kompute-builder:0.2
+	docker build .. -f KomputeBuilder.Dockerfile -t axsauze/kompute-builder:0.3
 
 push_kompute_builder: build_kompute_builder
-	docker push axsauze/kompute-builder:0.2
+	docker push axsauze/kompute-builder:0.3
 
 build_swiftshader:
 	docker build .. -f Swiftshader.Dockerfile -t axsauze/swiftshader:0.1

docs/overview/advanced-examples.rst

@@ -55,7 +55,8 @@ The example below shows how you can enable the "VK_EXT_shader_atomic_float" exte
                  atomicAdd(pa[2], pcs.z);
              })");
 
-       std::vector<uint32_t> spirv = kp::Shader::compileSource(shader);
+       // See shader documentation section for compileSource
+       std::vector<uint32_t> spirv = compileSource(shader);
 
        std::shared_ptr<kp::Sequence> sq = nullptr;
 
@@ -102,7 +103,8 @@ We also provide tools that allow you to `convert shaders into C++ headers <https
                 throw std::runtime_error("Kompute OpMult expected 3 tensors but got " + tensors.size());
             }
 
-            std::vector<uint32_t> spirv = kp::Shader::compileSource(R"(
+           // See shader documentation section for compileSource
+            std::vector<uint32_t> spirv = compileSource(R"(
                 #version 450
 
                 layout(set = 0, binding = 0) buffer tensorLhs {
@@ -215,7 +217,8 @@ In this case we create a shader that should take a couple of milliseconds to run
         }
     )");
 
-    auto algo = mgr.algorithm({tensor}, kp::Shader::compileSource(shader));
+    // See shader documentation section for compileSource
+    auto algo = mgr.algorithm({tensor}, compileSource(shader));
 
 Now we are able to run the await function on the default sequence. 
 
@@ -361,7 +364,8 @@ Similar to the asyncrhonous usecase above, we can still run synchronous commands
            }
        )");
 
-       std::vector<uint32_t> spirv = kp::Shader::compileSource(shader);
+       // See shader documentation section for compileSource
+       std::vector<uint32_t> spirv = compileSource(shader);
 
        std::shared_ptr<kp::Algorithm> algo = mgr.algorithm({tensorA, tenssorB}, spirv);
 

docs/overview/build-system.rst

@@ -69,8 +69,6 @@ Compile Flags
      - Enable debug build including debug flags (enabled by cmake debug build)
    * - -DKOMPUTE_DISABLE_VK_DEBUG_LAYERS
      - Disable the debug Vulkan SDK Layers, mainly used for android builds
-   * - -DKOMPUTE_DISABLE_SHADER_UTILS
-     - Disable the shader utils and skip adding glslang as dependency
 
 Other CMake Flags
 ~~~~~~~~~~~~~~~~~

docs/overview/ci-tests.rst

@@ -38,10 +38,7 @@ Running on the CPU
 
 We use `Swiftshader <https://github.com/google/swiftshader>`_ to enable us to run the Kompute framework directly on the CPU for the CI tests.
 
-Even though Swiftshader is optimized to function as a high-performance CPU backend for the Vulkan SDK, there are several limitations, the most notable in context of Kompute are:
-
-* Loading files (spirv or text) leads to segfault
-* Loading raw text string shaders leads to segfault
+Even though Swiftshader is optimized to function as a high-performance CPU backend for the Vulkan SDK, there are several limitations, the most notable are limitations in extensions.
 
 This is one of the main reason why only a subset of the tests are run in the CI.
 

docs/overview/reference.rst

@@ -119,13 +119,4 @@ The :class:`kp::OpMemoryBarrier` is a tensor only operation which adds memory ba
 .. doxygenclass:: kp::OpTensorSyncDevice
    :members:
 
-Shader
---------
-
-The :class:`kp::Shader` class contains a set of utilities to compile and process shaders.
-
-.. doxygenclass:: kp::Shader
-   :members:
-
-
 

docs/overview/shaders-to-headers.rst

@@ -3,21 +3,29 @@
 Processing Shaders with Kompute
 =====================
 
-Kompute allows for two main ways of interacting with shaders - namely:
-
-* Integration with [glslang](https://github.com/KhronosGroup/glslang) for online/runtime shader compilation
-* A CLI that coverts shaders into C++ header files
-
-Processing Shaders Online via Kompute Shader Utils
----------------
+Demo / testing function to compile shaders
+----------------------------------
 
-Kompute provides a set of helper functions that expose the C++ functionality of the glslang Khronos framework to process shader sources online during runtime.
+GLSLANG was initially integrated as part of the framework but it now has been removed due to the license of the glslang pre-processor being under a custom NVIDIA license which explicitly excludes grant of any licenses to NVIDIA's patents in the preprocessor. This is covered in more detail here: https://github.com/EthicalML/vulkan-kompute/pull/235
 
-It's worth emphasising that the suggested approach is to process shaders offline, so the section below is suggested to convert shaders to either their respective SPV format, or convert them into C++ sources that would be embedded as part of the resulting binary.
+For users that are looking to quickly test the processors it is possible to use the function that is provided in the examples which provides a (non-thread-safe / non-robust) implementation that compiles a shader string into spirv bytes. It is not recommended to use in production but it does enable for faster iteration cycles during development.
 
-The Shader utility function can be skipped on build time through compiler flags - for more information on this you should read the `build section <build-system.rst>`_.
+.. code-block:: cpp
+    :linenos:
 
-More details on the shader utils can be found in the :class:`kp::Shader` section of the `C++ reference page <reference.rst>`_.
+    static std::vector<uint32_t>
+    compileSource(
+      const std::string& source)
+    {
+        if (system(std::string("glslangValidator --stdin -S comp -V -o tmp_kp_shader.comp.spv << END
+    " + source + "
+    END").c_str()))
+            throw std::runtime_error("Error running glslangValidator command");
+        std::ifstream fileStream("tmp_kp_shader.comp.spv", std::ios::binary);
+        std::vector<char> buffer;
+        buffer.insert(buffer.begin(), std::istreambuf_iterator<char>(fileStream), {});
+        return {(uint32_t*)buffer.data(), (uint32_t*)(buffer.data() + buffer.size())};
+    }
 
 Converting Shaders into C / C++ Header Files
 ----------------------------------

examples/array_multiplication/src/Main.cpp

@@ -5,6 +5,18 @@
 
 #include "kompute/Kompute.hpp"
 
+static std::vector<uint32_t>
+compileSource(
+  const std::string& source)
+{
+    if (system(std::string("glslangValidator --stdin -S comp -V -o tmp_kp_shader.comp.spv << END\n" + source + "\nEND").c_str()))
+        throw std::runtime_error("Error running glslangValidator command");
+    std::ifstream fileStream("tmp_kp_shader.comp.spv", std::ios::binary);
+    std::vector<char> buffer;
+    buffer.insert(buffer.begin(), std::istreambuf_iterator<char>(fileStream), {});
+    return {(uint32_t*)buffer.data(), (uint32_t*)(buffer.data() + buffer.size())};
+}
+
 int main()
 {
 #if KOMPUTE_ENABLE_SPDLOG
@@ -39,7 +51,7 @@ int main()
 
     std::vector<std::shared_ptr<kp::Tensor>> params = { tensorInA, tensorInB, tensorOut };
 
-    std::shared_ptr<kp::Algorithm> algo = mgr.algorithm(params, kp::Shader::compileSource(shader));
+    std::shared_ptr<kp::Algorithm> algo = mgr.algorithm(params, kp_test_utils::compileSource(shader));
 
     mgr.sequence()
         ->record<kp::OpTensorSyncDevice>(params)

examples/godot_examples/custom_module/kompute_summator/KomputeSummatorNode.cpp

@@ -4,6 +4,18 @@
 
 #include "KomputeSummatorNode.h"
 
+static std::vector<uint32_t>
+compileSource(
+  const std::string& source)
+{
+    if (system(std::string("glslangValidator --stdin -S comp -V -o tmp_kp_shader.comp.spv << END\n" + source + "\nEND").c_str()))
+        throw std::runtime_error("Error running glslangValidator command");
+    std::ifstream fileStream("tmp_kp_shader.comp.spv", std::ios::binary);
+    std::vector<char> buffer;
+    buffer.insert(buffer.begin(), std::istreambuf_iterator<char>(fileStream), {});
+    return {(uint32_t*)buffer.data(), (uint32_t*)(buffer.data() + buffer.size())};
+}
+
 KomputeSummatorNode::KomputeSummatorNode() {
     this->_init();
 }
@@ -52,9 +64,9 @@ void KomputeSummatorNode::_init() {
         )");
 
         std::shared_ptr<kp::Algorithm> algo =
-          mgr.algorithm(
+          this->mManager.algorithm(
                 { this->mPrimaryTensor, this->mSecondaryTensor },
-                kp::Shader::compileSource(shader));
+                compileSource(shader));
 
 
         // First we ensure secondary tensor loads to GPU
@@ -63,7 +75,7 @@ void KomputeSummatorNode::_init() {
                 { this->mSecondaryTensor });
 
         // Then we run the operation with both tensors
-        sq->record<kp::OpAlgoDispatch>(algo)
+        sq->record<kp::OpAlgoDispatch>(algo);
 
         // We map the result back to local 
         sq->record<kp::OpTensorSyncLocal>(

examples/godot_examples/gdnative_shared/src/KomputeSummator.cpp

@@ -5,6 +5,18 @@
 
 #include "KomputeSummator.hpp"
 
+static std::vector<uint32_t>
+compileSource(
+  const std::string& source)
+{
+    if (system(std::string("glslangValidator --stdin -S comp -V -o tmp_kp_shader.comp.spv << END\n" + source + "\nEND").c_str()))
+        throw std::runtime_error("Error running glslangValidator command");
+    std::ifstream fileStream("tmp_kp_shader.comp.spv", std::ios::binary);
+    std::vector<char> buffer;
+    buffer.insert(buffer.begin(), std::istreambuf_iterator<char>(fileStream), {});
+    return {(uint32_t*)buffer.data(), (uint32_t*)(buffer.data() + buffer.size())};
+}
+
 namespace godot {
 
 KomputeSummator::KomputeSummator() {
@@ -58,7 +70,7 @@ void KomputeSummator::_init() {
         // Then we run the operation with both tensors
         this->mSequence->record<kp::OpAlgoCreate>(
             { this->mPrimaryTensor, this->mSecondaryTensor }, 
-            kp::Shader::compileSource(shader));
+            compileSource(shader));
 
         // We map the result back to local 
         this->mSequence->record<kp::OpTensorSyncLocal>(

examples/neural_network_vgg7/sh_conv.py

@@ -1,8 +1,15 @@
+import os
+
 import kp
 
+def compile_source(source):
+    os.system("glslangValidator --stdin -S comp -V -o tmp_kp_shader.comp.spv << END\n" + source + "\nEND")
+    return open("tmp_kp_shader.comp.spv", "rb").read()
+
+
 # This is the convolution & leakyrelu shader.
 global conv_shader
-conv_shader = kp.Shader.compile_source("""
+conv_shader = compile_source("""
 #version 450
 
 layout (local_size_x = 8, local_size_y = 2) in;