GLSL Compiler

.gitmodules

@@ -14,3 +14,6 @@
 	path = python/pybind11
 	url = https://github.com/pybind/pybind11
     branch = v2.6.1
+[submodule "external/glslang"]
+	path = external/glslang
+	url = https://github.com/KhronosGroup/glslang.git

CMakeLists.txt

@@ -18,6 +18,7 @@ option(KOMPUTE_OPT_ENABLE_SPDLOG "Extra compile flags for Kompute, see docs for
 option(KOMPUTE_OPT_REPO_SUBMODULE_BUILD, "Use the submodule repos instead of external package manager" 0)
 option(KOMPUTE_OPT_ANDOID_BUILD "Enable android compilation flags required" 0)
 option(KOMPUTE_OPT_DISABLE_VK_DEBUG_LAYERS "Explicitly disable debug layers even on debug" 0)
+option(KOMPUTE_OPT_INCLUDE_GLSLANG "Include GLSLang for runtime shader compilation")
 # Build flags
 set(KOMPUTE_EXTRA_CXX_FLAGS "" CACHE STRING "Extra compile flags for Kompute, see docs for full list")
 
@@ -26,6 +27,10 @@ if(KOMPUTE_OPT_ENABLE_SPDLOG)
     set(SPDLOG_INSTALL, 1)
 endif()
 
+if(KOMPUTE_OPT_INCLUDE_GLSLANG)
+    set(KOMPUTE_EXTRA_CXX_FLAGS "${KOMPUTE_EXTRA_CXX_FLAGS} -DKOMPUTE_INCLUDE_GLSLANG=1")
+endif()
+
 if(KOMPUTE_OPT_ANDOID_BUILD)
     set(KOMPUTE_EXTRA_CXX_FLAGS "${KOMPUTE_EXTRA_CXX_FLAGS} -DVK_USE_PLATFORM_ANDROID_KHR")
 endif()

python/src/main.cpp

@@ -3,6 +3,9 @@
 #include <pybind11/numpy.h>
 
 #include <kompute/Kompute.hpp>
+#if KOMPUTE_INCLUDE_GLSLANG
+    #include <kompute/GLSLCompiler.hpp>
+#endif
 
 #include "docstrings.hpp"
 
@@ -299,6 +302,14 @@ PYBIND11_MODULE(kp, m) {
             "Evaluates asynchronously an operation using a custom shader provided as raw string or spirv bytes with explicitly named Sequence")
         .def("eval_async_algo_lro", &kp::Manager::evalOpAsync<kp::OpAlgoLhsRhsOut>,
             "Evaluates asynchronously operation to run left right out operation with custom shader with explicitly named Sequence");
+
+    #if KOMPUTE_INCLUDE_GLSLANG
+    m.def("compile_glsl_to_spirv", 
+    [](std::string glsl_source){
+        const auto spirv = GLSLCompiler::compile_to_spirv(glsl_source, "main");
+        return py::bytes(std::string(spirv.begin(), spirv.end()));
+    });
+    #endif //KOMPUTE_INCLUDE_GLSLANG
 
 #ifdef VERSION_INFO
     m.attr("__version__") = VERSION_INFO;

python/test/test_kompute.py

@@ -54,7 +54,8 @@ def test_opalgobase_data():
 
     mgr.eval_tensor_create_def([tensor_in_a, tensor_in_b, tensor_out])
 
-    mgr.eval_algo_str_def([tensor_in_a, tensor_in_b, tensor_out], shaderData)
+    spirv = kp.compile_glsl_to_spirv(shaderData)
+    mgr.eval_algo_str_def([tensor_in_a, tensor_in_b, tensor_out], spirv)
 
     mgr.eval_tensor_sync_local_def([tensor_out])
 

setup.py

@@ -43,6 +43,7 @@ def build_extension(self, ext):
                       '-DKOMPUTE_OPT_BUILD_PYTHON=1',
                       '-DKOMPUTE_OPT_ENABLE_SPDLOG=1',
                       '-DKOMPUTE_OPT_REPO_SUBMODULE_BUILD=1',
+                      '-DKOMPUTE_OPT_INCLUDE_GLSLANG=1',
                       '-DPYTHON_EXECUTABLE=' + sys.executable]
 
         cfg = 'Debug' if self.debug else 'Release'

src/CMakeLists.txt

@@ -124,3 +124,14 @@ if(KOMPUTE_OPT_INSTALL)
         DESTINATION lib/cmake/kompute)
 endif()
 
+
+
+if(KOMPUTE_OPT_INCLUDE_GLSLANG)
+    add_subdirectory(${PROJECT_SOURCE_DIR}/external/glslang ${CMAKE_CURRENT_BINARY_DIR}/kompute_glslang)
+    target_include_directories(kompute PUBLIC $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/external/glslang>)
+
+    target_link_libraries(kompute glslang)
+    target_link_libraries(kompute MachineIndependent)
+    target_link_libraries(kompute glslang-default-resource-limits)
+    target_link_libraries(kompute SPIRV)
+endif()

src/GLSLCompiler.cpp

@@ -0,0 +1,76 @@
+#if KOMPUTE_INCLUDE_GLSLANG
+
+#include "kompute/GLSLCompiler.hpp"
+
+#include <StandAlone/ResourceLimits.h>
+#include <SPIRV/GlslangToSpv.h>
+
+
+std::vector<char> GLSLCompiler::compile_to_spirv(
+    const std::string& glsl_source,
+    const std::string& entry_point
+){
+
+    // Initialize glslang library.
+	glslang::InitializeProcess();
+
+    const EShLanguage language = EShLangCompute;
+    glslang::TShader shader(language);
+
+	const char *file_name_list[1] = {""};
+	const char *shader_source     = reinterpret_cast<const char *>(glsl_source.data());
+	shader.setStringsWithLengthsAndNames(&shader_source, nullptr, file_name_list, 1);
+    shader.setEntryPoint(entry_point.c_str());
+	shader.setSourceEntryPoint(entry_point.c_str());
+
+	std::string info_log = "";
+    const EShMessages messages = static_cast<EShMessages>(EShMsgDefault | EShMsgVulkanRules | EShMsgSpvRules);
+    if (!shader.parse(&glslang::DefaultTBuiltInResource, 100, false, messages))
+	{
+		info_log = std::string(shader.getInfoLog()) + "\n" + std::string(shader.getInfoDebugLog());
+		throw std::runtime_error(info_log);
+	}
+
+
+    // Add shader to new program object.
+	glslang::TProgram program;
+	program.addShader(&shader);
+    // Link program.
+	if (!program.link(messages))
+	{
+		info_log = std::string(program.getInfoLog()) + "\n" + std::string(program.getInfoDebugLog());
+		throw std::runtime_error(info_log);
+	}
+
+    // Save any info log that was generated.
+	if (shader.getInfoLog())
+	{
+		info_log += std::string(shader.getInfoLog()) + "\n" + std::string(shader.getInfoDebugLog()) + "\n";
+	}
+
+    if (program.getInfoLog())
+	{
+		info_log += std::string(program.getInfoLog()) + "\n" + std::string(program.getInfoDebugLog());
+	}
+
+    glslang::TIntermediate *intermediate = program.getIntermediate(language);
+    // Translate to SPIRV.
+	if (!intermediate)
+	{
+		info_log += "Failed to get shared intermediate code.\n";
+		throw std::runtime_error(info_log);
+	}
+
+    spv::SpvBuildLogger logger;
+	std::vector<std::uint32_t> spirv;
+	glslang::GlslangToSpv(*intermediate, spirv, &logger);
+	info_log += logger.getAllMessages() + "\n";
+
+	// Shutdown glslang library.
+	glslang::FinalizeProcess();
+
+
+	return std::vector<char>((char*)spirv.data(), (char*)(spirv.data()+spirv.size()) );
+}
+
+#endif //KOMPUTE_INCLUDE_GLSLANG

src/include/kompute/GLSLCompiler.hpp

@@ -0,0 +1,32 @@
+#pragma once
+
+#if KOMPUTE_INCLUDE_GLSLANG
+
+#include <string>
+#include <vector>
+
+#include <glslang/Public/ShaderLang.h>
+
+/// Adapted from Vulkan-Samples
+/// A very simple version of the glslValidator application
+namespace GLSLCompiler
+{
+
+
+/**
+	* @brief Compiles GLSL to SPIRV code
+	* @param glsl_source The GLSL source code to be compiled
+	* @param entry_point The entrypoint function name of the shader stage
+	*/
+std::vector<char> compile_to_spirv(
+	const std::string& glsl_source,
+	const std::string& entry_point
+);
+
+};
+
+
+
+#endif //KOMPUTE_INCLUDE_GLSLANG
+
+

test/TestOpShadersFromStringAndFile.cpp

@@ -2,6 +2,9 @@
 #include "gtest/gtest.h"
 
 #include "kompute/Kompute.hpp"
+#if KOMPUTE_INCLUDE_GLSLANG
+  #include "kompute/GLSLCompiler.hpp"
+#endif
 
 #include "kompute_test/shaders/shadertest_op_custom_shader.hpp"
 
@@ -28,8 +31,13 @@ TEST(TestOpAlgoBase, ShaderRawDataFromConstructor)
         }
     )");
 
-    mgr.evalOpDefault<kp::OpAlgoBase>(
-      { tensorA, tensorB }, std::vector<char>(shader.begin(), shader.end()));
+    #if KOMPUTE_INCLUDE_GLSLANG
+      const std::vector<char> spirv = GLSLCompiler::compile_to_spirv(shader, "main");
+      mgr.evalOpDefault<kp::OpAlgoBase>( { tensorA, tensorB }, spirv );
+    #else
+      mgr.evalOpDefault<kp::OpAlgoBase>(
+        { tensorA, tensorB }, std::vector<char>(shader.begin(), shader.end()));
+    #endif
 
     mgr.evalOpDefault<kp::OpTensorSyncLocal>({ tensorA, tensorB });