diff --git a/.gitmodules b/.gitmodules index 7b7ada44..09b92d06 100644 --- a/.gitmodules +++ b/.gitmodules @@ -14,3 +14,6 @@ path = python/pybind11 url = https://github.com/pybind/pybind11 branch = v2.6.1 +[submodule "external/glslang"] + path = external/glslang + url = https://github.com/KhronosGroup/glslang.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 7be7a3f9..0c834ba8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,6 +18,7 @@ option(KOMPUTE_OPT_ENABLE_SPDLOG "Extra compile flags for Kompute, see docs for option(KOMPUTE_OPT_REPO_SUBMODULE_BUILD, "Use the submodule repos instead of external package manager" 0) option(KOMPUTE_OPT_ANDOID_BUILD "Enable android compilation flags required" 0) option(KOMPUTE_OPT_DISABLE_VK_DEBUG_LAYERS "Explicitly disable debug layers even on debug" 0) +option(KOMPUTE_OPT_INCLUDE_GLSLANG "Include GLSLang for runtime shader compilation") # Build flags set(KOMPUTE_EXTRA_CXX_FLAGS "" CACHE STRING "Extra compile flags for Kompute, see docs for full list") @@ -26,6 +27,10 @@ if(KOMPUTE_OPT_ENABLE_SPDLOG) set(SPDLOG_INSTALL, 1) endif() +if(KOMPUTE_OPT_INCLUDE_GLSLANG) + set(KOMPUTE_EXTRA_CXX_FLAGS "${KOMPUTE_EXTRA_CXX_FLAGS} -DKOMPUTE_INCLUDE_GLSLANG=1") +endif() + if(KOMPUTE_OPT_ANDOID_BUILD) set(KOMPUTE_EXTRA_CXX_FLAGS "${KOMPUTE_EXTRA_CXX_FLAGS} -DVK_USE_PLATFORM_ANDROID_KHR") endif() diff --git a/external/glslang b/external/glslang new file mode 160000 index 00000000..3de5cfe5 --- /dev/null +++ b/external/glslang @@ -0,0 +1 @@ +Subproject commit 3de5cfe50edecd001e6d703555284d9b10b3dd57 diff --git a/python/src/main.cpp b/python/src/main.cpp index 77924581..6bb074f6 100644 --- a/python/src/main.cpp +++ b/python/src/main.cpp @@ -3,6 +3,9 @@ #include #include +#if KOMPUTE_INCLUDE_GLSLANG + #include +#endif #include "docstrings.hpp" @@ -299,6 +302,14 @@ PYBIND11_MODULE(kp, m) { "Evaluates asynchronously an operation using a custom shader provided as raw string or spirv bytes with explicitly named Sequence") .def("eval_async_algo_lro", &kp::Manager::evalOpAsync, "Evaluates asynchronously operation to run left right out operation with custom shader with explicitly named Sequence"); + + #if KOMPUTE_INCLUDE_GLSLANG + m.def("compile_glsl_to_spirv", + [](std::string glsl_source){ + const auto spirv = GLSLCompiler::compile_to_spirv(glsl_source, "main"); + return py::bytes(std::string(spirv.begin(), spirv.end())); + }); + #endif //KOMPUTE_INCLUDE_GLSLANG #ifdef VERSION_INFO m.attr("__version__") = VERSION_INFO; diff --git a/python/test/test_kompute.py b/python/test/test_kompute.py index 2232beae..fb033189 100644 --- a/python/test/test_kompute.py +++ b/python/test/test_kompute.py @@ -54,7 +54,8 @@ def test_opalgobase_data(): mgr.eval_tensor_create_def([tensor_in_a, tensor_in_b, tensor_out]) - mgr.eval_algo_str_def([tensor_in_a, tensor_in_b, tensor_out], shaderData) + spirv = kp.compile_glsl_to_spirv(shaderData) + mgr.eval_algo_str_def([tensor_in_a, tensor_in_b, tensor_out], spirv) mgr.eval_tensor_sync_local_def([tensor_out]) diff --git a/setup.py b/setup.py index 1896b8fb..ad0cfcce 100644 --- a/setup.py +++ b/setup.py @@ -43,6 +43,7 @@ def build_extension(self, ext): '-DKOMPUTE_OPT_BUILD_PYTHON=1', '-DKOMPUTE_OPT_ENABLE_SPDLOG=1', '-DKOMPUTE_OPT_REPO_SUBMODULE_BUILD=1', + '-DKOMPUTE_OPT_INCLUDE_GLSLANG=1', '-DPYTHON_EXECUTABLE=' + sys.executable] cfg = 'Debug' if self.debug else 'Release' diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 348c0536..f5d46de1 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -124,3 +124,14 @@ if(KOMPUTE_OPT_INSTALL) DESTINATION lib/cmake/kompute) endif() + + +if(KOMPUTE_OPT_INCLUDE_GLSLANG) + add_subdirectory(${PROJECT_SOURCE_DIR}/external/glslang ${CMAKE_CURRENT_BINARY_DIR}/kompute_glslang) + target_include_directories(kompute PUBLIC $) + + target_link_libraries(kompute glslang) + target_link_libraries(kompute MachineIndependent) + target_link_libraries(kompute glslang-default-resource-limits) + target_link_libraries(kompute SPIRV) +endif() diff --git a/src/GLSLCompiler.cpp b/src/GLSLCompiler.cpp new file mode 100644 index 00000000..2e8b39f0 --- /dev/null +++ b/src/GLSLCompiler.cpp @@ -0,0 +1,76 @@ +#if KOMPUTE_INCLUDE_GLSLANG + +#include "kompute/GLSLCompiler.hpp" + +#include +#include + + +std::vector GLSLCompiler::compile_to_spirv( + const std::string& glsl_source, + const std::string& entry_point +){ + + // Initialize glslang library. + glslang::InitializeProcess(); + + const EShLanguage language = EShLangCompute; + glslang::TShader shader(language); + + const char *file_name_list[1] = {""}; + const char *shader_source = reinterpret_cast(glsl_source.data()); + shader.setStringsWithLengthsAndNames(&shader_source, nullptr, file_name_list, 1); + shader.setEntryPoint(entry_point.c_str()); + shader.setSourceEntryPoint(entry_point.c_str()); + + std::string info_log = ""; + const EShMessages messages = static_cast(EShMsgDefault | EShMsgVulkanRules | EShMsgSpvRules); + if (!shader.parse(&glslang::DefaultTBuiltInResource, 100, false, messages)) + { + info_log = std::string(shader.getInfoLog()) + "\n" + std::string(shader.getInfoDebugLog()); + throw std::runtime_error(info_log); + } + + + // Add shader to new program object. + glslang::TProgram program; + program.addShader(&shader); + // Link program. + if (!program.link(messages)) + { + info_log = std::string(program.getInfoLog()) + "\n" + std::string(program.getInfoDebugLog()); + throw std::runtime_error(info_log); + } + + // Save any info log that was generated. + if (shader.getInfoLog()) + { + info_log += std::string(shader.getInfoLog()) + "\n" + std::string(shader.getInfoDebugLog()) + "\n"; + } + + if (program.getInfoLog()) + { + info_log += std::string(program.getInfoLog()) + "\n" + std::string(program.getInfoDebugLog()); + } + + glslang::TIntermediate *intermediate = program.getIntermediate(language); + // Translate to SPIRV. + if (!intermediate) + { + info_log += "Failed to get shared intermediate code.\n"; + throw std::runtime_error(info_log); + } + + spv::SpvBuildLogger logger; + std::vector spirv; + glslang::GlslangToSpv(*intermediate, spirv, &logger); + info_log += logger.getAllMessages() + "\n"; + + // Shutdown glslang library. + glslang::FinalizeProcess(); + + + return std::vector((char*)spirv.data(), (char*)(spirv.data()+spirv.size()) ); +} + +#endif //KOMPUTE_INCLUDE_GLSLANG \ No newline at end of file diff --git a/src/include/kompute/GLSLCompiler.hpp b/src/include/kompute/GLSLCompiler.hpp new file mode 100644 index 00000000..ea05dd54 --- /dev/null +++ b/src/include/kompute/GLSLCompiler.hpp @@ -0,0 +1,32 @@ +#pragma once + +#if KOMPUTE_INCLUDE_GLSLANG + +#include +#include + +#include + +/// Adapted from Vulkan-Samples +/// A very simple version of the glslValidator application +namespace GLSLCompiler +{ + + +/** + * @brief Compiles GLSL to SPIRV code + * @param glsl_source The GLSL source code to be compiled + * @param entry_point The entrypoint function name of the shader stage + */ +std::vector compile_to_spirv( + const std::string& glsl_source, + const std::string& entry_point +); + +}; + + + +#endif //KOMPUTE_INCLUDE_GLSLANG + + diff --git a/test/TestOpShadersFromStringAndFile.cpp b/test/TestOpShadersFromStringAndFile.cpp index 273421b2..d7f3b9b4 100644 --- a/test/TestOpShadersFromStringAndFile.cpp +++ b/test/TestOpShadersFromStringAndFile.cpp @@ -2,6 +2,9 @@ #include "gtest/gtest.h" #include "kompute/Kompute.hpp" +#if KOMPUTE_INCLUDE_GLSLANG + #include "kompute/GLSLCompiler.hpp" +#endif #include "kompute_test/shaders/shadertest_op_custom_shader.hpp" @@ -28,8 +31,13 @@ TEST(TestOpAlgoBase, ShaderRawDataFromConstructor) } )"); - mgr.evalOpDefault( - { tensorA, tensorB }, std::vector(shader.begin(), shader.end())); + #if KOMPUTE_INCLUDE_GLSLANG + const std::vector spirv = GLSLCompiler::compile_to_spirv(shader, "main"); + mgr.evalOpDefault( { tensorA, tensorB }, spirv ); + #else + mgr.evalOpDefault( + { tensorA, tensorB }, std::vector(shader.begin(), shader.end())); + #endif mgr.evalOpDefault({ tensorA, tensorB });