Added Shaderc to tests in order to enable online shader builds

.ccls

@@ -17,6 +17,8 @@
 -I./python/pybind11/include/
 -I./external/Vulkan-Headers/include/
 -I./external/googletest/googletest/include/
+-I./external/shaderc/libshaderc/include/
+-I./external/shaderc/libshaderc_util/include/
 -I./external/spdlog/include/
 -I./src/include/
 -I./single_include/

.github/workflows/cpp_tests.yml

@@ -30,5 +30,5 @@ jobs:
     - name: test-cpp
       run: |
         export VK_ICD_FILENAMES=/swiftshader/vk_swiftshader_icd.json
-        make mk_run_tests_cpu_only
+        make mk_run_tests
 

.gitmodules

@@ -14,3 +14,7 @@
 	path = python/pybind11
 	url = https://github.com/pybind/pybind11
     branch = v2.6.1
+[submodule "external/shaderc"]
+	path = external/shaderc
+	url = https://github.com/google/shaderc
+    branch = v2020.4

Makefile

@@ -13,7 +13,7 @@ VCPKG_WIN_PATH ?= "C:\\Users\\axsau\\Programming\\lib\\vcpkg\\scripts\\buildsyst
 VCPKG_UNIX_PATH ?= "/c/Users/axsau/Programming/lib/vcpkg/scripts/buildsystems/vcpkg.cmake"
 
 # Regext to pass to catch2 to filter tests
-FILTER_TESTS ?= "*"
+FILTER_TESTS ?= "-TestAsyncOperations.TestManagerParallelExecution"
 
 ifeq ($(OS),Windows_NT)     # is Windows_NT on XP, 2000, 7, Vista, 10...
 	CMAKE_BIN ?= "C:\Program Files\CMake\bin\cmake.exe"
@@ -70,25 +70,26 @@ mk_cmake:
 		-DKOMPUTE_OPT_ENABLE_SPDLOG=1 \
 		-DSPDLOG_INSTALL=1 \
 		-DKOMPUTE_OPT_CODE_COVERAGE=1 \
+		-DSHADERC_SKIP_TESTS=1 \
 		-G "Unix Makefiles"
 
 mk_build_all:
-	cmake --build build/. --parallel
+	cmake --build build/. --parallel $(($(nproc)-2))
 
 mk_build_docs:
-	cmake --build build/. --target gendocsall --parallel
+	cmake --build build/. --target gendocsall --parallel $(($(nproc)-2))
 
 mk_build_kompute:
-	cmake --build build/. --target kompute --parallel
+	cmake --build build/. --target kompute --parallel $(($(nproc)-2))
 
 mk_build_tests:
-	cmake --build build/ --target test_kompute --parallel
+	cmake --build build/ --target test_kompute --parallel $(($(nproc)-2))
 
 mk_run_docs: mk_build_docs
 	(cd build/docs/sphinx && python2.7 -m SimpleHTTPServer)
 
 mk_run_tests: mk_build_tests
-	./build/test/test_kompute $(FILTER_TESTS)
+	./build/test/test_kompute --gtest_filter=$(FILTER_TESTS)
 
 mk_build_swiftshader_library:
 	git clone https://github.com/google/swiftshader || echo "Assuming already cloned"
@@ -99,16 +100,6 @@ mk_build_swiftshader_library:
 mk_run_tests_cpu: export VK_ICD_FILENAMES=$(PWD)/swiftshader/build/vk_swiftshader_icd.json
 mk_run_tests_cpu: mk_build_swiftshader_library mk_build_tests mk_run_tests_cpu_only
 
-mk_run_tests_cpu_only:
-	./build/test/test_kompute --gtest_filter="TestLogisticRegressionAlgorithm.*"
-	./build/test/test_kompute --gtest_filter="TestManager.*"
-	./build/test/test_kompute --gtest_filter="TestOpAlgoBase.ShaderCompiledDataFromConstructor"
-	./build/test/test_kompute --gtest_filter="TestOpTensorCopy.*"
-	./build/test/test_kompute --gtest_filter="TestOpTensorCreate.*"
-	./build/test/test_kompute --gtest_filter="TestOpTensorSync.*"
-	./build/test/test_kompute --gtest_filter="TestSequence.*"
-	./build/test/test_kompute --gtest_filter="TestTensor.*"
-
 
 ####### Visual studio build shortcut commands #######
 
@@ -133,6 +124,7 @@ vs_cmake:
 		-DKOMPUTE_OPT_BUILD_SINGLE_HEADER=1 \
 		-DKOMPUTE_OPT_ENABLE_SPDLOG=1 \
 		-DSPDLOG_INSTALL=1 \
+		-DSHADERC_SKIP_TESTS=1 \
 		-G "Visual Studio 16 2019"
 
 vs_build_all:

test/CMakeLists.txt

@@ -1,4 +1,7 @@
 
+#####################################################
+#################### GETEST #######################
+#####################################################
 enable_testing()
 if(KOMPUTE_OPT_REPO_SUBMODULE_BUILD)
     add_subdirectory(${PROJECT_SOURCE_DIR}/external/googletest EXCLUDE_FROM_ALL
@@ -35,6 +38,42 @@ target_link_libraries(test_kompute PRIVATE kompute)
 
 add_test(NAME test_kompute COMMAND test_kompute)
 
+#####################################################
+#################### PYSHADERC #######################
+#####################################################
+
+set(SHADERC_SKIP_TESTS 1)
+
+if(KOMPUTE_OPT_REPO_SUBMODULE_BUILD)
+    # Ensure shaderc is initialised by cloning all dependencies
+    message(STATUS "Cloning all repositories for shaderc dependency")
+    execute_process(COMMAND
+        python3 ${PROJECT_SOURCE_DIR}/external/shaderc/utils/git-sync-deps)
+
+    add_subdirectory(${PROJECT_SOURCE_DIR}/external/shaderc
+        ${CMAKE_CURRENT_BINARY_DIR}/kompute_shaderc)
+
+    target_include_directories(
+        test_kompute PRIVATE
+        ${PROJECT_SOURCE_DIR}/external/shaderc/libshaderc/include)
+
+    target_link_libraries(test_kompute PRIVATE
+        shaderc)
+
+    message(STATUS "Shaderc lib:" shaderc)
+else()
+    find_library(SHADERC_LIB shaderc_combined)
+
+    find_path(SHADERC_INCLUDE_DIRS shaderc)
+
+    target_include_directories(
+        test_kompute PRIVATE
+        ${SHADERC_INCLUDE_DIRS})
+
+    target_link_libraries(test_kompute PRIVATE
+        ${SHADERC_LIB})
+endif()
+
 #####################################################
 #################### CODECOV  #######################
 #####################################################
@@ -67,25 +106,28 @@ if (KOMPUTE_OPT_CODE_COVERAGE)
         WORKING_DIRECTORY ${CODECOV_DIR}
         DEPENDS codecov_copy_files)
 
-    add_custom_target(codecov_lcov
+    add_custom_target(codecov_lcov_capture
         COMMAND lcov
             --capture
             -o ${CODECOV_FILENAME_LCOV_INFO_FULL}
             -d .
+        WORKING_DIRECTORY ${CODECOV_DIR}
+        DEPENDS codecov_gcov)
+    add_custom_target(codecov_lcov_extract
         COMMAND lcov
             --extract
             ${CODECOV_FILENAME_LCOV_INFO_FULL}
             -o ${CODECOV_FILENAME_LCOV_INFO}
             -d .
-            "*/src/*" "*/test/*"
+            "*/src/*"
         WORKING_DIRECTORY ${CODECOV_DIR}
-        DEPENDS codecov_gcov)
+        DEPENDS codecov_lcov_capture)
 
     add_custom_target(codecov_genhtml
         COMMAND genhtml
             ${CODECOV_FILENAME_LCOV_INFO}
             --output-directory ${CODECOV_DIR_HTML}
         WORKING_DIRECTORY ${CODECOV_DIR}
-        DEPENDS codecov_lcov)
+        DEPENDS codecov_lcov_extract)
 endif()
 

test/TestAsyncOperations.cpp

@@ -5,6 +5,8 @@
 
 #include "kompute/Kompute.hpp"
 
+#include "TestUtils.cpp"
+
 TEST(TestAsyncOperations, TestManagerParallelExecution)
 {
     // This test is built for NVIDIA 1650. It assumes:
@@ -55,7 +57,7 @@ TEST(TestAsyncOperations, TestManagerParallelExecution)
 
     for (uint32_t i = 0; i < numParallel; i++) {
         mgr.evalOpDefault<kp::OpAlgoBase>(
-          { inputsSyncB[i] }, std::vector<char>(shader.begin(), shader.end()));
+          { inputsSyncB[i] }, spirv_from_string(shader));
     }
 
     auto endSync = std::chrono::high_resolution_clock::now();
@@ -89,7 +91,7 @@ TEST(TestAsyncOperations, TestManagerParallelExecution)
         mgrAsync.evalOpAsync<kp::OpAlgoBase>(
           { inputsAsyncB[i] },
           "async" + std::to_string(i),
-          std::vector<char>(shader.begin(), shader.end()));
+          spirv_from_string(shader));
     }
 
     for (uint32_t i = 0; i < numParallel; i++) {
@@ -152,10 +154,10 @@ TEST(TestAsyncOperations, TestManagerAsyncExecution)
     mgr.rebuild({ tensorA, tensorB });
 
     mgr.evalOpAsync<kp::OpAlgoBase>(
-      { tensorA }, "asyncOne", std::vector<char>(shader.begin(), shader.end()));
+      { tensorA }, "asyncOne", spirv_from_string(shader));
 
     mgr.evalOpAsync<kp::OpAlgoBase>(
-      { tensorB }, "asyncTwo", std::vector<char>(shader.begin(), shader.end()));
+      { tensorB }, "asyncTwo", spirv_from_string(shader));
 
     mgr.evalOpAwait("asyncOne");
     mgr.evalOpAwait("asyncTwo");

test/TestDestroy.cpp

@@ -3,6 +3,8 @@
 
 #include "kompute/Kompute.hpp"
 
+#include "TestUtils.cpp"
+
 TEST(TestDestroy, TestDestroyTensorSingle)
 {
     std::shared_ptr<kp::Tensor> tensorA{ new kp::Tensor({ 0, 0, 0 }) };
@@ -28,7 +30,7 @@ TEST(TestDestroy, TestDestroyTensorSingle)
 
             sq->begin();
             sq->record<kp::OpAlgoBase>(
-              { tensorA }, std::vector<char>(shader.begin(), shader.end()));
+              { tensorA }, spirv_from_string(shader));
             sq->end();
 
             sq->eval();
@@ -71,7 +73,7 @@ TEST(TestDestroy, TestDestroyTensorVector)
 
             sq->begin();
             sq->record<kp::OpAlgoBase>(
-              { tensorA, tensorB }, std::vector<char>(shader.begin(), shader.end()));
+              { tensorA, tensorB }, spirv_from_string(shader));
             sq->end();
 
             sq->eval();
@@ -136,7 +138,7 @@ TEST(TestDestroy, TestDestroySequenceSingle)
 
             sq->begin();
             sq->record<kp::OpAlgoBase>(
-              { tensorA }, std::vector<char>(shader.begin(), shader.end()));
+              { tensorA }, spirv_from_string(shader));
             sq->end();
 
             sq->eval();
@@ -176,14 +178,14 @@ TEST(TestDestroy, TestDestroySequenceVector)
             sq1 = mgr.sequence("One");
             sq1->begin();
             sq1->record<kp::OpAlgoBase>(
-              { tensorA }, std::vector<char>(shader.begin(), shader.end()));
+              { tensorA }, spirv_from_string(shader));
             sq1->end();
             sq1->eval();
 
             sq2 = mgr.sequence("Two");
             sq2->begin();
             sq2->record<kp::OpAlgoBase>(
-              { tensorA }, std::vector<char>(shader.begin(), shader.end()));
+              { tensorA }, spirv_from_string(shader));
             sq2->end();
             sq2->eval();
 
@@ -218,11 +220,11 @@ TEST(TestDestroy, TestDestroySequenceNameSingleInsideManager)
 
             mgr.evalOp<kp::OpAlgoBase>(
               { tensorA }, "one",
-              std::vector<char>(shader.begin(), shader.end()));
+              spirv_from_string(shader));
 
             mgr.evalOp<kp::OpAlgoBase>(
               { tensorA }, "two",
-              std::vector<char>(shader.begin(), shader.end()));
+              spirv_from_string(shader));
 
             mgr.evalOpDefault<kp::OpTensorSyncLocal>({ tensorA });
 
@@ -257,7 +259,7 @@ TEST(TestDestroy, TestDestroySequenceNameSingleOutsideManager)
             sq1 = mgr.sequence("One");
             sq1->begin();
             sq1->record<kp::OpAlgoBase>(
-              { tensorA }, std::vector<char>(shader.begin(), shader.end()));
+              { tensorA }, spirv_from_string(shader));
             sq1->end();
             sq1->eval();
 
@@ -291,11 +293,11 @@ TEST(TestDestroy, TestDestroySequenceNameVectorInsideManager)
 
             mgr.evalOp<kp::OpAlgoBase>(
               { tensorA }, "one",
-              std::vector<char>(shader.begin(), shader.end()));
+              spirv_from_string(shader));
 
             mgr.evalOp<kp::OpAlgoBase>(
               { tensorA }, "two",
-              std::vector<char>(shader.begin(), shader.end()));
+              spirv_from_string(shader));
 
             mgr.evalOpDefault<kp::OpTensorSyncLocal>({ tensorA });
 
@@ -325,11 +327,11 @@ TEST(TestDestroy, TestDestroySequenceNameVectorOutsideManager)
 
             mgr.evalOp<kp::OpAlgoBase>(
               { tensorA }, "one",
-              std::vector<char>(shader.begin(), shader.end()));
+              spirv_from_string(shader));
 
             mgr.evalOp<kp::OpAlgoBase>(
               { tensorA }, "two",
-              std::vector<char>(shader.begin(), shader.end()));
+              spirv_from_string(shader));
 
             mgr.evalOpDefault<kp::OpTensorSyncLocal>({ tensorA });
 
@@ -359,7 +361,7 @@ TEST(TestDestroy, TestDestroySequenceNameDefaultOutsideManager)
 
             mgr.evalOpDefault<kp::OpAlgoBase>(
               { tensorA },
-              std::vector<char>(shader.begin(), shader.end()));
+              spirv_from_string(shader));
 
             mgr.evalOpDefault<kp::OpTensorSyncLocal>({ tensorA });
 

test/TestLogisticRegression.cpp

@@ -5,7 +5,7 @@
 
 #include "kompute_test/shaders/shadertest_logistic_regression.hpp"
 
-TEST(TestLogisticRegressionAlgorithm, TestMainLogisticRegression)
+TEST(TestLogisticRegression, TestMainLogisticRegression)
 {
 
     uint32_t ITERATIONS = 100;
@@ -41,19 +41,13 @@ TEST(TestLogisticRegressionAlgorithm, TestMainLogisticRegression)
 
         sq->record<kp::OpTensorSyncDevice>({ wIn, bIn });
 
-#ifdef KOMPUTE_SHADER_FROM_STRING
-        sq->record<kp::OpAlgoBase>(
-          params, "test/shaders/glsl/test_logistic_regression.comp",
-          kp::Workgroup(), kp::Constants({5.0}));
-#else
         sq->record<kp::OpAlgoBase>(
           params,
           std::vector<char>(
             kp::shader_data::shaders_glsl_logisticregression_comp_spv,
             kp::shader_data::shaders_glsl_logisticregression_comp_spv +
               kp::shader_data::shaders_glsl_logisticregression_comp_spv_len),
           kp::Workgroup(), kp::Constants({5.0}));
-#endif
 
         sq->record<kp::OpTensorSyncLocal>({ wOutI, wOutJ, bOut, lOut });