Updated python and cpp with working gslslangvalidator

.github/workflows/cpp_tests.yml

@@ -10,7 +10,7 @@ jobs:
   cpp-tests:
 
     runs-on: ubuntu-18.04
-    container: axsauze/kompute-builder:0.2
+    container: axsauze/kompute-builder:0.3
 
     steps:
     - uses: actions/checkout@v2

.github/workflows/python_tests.yml

@@ -10,7 +10,7 @@ jobs:
   python-tests:
 
     runs-on: ubuntu-18.04
-    container: axsauze/kompute-builder:0.2
+    container: axsauze/kompute-builder:0.3
 
     steps:
     - uses: actions/checkout@v2

docker-builders/KomputeBuilder.Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
+FROM ubuntu:20.04
 
 # Base packages from default ppa
 RUN apt-get update -y
@@ -26,9 +26,14 @@ RUN apt-get install -y libxext-dev
 
 COPY --from=axsauze/swiftshader:0.1 /swiftshader/ /swiftshader/
 
+# GLSLANG tools for tests
+RUN apt-get install -y glslang-tools
+
 # Setup Python
 RUN apt-get install -y python3-pip
 
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10
+
 RUN mkdir builder
 WORKDIR /builder
 

docker-builders/Makefile

@@ -1,9 +1,9 @@
 
 build_kompute_builder:
-	docker build .. -f KomputeBuilder.Dockerfile -t axsauze/kompute-builder:0.2
+	docker build .. -f KomputeBuilder.Dockerfile -t axsauze/kompute-builder:0.3
 
 push_kompute_builder: build_kompute_builder
-	docker push axsauze/kompute-builder:0.2
+	docker push axsauze/kompute-builder:0.3
 
 build_swiftshader:
 	docker build .. -f Swiftshader.Dockerfile -t axsauze/swiftshader:0.1

python/test/requirements-dev.txt

@@ -1,4 +1,3 @@
 pyshader==0.7.0
 numpy==1.19.5
 pytest==6.2.1
-pyshaderc==1.1.2

python/test/test_kompute.py

@@ -5,7 +5,7 @@
 import logging
 import pyshader as ps
 
-import pyshaderc
+from .utils import compile_source
 
 DIRNAME = os.path.dirname(os.path.abspath(__file__))
 
@@ -55,7 +55,7 @@ def test_end_to_end():
     push_consts_a = [2]
     push_consts_b = [3]
 
-    algo = mgr.algorithm(params, pyshaderc.compile_into_spirv(shader.encode("utf-8"), "comp", "shader.comp"), workgroup, spec_consts, push_consts_a)
+    algo = mgr.algorithm(params, compile_source(shader), workgroup, spec_consts, push_consts_a)
 
     (mgr.sequence()
         .record(kp.OpTensorSyncDevice(params))
@@ -91,7 +91,7 @@ def test_shader_str():
 }
     """
 
-    spirv = pyshaderc.compile_into_spirv(shader.encode("utf-8"), "comp", "shader.comp")
+    spirv = compile_source(shader)
 
     mgr = kp.Manager()
 
@@ -131,7 +131,7 @@ def test_sequence():
         }
     """
 
-    spirv = pyshaderc.compile_into_spirv(shader.encode("utf-8"), "comp", "shader.comp")
+    spirv = compile_source(shader)
 
     mgr = kp.Manager(0)
 
@@ -171,7 +171,7 @@ def test_sequence():
 
 def test_pushconsts():
 
-    spirv = pyshaderc.compile_into_spirv("""
+    spirv = compile_source("""
           #version 450
           layout(push_constant) uniform PushConstants {
             float x;
@@ -185,7 +185,7 @@ def test_pushconsts():
               pa[1] += pcs.y;
               pa[2] += pcs.z;
           }
-    """.encode("utf-8"), "comp", "shader.comp")
+    """)
 
     mgr = kp.Manager()
 

python/test/test_tensor_types.py

@@ -1,10 +1,10 @@
-import pyshader as ps
-import pyshaderc
 import os
 import pytest
 import kp
 import numpy as np
 
+from .utils import compile_source
+
 VK_ICD_FILENAMES = os.environ.get("VK_ICD_FILENAMES", "")
 
 def test_type_float():
@@ -23,7 +23,7 @@ def test_type_float():
         }
     """
 
-    spirv = pyshaderc.compile_into_spirv(shader.encode("utf-8"), "comp", "shader.comp")
+    spirv = compile_source(shader)
 
     arr_in_a = np.array([123., 153., 231.], dtype=np.float32)
     arr_in_b = np.array([9482, 1208, 1238], dtype=np.float32)
@@ -62,7 +62,7 @@ def test_type_float_double_incorrect():
         }
     """
 
-    spirv = pyshaderc.compile_into_spirv(shader.encode("utf-8"), "comp", "shader.comp")
+    spirv = compile_source(shader)
 
     arr_in_a = np.array([123., 153., 231.], dtype=np.float32)
     arr_in_b = np.array([9482, 1208, 1238], dtype=np.uint32)
@@ -104,7 +104,7 @@ def test_type_double():
         }
     """
 
-    spirv = pyshaderc.compile_into_spirv(shader.encode("utf-8"), "comp", "shader.comp")
+    spirv = compile_source(shader)
 
     arr_in_a = np.array([123., 153., 231.], dtype=np.float64)
     arr_in_b = np.array([9482, 1208, 1238], dtype=np.float64)
@@ -144,7 +144,7 @@ def test_type_int():
         }
     """
 
-    spirv = pyshaderc.compile_into_spirv(shader.encode("utf-8"), "comp", "shader.comp")
+    spirv = compile_source(shader)
 
     arr_in_a = np.array([123, 153, 231], dtype=np.int32)
     arr_in_b = np.array([9482, 1208, 1238], dtype=np.int32)
@@ -184,7 +184,7 @@ def test_type_unsigned_int():
         }
     """
 
-    spirv = pyshaderc.compile_into_spirv(shader.encode("utf-8"), "comp", "shader.comp")
+    spirv = compile_source(shader)
 
     arr_in_a = np.array([123, 153, 231], dtype=np.uint32)
     arr_in_b = np.array([9482, 1208, 1238], dtype=np.uint32)

python/test/utils.py

@@ -0,0 +1,7 @@
+import os
+
+
+def compile_source(source):
+    os.system("glslangValidator --stdin -S comp -V -o tmp_kp_shader.comp.spv << END\n" + source + "\nEND")
+    return open("tmp_kp_shader.comp.spv", "rb").read()
+

test/utils/kompute_test/Shader.cpp

@@ -9,7 +9,8 @@ std::vector<uint32_t>
 Shader::compileSource(
   const std::string& source)
 {
-    system(std::string("glslc -fshader-stage=compute -o tmp_kp_shader.comp.spv - << END\n" + source + "\nEND").c_str());
+    if (system(std::string("glslangValidator --stdin -S comp -V -o tmp_kp_shader.comp.spv << END\n" + source + "\nEND").c_str()))
+        throw std::runtime_error("Error running glslangValidator command");
     std::ifstream fileStream("tmp_kp_shader.comp.spv", std::ios::binary);
     std::vector<char> buffer;
     buffer.insert(buffer.begin(), std::istreambuf_iterator<char>(fileStream), {});