Skip to content

Commit 4b7ae34

Browse files
lhezquic-sszotshawngu-quicquic-aanguswanghqc
authored and
arthw
committed
Introducing experimental OpenCL backend with support for Qualcomm Adreno GPUs (ggml-org#10693)
* [cl][adreno] Add Adreno GPU support Add new OpenCL backend to support Adreno GPUs --------- Co-authored-by: Skyler Szot <quic_sszot@quicinc.com> Co-authored-by: Shangqing Gu <quic_shawngu@quicinc.com> Co-authored-by: Alexander Angus <quic_aangus@quicinc.com> Co-authored-by: Hongqiang Wang <quic_wangh@quicinc.com> Co-authored-by: Max Krasnyansky <quic_maxk@quicinc.com> * [cl][ci] Add workflow for CL * [cl][adreno] Fix memory leak for non SMALL_ALLOC path * opencl: integrate backend dyn.load interface and fix compiler and format warnings * opencl: remove small-alloc support and fix build errors for non-opencl platforms * opencl: fixed merge conflict (MUSA added twice in cmake) * opencl-ci: use RUNNER_TEMP instead of github.workspace * opencl: fix embed tool invocation with python3 * opencl: CI workflow fixes * opencl: Clean up small-alloc in CMake files * opencl: cleanup ggml-opencl2 header file * opencl: use ulong for offsets and strides in ADD kernel * opencl: use cl_ulong for all offsets * opencl: use cl_ulong for sizes and strides * opencl: use `GGML_LOG_xxx` instead of `fprintf(stderr, ...)` * opencl: rename backend `opencl2` -> `opencl` * opencl: rename kernel files `ggml-opencl2` -> `ggml-opencl` * opencl: make OpenCL required, remove redundant lib and inc directories * `ggml-base`, `..` and `.` are added by `ggml_add_backend_library` * opencl: rename backend - funcs, structs, etc `opencl2` -> `opencl` * opencl: remove copyright marker since main license already covers * opencl: replace some more OPENCL2 leftovers * opencl: remove limits on `tensor_extra` * opencl: use pools for `tensor_extra` * opencl: fix compiler warnings with GCC and Clang Still getting the warning about clCreateCmdQueue being obsolete. Will fix that separately. * opencl: fail gracefully if opencl devices are not available Also for unsupported GPUs. * opencl: fix MSVC builds (string length error) * opencl: check for various requirements, allow deprecated API * opencl: update log message for unsupported GPUs --------- Co-authored-by: Skyler Szot <quic_sszot@quicinc.com> Co-authored-by: Shangqing Gu <quic_shawngu@quicinc.com> Co-authored-by: Alexander Angus <quic_aangus@quicinc.com> Co-authored-by: Hongqiang Wang <quic_wangh@quicinc.com> Co-authored-by: Max Krasnyansky <quic_maxk@quicinc.com>
1 parent e7c0f17 commit 4b7ae34

17 files changed

+9014
-1
lines changed

‎.github/workflows/build.yml

+25-1
Original file line numberDiff line numberDiff line change
@@ -662,6 +662,8 @@ jobs:
662662
defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DBUILD_SHARED_LIBS=ON'
663663
- build: 'msvc-arm64'
664664
defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DBUILD_SHARED_LIBS=ON'
665+
- build: 'llvm-arm64-opencl-adreno'
666+
defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DCMAKE_PREFIX_PATH="$env:RUNNER_TEMP/opencl-arm64-release" -DGGML_OPENCL=ON -DGGML_OPENCL_USE_ADRENO_KERNELS=ON'
665667

666668
steps:
667669
- name: Clone
@@ -703,6 +705,28 @@ jobs:
703705
run: |
704706
choco install ninja
705707
708+
- name: Install OpenCL Headers and Libs
709+
id: install_opencl
710+
if: ${{ matrix.build == 'llvm-arm64-opencl-adreno' }}
711+
run: |
712+
git clone https://github.com/KhronosGroup/OpenCL-Headers
713+
cd OpenCL-Headers
714+
mkdir build && cd build
715+
cmake .. `
716+
-DBUILD_TESTING=OFF `
717+
-DOPENCL_HEADERS_BUILD_TESTING=OFF `
718+
-DOPENCL_HEADERS_BUILD_CXX_TESTS=OFF `
719+
-DCMAKE_INSTALL_PREFIX="$env:RUNNER_TEMP/opencl-arm64-release"
720+
cmake --build . --target install
721+
git clone https://github.com/KhronosGroup/OpenCL-ICD-Loader
722+
cd OpenCL-ICD-Loader
723+
mkdir build-arm64-release && cd build-arm64-release
724+
cmake .. `
725+
-A arm64 `
726+
-DCMAKE_PREFIX_PATH="$env:RUNNER_TEMP/opencl-arm64-release" `
727+
-DCMAKE_INSTALL_PREFIX="$env:RUNNER_TEMP/opencl-arm64-release"
728+
cmake --build . --target install --config release
729+
706730
- name: Build
707731
id: cmake_build
708732
run: |
@@ -732,7 +756,7 @@ jobs:
732756
- name: Test
733757
id: cmake_test
734758
# not all machines have native AVX-512
735-
if: ${{ matrix.build != 'msvc-arm64' && matrix.build != 'llvm-arm64' && matrix.build != 'kompute-x64' && matrix.build != 'vulkan-x64' && (matrix.build != 'avx512-x64' || env.HAS_AVX512F == '1') }}
759+
if: ${{ matrix.build != 'msvc-arm64' && matrix.build != 'llvm-arm64' && matrix.build != 'llvm-arm64-opencl-adreno' && matrix.build != 'kompute-x64' && matrix.build != 'vulkan-x64' && (matrix.build != 'avx512-x64' || env.HAS_AVX512F == '1') }}
736760
run: |
737761
cd build
738762
ctest -L main -C Release --verbose --timeout 900

‎ggml/CMakeLists.txt

+5
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,11 @@ set (GGML_SYCL_TARGET "INTEL" CACHE STRING
179179
set (GGML_SYCL_DEVICE_ARCH "" CACHE STRING
180180
"ggml: sycl device architecture")
181181

182+
option(GGML_OPENCL "ggml: use OpenCL" OFF)
183+
option(GGML_OPENCL_PROFILING "ggml: use OpenCL profiling (increases overhead)" OFF)
184+
option(GGML_OPENCL_EMBED_KERNELS "ggml: embed kernels" ON)
185+
option(GGML_OPENCL_USE_ADRENO_KERNELS "ggml: use optimized kernels for Adreno" ON)
186+
182187
# extra artifacts
183188
option(GGML_BUILD_TESTS "ggml: build tests" ${GGML_STANDALONE})
184189
option(GGML_BUILD_EXAMPLES "ggml: build examples" ${GGML_STANDALONE})

‎ggml/include/ggml-opencl.h

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#ifndef GGML_OPENCL_H
2+
#define GGML_OPENCL_H
3+
4+
#include "ggml.h"
5+
#include "ggml-backend.h"
6+
7+
#ifdef __cplusplus
8+
extern "C" {
9+
#endif
10+
11+
//
12+
// backend API
13+
//
14+
GGML_BACKEND_API ggml_backend_t ggml_backend_opencl_init(void);
15+
GGML_BACKEND_API bool ggml_backend_is_opencl(ggml_backend_t backend);
16+
17+
GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_opencl_buffer_type(void);
18+
GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_opencl_host_buffer_type(void);
19+
20+
GGML_BACKEND_API ggml_backend_reg_t ggml_backend_opencl_reg(void);
21+
22+
#ifdef __cplusplus
23+
}
24+
#endif
25+
26+
#endif // GGML_OPENCL_H

‎ggml/src/CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -308,6 +308,7 @@ ggml_add_backend(MUSA)
308308
ggml_add_backend(RPC)
309309
ggml_add_backend(SYCL)
310310
ggml_add_backend(Vulkan)
311+
ggml_add_backend(OpenCL)
311312

312313
foreach (target ggml-base ggml)
313314
target_include_directories(${target} PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../include> $<INSTALL_INTERFACE:include>)

‎ggml/src/ggml-backend-reg.cpp

+8
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,10 @@
4646
#include "ggml-vulkan.h"
4747
#endif
4848

49+
#ifdef GGML_USE_OPENCL
50+
#include "ggml-opencl.h"
51+
#endif
52+
4953
#ifdef GGML_USE_BLAS
5054
#include "ggml-blas.h"
5155
#endif
@@ -146,6 +150,9 @@ struct ggml_backend_registry {
146150
#ifdef GGML_USE_VULKAN
147151
register_backend(ggml_backend_vk_reg());
148152
#endif
153+
#ifdef GGML_USE_OPENCL
154+
register_backend(ggml_backend_opencl_reg());
155+
#endif
149156
#ifdef GGML_USE_CANN
150157
register_backend(ggml_backend_cann_reg());
151158
#endif
@@ -539,6 +546,7 @@ void ggml_backend_load_all_from_path(const char * dir_path) {
539546
ggml_backend_load_best("rpc", silent, dir_path);
540547
ggml_backend_load_best("sycl", silent, dir_path);
541548
ggml_backend_load_best("vulkan", silent, dir_path);
549+
ggml_backend_load_best("opencl", silent, dir_path);
542550
ggml_backend_load_best("musa", silent, dir_path);
543551
ggml_backend_load_best("cpu", silent, dir_path);
544552
}

‎ggml/src/ggml-opencl/CMakeLists.txt

+147
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
find_package(OpenCL REQUIRED)
2+
find_package(Python3 REQUIRED)
3+
4+
set(TARGET_NAME ggml-opencl)
5+
6+
ggml_add_backend_library(${TARGET_NAME}
7+
ggml-opencl.cpp
8+
../../include/ggml-opencl.h)
9+
target_link_libraries(${TARGET_NAME} PRIVATE ${OpenCL_LIBRARIES})
10+
target_include_directories(${TARGET_NAME} PRIVATE ${OpenCL_INCLUDE_DIRS})
11+
12+
if (GGML_OPENCL_PROFILING)
13+
message(STATUS "OpenCL profiling enabled (increases CPU overhead)")
14+
add_compile_definitions(GGML_OPENCL_PROFILING)
15+
endif ()
16+
17+
add_compile_definitions(GGML_OPENCL_SOA_Q)
18+
19+
if (GGML_OPENCL_USE_ADRENO_KERNELS)
20+
message(STATUS "OpenCL will use matmul kernels optimized for Adreno")
21+
add_compile_definitions(GGML_OPENCL_USE_ADRENO_KERNELS)
22+
endif ()
23+
24+
if (GGML_OPENCL_EMBED_KERNELS)
25+
add_compile_definitions(GGML_OPENCL_EMBED_KERNELS)
26+
27+
set(OPENCL_CL_SOURCE_EMBED "${CMAKE_BINARY_DIR}/autogenerated/ggml-opencl.cl.h")
28+
set(OPENCL_MM_CL_SOURCE_EMBED "${CMAKE_BINARY_DIR}/autogenerated/ggml-opencl_mm.cl.h")
29+
set(OPENCL_CVT_CL_SOURCE_EMBED "${CMAKE_BINARY_DIR}/autogenerated/ggml-opencl_cvt.cl.h")
30+
31+
set(OPENCL_GEMV_NOSHUFFLE_SOURCE_EMBED "${CMAKE_BINARY_DIR}/autogenerated/ggml-opencl_gemv_noshuffle.cl.h")
32+
set(OPENCL_GEMV_NOSHUFFLE_GENERAL_SOURCE_EMBED "${CMAKE_BINARY_DIR}/autogenerated/ggml-opencl_gemv_noshuffle_general.cl.h")
33+
set(OPENCL_MUL_MAT_Ab_Bi_8x4_SOURCE_EMBED "${CMAKE_BINARY_DIR}/autogenerated/ggml-opencl_mul_mat_Ab_Bi_8x4.cl.h")
34+
set(OPENCL_TRANSPOSE_16_SOURCE_EMBED "${CMAKE_BINARY_DIR}/autogenerated/ggml-opencl_transpose_16.cl.h")
35+
set(OPENCL_TRANSPOSE_32_SOURCE_EMBED "${CMAKE_BINARY_DIR}/autogenerated/ggml-opencl_transpose_32.cl.h")
36+
set(OPENCL_TRANSPOSE_32_16_SOURCE_EMBED "${CMAKE_BINARY_DIR}/autogenerated/ggml-opencl_transpose_32_16.cl.h")
37+
38+
set(EMBED_KERNEL_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/kernels/embed_kernel.py")
39+
file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/autogenerated")
40+
41+
include_directories("${CMAKE_BINARY_DIR}/autogenerated")
42+
43+
# Python must be accessible from command line
44+
add_custom_command(
45+
OUTPUT ${OPENCL_CL_SOURCE_EMBED}
46+
COMMAND ${Python3_EXECUTABLE} ${EMBED_KERNEL_SCRIPT}
47+
${CMAKE_CURRENT_SOURCE_DIR}/kernels/ggml-opencl.cl
48+
${OPENCL_CL_SOURCE_EMBED}
49+
DEPENDS kernels/ggml-opencl.cl ${EMBED_KERNEL_SCRIPT}
50+
COMMENT "Generate ggml-opencl.cl.h"
51+
)
52+
53+
add_custom_command(
54+
OUTPUT ${OPENCL_MM_CL_SOURCE_EMBED}
55+
COMMAND ${Python3_EXECUTABLE} ${EMBED_KERNEL_SCRIPT}
56+
${CMAKE_CURRENT_SOURCE_DIR}/kernels/ggml-opencl_mm.cl
57+
${OPENCL_MM_CL_SOURCE_EMBED}
58+
DEPENDS kernels/ggml-opencl_mm.cl ${EMBED_KERNEL_SCRIPT}
59+
COMMENT "Generate ggml-opencl_mm.cl.h"
60+
)
61+
62+
add_custom_command(
63+
OUTPUT ${OPENCL_CVT_CL_SOURCE_EMBED}
64+
COMMAND ${Python3_EXECUTABLE} ${EMBED_KERNEL_SCRIPT}
65+
${CMAKE_CURRENT_SOURCE_DIR}/kernels/ggml-opencl_cvt.cl
66+
${OPENCL_CVT_CL_SOURCE_EMBED}
67+
DEPENDS kernels/ggml-opencl_cvt.cl ${EMBED_KERNEL_SCRIPT}
68+
COMMENT "Generate ggml-opencl_cvt.cl.h"
69+
)
70+
71+
add_custom_command(
72+
OUTPUT ${OPENCL_GEMV_NOSHUFFLE_SOURCE_EMBED}
73+
COMMAND ${Python3_EXECUTABLE} ${EMBED_KERNEL_SCRIPT}
74+
${CMAKE_CURRENT_SOURCE_DIR}/kernels/ggml-opencl_gemv_noshuffle.cl
75+
${OPENCL_GEMV_NOSHUFFLE_SOURCE_EMBED}
76+
DEPENDS kernels/ggml-opencl_gemv_noshuffle.cl ${EMBED_KERNEL_SCRIPT}
77+
COMMENT "Generate ggml-opencl_gemv_noshuffle.cl.h"
78+
)
79+
80+
add_custom_command(
81+
OUTPUT ${OPENCL_GEMV_NOSHUFFLE_GENERAL_SOURCE_EMBED}
82+
COMMAND ${Python3_EXECUTABLE} ${EMBED_KERNEL_SCRIPT}
83+
${CMAKE_CURRENT_SOURCE_DIR}/kernels/ggml-opencl_gemv_noshuffle_general.cl
84+
${OPENCL_GEMV_NOSHUFFLE_GENERAL_SOURCE_EMBED}
85+
DEPENDS kernels/ggml-opencl_gemv_noshuffle_general.cl ${EMBED_KERNEL_SCRIPT}
86+
COMMENT "Generate ggml-opencl_gemv_noshuffle_general.cl.h"
87+
)
88+
89+
add_custom_command(
90+
OUTPUT ${OPENCL_MUL_MAT_Ab_Bi_8x4_SOURCE_EMBED}
91+
COMMAND ${Python3_EXECUTABLE} ${EMBED_KERNEL_SCRIPT}
92+
${CMAKE_CURRENT_SOURCE_DIR}/kernels/ggml-opencl_mul_mat_Ab_Bi_8x4.cl
93+
${OPENCL_MUL_MAT_Ab_Bi_8x4_SOURCE_EMBED}
94+
DEPENDS kernels/ggml-opencl_mul_mat_Ab_Bi_8x4.cl ${EMBED_KERNEL_SCRIPT}
95+
COMMENT "Generate ggml-opencl_mul_mat_Ab_Bi_8x4.cl.cl.h"
96+
)
97+
98+
add_custom_command(
99+
OUTPUT ${OPENCL_TRANSPOSE_16_SOURCE_EMBED}
100+
COMMAND ${Python3_EXECUTABLE} ${EMBED_KERNEL_SCRIPT}
101+
${CMAKE_CURRENT_SOURCE_DIR}/kernels/ggml-opencl_transpose_16.cl
102+
${OPENCL_TRANSPOSE_16_SOURCE_EMBED}
103+
DEPENDS kernels/ggml-opencl_transpose_16.cl ${EMBED_KERNEL_SCRIPT}
104+
COMMENT "Generate ggml-opencl_transpose_16.cl.h"
105+
)
106+
107+
add_custom_command(
108+
OUTPUT ${OPENCL_TRANSPOSE_32_SOURCE_EMBED}
109+
COMMAND ${Python3_EXECUTABLE} ${EMBED_KERNEL_SCRIPT}
110+
${CMAKE_CURRENT_SOURCE_DIR}/kernels/ggml-opencl_transpose_32.cl
111+
${OPENCL_TRANSPOSE_32_SOURCE_EMBED}
112+
DEPENDS kernels/ggml-opencl_transpose_32.cl ${EMBED_KERNEL_SCRIPT}
113+
COMMENT "Generate ggml-opencl_transpose_32.cl.h"
114+
)
115+
116+
add_custom_command(
117+
OUTPUT ${OPENCL_TRANSPOSE_32_16_SOURCE_EMBED}
118+
COMMAND ${Python3_EXECUTABLE} ${EMBED_KERNEL_SCRIPT}
119+
${CMAKE_CURRENT_SOURCE_DIR}/kernels/ggml-opencl_transpose_32_16.cl
120+
${OPENCL_TRANSPOSE_32_16_SOURCE_EMBED}
121+
DEPENDS kernels/ggml-opencl_transpose_32_16.cl ${EMBED_KERNEL_SCRIPT}
122+
COMMENT "Generate ggml-opencl_transpose_32_16.cl.h"
123+
)
124+
125+
target_sources(${TARGET_NAME} PRIVATE
126+
${OPENCL_CL_SOURCE_EMBED}
127+
${OPENCL_MM_CL_SOURCE_EMBED}
128+
${OPENCL_CVT_CL_SOURCE_EMBED}
129+
${OPENCL_GEMV_NOSHUFFLE_SOURCE_EMBED}
130+
${OPENCL_GEMV_NOSHUFFLE_GENERAL_SOURCE_EMBED}
131+
${OPENCL_MUL_MAT_Ab_Bi_8x4_SOURCE_EMBED}
132+
${OPENCL_TRANSPOSE_16_SOURCE_EMBED}
133+
${OPENCL_TRANSPOSE_32_SOURCE_EMBED}
134+
${OPENCL_TRANSPOSE_32_16_SOURCE_EMBED})
135+
else ()
136+
# copy ggml-opencl.cl to bin directory
137+
configure_file(kernels/ggml-opencl.cl ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-opencl.cl COPYONLY)
138+
configure_file(kernels/ggml-opencl_mm.cl ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-opencl_mm.cl COPYONLY)
139+
configure_file(kernels/ggml-opencl_cvt.cl ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-opencl_cvt.cl COPYONLY)
140+
141+
configure_file(kernels/ggml-opencl_gemv_noshuffle.cl ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-opencl_gemv_noshuffle.cl COPYONLY)
142+
configure_file(kernels/ggml-opencl_gemv_noshuffle_general.cl ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-opencl_gemv_noshuffle_general.cl COPYONLY)
143+
configure_file(kernels/ggml-opencl_mul_mat_Ab_Bi_8x4.cl ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-opencl_mul_mat_Ab_Bi_8x4.cl COPYONLY)
144+
configure_file(kernels/ggml-opencl_transpose_16.cl ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-opencl_transpose_16.cl COPYONLY)
145+
configure_file(kernels/ggml-opencl_transpose_32.cl ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-opencl_transpose_32.cl COPYONLY)
146+
configure_file(kernels/ggml-opencl_transpose_32_16.cl ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-opencl_transpose_32_16.cl COPYONLY)
147+
endif ()

0 commit comments

Comments
 (0)