Skip to content

Commit b913e89

Browse files
LETS-BEELETS-BEE
committed
Merge remote-tracking branch 'jJJYmmm/qwen3vl-1022' into qwen3vl.
2 parents 9971912 + ea677dc commit b913e89

File tree

24 files changed

+958
-90
lines changed

24 files changed

+958
-90
lines changed

‎.github/actions/windows-setup-cuda/action.yml‎

Lines changed: 26 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -36,32 +36,32 @@ runs:
3636
echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
3737
echo "CUDA_PATH_V11_7=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
3838
39-
- name: Install Cuda Toolkit 12.4
40-
if: ${{ inputs.cuda_version == '12.4' }}
39+
- name: Install Cuda Toolkit 12.8
40+
if: ${{ inputs.cuda_version == '12.8' }}
4141
shell: pwsh
4242
run: |
43-
mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4"
43+
mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8"
4444
choco install unzip -y
45-
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-12.4.127-archive.zip"
46-
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-12.4.131-archive.zip"
47-
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-12.4.127-archive.zip"
48-
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-12.4.5.8-archive.zip"
49-
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-12.4.127-archive.zip"
50-
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-12.4.127-archive.zip"
51-
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-12.4.127-archive.zip"
52-
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvprof/windows-x86_64/cuda_nvprof-windows-x86_64-12.4.127-archive.zip"
53-
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-12.4.127-archive.zip"
54-
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4"
55-
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_cudart-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
56-
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_nvcc-windows-x86_64-12.4.131-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
57-
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_nvrtc-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
58-
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\libcublas-windows-x86_64-12.4.5.8-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
59-
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_nvtx-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
60-
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_profiler_api-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
61-
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\visual_studio_integration-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
62-
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_nvprof-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
63-
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_cccl-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
64-
echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
65-
echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\libnvvp" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
66-
echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
67-
echo "CUDA_PATH_V12_4=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
45+
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-12.8.90-archive.zip"
46+
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-12.8.93-archive.zip"
47+
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-12.8.93-archive.zip"
48+
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-12.8.4.1-archive.zip"
49+
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-12.8.90-archive.zip"
50+
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-12.8.90-archive.zip"
51+
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-12.8.90-archive.zip"
52+
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvprof/windows-x86_64/cuda_nvprof-windows-x86_64-12.8.90-archive.zip"
53+
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-12.8.90-archive.zip"
54+
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8"
55+
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8\cuda_cudart-windows-x86_64-12.8.90-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8" /E /I /H /Y
56+
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8\cuda_nvcc-windows-x86_64-12.8.93-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8" /E /I /H /Y
57+
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8\cuda_nvrtc-windows-x86_64-12.8.93-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8" /E /I /H /Y
58+
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8\libcublas-windows-x86_64-12.8.4.1-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8" /E /I /H /Y
59+
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8\cuda_nvtx-windows-x86_64-12.8.90-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8" /E /I /H /Y
60+
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8\cuda_profiler_api-windows-x86_64-12.8.90-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8" /E /I /H /Y
61+
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8\visual_studio_integration-windows-x86_64-12.8.90-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8" /E /I /H /Y
62+
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8\cuda_nvprof-windows-x86_64-12.8.90-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8" /E /I /H /Y
63+
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8\cuda_cccl-windows-x86_64-12.8.90-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8" /E /I /H /Y
64+
echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
65+
echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8\libnvvp" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
66+
echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
67+
echo "CUDA_PATH_V12_8=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8

‎.github/workflows/build.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1075,7 +1075,7 @@ jobs:
10751075

10761076
strategy:
10771077
matrix:
1078-
cuda: ['12.4']
1078+
cuda: ['12.8']
10791079

10801080
steps:
10811081
- name: Clone

‎.github/workflows/release.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ concurrency:
1818

1919
env:
2020
BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
21-
CMAKE_ARGS: "-DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_TOOLS=ON -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON"
21+
CMAKE_ARGS: "-DLLAMA_BUILD_EXAMPLES=ON -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_TOOLS=ON -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON"
2222

2323
jobs:
2424
macOS-arm64:
@@ -391,7 +391,7 @@ jobs:
391391

392392
strategy:
393393
matrix:
394-
cuda: ['12.4']
394+
cuda: ['12.8']
395395

396396
steps:
397397
- name: Clone

‎convert_hf_to_gguf.py‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3538,16 +3538,25 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter
35383538
return super().modify_tensors(data_torch, name, bid)
35393539

35403540

3541+
<<<<<<< HEAD
35413542
@ModelBase.register("Qwen3VLForConditionalGeneration", "Qwen3VLMoeForConditionalGeneration")
35423543
class Qwen3VLVisionModel(MmprojModel):
3544+
=======
3545+
@ModelBase.register("Qwen3VLMoeForConditionalGeneration")
3546+
class Qwen3VLMoeVisionModel(MmprojModel):
3547+
>>>>>>> remote-JJJYmmm/qwen3vl-1022
35433548
def __init__(self, *args, **kwargs):
35443549
super().__init__(*args, **kwargs)
35453550
assert self.has_vision_encoder
35463551
assert self.hparams_vision is not None
35473552

35483553
# Compute image_size if not present
35493554
if "image_size" not in self.hparams_vision:
3555+
<<<<<<< HEAD
35503556
# For Qwen3VL/Qwen3VLMoe, compute from num_position_embeddings
3557+
=======
3558+
# For Qwen3VLMoe, compute from num_position_embeddings
3559+
>>>>>>> remote-JJJYmmm/qwen3vl-1022
35513560
num_pos = self.hparams_vision.get("num_position_embeddings", 2304)
35523561
patch_size = self.hparams_vision.get("patch_size", 16)
35533562
# num_position_embeddings = (image_size / patch_size) ** 2
@@ -3592,10 +3601,15 @@ def set_gguf_parameters(self):
35923601
self.gguf_writer.add_vision_deepstack_layers(self.deepstack_layers)
35933602

35943603
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
3604+
<<<<<<< HEAD
35953605
# Skip text model tensors - they go in the text model file
35963606
if name.startswith("model.language_model.") or name.startswith("lm_head."):
35973607
return []
35983608

3609+
=======
3610+
del bid # unused
3611+
3612+
>>>>>>> remote-JJJYmmm/qwen3vl-1022
35993613
if name.startswith("model.visual."):
36003614
name = name.replace("model.visual.", "visual.", 1)
36013615

@@ -3652,8 +3666,13 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter
36523666
]
36533667

36543668
if name == "visual.patch_embed.proj.bias":
3669+
<<<<<<< HEAD
36553670
# Include the bias - it's used by the C++ code
36563671
return [(gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".bias", data_torch)]
3672+
=======
3673+
# Skip bias for Qwen3VL - the C++ code expects it to be null
3674+
return []
3675+
>>>>>>> remote-JJJYmmm/qwen3vl-1022
36573676

36583677
if name.startswith("visual."):
36593678
if ".qkv." in name:
@@ -3676,8 +3695,12 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter
36763695

36773696
return [(self.map_tensor_name(name), data_torch)]
36783697

3698+
<<<<<<< HEAD
36793699
# Fall back to parent class for other tensors
36803700
return super().modify_tensors(data_torch, name, bid)
3701+
=======
3702+
return []
3703+
>>>>>>> remote-JJJYmmm/qwen3vl-1022
36813704

36823705
@ModelBase.register("InternVisionModel")
36833706
class InternVisionModel(MmprojModel):
@@ -4003,6 +4026,7 @@ def set_vocab(self):
40034026
super().set_vocab()
40044027

40054028

4029+
<<<<<<< HEAD
40064030
@ModelBase.register("Qwen3VLForConditionalGeneration")
40074031
class Qwen3VLTextModel(Qwen3Model):
40084032
model_arch = gguf.MODEL_ARCH.QWEN3VL
@@ -4032,6 +4056,8 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter
40324056
return super().modify_tensors(data_torch, name, bid)
40334057

40344058

4059+
=======
4060+
>>>>>>> remote-JJJYmmm/qwen3vl-1022
40354061
@ModelBase.register("Qwen3VLMoeForConditionalGeneration")
40364062
class Qwen3VLMoeTextModel(Qwen3MoeModel):
40374063
model_arch = gguf.MODEL_ARCH.QWEN3VLMOE
@@ -4053,13 +4079,16 @@ def set_gguf_parameters(self):
40534079

40544080
logger.info(f"MRoPE sections: {mrope_section[:4]}")
40554081

4082+
<<<<<<< HEAD
40564083
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
40574084
# Skip vision tensors - they go in the mmproj file
40584085
if name.startswith("model.visual."):
40594086
return []
40604087

40614088
return super().modify_tensors(data_torch, name, bid)
40624089

4090+
=======
4091+
>>>>>>> remote-JJJYmmm/qwen3vl-1022
40634092

40644093
@ModelBase.register("GPT2LMHeadModel")
40654094
class GPT2Model(TextModel):

‎examples/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ else()
3030
add_subdirectory(simple-chat)
3131
add_subdirectory(speculative)
3232
add_subdirectory(speculative-simple)
33+
add_subdirectory(sweep-bench)
3334
add_subdirectory(gen-docs)
3435
add_subdirectory(training)
3536
add_subdirectory(diffusion)

‎examples/sweep-bench/CMakeLists.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
set(TARGET llama-sweep-bench)
2+
add_executable(${TARGET} sweep-bench.cpp)
3+
install(TARGETS ${TARGET} RUNTIME)
4+
target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT})
5+
target_compile_features(${TARGET} PRIVATE cxx_std_17)

‎examples/sweep-bench/README.md‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
# ik_llama.cpp/example/sweep-bench
2+
3+
Benchmark the prompt processing and token generation performance of `ik_llama.cpp`
4+
by doing a sweep over a whole context size and gathering performance metrics
5+
in each ubatch-sized window. Only a single token sequence is used.
6+
7+
The benchmark steps are:
8+
9+
for each ubatch-sized window in context:
10+
11+
1. generate ubatch/4 tokens (not the whole window to save some time)
12+
2. measure generation performance
13+
3. remove generated tokens from KV cache
14+
4. prepare a ubatch-sized batch of random tokens
15+
4. process prepated batch
16+
5. measure prompt processing performance
17+
18+
The purpose of the benchmark is to visualize how the performance changes with
19+
the context size without averaging the metrics values over the whole context.
20+
21+
## Usage
22+
23+
./llama-sweep-bench -c 8704 -ub 512 -m models/Meta-Llama-3.2-3B-Instruct-Q8_0.gguf
24+
25+
## Sample results
26+
27+
- `PP` - prompt tokens per ubatch
28+
- `TG` - generated tokens per ubatch
29+
- `N_KV` - current KV cache size
30+
- `T_PP` - prompt processing time (i.e. time to first token)
31+
- `S_PP` - prompt processing speed (`(B*PP)/T_PP` or `PP/T_PP`)
32+
- `T_TG` - time to generate all batches
33+
- `S_TG` - text generation speed (`(B*TG)/T_TG`)
34+
35+
| PP | TG | N_KV | T_PP s | S_PP t/s | T_TG s | S_TG t/s |
36+
|-------|--------|--------|----------|----------|----------|----------|
37+
| 512 | 128 | 0 | 1.100 | 465.51 | 2.311 | 55.38 |
38+
| 512 | 128 | 512 | 1.183 | 432.97 | 1.895 | 67.55 |
39+
| 512 | 128 | 1024 | 1.305 | 392.38 | 2.071 | 61.81 |
40+
| 512 | 128 | 1536 | 1.279 | 400.42 | 2.164 | 59.14 |
41+
| 512 | 128 | 2048 | 1.571 | 325.96 | 2.280 | 56.14 |
42+
| 512 | 128 | 2560 | 1.431 | 357.87 | 2.418 | 52.94 |
43+
| 512 | 128 | 3072 | 1.515 | 337.93 | 2.566 | 49.88 |
44+
| 512 | 128 | 3584 | 1.588 | 322.34 | 2.722 | 47.03 |
45+
| 512 | 128 | 4096 | 1.675 | 305.70 | 2.864 | 44.69 |
46+
| 512 | 128 | 4608 | 1.769 | 289.50 | 2.999 | 42.68 |
47+
| 512 | 128 | 5120 | 1.845 | 277.48 | 3.102 | 41.26 |
48+
| 512 | 128 | 5632 | 1.893 | 270.46 | 3.219 | 39.76 |
49+
| 512 | 128 | 6144 | 1.953 | 262.20 | 3.348 | 38.23 |
50+
| 512 | 128 | 6656 | 2.018 | 253.71 | 3.474 | 36.84 |
51+
| 512 | 128 | 7168 | 2.078 | 246.34 | 3.589 | 35.66 |
52+
| 512 | 128 | 7680 | 2.140 | 239.22 | 3.717 | 34.43 |
53+
| 512 | 128 | 8192 | 2.196 | 233.15 | 3.854 | 33.21 |
54+
55+
### JSONL output
56+
57+
Pass `--output-format jsonl` to output JSONL instead of Markdown, á la
58+
59+
```json lines
60+
{"n_kv_max": 8704, "n_batch": 2048, "n_ubatch": 512, "flash_attn": 0, "n_gpu_layers": -1, "n_threads": 32, "n_threads_batch": 32, "pp": 512, "tg": 128, "n_kv": 0, "t_pp": 1.093814, "speed_pp": 468.086884, "t_tg": 1.780312, "speed_tg": 71.897514 }
61+
{"n_kv_max": 8704, "n_batch": 2048, "n_ubatch": 512, "flash_attn": 0, "n_gpu_layers": -1, "n_threads": 32, "n_threads_batch": 32, "pp": 512, "tg": 128, "n_kv": 512, "t_pp": 1.169302, "speed_pp": 437.868073, "t_tg": 1.897474, "speed_tg": 67.458099 }
62+
{"n_kv_max": 8704, "n_batch": 2048, "n_ubatch": 512, "flash_attn": 0, "n_gpu_layers": -1, "n_threads": 32, "n_threads_batch": 32, "pp": 512, "tg": 128, "n_kv": 1024, "t_pp": 1.183700, "speed_pp": 432.542053, "t_tg": 2.059179, "speed_tg": 62.160694 }
63+
{"n_kv_max": 8704, "n_batch": 2048, "n_ubatch": 512, "flash_attn": 0, "n_gpu_layers": -1, "n_threads": 32, "n_threads_batch": 32, "pp": 512, "tg": 128, "n_kv": 1536, "t_pp": 1.428625, "speed_pp": 358.386566, "t_tg": 2.160639, "speed_tg": 59.241734 }
64+
{"n_kv_max": 8704, "n_batch": 2048, "n_ubatch": 512, "flash_attn": 0, "n_gpu_layers": -1, "n_threads": 32, "n_threads_batch": 32, "pp": 512, "tg": 128, "n_kv": 2048, "t_pp": 1.360647, "speed_pp": 376.291595, "t_tg": 2.274003, "speed_tg": 56.288403 }
65+
```

0 commit comments

Comments
 (0)