From 2dfa744182a9ae426171341ae5c89f438513f039 Mon Sep 17 00:00:00 2001 From: dbsid Date: Fri, 30 May 2025 16:23:58 +0800 Subject: [PATCH 01/16] add tikv compaction and flow control setting --- tidb-performance-tuning-config.md | 34 +++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tidb-performance-tuning-config.md b/tidb-performance-tuning-config.md index 4449cf4d413bc..d8819d8919a2f 100644 --- a/tidb-performance-tuning-config.md +++ b/tidb-performance-tuning-config.md @@ -29,6 +29,7 @@ The following settings are commonly used to optimize TiDB performance: - Enhance execution plan cache, such as [SQL Prepared Execution Plan Cache](/sql-prepared-plan-cache.md), [Non-prepared plan cache](/sql-non-prepared-plan-cache.md), and [Instance-level execution plan cache](/system-variables.md#tidb_enable_instance_plan_cache-new-in-v840). - Optimize the behavior of the TiDB optimizer by using [Optimizer Fix Controls](/optimizer-fix-controls.md). - Use the [Titan](/storage-engine/titan-overview.md) storage engine more aggressively. +- Fine-tune TiKV's compaction and flow control configurations, to ensure optimal and stable performance under write-intensive workloads. These settings can significantly improve performance for many workloads. However, as with any optimization, thoroughly test them in your environment before deploying to production. @@ -92,22 +93,55 @@ snap-io-max-bytes-per-sec = "400MiB" in-memory-peer-size-limit = "32MiB" in-memory-instance-size-limit = "512MiB" +[rocksdb] +rate-bytes-per-sec = "600MiB" +max-manifest-file-size = "256MiB" [rocksdb.titan] enabled = true [rocksdb.defaultcf.titan] min-blob-size = "1KB" blob-file-compression = "zstd" +[storage] +scheduler-pending-write-threshold = "512MiB" [storage.flow-control] l0-files-threshold = 60 +soft-pending-compaction-bytes-limit = "1TiB" +hard-pending-compaction-bytes-limit = "2TiB" ``` | Configuration item | Description | Note | | ---------| ---- | ----| | [`concurrent-send-snap-limit`](/tikv-configuration-file.md#concurrent-send-snap-limit), [`concurrent-recv-snap-limit`](/tikv-configuration-file.md#concurrent-recv-snap-limit), and [`snap-io-max-bytes-per-sec`](/tikv-configuration-file.md#snap-io-max-bytes-per-sec) | Set limits for concurrent snapshot transfer and I/O bandwidth during TiKV scaling operations. Higher limits reduce scaling time by allowing faster data migration. | Adjusting these limits affects the trade-off between scaling speed and online transaction performance. | | [`in-memory-peer-size-limit`](/tikv-configuration-file.md#in-memory-peer-size-limit-new-in-v840) and [`in-memory-instance-size-limit`](/tikv-configuration-file.md#in-memory-instance-size-limit-new-in-v840) | Control the memory allocation for pessimistic lock caching at the Region and TiKV instance levels. Storing locks in memory reduces disk I/O and improves transaction performance. | Monitor memory usage carefully. Higher limits improve performance but increase memory consumption. | +|[`rocksdb.rate-bytes-per-sec`](/tikv-configuration-file.md#rate-bytes-per-sec) | Limits the I/O rate of background compaction and flush operations to prevent them from saturating disk bandwidth, ensuring stable performance. | It's recommended to set this to around 60% of the disk's maximum throughput. Overly high settings can lead to disk saturation, impacting foreground operations. Conversely, setting it too low may slow down compaction, causing write stalls and flow control activations. | +|[`rocksdb.max-manifest-file-size`](/tikv-configuration-file.md#max-manifest-file-size) | Sets the maximum size of the RocksDB MANIFEST file, which logs metadata about SST files and database state changes. Increasing this size reduces the frequency of MANIFEST file rewrites, thereby minimizing their impact on foreground write performance. | The default value is 128 MiB. In environments with a large number of SST files (e.g., hundreds of thousands), frequent MANIFEST rewrites can degrade write performance. Adjusting this parameter to a higher value, such as 256 MiB or more, can help maintain optimal performance. | | [`rocksdb.titan`](/tikv-configuration-file.md#rocksdbtitan), [`rocksdb.defaultcf.titan`](/tikv-configuration-file.md#rocksdbdefaultcftitan), [`min-blob-size`](/tikv-configuration-file.md#min-blob-size), and [`blob-file-compression`](/tikv-configuration-file.md#blob-file-compression) | Enable the Titan storage engine to reduce write amplification and alleviate disk I/O bottlenecks. Particularly useful when RocksDB compaction cannot keep up with write workloads, resulting in accumulated pending compaction bytes. | Enable it when write amplification is the primary bottleneck. Trade-offs include: 1. Potential performance impact on primary key range scans. 2. Increased space amplification (up to 2x in the worst case). 3. Additional memory usage for blob cache. | +| [`storage.scheduler-pending-write-threshold`](/tikv-configuration-file.md#scheduler-pending-write-threshold) | Sets the maximum size of the write queue in TiKV's scheduler. When the total size of pending write tasks exceeds this threshold, TiKV returns a `Server Is Busy` error for new write requests. | The default value is 100 MiB. In scenarios with high write concurrency or temporary write spikes, increasing this threshold (e.g., to 512 MiB) can help accommodate the load. However, if the write queue continues to accumulate and exceeds this threshold persistently, it may indicate underlying performance issues that require further investigation. | | [`storage.flow-control.l0-files-threshold`](/tikv-configuration-file.md#l0-files-threshold) | Control when write flow control is triggered based on the number of kvDB L0 files. Increasing the threshold reduces write stalls during high write workloads. | Higher thresholds might lead to more aggressive compactions when many L0 files exist. | +| [`storage.flow-control.soft-pending-compaction-bytes-limit`](/tikv-configuration-file.md#soft-pending-compaction-bytes-limit) and [`storage.flow-control.hard-pending-compaction-bytes-limit`](/tikv-configuration-file.md#hard-pending-compaction-bytes-limit) | Define thresholds for pending compaction bytes to manage write flow control. The soft limit triggers partial write rejections, while the hard limit results in full write rejections when exceeded. | The default soft limit is 192 GiB, and the hard limit is 256 GiB. In write-intensive scenarios, if compaction processes cannot keep up, pending compaction bytes accumulate, potentially triggering flow control. Adjusting these limits can provide more buffer space, but persistent accumulation indicates underlying issues that require further investigation. | + +Be noted that the compaction and flow control configuration adjustments outlined above are tailored for TiKV deployments on instances with the following specifications: + + - CPU: 32 cores + - Memory: 128 GiB + - Storage: 5 TiB EBS + - Disk Throughput: 1 GiB/s + +#### Recommended Configuration Adjustments for Write-Intensive Workloads + +To optimize TiKV's performance and stability under write-intensive workloads, it's advisable to adjust certain compaction and flow control parameters based on the instance's hardware specifications. For example: + + - `rocksdb.rate-bytes-per-sec`: Set to approximately 60% of the disk's maximum throughput (e.g., ~600 MiB/s for a 1 GiB/s disk) to balance compaction I/O and prevent disk saturation. + - `storage.flow-control.soft-pending-compaction-bytes-limit` and `storage.flow-control.hard-pending-compaction-bytes-limit`: Increase these limits proportionally to the available disk space (e.g., 1 TiB and 2 TiB, respectively) to provide more buffer for compaction processes. + +These settings help ensure efficient resource utilization and minimize potential bottlenecks during peak write loads. + + +> **Note:** +> +> TiKV employs a flow control mechanism at the scheduler layer to maintain system stability. When certain thresholds are exceeded—such as pending compaction bytes or write queue sizes—TiKV may start rejecting some or all write requests, returning a `ServerIsBusy` error. This indicates that the system is under pressure and cannot process writes at the current rate. To avoid such scenarios, it's crucial to perform thorough capacity planning and configure compaction and storage settings appropriately. Monitoring key metrics and adjusting configurations proactively can help maintain optimal performance and prevent flow controlss. + ### TiFlash configurations From 7876c49ce4414de6e82c08ea640c47a8105e4348 Mon Sep 17 00:00:00 2001 From: dbsid Date: Fri, 30 May 2025 16:40:13 +0800 Subject: [PATCH 02/16] Update tidb-performance-tuning-config.md Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- tidb-performance-tuning-config.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tidb-performance-tuning-config.md b/tidb-performance-tuning-config.md index d8819d8919a2f..9d71dc52df15f 100644 --- a/tidb-performance-tuning-config.md +++ b/tidb-performance-tuning-config.md @@ -140,7 +140,7 @@ These settings help ensure efficient resource utilization and minimize potential > **Note:** > -> TiKV employs a flow control mechanism at the scheduler layer to maintain system stability. When certain thresholds are exceeded—such as pending compaction bytes or write queue sizes—TiKV may start rejecting some or all write requests, returning a `ServerIsBusy` error. This indicates that the system is under pressure and cannot process writes at the current rate. To avoid such scenarios, it's crucial to perform thorough capacity planning and configure compaction and storage settings appropriately. Monitoring key metrics and adjusting configurations proactively can help maintain optimal performance and prevent flow controlss. +Monitoring key metrics and adjusting configurations proactively can help maintain optimal performance and prevent flow controls. ### TiFlash configurations From ffdef66fb84ef1910ce9200d764fef528b0185c2 Mon Sep 17 00:00:00 2001 From: dbsid Date: Wed, 4 Jun 2025 13:59:50 +0800 Subject: [PATCH 03/16] Update tidb-performance-tuning-config.md --- tidb-performance-tuning-config.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tidb-performance-tuning-config.md b/tidb-performance-tuning-config.md index 9d71dc52df15f..b7dd5cb7e13c7 100644 --- a/tidb-performance-tuning-config.md +++ b/tidb-performance-tuning-config.md @@ -137,10 +137,10 @@ To optimize TiKV's performance and stability under write-intensive workloads, it These settings help ensure efficient resource utilization and minimize potential bottlenecks during peak write loads. - > **Note:** > -Monitoring key metrics and adjusting configurations proactively can help maintain optimal performance and prevent flow controls. +> TiKV implements flow control at the scheduler layer to ensure system stability. When critical thresholds are breached—including pending compaction bytes or write queue sizes—TiKV begins rejecting write requests and returns a ServerIsBusy error. This error signals that background compaction processes cannot keep pace with the current rate of foreground write operations. +> Flow control activation typically results in latency spikes and reduced query throughput (QPS drops). To prevent these performance degradations, comprehensive capacity planning is essential, along with proper configuration of compaction parameters and storage settings. ### TiFlash configurations From 0f2cbd7a7d41ddb6625afdb930b2c5850d57b6cd Mon Sep 17 00:00:00 2001 From: dbsid Date: Wed, 4 Jun 2025 14:01:15 +0800 Subject: [PATCH 04/16] Update tidb-performance-tuning-config.md --- tidb-performance-tuning-config.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tidb-performance-tuning-config.md b/tidb-performance-tuning-config.md index b7dd5cb7e13c7..9ba7983f2d74c 100644 --- a/tidb-performance-tuning-config.md +++ b/tidb-performance-tuning-config.md @@ -139,8 +139,7 @@ These settings help ensure efficient resource utilization and minimize potential > **Note:** > -> TiKV implements flow control at the scheduler layer to ensure system stability. When critical thresholds are breached—including pending compaction bytes or write queue sizes—TiKV begins rejecting write requests and returns a ServerIsBusy error. This error signals that background compaction processes cannot keep pace with the current rate of foreground write operations. -> Flow control activation typically results in latency spikes and reduced query throughput (QPS drops). To prevent these performance degradations, comprehensive capacity planning is essential, along with proper configuration of compaction parameters and storage settings. +> TiKV implements flow control at the scheduler layer to ensure system stability. When critical thresholds are breached—including pending compaction bytes or write queue sizes—TiKV begins rejecting write requests and returns a ServerIsBusy error. This error signals that background compaction processes cannot keep pace with the current rate of foreground write operations. Flow control activation typically results in latency spikes and reduced query throughput (QPS drops). To prevent these performance degradations, comprehensive capacity planning is essential, along with proper configuration of compaction parameters and storage settings. ### TiFlash configurations From b38b73eade7bf75092b023838b17e053707952f0 Mon Sep 17 00:00:00 2001 From: xixirangrang Date: Thu, 5 Jun 2025 09:38:54 +0800 Subject: [PATCH 05/16] Apply suggestions from code review Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- tidb-performance-tuning-config.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tidb-performance-tuning-config.md b/tidb-performance-tuning-config.md index 9ba7983f2d74c..aa6a5f6285593 100644 --- a/tidb-performance-tuning-config.md +++ b/tidb-performance-tuning-config.md @@ -121,7 +121,7 @@ hard-pending-compaction-bytes-limit = "2TiB" | [`storage.flow-control.l0-files-threshold`](/tikv-configuration-file.md#l0-files-threshold) | Control when write flow control is triggered based on the number of kvDB L0 files. Increasing the threshold reduces write stalls during high write workloads. | Higher thresholds might lead to more aggressive compactions when many L0 files exist. | | [`storage.flow-control.soft-pending-compaction-bytes-limit`](/tikv-configuration-file.md#soft-pending-compaction-bytes-limit) and [`storage.flow-control.hard-pending-compaction-bytes-limit`](/tikv-configuration-file.md#hard-pending-compaction-bytes-limit) | Define thresholds for pending compaction bytes to manage write flow control. The soft limit triggers partial write rejections, while the hard limit results in full write rejections when exceeded. | The default soft limit is 192 GiB, and the hard limit is 256 GiB. In write-intensive scenarios, if compaction processes cannot keep up, pending compaction bytes accumulate, potentially triggering flow control. Adjusting these limits can provide more buffer space, but persistent accumulation indicates underlying issues that require further investigation. | -Be noted that the compaction and flow control configuration adjustments outlined above are tailored for TiKV deployments on instances with the following specifications: +Note that the compaction and flow control configuration adjustments outlined above are tailored for TiKV deployments on instances with the following specifications: - CPU: 32 cores - Memory: 128 GiB @@ -130,7 +130,7 @@ Be noted that the compaction and flow control configuration adjustments outlined #### Recommended Configuration Adjustments for Write-Intensive Workloads -To optimize TiKV's performance and stability under write-intensive workloads, it's advisable to adjust certain compaction and flow control parameters based on the instance's hardware specifications. For example: +To optimize TiKV's performance and stability under write-intensive workloads, it is recommended that you adjust certain compaction and flow control parameters based on the instance's hardware specifications. For example: - `rocksdb.rate-bytes-per-sec`: Set to approximately 60% of the disk's maximum throughput (e.g., ~600 MiB/s for a 1 GiB/s disk) to balance compaction I/O and prevent disk saturation. - `storage.flow-control.soft-pending-compaction-bytes-limit` and `storage.flow-control.hard-pending-compaction-bytes-limit`: Increase these limits proportionally to the available disk space (e.g., 1 TiB and 2 TiB, respectively) to provide more buffer for compaction processes. @@ -139,7 +139,7 @@ These settings help ensure efficient resource utilization and minimize potential > **Note:** > -> TiKV implements flow control at the scheduler layer to ensure system stability. When critical thresholds are breached—including pending compaction bytes or write queue sizes—TiKV begins rejecting write requests and returns a ServerIsBusy error. This error signals that background compaction processes cannot keep pace with the current rate of foreground write operations. Flow control activation typically results in latency spikes and reduced query throughput (QPS drops). To prevent these performance degradations, comprehensive capacity planning is essential, along with proper configuration of compaction parameters and storage settings. +> TiKV implements flow control at the scheduler layer to ensure system stability. When critical thresholds are breached, including those for pending compaction bytes or write queue sizes, TiKV begins rejecting write requests and returns a ServerIsBusy error. This error signals that background compaction processes cannot keep pace with the current rate of foreground write operations. Flow control activation typically results in latency spikes and reduced query throughput (QPS drops). To prevent these performance degradations, comprehensive capacity planning is essential, along with proper configuration of compaction parameters and storage settings. ### TiFlash configurations From 9699be98182d957cca9eb3ff0374c784d69c1de0 Mon Sep 17 00:00:00 2001 From: dbsid Date: Thu, 5 Jun 2025 16:23:17 +0800 Subject: [PATCH 06/16] Update tidb-performance-tuning-config.md --- tidb-performance-tuning-config.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tidb-performance-tuning-config.md b/tidb-performance-tuning-config.md index 9ba7983f2d74c..21a7ae70abc8b 100644 --- a/tidb-performance-tuning-config.md +++ b/tidb-performance-tuning-config.md @@ -89,6 +89,9 @@ concurrent-send-snap-limit = 64 concurrent-recv-snap-limit = 64 snap-io-max-bytes-per-sec = "400MiB" +[quota] +background-read-bandwidth = "400Mib" + [pessimistic-txn] in-memory-peer-size-limit = "32MiB" in-memory-instance-size-limit = "512MiB" @@ -113,6 +116,7 @@ hard-pending-compaction-bytes-limit = "2TiB" | Configuration item | Description | Note | | ---------| ---- | ----| | [`concurrent-send-snap-limit`](/tikv-configuration-file.md#concurrent-send-snap-limit), [`concurrent-recv-snap-limit`](/tikv-configuration-file.md#concurrent-recv-snap-limit), and [`snap-io-max-bytes-per-sec`](/tikv-configuration-file.md#snap-io-max-bytes-per-sec) | Set limits for concurrent snapshot transfer and I/O bandwidth during TiKV scaling operations. Higher limits reduce scaling time by allowing faster data migration. | Adjusting these limits affects the trade-off between scaling speed and online transaction performance. | +| [`quota.background-read-bandwidth`](/tikv-configuration-file.md#background-read-bandwidth-new-in-v620) | Sets a soft limit on read bandwidth for background transactions and Coprocessor operations. Recommended to configure at 40% of disk bandwidth to prevent interference with foreground traffic. | Default: unlimited. Without this limit, analyze jobs on large tables may significantly degrade application performance. | | [`in-memory-peer-size-limit`](/tikv-configuration-file.md#in-memory-peer-size-limit-new-in-v840) and [`in-memory-instance-size-limit`](/tikv-configuration-file.md#in-memory-instance-size-limit-new-in-v840) | Control the memory allocation for pessimistic lock caching at the Region and TiKV instance levels. Storing locks in memory reduces disk I/O and improves transaction performance. | Monitor memory usage carefully. Higher limits improve performance but increase memory consumption. | |[`rocksdb.rate-bytes-per-sec`](/tikv-configuration-file.md#rate-bytes-per-sec) | Limits the I/O rate of background compaction and flush operations to prevent them from saturating disk bandwidth, ensuring stable performance. | It's recommended to set this to around 60% of the disk's maximum throughput. Overly high settings can lead to disk saturation, impacting foreground operations. Conversely, setting it too low may slow down compaction, causing write stalls and flow control activations. | |[`rocksdb.max-manifest-file-size`](/tikv-configuration-file.md#max-manifest-file-size) | Sets the maximum size of the RocksDB MANIFEST file, which logs metadata about SST files and database state changes. Increasing this size reduces the frequency of MANIFEST file rewrites, thereby minimizing their impact on foreground write performance. | The default value is 128 MiB. In environments with a large number of SST files (e.g., hundreds of thousands), frequent MANIFEST rewrites can degrade write performance. Adjusting this parameter to a higher value, such as 256 MiB or more, can help maintain optimal performance. | From 33d94f4dd12133589038c41e8d3c240b412ee01b Mon Sep 17 00:00:00 2001 From: dbsid Date: Fri, 6 Jun 2025 17:41:07 +0800 Subject: [PATCH 07/16] Apply suggestions from code review Co-authored-by: xixirangrang --- tidb-performance-tuning-config.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tidb-performance-tuning-config.md b/tidb-performance-tuning-config.md index 07875a0cca603..95ce0b63c7be7 100644 --- a/tidb-performance-tuning-config.md +++ b/tidb-performance-tuning-config.md @@ -116,34 +116,34 @@ hard-pending-compaction-bytes-limit = "2TiB" | Configuration item | Description | Note | | ---------| ---- | ----| | [`concurrent-send-snap-limit`](/tikv-configuration-file.md#concurrent-send-snap-limit), [`concurrent-recv-snap-limit`](/tikv-configuration-file.md#concurrent-recv-snap-limit), and [`snap-io-max-bytes-per-sec`](/tikv-configuration-file.md#snap-io-max-bytes-per-sec) | Set limits for concurrent snapshot transfer and I/O bandwidth during TiKV scaling operations. Higher limits reduce scaling time by allowing faster data migration. | Adjusting these limits affects the trade-off between scaling speed and online transaction performance. | -| [`quota.background-read-bandwidth`](/tikv-configuration-file.md#background-read-bandwidth-new-in-v620) | Sets a soft limit on read bandwidth for background transactions and Coprocessor operations. Recommended to configure at 40% of disk bandwidth to prevent interference with foreground traffic. | Default: unlimited. Without this limit, analyze jobs on large tables may significantly degrade application performance. | +| [`quota.background-read-bandwidth`](/tikv-configuration-file.md#background-read-bandwidth-new-in-v620) | Set a soft limit on read bandwidth for background transactions and Coprocessor operations. It is recommended that you configure at 40% of disk bandwidth to prevent interference with foreground traffic. | Default value: `0KiB` (which means no limit). Without this limit, analyze jobs on large tables might significantly degrade application performance. | | [`in-memory-peer-size-limit`](/tikv-configuration-file.md#in-memory-peer-size-limit-new-in-v840) and [`in-memory-instance-size-limit`](/tikv-configuration-file.md#in-memory-instance-size-limit-new-in-v840) | Control the memory allocation for pessimistic lock caching at the Region and TiKV instance levels. Storing locks in memory reduces disk I/O and improves transaction performance. | Monitor memory usage carefully. Higher limits improve performance but increase memory consumption. | -|[`rocksdb.rate-bytes-per-sec`](/tikv-configuration-file.md#rate-bytes-per-sec) | Limits the I/O rate of background compaction and flush operations to prevent them from saturating disk bandwidth, ensuring stable performance. | It's recommended to set this to around 60% of the disk's maximum throughput. Overly high settings can lead to disk saturation, impacting foreground operations. Conversely, setting it too low may slow down compaction, causing write stalls and flow control activations. | -|[`rocksdb.max-manifest-file-size`](/tikv-configuration-file.md#max-manifest-file-size) | Sets the maximum size of the RocksDB MANIFEST file, which logs metadata about SST files and database state changes. Increasing this size reduces the frequency of MANIFEST file rewrites, thereby minimizing their impact on foreground write performance. | The default value is 128 MiB. In environments with a large number of SST files (e.g., hundreds of thousands), frequent MANIFEST rewrites can degrade write performance. Adjusting this parameter to a higher value, such as 256 MiB or more, can help maintain optimal performance. | +|[`rocksdb.rate-bytes-per-sec`](/tikv-configuration-file.md#rate-bytes-per-sec) | Limit the I/O rate of background compaction and flush operations to prevent them from saturating disk bandwidth, ensuring stable performance. | It is recommended that you set it to around 60% of the disk's maximum throughput. Overly high settings can lead to disk saturation, impacting foreground operations. Conversely, setting it too low might slow down compaction, causing write stalls and flow control activations. | +|[`rocksdb.max-manifest-file-size`](/tikv-configuration-file.md#max-manifest-file-size) | Set the maximum size of the RocksDB MANIFEST file, which logs metadata about SST files and database state changes. Increasing this size reduces the frequency of MANIFEST file rewrites, thereby minimizing their impact on foreground write performance. | The default value is `128MiB`. In environments with a large number of SST files (for example, hundreds of thousands), frequent MANIFEST rewrites can degrade write performance. Adjusting this parameter to a higher value, such as `256MiB` or larger, can help maintain optimal performance. | | [`rocksdb.titan`](/tikv-configuration-file.md#rocksdbtitan), [`rocksdb.defaultcf.titan`](/tikv-configuration-file.md#rocksdbdefaultcftitan), [`min-blob-size`](/tikv-configuration-file.md#min-blob-size), and [`blob-file-compression`](/tikv-configuration-file.md#blob-file-compression) | Enable the Titan storage engine to reduce write amplification and alleviate disk I/O bottlenecks. Particularly useful when RocksDB compaction cannot keep up with write workloads, resulting in accumulated pending compaction bytes. | Enable it when write amplification is the primary bottleneck. Trade-offs include: 1. Potential performance impact on primary key range scans. 2. Increased space amplification (up to 2x in the worst case). 3. Additional memory usage for blob cache. | -| [`storage.scheduler-pending-write-threshold`](/tikv-configuration-file.md#scheduler-pending-write-threshold) | Sets the maximum size of the write queue in TiKV's scheduler. When the total size of pending write tasks exceeds this threshold, TiKV returns a `Server Is Busy` error for new write requests. | The default value is 100 MiB. In scenarios with high write concurrency or temporary write spikes, increasing this threshold (e.g., to 512 MiB) can help accommodate the load. However, if the write queue continues to accumulate and exceeds this threshold persistently, it may indicate underlying performance issues that require further investigation. | +| [`storage.scheduler-pending-write-threshold`](/tikv-configuration-file.md#scheduler-pending-write-threshold) | Set the maximum size of the write queue in TiKV's scheduler. When the total size of pending write tasks exceeds this threshold, TiKV returns a `Server Is Busy` error for new write requests. | The default value is `100MiB`. In scenarios with high write concurrency or temporary write spikes, increasing this threshold (for example, to `512MiB`) can help accommodate the load. However, if the write queue continues to accumulate and exceeds this threshold persistently, it might indicate underlying performance issues that require further investigation. | | [`storage.flow-control.l0-files-threshold`](/tikv-configuration-file.md#l0-files-threshold) | Control when write flow control is triggered based on the number of kvDB L0 files. Increasing the threshold reduces write stalls during high write workloads. | Higher thresholds might lead to more aggressive compactions when many L0 files exist. | -| [`storage.flow-control.soft-pending-compaction-bytes-limit`](/tikv-configuration-file.md#soft-pending-compaction-bytes-limit) and [`storage.flow-control.hard-pending-compaction-bytes-limit`](/tikv-configuration-file.md#hard-pending-compaction-bytes-limit) | Define thresholds for pending compaction bytes to manage write flow control. The soft limit triggers partial write rejections, while the hard limit results in full write rejections when exceeded. | The default soft limit is 192 GiB, and the hard limit is 256 GiB. In write-intensive scenarios, if compaction processes cannot keep up, pending compaction bytes accumulate, potentially triggering flow control. Adjusting these limits can provide more buffer space, but persistent accumulation indicates underlying issues that require further investigation. | +| [`storage.flow-control.soft-pending-compaction-bytes-limit`](/tikv-configuration-file.md#soft-pending-compaction-bytes-limit) and [`storage.flow-control.hard-pending-compaction-bytes-limit`](/tikv-configuration-file.md#hard-pending-compaction-bytes-limit) | Define thresholds for pending compaction bytes to manage write flow control. The soft limit triggers partial write rejections, while the hard limit results in full write rejections when exceeded. | The default soft limit is `192GiB`, and the hard limit is `256GiB`. In write-intensive scenarios, if compaction processes cannot keep up, pending compaction bytes accumulate, potentially triggering flow control. Adjusting these limits can provide more buffer space, but persistent accumulation indicates underlying issues that require further investigation. | -Note that the compaction and flow control configuration adjustments outlined above are tailored for TiKV deployments on instances with the following specifications: +Note that the compaction and flow control configuration adjustments outlined in the preceding table are tailored for TiKV deployments on instances with the following specifications: - CPU: 32 cores - Memory: 128 GiB - Storage: 5 TiB EBS - Disk Throughput: 1 GiB/s -#### Recommended Configuration Adjustments for Write-Intensive Workloads +#### Recommended configuration adjustments for write-intensive workloads -To optimize TiKV's performance and stability under write-intensive workloads, it is recommended that you adjust certain compaction and flow control parameters based on the instance's hardware specifications. For example: +To optimize TiKV performance and stability under write-intensive workloads, it is recommended that you adjust certain compaction and flow control parameters based on the hardware specifications of the instance. For example: - - `rocksdb.rate-bytes-per-sec`: Set to approximately 60% of the disk's maximum throughput (e.g., ~600 MiB/s for a 1 GiB/s disk) to balance compaction I/O and prevent disk saturation. - - `storage.flow-control.soft-pending-compaction-bytes-limit` and `storage.flow-control.hard-pending-compaction-bytes-limit`: Increase these limits proportionally to the available disk space (e.g., 1 TiB and 2 TiB, respectively) to provide more buffer for compaction processes. + - `rocksdb.rate-bytes-per-sec`: Set to approximately 60% of the disk's maximum throughput (for example, ~600 MiB/s for a 1 GiB/s disk) to balance compaction I/O and prevent disk saturation. + - `storage.flow-control.soft-pending-compaction-bytes-limit` and `storage.flow-control.hard-pending-compaction-bytes-limit`: Increase these limits proportionally to the available disk space (for example, 1 TiB and 2 TiB, respectively) to provide more buffer for compaction processes. These settings help ensure efficient resource utilization and minimize potential bottlenecks during peak write loads. > **Note:** > -> TiKV implements flow control at the scheduler layer to ensure system stability. When critical thresholds are breached, including those for pending compaction bytes or write queue sizes, TiKV begins rejecting write requests and returns a ServerIsBusy error. This error signals that background compaction processes cannot keep pace with the current rate of foreground write operations. Flow control activation typically results in latency spikes and reduced query throughput (QPS drops). To prevent these performance degradations, comprehensive capacity planning is essential, along with proper configuration of compaction parameters and storage settings. +> TiKV implements flow control at the scheduler layer to ensure system stability. When critical thresholds are breached, including those for pending compaction bytes or write queue sizes, TiKV begins rejecting write requests and returns a ServerIsBusy error. This error indicates that the background compaction processes cannot keep pace with the current rate of foreground write operations. Flow control activation typically results in latency spikes and reduced query throughput (QPS drops). To prevent these performance degradations, comprehensive capacity planning is essential, along with proper configuration of compaction parameters and storage settings. ### TiFlash configurations From b61548de1165cf1aee4ecd78af4a14efe0f46a3e Mon Sep 17 00:00:00 2001 From: xixirangrang Date: Mon, 9 Jun 2025 09:21:24 +0800 Subject: [PATCH 08/16] Update tidb-performance-tuning-config.md --- tidb-performance-tuning-config.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tidb-performance-tuning-config.md b/tidb-performance-tuning-config.md index 95ce0b63c7be7..4134473395144 100644 --- a/tidb-performance-tuning-config.md +++ b/tidb-performance-tuning-config.md @@ -123,7 +123,7 @@ hard-pending-compaction-bytes-limit = "2TiB" | [`rocksdb.titan`](/tikv-configuration-file.md#rocksdbtitan), [`rocksdb.defaultcf.titan`](/tikv-configuration-file.md#rocksdbdefaultcftitan), [`min-blob-size`](/tikv-configuration-file.md#min-blob-size), and [`blob-file-compression`](/tikv-configuration-file.md#blob-file-compression) | Enable the Titan storage engine to reduce write amplification and alleviate disk I/O bottlenecks. Particularly useful when RocksDB compaction cannot keep up with write workloads, resulting in accumulated pending compaction bytes. | Enable it when write amplification is the primary bottleneck. Trade-offs include: 1. Potential performance impact on primary key range scans. 2. Increased space amplification (up to 2x in the worst case). 3. Additional memory usage for blob cache. | | [`storage.scheduler-pending-write-threshold`](/tikv-configuration-file.md#scheduler-pending-write-threshold) | Set the maximum size of the write queue in TiKV's scheduler. When the total size of pending write tasks exceeds this threshold, TiKV returns a `Server Is Busy` error for new write requests. | The default value is `100MiB`. In scenarios with high write concurrency or temporary write spikes, increasing this threshold (for example, to `512MiB`) can help accommodate the load. However, if the write queue continues to accumulate and exceeds this threshold persistently, it might indicate underlying performance issues that require further investigation. | | [`storage.flow-control.l0-files-threshold`](/tikv-configuration-file.md#l0-files-threshold) | Control when write flow control is triggered based on the number of kvDB L0 files. Increasing the threshold reduces write stalls during high write workloads. | Higher thresholds might lead to more aggressive compactions when many L0 files exist. | -| [`storage.flow-control.soft-pending-compaction-bytes-limit`](/tikv-configuration-file.md#soft-pending-compaction-bytes-limit) and [`storage.flow-control.hard-pending-compaction-bytes-limit`](/tikv-configuration-file.md#hard-pending-compaction-bytes-limit) | Define thresholds for pending compaction bytes to manage write flow control. The soft limit triggers partial write rejections, while the hard limit results in full write rejections when exceeded. | The default soft limit is `192GiB`, and the hard limit is `256GiB`. In write-intensive scenarios, if compaction processes cannot keep up, pending compaction bytes accumulate, potentially triggering flow control. Adjusting these limits can provide more buffer space, but persistent accumulation indicates underlying issues that require further investigation. | +| [`storage.flow-control.soft-pending-compaction-bytes-limit`](/tikv-configuration-file.md#soft-pending-compaction-bytes-limit) and [`storage.flow-control.hard-pending-compaction-bytes-limit`](/tikv-configuration-file.md#hard-pending-compaction-bytes-limit) | Define thresholds for pending compaction bytes to manage write flow control. The soft limit triggers partial write rejections, while the hard limit results in full write rejections when exceeded. | The default soft limit is `192GiB`, and the hard limit is `256GiB`. In write-intensive scenarios, if compaction processes cannot keep up, pending compaction bytes accumulate, potentially triggering flow control. Adjusting these limits can provide more buffer space, but persistent accumulation indicates underlying issues that require further investigation. | Note that the compaction and flow control configuration adjustments outlined in the preceding table are tailored for TiKV deployments on instances with the following specifications: From ffc39418d7832c5b2b119445a3011715a8581a00 Mon Sep 17 00:00:00 2001 From: dbsid Date: Tue, 10 Jun 2025 19:17:01 +0800 Subject: [PATCH 09/16] remove background-read-bandwidth and rate-bytes-per-sec --- tidb-performance-tuning-config.md | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tidb-performance-tuning-config.md b/tidb-performance-tuning-config.md index 4134473395144..ccf5d3a7f6889 100644 --- a/tidb-performance-tuning-config.md +++ b/tidb-performance-tuning-config.md @@ -89,15 +89,11 @@ concurrent-send-snap-limit = 64 concurrent-recv-snap-limit = 64 snap-io-max-bytes-per-sec = "400MiB" -[quota] -background-read-bandwidth = "400Mib" - [pessimistic-txn] in-memory-peer-size-limit = "32MiB" in-memory-instance-size-limit = "512MiB" [rocksdb] -rate-bytes-per-sec = "600MiB" max-manifest-file-size = "256MiB" [rocksdb.titan] enabled = true @@ -116,9 +112,7 @@ hard-pending-compaction-bytes-limit = "2TiB" | Configuration item | Description | Note | | ---------| ---- | ----| | [`concurrent-send-snap-limit`](/tikv-configuration-file.md#concurrent-send-snap-limit), [`concurrent-recv-snap-limit`](/tikv-configuration-file.md#concurrent-recv-snap-limit), and [`snap-io-max-bytes-per-sec`](/tikv-configuration-file.md#snap-io-max-bytes-per-sec) | Set limits for concurrent snapshot transfer and I/O bandwidth during TiKV scaling operations. Higher limits reduce scaling time by allowing faster data migration. | Adjusting these limits affects the trade-off between scaling speed and online transaction performance. | -| [`quota.background-read-bandwidth`](/tikv-configuration-file.md#background-read-bandwidth-new-in-v620) | Set a soft limit on read bandwidth for background transactions and Coprocessor operations. It is recommended that you configure at 40% of disk bandwidth to prevent interference with foreground traffic. | Default value: `0KiB` (which means no limit). Without this limit, analyze jobs on large tables might significantly degrade application performance. | | [`in-memory-peer-size-limit`](/tikv-configuration-file.md#in-memory-peer-size-limit-new-in-v840) and [`in-memory-instance-size-limit`](/tikv-configuration-file.md#in-memory-instance-size-limit-new-in-v840) | Control the memory allocation for pessimistic lock caching at the Region and TiKV instance levels. Storing locks in memory reduces disk I/O and improves transaction performance. | Monitor memory usage carefully. Higher limits improve performance but increase memory consumption. | -|[`rocksdb.rate-bytes-per-sec`](/tikv-configuration-file.md#rate-bytes-per-sec) | Limit the I/O rate of background compaction and flush operations to prevent them from saturating disk bandwidth, ensuring stable performance. | It is recommended that you set it to around 60% of the disk's maximum throughput. Overly high settings can lead to disk saturation, impacting foreground operations. Conversely, setting it too low might slow down compaction, causing write stalls and flow control activations. | |[`rocksdb.max-manifest-file-size`](/tikv-configuration-file.md#max-manifest-file-size) | Set the maximum size of the RocksDB MANIFEST file, which logs metadata about SST files and database state changes. Increasing this size reduces the frequency of MANIFEST file rewrites, thereby minimizing their impact on foreground write performance. | The default value is `128MiB`. In environments with a large number of SST files (for example, hundreds of thousands), frequent MANIFEST rewrites can degrade write performance. Adjusting this parameter to a higher value, such as `256MiB` or larger, can help maintain optimal performance. | | [`rocksdb.titan`](/tikv-configuration-file.md#rocksdbtitan), [`rocksdb.defaultcf.titan`](/tikv-configuration-file.md#rocksdbdefaultcftitan), [`min-blob-size`](/tikv-configuration-file.md#min-blob-size), and [`blob-file-compression`](/tikv-configuration-file.md#blob-file-compression) | Enable the Titan storage engine to reduce write amplification and alleviate disk I/O bottlenecks. Particularly useful when RocksDB compaction cannot keep up with write workloads, resulting in accumulated pending compaction bytes. | Enable it when write amplification is the primary bottleneck. Trade-offs include: 1. Potential performance impact on primary key range scans. 2. Increased space amplification (up to 2x in the worst case). 3. Additional memory usage for blob cache. | | [`storage.scheduler-pending-write-threshold`](/tikv-configuration-file.md#scheduler-pending-write-threshold) | Set the maximum size of the write queue in TiKV's scheduler. When the total size of pending write tasks exceeds this threshold, TiKV returns a `Server Is Busy` error for new write requests. | The default value is `100MiB`. In scenarios with high write concurrency or temporary write spikes, increasing this threshold (for example, to `512MiB`) can help accommodate the load. However, if the write queue continues to accumulate and exceeds this threshold persistently, it might indicate underlying performance issues that require further investigation. | @@ -136,7 +130,7 @@ Note that the compaction and flow control configuration adjustments outlined in To optimize TiKV performance and stability under write-intensive workloads, it is recommended that you adjust certain compaction and flow control parameters based on the hardware specifications of the instance. For example: - - `rocksdb.rate-bytes-per-sec`: Set to approximately 60% of the disk's maximum throughput (for example, ~600 MiB/s for a 1 GiB/s disk) to balance compaction I/O and prevent disk saturation. + - `rocksdb.rate-bytes-per-sec`: Usually it’s best to leave this at its default value. If you notice compaction I/O consuming a significant share of the disk bandwidth, consider capping the rate to about 60% of your disk’s maximum throughput. This helps balance compaction work and ensures the disk isn’t saturated—for instance, on a disk rated at 1 GiB/s, set this to roughly 600 MiB/s. - `storage.flow-control.soft-pending-compaction-bytes-limit` and `storage.flow-control.hard-pending-compaction-bytes-limit`: Increase these limits proportionally to the available disk space (for example, 1 TiB and 2 TiB, respectively) to provide more buffer for compaction processes. These settings help ensure efficient resource utilization and minimize potential bottlenecks during peak write loads. From a39fe334df2b0a5aec35bdf7e26d6f46a6299fc6 Mon Sep 17 00:00:00 2001 From: dbsid Date: Fri, 20 Jun 2025 16:13:33 +0800 Subject: [PATCH 10/16] Update tidb-performance-tuning-config.md --- tidb-performance-tuning-config.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tidb-performance-tuning-config.md b/tidb-performance-tuning-config.md index ccf5d3a7f6889..ccdc6ea98af28 100644 --- a/tidb-performance-tuning-config.md +++ b/tidb-performance-tuning-config.md @@ -44,6 +44,7 @@ SET GLOBAL tidb_enable_non_prepared_plan_cache=on; SET GLOBAL tidb_ignore_prepared_cache_close_stmt=on; SET GLOBAL tidb_analyze_column_options='ALL'; SET GLOBAL tidb_stats_load_sync_wait=2000; +SET GLOBAL tidb_opt_limit_push_down_threshold=10000; SET GLOBAL tidb_opt_derive_topn=on; SET GLOBAL tidb_runtime_filter_mode=LOCAL; SET GLOBAL tidb_opt_enable_mpp_shared_cte_execution=on; @@ -62,6 +63,7 @@ The following table outlines the impact of specific system variable configuratio | [`tidb_ignore_prepared_cache_close_stmt`](/system-variables.md#tidb_ignore_prepared_cache_close_stmt-new-in-v600)| Cache plans for applications that use prepared statements but close the plan after each execution. | N/A | | [`tidb_analyze_column_options`](/system-variables.md#tidb_analyze_column_options-new-in-v830)| Collect statistics for all columns to avoid suboptimal execution plans due to missing column statistics. By default, TiDB only collects statistics for [predicate columns](/statistics.md#collect-statistics-on-some-columns). | Setting this variable to `'ALL'` can cause more resource usage for the `ANALYZE TABLE` operation compared with the default value `'PREDICATE'`. | | [`tidb_stats_load_sync_wait`](/system-variables.md#tidb_stats_load_sync_wait-new-in-v540)| Increase the timeout for synchronously loading statistics from the default 100 milliseconds to 2 seconds. This ensures TiDB loads the necessary statistics before query compilation. | Increasing this value leads to a longer synchronization wait time before query compilation. | +| [`tidb_opt_limit_push_down_threshold`](/system-variables.md#tidb_opt_limit_push_down_threshold)| Increase the threshold that determines whether to push the `Limit` or `TopN` operator down to TiKV. | When multiple index options exist, increasing this variable will make the optimizer favor indexes, which can optimize the `ORDER BY` and `Limit`. | | [`tidb_opt_derive_topn`](/system-variables.md#tidb_opt_derive_topn-new-in-v700)| Enable the optimization rule of [Deriving TopN or Limit from window functions](/derive-topn-from-window.md). | This is limited to the `ROW_NUMBER()` window function. | | [`tidb_runtime_filter_mode`](/system-variables.md#tidb_runtime_filter_mode-new-in-v720)| Enable [Runtime Filter](/runtime-filter.md#runtime-filter-mode) in the local mode to improve hash join efficiency. | The variable is introduced in v7.2.0 and is disabled by default for safety. | | [`tidb_opt_enable_mpp_shared_cte_execution`](/system-variables.md#tidb_opt_enable_mpp_shared_cte_execution-new-in-v720)| Enable non-recursive [Common Table Expressions (CTE)](/sql-statements/sql-statement-with.md) pushdown to TiFlash. | This is an experimental feature. | @@ -104,9 +106,8 @@ blob-file-compression = "zstd" [storage] scheduler-pending-write-threshold = "512MiB" [storage.flow-control] -l0-files-threshold = 60 -soft-pending-compaction-bytes-limit = "1TiB" -hard-pending-compaction-bytes-limit = "2TiB" +l0-files-threshold = 50 +soft-pending-compaction-bytes-limit = "512GiB" ``` | Configuration item | Description | Note | @@ -117,7 +118,7 @@ hard-pending-compaction-bytes-limit = "2TiB" | [`rocksdb.titan`](/tikv-configuration-file.md#rocksdbtitan), [`rocksdb.defaultcf.titan`](/tikv-configuration-file.md#rocksdbdefaultcftitan), [`min-blob-size`](/tikv-configuration-file.md#min-blob-size), and [`blob-file-compression`](/tikv-configuration-file.md#blob-file-compression) | Enable the Titan storage engine to reduce write amplification and alleviate disk I/O bottlenecks. Particularly useful when RocksDB compaction cannot keep up with write workloads, resulting in accumulated pending compaction bytes. | Enable it when write amplification is the primary bottleneck. Trade-offs include: 1. Potential performance impact on primary key range scans. 2. Increased space amplification (up to 2x in the worst case). 3. Additional memory usage for blob cache. | | [`storage.scheduler-pending-write-threshold`](/tikv-configuration-file.md#scheduler-pending-write-threshold) | Set the maximum size of the write queue in TiKV's scheduler. When the total size of pending write tasks exceeds this threshold, TiKV returns a `Server Is Busy` error for new write requests. | The default value is `100MiB`. In scenarios with high write concurrency or temporary write spikes, increasing this threshold (for example, to `512MiB`) can help accommodate the load. However, if the write queue continues to accumulate and exceeds this threshold persistently, it might indicate underlying performance issues that require further investigation. | | [`storage.flow-control.l0-files-threshold`](/tikv-configuration-file.md#l0-files-threshold) | Control when write flow control is triggered based on the number of kvDB L0 files. Increasing the threshold reduces write stalls during high write workloads. | Higher thresholds might lead to more aggressive compactions when many L0 files exist. | -| [`storage.flow-control.soft-pending-compaction-bytes-limit`](/tikv-configuration-file.md#soft-pending-compaction-bytes-limit) and [`storage.flow-control.hard-pending-compaction-bytes-limit`](/tikv-configuration-file.md#hard-pending-compaction-bytes-limit) | Define thresholds for pending compaction bytes to manage write flow control. The soft limit triggers partial write rejections, while the hard limit results in full write rejections when exceeded. | The default soft limit is `192GiB`, and the hard limit is `256GiB`. In write-intensive scenarios, if compaction processes cannot keep up, pending compaction bytes accumulate, potentially triggering flow control. Adjusting these limits can provide more buffer space, but persistent accumulation indicates underlying issues that require further investigation. | +| [`storage.flow-control.soft-pending-compaction-bytes-limit`](/tikv-configuration-file.md#soft-pending-compaction-bytes-limit) | Define thresholds for pending compaction bytes to manage write flow control. The soft limit triggers partial write rejections. | The default soft limit is `192GiB`. In write-intensive scenarios, if compaction processes cannot keep up, pending compaction bytes accumulate, potentially triggering flow control. Adjusting the limit can provide more buffer space, but persistent accumulation indicates underlying issues that require further investigation. | Note that the compaction and flow control configuration adjustments outlined in the preceding table are tailored for TiKV deployments on instances with the following specifications: From 62a4a1d59d10385e72bffe9013d5329fa160551d Mon Sep 17 00:00:00 2001 From: dbsid Date: Wed, 30 Jul 2025 14:15:24 +0800 Subject: [PATCH 11/16] ad rocksdb parameters --- tidb-performance-tuning-config.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tidb-performance-tuning-config.md b/tidb-performance-tuning-config.md index ccdc6ea98af28..dd0074e7cb115 100644 --- a/tidb-performance-tuning-config.md +++ b/tidb-performance-tuning-config.md @@ -108,6 +108,16 @@ scheduler-pending-write-threshold = "512MiB" [storage.flow-control] l0-files-threshold = 50 soft-pending-compaction-bytes-limit = "512GiB" + +[rocksdb.writecf] +level0-slowdown-writes-trigger = 20 +soft-pending-compaction-bytes-limit = "192GiB" +[rocksdb.defaultcf] +level0-slowdown-writes-trigger = 20 +soft-pending-compaction-bytes-limit = "192GiB" +[rocksdb.lockcf] +level0-slowdown-writes-trigger = 20 +soft-pending-compaction-bytes-limit = "192GiB" ``` | Configuration item | Description | Note | @@ -119,6 +129,7 @@ soft-pending-compaction-bytes-limit = "512GiB" | [`storage.scheduler-pending-write-threshold`](/tikv-configuration-file.md#scheduler-pending-write-threshold) | Set the maximum size of the write queue in TiKV's scheduler. When the total size of pending write tasks exceeds this threshold, TiKV returns a `Server Is Busy` error for new write requests. | The default value is `100MiB`. In scenarios with high write concurrency or temporary write spikes, increasing this threshold (for example, to `512MiB`) can help accommodate the load. However, if the write queue continues to accumulate and exceeds this threshold persistently, it might indicate underlying performance issues that require further investigation. | | [`storage.flow-control.l0-files-threshold`](/tikv-configuration-file.md#l0-files-threshold) | Control when write flow control is triggered based on the number of kvDB L0 files. Increasing the threshold reduces write stalls during high write workloads. | Higher thresholds might lead to more aggressive compactions when many L0 files exist. | | [`storage.flow-control.soft-pending-compaction-bytes-limit`](/tikv-configuration-file.md#soft-pending-compaction-bytes-limit) | Define thresholds for pending compaction bytes to manage write flow control. The soft limit triggers partial write rejections. | The default soft limit is `192GiB`. In write-intensive scenarios, if compaction processes cannot keep up, pending compaction bytes accumulate, potentially triggering flow control. Adjusting the limit can provide more buffer space, but persistent accumulation indicates underlying issues that require further investigation. | +| [`rocksdb.(defaultcf\|writecf\|lockcf).level0-slowdown-writes-trigger`](/tikv-configuration-file.md#level0-slowdown-writes-trigger) [`rocksdb.(defaultcf\|writecf\|lockcf).soft-pending-compaction-bytes-limit`](/tikv-configuration-file.md#level0-slowdown-writes-trigger) | Need to manually set Rocksdb's level0-slowdown-writes-trigger and soft-pending-compaction-bytes-limit back to their original default values. This way they won't be affected by flow control parameters. Need to additionally set the Rocksdb parameters to maintain the same compaction efficiency as the default parameters. | more details is documented in the issue https://github.com/tikv/tikv/issues/18708. | Note that the compaction and flow control configuration adjustments outlined in the preceding table are tailored for TiKV deployments on instances with the following specifications: From 177301a411b7e2d29328767f35f19be7fbc9aa3c Mon Sep 17 00:00:00 2001 From: dbsid Date: Wed, 30 Jul 2025 14:24:38 +0800 Subject: [PATCH 12/16] fix format --- tidb-performance-tuning-config.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tidb-performance-tuning-config.md b/tidb-performance-tuning-config.md index dd0074e7cb115..d5f6e1e084419 100644 --- a/tidb-performance-tuning-config.md +++ b/tidb-performance-tuning-config.md @@ -129,7 +129,7 @@ soft-pending-compaction-bytes-limit = "192GiB" | [`storage.scheduler-pending-write-threshold`](/tikv-configuration-file.md#scheduler-pending-write-threshold) | Set the maximum size of the write queue in TiKV's scheduler. When the total size of pending write tasks exceeds this threshold, TiKV returns a `Server Is Busy` error for new write requests. | The default value is `100MiB`. In scenarios with high write concurrency or temporary write spikes, increasing this threshold (for example, to `512MiB`) can help accommodate the load. However, if the write queue continues to accumulate and exceeds this threshold persistently, it might indicate underlying performance issues that require further investigation. | | [`storage.flow-control.l0-files-threshold`](/tikv-configuration-file.md#l0-files-threshold) | Control when write flow control is triggered based on the number of kvDB L0 files. Increasing the threshold reduces write stalls during high write workloads. | Higher thresholds might lead to more aggressive compactions when many L0 files exist. | | [`storage.flow-control.soft-pending-compaction-bytes-limit`](/tikv-configuration-file.md#soft-pending-compaction-bytes-limit) | Define thresholds for pending compaction bytes to manage write flow control. The soft limit triggers partial write rejections. | The default soft limit is `192GiB`. In write-intensive scenarios, if compaction processes cannot keep up, pending compaction bytes accumulate, potentially triggering flow control. Adjusting the limit can provide more buffer space, but persistent accumulation indicates underlying issues that require further investigation. | -| [`rocksdb.(defaultcf\|writecf\|lockcf).level0-slowdown-writes-trigger`](/tikv-configuration-file.md#level0-slowdown-writes-trigger) [`rocksdb.(defaultcf\|writecf\|lockcf).soft-pending-compaction-bytes-limit`](/tikv-configuration-file.md#level0-slowdown-writes-trigger) | Need to manually set Rocksdb's level0-slowdown-writes-trigger and soft-pending-compaction-bytes-limit back to their original default values. This way they won't be affected by flow control parameters. Need to additionally set the Rocksdb parameters to maintain the same compaction efficiency as the default parameters. | more details is documented in the issue https://github.com/tikv/tikv/issues/18708. | +| [`rocksdb.(defaultcf\|writecf\|lockcf).level0-slowdown-writes-trigger`](/tikv-configuration-file.md#level0-slowdown-writes-trigger) [`rocksdb.(defaultcf\|writecf\|lockcf).soft-pending-compaction-bytes-limit`](/tikv-configuration-file.md#level0-slowdown-writes-trigger) | Need to manually set Rocksdb's level0-slowdown-writes-trigger and soft-pending-compaction-bytes-limit back to their original default values. This way they won't be affected by flow control parameters. Need to additionally set the Rocksdb parameters to maintain the same compaction efficiency as the default parameters. | more details is documented in the issue [18708](https://github.com/tikv/tikv/issues/18708). | Note that the compaction and flow control configuration adjustments outlined in the preceding table are tailored for TiKV deployments on instances with the following specifications: From 52cf37bf45099db84d2988e00eb1c200253af51a Mon Sep 17 00:00:00 2001 From: houfaxin Date: Wed, 30 Jul 2025 17:05:34 +0800 Subject: [PATCH 13/16] Refine descriptions and clarify config recommendations Improved clarity and consistency in configuration descriptions for TiDB and TiKV performance tuning. Updated terminology, fixed grammar, and provided clearer instructions and references for compaction and flow control parameters. Enhanced guidance for setting rate limits and compaction thresholds based on hardware specifications. --- tidb-performance-tuning-config.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tidb-performance-tuning-config.md b/tidb-performance-tuning-config.md index d5f6e1e084419..eb3470e34b7ba 100644 --- a/tidb-performance-tuning-config.md +++ b/tidb-performance-tuning-config.md @@ -63,7 +63,7 @@ The following table outlines the impact of specific system variable configuratio | [`tidb_ignore_prepared_cache_close_stmt`](/system-variables.md#tidb_ignore_prepared_cache_close_stmt-new-in-v600)| Cache plans for applications that use prepared statements but close the plan after each execution. | N/A | | [`tidb_analyze_column_options`](/system-variables.md#tidb_analyze_column_options-new-in-v830)| Collect statistics for all columns to avoid suboptimal execution plans due to missing column statistics. By default, TiDB only collects statistics for [predicate columns](/statistics.md#collect-statistics-on-some-columns). | Setting this variable to `'ALL'` can cause more resource usage for the `ANALYZE TABLE` operation compared with the default value `'PREDICATE'`. | | [`tidb_stats_load_sync_wait`](/system-variables.md#tidb_stats_load_sync_wait-new-in-v540)| Increase the timeout for synchronously loading statistics from the default 100 milliseconds to 2 seconds. This ensures TiDB loads the necessary statistics before query compilation. | Increasing this value leads to a longer synchronization wait time before query compilation. | -| [`tidb_opt_limit_push_down_threshold`](/system-variables.md#tidb_opt_limit_push_down_threshold)| Increase the threshold that determines whether to push the `Limit` or `TopN` operator down to TiKV. | When multiple index options exist, increasing this variable will make the optimizer favor indexes, which can optimize the `ORDER BY` and `Limit`. | +| [`tidb_opt_limit_push_down_threshold`](/system-variables.md#tidb_opt_limit_push_down_threshold)| Increase the threshold that determines whether to push the `Limit` or `TopN` operator down to TiKV. | When multiple index options exist, increasing this variable makes the optimizer favor indexes, which can optimize the `ORDER BY` and `Limit` operators. | | [`tidb_opt_derive_topn`](/system-variables.md#tidb_opt_derive_topn-new-in-v700)| Enable the optimization rule of [Deriving TopN or Limit from window functions](/derive-topn-from-window.md). | This is limited to the `ROW_NUMBER()` window function. | | [`tidb_runtime_filter_mode`](/system-variables.md#tidb_runtime_filter_mode-new-in-v720)| Enable [Runtime Filter](/runtime-filter.md#runtime-filter-mode) in the local mode to improve hash join efficiency. | The variable is introduced in v7.2.0 and is disabled by default for safety. | | [`tidb_opt_enable_mpp_shared_cte_execution`](/system-variables.md#tidb_opt_enable_mpp_shared_cte_execution-new-in-v720)| Enable non-recursive [Common Table Expressions (CTE)](/sql-statements/sql-statement-with.md) pushdown to TiFlash. | This is an experimental feature. | @@ -124,12 +124,12 @@ soft-pending-compaction-bytes-limit = "192GiB" | ---------| ---- | ----| | [`concurrent-send-snap-limit`](/tikv-configuration-file.md#concurrent-send-snap-limit), [`concurrent-recv-snap-limit`](/tikv-configuration-file.md#concurrent-recv-snap-limit), and [`snap-io-max-bytes-per-sec`](/tikv-configuration-file.md#snap-io-max-bytes-per-sec) | Set limits for concurrent snapshot transfer and I/O bandwidth during TiKV scaling operations. Higher limits reduce scaling time by allowing faster data migration. | Adjusting these limits affects the trade-off between scaling speed and online transaction performance. | | [`in-memory-peer-size-limit`](/tikv-configuration-file.md#in-memory-peer-size-limit-new-in-v840) and [`in-memory-instance-size-limit`](/tikv-configuration-file.md#in-memory-instance-size-limit-new-in-v840) | Control the memory allocation for pessimistic lock caching at the Region and TiKV instance levels. Storing locks in memory reduces disk I/O and improves transaction performance. | Monitor memory usage carefully. Higher limits improve performance but increase memory consumption. | -|[`rocksdb.max-manifest-file-size`](/tikv-configuration-file.md#max-manifest-file-size) | Set the maximum size of the RocksDB MANIFEST file, which logs metadata about SST files and database state changes. Increasing this size reduces the frequency of MANIFEST file rewrites, thereby minimizing their impact on foreground write performance. | The default value is `128MiB`. In environments with a large number of SST files (for example, hundreds of thousands), frequent MANIFEST rewrites can degrade write performance. Adjusting this parameter to a higher value, such as `256MiB` or larger, can help maintain optimal performance. | +| [`rocksdb.max-manifest-file-size`](/tikv-configuration-file.md#max-manifest-file-size) | Set the maximum size of the RocksDB Manifest file, which logs the metadata about SST files and database state changes. Increasing this size reduces the frequency of Manifest file rewrites, thereby minimizing their impact on foreground write performance. | The default value is `128MiB`. In environments with a large number of SST files (for example, hundreds of thousands), frequent Manifest rewrites can degrade write performance. Adjusting this parameter to a higher value, such as `256MiB` or larger, can help maintain optimal performance. | | [`rocksdb.titan`](/tikv-configuration-file.md#rocksdbtitan), [`rocksdb.defaultcf.titan`](/tikv-configuration-file.md#rocksdbdefaultcftitan), [`min-blob-size`](/tikv-configuration-file.md#min-blob-size), and [`blob-file-compression`](/tikv-configuration-file.md#blob-file-compression) | Enable the Titan storage engine to reduce write amplification and alleviate disk I/O bottlenecks. Particularly useful when RocksDB compaction cannot keep up with write workloads, resulting in accumulated pending compaction bytes. | Enable it when write amplification is the primary bottleneck. Trade-offs include: 1. Potential performance impact on primary key range scans. 2. Increased space amplification (up to 2x in the worst case). 3. Additional memory usage for blob cache. | -| [`storage.scheduler-pending-write-threshold`](/tikv-configuration-file.md#scheduler-pending-write-threshold) | Set the maximum size of the write queue in TiKV's scheduler. When the total size of pending write tasks exceeds this threshold, TiKV returns a `Server Is Busy` error for new write requests. | The default value is `100MiB`. In scenarios with high write concurrency or temporary write spikes, increasing this threshold (for example, to `512MiB`) can help accommodate the load. However, if the write queue continues to accumulate and exceeds this threshold persistently, it might indicate underlying performance issues that require further investigation. | +| [`storage.scheduler-pending-write-threshold`](/tikv-configuration-file.md#scheduler-pending-write-threshold) | Set the maximum size of the write queue in the TiKV scheduler. When the total size of pending write tasks exceeds this threshold, TiKV returns a `Server Is Busy` error for new write requests. | The default value is `100MiB`. In scenarios with high write concurrency or temporary write spikes, increasing this threshold (for example, to `512MiB`) can help accommodate the load. However, if the write queue continues to accumulate and exceeds this threshold persistently, it might indicate underlying performance issues that require further investigation. | | [`storage.flow-control.l0-files-threshold`](/tikv-configuration-file.md#l0-files-threshold) | Control when write flow control is triggered based on the number of kvDB L0 files. Increasing the threshold reduces write stalls during high write workloads. | Higher thresholds might lead to more aggressive compactions when many L0 files exist. | -| [`storage.flow-control.soft-pending-compaction-bytes-limit`](/tikv-configuration-file.md#soft-pending-compaction-bytes-limit) | Define thresholds for pending compaction bytes to manage write flow control. The soft limit triggers partial write rejections. | The default soft limit is `192GiB`. In write-intensive scenarios, if compaction processes cannot keep up, pending compaction bytes accumulate, potentially triggering flow control. Adjusting the limit can provide more buffer space, but persistent accumulation indicates underlying issues that require further investigation. | -| [`rocksdb.(defaultcf\|writecf\|lockcf).level0-slowdown-writes-trigger`](/tikv-configuration-file.md#level0-slowdown-writes-trigger) [`rocksdb.(defaultcf\|writecf\|lockcf).soft-pending-compaction-bytes-limit`](/tikv-configuration-file.md#level0-slowdown-writes-trigger) | Need to manually set Rocksdb's level0-slowdown-writes-trigger and soft-pending-compaction-bytes-limit back to their original default values. This way they won't be affected by flow control parameters. Need to additionally set the Rocksdb parameters to maintain the same compaction efficiency as the default parameters. | more details is documented in the issue [18708](https://github.com/tikv/tikv/issues/18708). | +| [`storage.flow-control.soft-pending-compaction-bytes-limit`](/tikv-configuration-file.md#soft-pending-compaction-bytes-limit) | Control the threshold for pending compaction bytes to manage write flow control. The soft limit triggers partial write rejections. | The default soft limit is `192GiB`. In write-intensive scenarios, if compaction processes cannot keep up, pending compaction bytes accumulate, potentially triggering flow control. Adjusting the limit can provide more buffer space, but persistent accumulation indicates underlying issues that require further investigation. | +| [`rocksdb.(defaultcf\|writecf\|lockcf).level0-slowdown-writes-trigger`](/tikv-configuration-file.md#level0-slowdown-writes-trigger) [`rocksdb.(defaultcf\|writecf\|lockcf).soft-pending-compaction-bytes-limit`](/tikv-configuration-file.md#level0-slowdown-writes-trigger) | You need to manually set `level0-slowdown-writes-trigger` and `soft-pending-compaction-bytes-limit` back to their default values. This way they will not be affected by flow control parameters. In addition, set the Rocksdb parameters to maintain the same compaction efficiency as the default parameters. | For more information, see [Issue 18708](https://github.com/tikv/tikv/issues/18708). | Note that the compaction and flow control configuration adjustments outlined in the preceding table are tailored for TiKV deployments on instances with the following specifications: @@ -142,8 +142,9 @@ Note that the compaction and flow control configuration adjustments outlined in To optimize TiKV performance and stability under write-intensive workloads, it is recommended that you adjust certain compaction and flow control parameters based on the hardware specifications of the instance. For example: - - `rocksdb.rate-bytes-per-sec`: Usually it’s best to leave this at its default value. If you notice compaction I/O consuming a significant share of the disk bandwidth, consider capping the rate to about 60% of your disk’s maximum throughput. This helps balance compaction work and ensures the disk isn’t saturated—for instance, on a disk rated at 1 GiB/s, set this to roughly 600 MiB/s. - - `storage.flow-control.soft-pending-compaction-bytes-limit` and `storage.flow-control.hard-pending-compaction-bytes-limit`: Increase these limits proportionally to the available disk space (for example, 1 TiB and 2 TiB, respectively) to provide more buffer for compaction processes. +- [`rocksdb.rate-bytes-per-sec`](/tikv-configuration-file.md#rate-bytes-per-sec): usually use the default value. If you notice that compaction I/O is consuming a significant share of the disk bandwidth, consider capping the rate to about 60% of your disk’s maximum throughput. This helps balance compaction work and ensures the disk is not saturated. For example, on a disk rated at **1 GiB/s**, set this to roughly `600MiB`. + +- [`storage.flow-control.soft-pending-compaction-bytes-limit`](/tikv-configuration-file.md#soft-pending-compaction-bytes-limit-1) and [`storage.flow-control.hard-pending-compaction-bytes-limit`](/tikv-configuration-file.md#hard-pending-compaction-bytes-limit-1): increase these limits proportionally to the available disk space (for example, 1 TiB and 2 TiB, respectively) to provide more buffer for compaction processes. These settings help ensure efficient resource utilization and minimize potential bottlenecks during peak write loads. @@ -151,7 +152,6 @@ These settings help ensure efficient resource utilization and minimize potential > > TiKV implements flow control at the scheduler layer to ensure system stability. When critical thresholds are breached, including those for pending compaction bytes or write queue sizes, TiKV begins rejecting write requests and returns a ServerIsBusy error. This error indicates that the background compaction processes cannot keep pace with the current rate of foreground write operations. Flow control activation typically results in latency spikes and reduced query throughput (QPS drops). To prevent these performance degradations, comprehensive capacity planning is essential, along with proper configuration of compaction parameters and storage settings. - ### TiFlash configurations Add the following configuration items to the TiFlash configuration file: From 93c82ee48b86d17c44807b54a0095fb689d7e663 Mon Sep 17 00:00:00 2001 From: dbsid Date: Wed, 30 Jul 2025 17:15:53 +0800 Subject: [PATCH 14/16] Update tidb-performance-tuning-config.md Co-authored-by: xixirangrang --- tidb-performance-tuning-config.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tidb-performance-tuning-config.md b/tidb-performance-tuning-config.md index eb3470e34b7ba..367ee67a89eea 100644 --- a/tidb-performance-tuning-config.md +++ b/tidb-performance-tuning-config.md @@ -63,7 +63,7 @@ The following table outlines the impact of specific system variable configuratio | [`tidb_ignore_prepared_cache_close_stmt`](/system-variables.md#tidb_ignore_prepared_cache_close_stmt-new-in-v600)| Cache plans for applications that use prepared statements but close the plan after each execution. | N/A | | [`tidb_analyze_column_options`](/system-variables.md#tidb_analyze_column_options-new-in-v830)| Collect statistics for all columns to avoid suboptimal execution plans due to missing column statistics. By default, TiDB only collects statistics for [predicate columns](/statistics.md#collect-statistics-on-some-columns). | Setting this variable to `'ALL'` can cause more resource usage for the `ANALYZE TABLE` operation compared with the default value `'PREDICATE'`. | | [`tidb_stats_load_sync_wait`](/system-variables.md#tidb_stats_load_sync_wait-new-in-v540)| Increase the timeout for synchronously loading statistics from the default 100 milliseconds to 2 seconds. This ensures TiDB loads the necessary statistics before query compilation. | Increasing this value leads to a longer synchronization wait time before query compilation. | -| [`tidb_opt_limit_push_down_threshold`](/system-variables.md#tidb_opt_limit_push_down_threshold)| Increase the threshold that determines whether to push the `Limit` or `TopN` operator down to TiKV. | When multiple index options exist, increasing this variable makes the optimizer favor indexes, which can optimize the `ORDER BY` and `Limit` operators. | +| [`tidb_opt_limit_push_down_threshold`](/system-variables.md#tidb_opt_limit_push_down_threshold)| Increase the threshold that determines whether to push the `Limit` or `TopN` operator down to TiKV. | When multiple index options exist, increasing this variable makes the optimizer favor indexes that can optimize the `ORDER BY` and `Limit` operators. | | [`tidb_opt_derive_topn`](/system-variables.md#tidb_opt_derive_topn-new-in-v700)| Enable the optimization rule of [Deriving TopN or Limit from window functions](/derive-topn-from-window.md). | This is limited to the `ROW_NUMBER()` window function. | | [`tidb_runtime_filter_mode`](/system-variables.md#tidb_runtime_filter_mode-new-in-v720)| Enable [Runtime Filter](/runtime-filter.md#runtime-filter-mode) in the local mode to improve hash join efficiency. | The variable is introduced in v7.2.0 and is disabled by default for safety. | | [`tidb_opt_enable_mpp_shared_cte_execution`](/system-variables.md#tidb_opt_enable_mpp_shared_cte_execution-new-in-v720)| Enable non-recursive [Common Table Expressions (CTE)](/sql-statements/sql-statement-with.md) pushdown to TiFlash. | This is an experimental feature. | From 9474f55af6fc77f9ed060730f7b8fea194d22f45 Mon Sep 17 00:00:00 2001 From: xixirangrang Date: Fri, 1 Aug 2025 08:49:05 +0800 Subject: [PATCH 15/16] Update tidb-performance-tuning-config.md --- tidb-performance-tuning-config.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tidb-performance-tuning-config.md b/tidb-performance-tuning-config.md index 367ee67a89eea..6cc6875a86242 100644 --- a/tidb-performance-tuning-config.md +++ b/tidb-performance-tuning-config.md @@ -133,10 +133,10 @@ soft-pending-compaction-bytes-limit = "192GiB" Note that the compaction and flow control configuration adjustments outlined in the preceding table are tailored for TiKV deployments on instances with the following specifications: - - CPU: 32 cores - - Memory: 128 GiB - - Storage: 5 TiB EBS - - Disk Throughput: 1 GiB/s + - CPU: 32 cores + - Memory: 128 GiB + - Storage: 5 TiB EBS + - Disk Throughput: 1 GiB/s #### Recommended configuration adjustments for write-intensive workloads From 6f580fdb5bc6195524b10c57dee89e5e81362bdb Mon Sep 17 00:00:00 2001 From: xixirangrang Date: Fri, 1 Aug 2025 11:40:02 +0800 Subject: [PATCH 16/16] Update tidb-performance-tuning-config.md --- tidb-performance-tuning-config.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tidb-performance-tuning-config.md b/tidb-performance-tuning-config.md index 6cc6875a86242..57e729626c1fe 100644 --- a/tidb-performance-tuning-config.md +++ b/tidb-performance-tuning-config.md @@ -133,10 +133,10 @@ soft-pending-compaction-bytes-limit = "192GiB" Note that the compaction and flow control configuration adjustments outlined in the preceding table are tailored for TiKV deployments on instances with the following specifications: - - CPU: 32 cores - - Memory: 128 GiB - - Storage: 5 TiB EBS - - Disk Throughput: 1 GiB/s +- CPU: 32 cores +- Memory: 128 GiB +- Storage: 5 TiB EBS +- Disk Throughput: 1 GiB/s #### Recommended configuration adjustments for write-intensive workloads