From ee56b4e4f04d2848edf1099d27b4294ecada9942 Mon Sep 17 00:00:00 2001 From: tabokie Date: Mon, 19 Apr 2021 21:57:10 +0800 Subject: [PATCH 1/4] update comments Signed-off-by: tabokie --- .../write_amp_based_rate_limiter.cc | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/utilities/rate_limiters/write_amp_based_rate_limiter.cc b/utilities/rate_limiters/write_amp_based_rate_limiter.cc index 18c556db6ed..cfcf1011d81 100644 --- a/utilities/rate_limiters/write_amp_based_rate_limiter.cc +++ b/utilities/rate_limiters/write_amp_based_rate_limiter.cc @@ -31,10 +31,11 @@ constexpr int kSecondsPerTune = 1; constexpr int kMillisPerTune = 1000 * kSecondsPerTune; constexpr int kMicrosPerTune = 1000 * 1000 * kSecondsPerTune; -// Two reasons for adding padding to baseline limit: -// 1. compaction cannot fully utilize the IO quota we set. -// 2. make it faster to digest unexpected burst of pending compaction bytes, -// generally this will help flatten IO waves. +// Due to the execution model of compaction, large waves of pending compactions +// could possibly be hidden behind a constant rate of I/O requests. It's then +// wise to raise the threshold slightly above estimation to ensure those +// pending compactions can contribute to the convergence of a new alternative +// threshold. // Padding is calculated through hyperbola based on empirical percentage of 10% // and special care for low-pressure domain. E.g. coordinates (5M, 18M) and // (10M, 16M) are on this curve. @@ -314,6 +315,13 @@ int64_t WriteAmpBasedRateLimiter::CalculateRefillBytesPerPeriod( } } +// The core function used to dynamically adjust the compaction rate limit, +// called **at most** once every `kSecondsPerTune`. +// A write amplification ratio is calculated based on history samples of +// compaction and flush flow, then it is used to deduce the appropriate +// threshold before next tune. This algorithm excels by taking into account +// the limiter's inability to estimate the pressure of pending compactions, +// and the possibility of foreground write fluctuation. Status WriteAmpBasedRateLimiter::Tune() { // computed rate limit will be larger than 10MB/s const int64_t kMinBytesPerSec = 10 << 20; From a5677d5fd2e314011b5b27ecfd80deb38a2dc1d2 Mon Sep 17 00:00:00 2001 From: tabokie Date: Mon, 19 Apr 2021 22:07:35 +0800 Subject: [PATCH 2/4] use write-amp in one way Signed-off-by: tabokie --- .../write_amp_based_rate_limiter.cc | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/utilities/rate_limiters/write_amp_based_rate_limiter.cc b/utilities/rate_limiters/write_amp_based_rate_limiter.cc index cfcf1011d81..2f6c5e13186 100644 --- a/utilities/rate_limiters/write_amp_based_rate_limiter.cc +++ b/utilities/rate_limiters/write_amp_based_rate_limiter.cc @@ -339,21 +339,32 @@ Status WriteAmpBasedRateLimiter::Tune() { int64_t prev_bytes_per_sec = GetBytesPerSecond(); - // Loop through the actual time slice to make sure bytes flow from long period - // of time is properly estimated when the compaction rate is low. + // This function can be called less frequent than we anticipate when + // compaction rate is low. Loop through the actual time slice to correct + // the estimation. for (uint32_t i = 0; i < duration_ms / kMillisPerTune; i++) { bytes_sampler_.AddSample(duration_bytes_through_ * 1000 / duration_ms); highpri_bytes_sampler_.AddSample(duration_highpri_bytes_through_ * 1000 / duration_ms); limit_bytes_sampler_.AddSample(prev_bytes_per_sec); } + int64_t new_bytes_per_sec = bytes_sampler_.GetFullValue(); int32_t ratio = std::max( kRatioLower, static_cast( bytes_sampler_.GetFullValue() * 10 / std::max(highpri_bytes_sampler_.GetFullValue(), kHighBytesLower))); - - // in case there are compaction bursts even when online writes are stable + // Only adjust threshold when foreground writes (flush flow) increases rather + // than decreases. + new_bytes_per_sec = std::max( + new_bytes_per_sec, + ratio * + std::max(highpri_bytes_sampler_.GetRecentValue(), kHighBytesLower) / + 10); + // Set the threshold higher to avoid write stalls caused by pending + // compactions. + int64_t padding = CalculatePadding(new_bytes_per_sec); + // Adjustment based on utilization. int64_t util = bytes_sampler_.GetRecentValue() * 1000 / limit_bytes_sampler_.GetRecentValue(); if (util >= 995) { @@ -363,11 +374,7 @@ Status WriteAmpBasedRateLimiter::Tune() { } else if (percent_delta_ > 0) { percent_delta_ -= 1; } - - int64_t new_bytes_per_sec = - ratio * - std::max(highpri_bytes_sampler_.GetRecentValue(), kHighBytesLower) / 10; - int64_t padding = CalculatePadding(new_bytes_per_sec); + // React to pace-up requests when LSM is out of shape. if (critical_pace_up_.load(std::memory_order_relaxed)) { percent_delta_ = 150; critical_pace_up_.store(false, std::memory_order_relaxed); From 19a9c699c62dcc21935eeb9d11687a000d472a35 Mon Sep 17 00:00:00 2001 From: tabokie Date: Thu, 29 Apr 2021 14:16:58 +0800 Subject: [PATCH 3/4] stashed comments Signed-off-by: tabokie --- utilities/rate_limiters/write_amp_based_rate_limiter.cc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/utilities/rate_limiters/write_amp_based_rate_limiter.cc b/utilities/rate_limiters/write_amp_based_rate_limiter.cc index 2f6c5e13186..7232759b5c1 100644 --- a/utilities/rate_limiters/write_amp_based_rate_limiter.cc +++ b/utilities/rate_limiters/write_amp_based_rate_limiter.cc @@ -317,11 +317,10 @@ int64_t WriteAmpBasedRateLimiter::CalculateRefillBytesPerPeriod( // The core function used to dynamically adjust the compaction rate limit, // called **at most** once every `kSecondsPerTune`. -// A write amplification ratio is calculated based on history samples of -// compaction and flush flow, then it is used to deduce the appropriate -// threshold before next tune. This algorithm excels by taking into account -// the limiter's inability to estimate the pressure of pending compactions, -// and the possibility of foreground write fluctuation. +// I/O throughput threshold is automatically tuned based on history samples of +// compaction and flush flow. This algorithm excels by taking into account the +// limiter's inability to estimate the pressure of pending compactions, and the +// possibility of foreground write fluctuation. Status WriteAmpBasedRateLimiter::Tune() { // computed rate limit will be larger than 10MB/s const int64_t kMinBytesPerSec = 10 << 20; From 6b553f98c1051cc94bb5e50040a7ab7eed85297b Mon Sep 17 00:00:00 2001 From: tabokie Date: Thu, 12 Aug 2021 11:41:12 +0800 Subject: [PATCH 4/4] address comment Signed-off-by: tabokie --- utilities/rate_limiters/write_amp_based_rate_limiter.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/utilities/rate_limiters/write_amp_based_rate_limiter.cc b/utilities/rate_limiters/write_amp_based_rate_limiter.cc index 7232759b5c1..487f6feb51b 100644 --- a/utilities/rate_limiters/write_amp_based_rate_limiter.cc +++ b/utilities/rate_limiters/write_amp_based_rate_limiter.cc @@ -353,8 +353,9 @@ Status WriteAmpBasedRateLimiter::Tune() { static_cast( bytes_sampler_.GetFullValue() * 10 / std::max(highpri_bytes_sampler_.GetFullValue(), kHighBytesLower))); - // Only adjust threshold when foreground writes (flush flow) increases rather - // than decreases. + // Only adjust threshold when foreground write (flush) flow increases, + // because decreasement could also be caused by manual flow control at + // application level to alleviate background pressure. new_bytes_per_sec = std::max( new_bytes_per_sec, ratio *