From cf40b9722e2a0c05d1ac228783b6da0d8e670010 Mon Sep 17 00:00:00 2001 From: Jackson Owens Date: Tue, 9 Sep 2025 16:56:42 -0400 Subject: [PATCH 1/2] go.mod: bump Pebble to 342d2e8b9a0b Changes: * [`342d2e8b`](https://github.com/cockroachdb/pebble/commit/342d2e8b) db: add high-priority blob file rewrite compactions * [`fff7d3ad`](https://github.com/cockroachdb/pebble/commit/fff7d3ad) db: fix MarkedForCompaction flake * [`d86f6dab`](https://github.com/cockroachdb/pebble/commit/d86f6dab) scripts: more improvements to stress.sh Release note: none. Epic: none. --- DEPS.bzl | 6 +++--- build/bazelutil/distdir_files.bzl | 2 +- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/DEPS.bzl b/DEPS.bzl index 89ebac987635..9139a2fa4d9d 100644 --- a/DEPS.bzl +++ b/DEPS.bzl @@ -1820,10 +1820,10 @@ def go_deps(): patches = [ "@com_github_cockroachdb_cockroach//build/patches:com_github_cockroachdb_pebble.patch", ], - sha256 = "5c8a6d6f5788c3817da3f80eae015ceccf9e465414053933a3fae10ca52eb515", - strip_prefix = "github.com/cockroachdb/pebble@v0.0.0-20250905164320-f9312e52bec6", + sha256 = "349899ce403cf2875a953fec3a21666bc8f94b1d8ca6804afde2468158fbdfaa", + strip_prefix = "github.com/cockroachdb/pebble@v0.0.0-20250908223345-342d2e8b9a0b", urls = [ - "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20250905164320-f9312e52bec6.zip", + "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20250908223345-342d2e8b9a0b.zip", ], ) go_repository( diff --git a/build/bazelutil/distdir_files.bzl b/build/bazelutil/distdir_files.bzl index dc62d098a248..bf5ea0550970 100644 --- a/build/bazelutil/distdir_files.bzl +++ b/build/bazelutil/distdir_files.bzl @@ -355,7 +355,7 @@ DISTDIR_FILES = { "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/gostdlib/com_github_cockroachdb_gostdlib-v1.19.0.zip": "c4d516bcfe8c07b6fc09b8a9a07a95065b36c2855627cb3514e40c98f872b69e", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/logtags/com_github_cockroachdb_logtags-v0.0.0-20241215232642-bb51bb14a506.zip": "920068af09e3846d9ebb4e4a7787ff1dd10f3989c5f940ad861b0f6a9f824f6e", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/metamorphic/com_github_cockroachdb_metamorphic-v0.0.0-20231108215700-4ba948b56895.zip": "28c8cf42192951b69378cf537be5a9a43f2aeb35542908cc4fe5f689505853ea", - "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20250905164320-f9312e52bec6.zip": "5c8a6d6f5788c3817da3f80eae015ceccf9e465414053933a3fae10ca52eb515", + "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20250908223345-342d2e8b9a0b.zip": "349899ce403cf2875a953fec3a21666bc8f94b1d8ca6804afde2468158fbdfaa", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/redact/com_github_cockroachdb_redact-v1.1.6.zip": "018eccb5fb9ca52d43ec9eaf213539d01c1f2b94e0e822406ebfb2e9321ef6cf", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/returncheck/com_github_cockroachdb_returncheck-v0.0.0-20200612231554-92cdbca611dd.zip": "ce92ba4352deec995b1f2eecf16eba7f5d51f5aa245a1c362dfe24c83d31f82b", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/stress/com_github_cockroachdb_stress-v0.0.0-20220803192808-1806698b1b7b.zip": "3fda531795c600daf25532a4f98be2a1335cd1e5e182c72789bca79f5f69fcc1", diff --git a/go.mod b/go.mod index b11f0137b247..a860a1cc4c18 100644 --- a/go.mod +++ b/go.mod @@ -141,7 +141,7 @@ require ( github.com/cockroachdb/go-test-teamcity v0.0.0-20191211140407-cff980ad0a55 github.com/cockroachdb/gostdlib v1.19.0 github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 - github.com/cockroachdb/pebble v0.0.0-20250905164320-f9312e52bec6 + github.com/cockroachdb/pebble v0.0.0-20250908223345-342d2e8b9a0b github.com/cockroachdb/redact v1.1.6 github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd github.com/cockroachdb/stress v0.0.0-20220803192808-1806698b1b7b diff --git a/go.sum b/go.sum index 38fc6839b5c1..7e173e5111dd 100644 --- a/go.sum +++ b/go.sum @@ -575,8 +575,8 @@ github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 h1:ASDL+UJcILM github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506/go.mod h1:Mw7HqKr2kdtu6aYGn3tPmAftiP3QPX63LdK/zcariIo= github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA= github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA= -github.com/cockroachdb/pebble v0.0.0-20250905164320-f9312e52bec6 h1:yg6yxxBmtMb+daJgTXHy8rjljWfZqgHDKxZTCumz9/U= -github.com/cockroachdb/pebble v0.0.0-20250905164320-f9312e52bec6/go.mod h1:zc33YirQWSE/Dw6WFmZSOAi3L0FcH23cXGbH6qZGF9k= +github.com/cockroachdb/pebble v0.0.0-20250908223345-342d2e8b9a0b h1:CTGQjhiyKYKYCBc5hz/do1fHKFJZg5l8TOnLQ7LgZCs= +github.com/cockroachdb/pebble v0.0.0-20250908223345-342d2e8b9a0b/go.mod h1:zc33YirQWSE/Dw6WFmZSOAi3L0FcH23cXGbH6qZGF9k= github.com/cockroachdb/redact v1.1.6 h1:zXJBwDZ84xJNlHl1rMyCojqyIxv+7YUpQiJLQ7n4314= github.com/cockroachdb/redact v1.1.6/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd h1:KFOt5I9nEKZgCnOSmy8r4Oykh8BYQO8bFOTgHDS8YZA= From 95d94759aea93c567fea0c1736f9b01e5757d587 Mon Sep 17 00:00:00 2001 From: Jackson Owens Date: Tue, 9 Sep 2025 16:59:44 -0400 Subject: [PATCH 2/2] storage: add storage.value_separation.compaction_garbage_threshold_high_priority Add a new cluster setting for configuring high-priority blob file rewrite compactions. One of these compactions is allowed to run ahead of a default compaction if the amount of garbage exceeds the configured percentage. Epic: none Release note: none --- pkg/storage/pebble.go | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/pkg/storage/pebble.go b/pkg/storage/pebble.go index 78046cc7572f..97babaaaec68 100644 --- a/pkg/storage/pebble.go +++ b/pkg/storage/pebble.go @@ -452,12 +452,21 @@ var ( settings.SystemVisible, "storage.value_separation.compaction_garbage_threshold", "the max garbage threshold configures the percentage of unreferenced value "+ - "bytes that trigger blob-file rewrite compactions; 100 disables these compactions", + "bytes that begin to trigger blob-file rewrite compactions; 100 disables these compactions", int64(metamorphic.ConstantWithTestRange("storage.value_separation.compaction_garbage_threshold", 10, /* default */ 1 /* min */, 80 /* max */)), settings.IntInRange(1, 100), ) + valueSeparationCompactionGarbageThresholdHighPriority = settings.RegisterIntSetting( + settings.SystemVisible, + "storage.value_separation.compaction_garbage_threshold_high_priority", + "configures the percentage of unreferenced value bytes that trigger high-priority blob-file rewrite compactions", + int64(metamorphic.ConstantWithTestRange("storage.value_separation.compaction_garbage_threshold", + 20, /* default */ + 1 /* min */, 80 /* max */)), + settings.IntInRange(1, 100), + ) ) // EngineComparer is a pebble.Comparer object that implements MVCC-specific @@ -936,12 +945,16 @@ func newPebble(ctx context.Context, cfg engineConfig) (p *Pebble, err error) { if !valueSeparationEnabled.Get(&cfg.settings.SV) { return pebble.ValueSeparationPolicy{} } + lowPri := float64(valueSeparationCompactionGarbageThreshold.Get(&cfg.settings.SV)) / 100.0 + highPri := float64(valueSeparationCompactionGarbageThresholdHighPriority.Get(&cfg.settings.SV)) / 100.0 + highPri = max(highPri, lowPri) return pebble.ValueSeparationPolicy{ - Enabled: true, - MinimumSize: int(valueSeparationMinimumSize.Get(&cfg.settings.SV)), - MaxBlobReferenceDepth: int(valueSeparationMaxReferenceDepth.Get(&cfg.settings.SV)), - RewriteMinimumAge: valueSeparationRewriteMinimumAge.Get(&cfg.settings.SV), - TargetGarbageRatio: float64(valueSeparationCompactionGarbageThreshold.Get(&cfg.settings.SV)) / 100.0, + Enabled: true, + MinimumSize: int(valueSeparationMinimumSize.Get(&cfg.settings.SV)), + MaxBlobReferenceDepth: int(valueSeparationMaxReferenceDepth.Get(&cfg.settings.SV)), + RewriteMinimumAge: valueSeparationRewriteMinimumAge.Get(&cfg.settings.SV), + GarbageRatioLowPriority: lowPri, + GarbageRatioHighPriority: highPri, } } cfg.opts.Experimental.MultiLevelCompactionHeuristic = func() pebble.MultiLevelHeuristic {