Skip to content

storage: bump pebble, add new blob rewrite cluster setting #153293

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
craig merged 2 commits into from
Sep 10, 2025
Merged

storage: bump pebble, add new blob rewrite cluster setting #153293

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cf40b97
go.mod: bump Pebble to 342d2e8b9a0b
jbowens Sep 9, 2025
95d9475
storage: add storage.value_separation.compaction_garbage_threshold_hi…
jbowens Sep 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions DEPS.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1820,10 +1820,10 @@ def go_deps():
patches = [
"@com_github_cockroachdb_cockroach//build/patches:com_github_cockroachdb_pebble.patch",
],
sha256 = "5c8a6d6f5788c3817da3f80eae015ceccf9e465414053933a3fae10ca52eb515",
strip_prefix = "github.com/cockroachdb/pebble@v0.0.0-20250905164320-f9312e52bec6",
sha256 = "349899ce403cf2875a953fec3a21666bc8f94b1d8ca6804afde2468158fbdfaa",
strip_prefix = "github.com/cockroachdb/pebble@v0.0.0-20250908223345-342d2e8b9a0b",
urls = [
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20250905164320-f9312e52bec6.zip",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20250908223345-342d2e8b9a0b.zip",
],
)
go_repository(
Expand Down
2 changes: 1 addition & 1 deletion build/bazelutil/distdir_files.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -355,7 +355,7 @@ DISTDIR_FILES = {
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/gostdlib/com_github_cockroachdb_gostdlib-v1.19.0.zip": "c4d516bcfe8c07b6fc09b8a9a07a95065b36c2855627cb3514e40c98f872b69e",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/logtags/com_github_cockroachdb_logtags-v0.0.0-20241215232642-bb51bb14a506.zip": "920068af09e3846d9ebb4e4a7787ff1dd10f3989c5f940ad861b0f6a9f824f6e",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/metamorphic/com_github_cockroachdb_metamorphic-v0.0.0-20231108215700-4ba948b56895.zip": "28c8cf42192951b69378cf537be5a9a43f2aeb35542908cc4fe5f689505853ea",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20250905164320-f9312e52bec6.zip": "5c8a6d6f5788c3817da3f80eae015ceccf9e465414053933a3fae10ca52eb515",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20250908223345-342d2e8b9a0b.zip": "349899ce403cf2875a953fec3a21666bc8f94b1d8ca6804afde2468158fbdfaa",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/redact/com_github_cockroachdb_redact-v1.1.6.zip": "018eccb5fb9ca52d43ec9eaf213539d01c1f2b94e0e822406ebfb2e9321ef6cf",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/returncheck/com_github_cockroachdb_returncheck-v0.0.0-20200612231554-92cdbca611dd.zip": "ce92ba4352deec995b1f2eecf16eba7f5d51f5aa245a1c362dfe24c83d31f82b",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/stress/com_github_cockroachdb_stress-v0.0.0-20220803192808-1806698b1b7b.zip": "3fda531795c600daf25532a4f98be2a1335cd1e5e182c72789bca79f5f69fcc1",
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ require (
github.com/cockroachdb/go-test-teamcity v0.0.0-20191211140407-cff980ad0a55
github.com/cockroachdb/gostdlib v1.19.0
github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506
github.com/cockroachdb/pebble v0.0.0-20250905164320-f9312e52bec6
github.com/cockroachdb/pebble v0.0.0-20250908223345-342d2e8b9a0b
github.com/cockroachdb/redact v1.1.6
github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd
github.com/cockroachdb/stress v0.0.0-20220803192808-1806698b1b7b
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -575,8 +575,8 @@ github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 h1:ASDL+UJcILM
github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506/go.mod h1:Mw7HqKr2kdtu6aYGn3tPmAftiP3QPX63LdK/zcariIo=
github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA=
github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA=
github.com/cockroachdb/pebble v0.0.0-20250905164320-f9312e52bec6 h1:yg6yxxBmtMb+daJgTXHy8rjljWfZqgHDKxZTCumz9/U=
github.com/cockroachdb/pebble v0.0.0-20250905164320-f9312e52bec6/go.mod h1:zc33YirQWSE/Dw6WFmZSOAi3L0FcH23cXGbH6qZGF9k=
github.com/cockroachdb/pebble v0.0.0-20250908223345-342d2e8b9a0b h1:CTGQjhiyKYKYCBc5hz/do1fHKFJZg5l8TOnLQ7LgZCs=
github.com/cockroachdb/pebble v0.0.0-20250908223345-342d2e8b9a0b/go.mod h1:zc33YirQWSE/Dw6WFmZSOAi3L0FcH23cXGbH6qZGF9k=
github.com/cockroachdb/redact v1.1.6 h1:zXJBwDZ84xJNlHl1rMyCojqyIxv+7YUpQiJLQ7n4314=
github.com/cockroachdb/redact v1.1.6/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd h1:KFOt5I9nEKZgCnOSmy8r4Oykh8BYQO8bFOTgHDS8YZA=
Expand Down
25 changes: 19 additions & 6 deletions pkg/storage/pebble.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -452,12 +452,21 @@ var (
settings.SystemVisible,
"storage.value_separation.compaction_garbage_threshold",
"the max garbage threshold configures the percentage of unreferenced value "+
"bytes that trigger blob-file rewrite compactions; 100 disables these compactions",
"bytes that begin to trigger blob-file rewrite compactions; 100 disables these compactions",
int64(metamorphic.ConstantWithTestRange("storage.value_separation.compaction_garbage_threshold",
10, /* default */
1 /* min */, 80 /* max */)),
settings.IntInRange(1, 100),
)
valueSeparationCompactionGarbageThresholdHighPriority = settings.RegisterIntSetting(
settings.SystemVisible,
"storage.value_separation.compaction_garbage_threshold_high_priority",
"configures the percentage of unreferenced value bytes that trigger high-priority blob-file rewrite compactions",
int64(metamorphic.ConstantWithTestRange("storage.value_separation.compaction_garbage_threshold",
20, /* default */
1 /* min */, 80 /* max */)),
settings.IntInRange(1, 100),
)
)

// EngineComparer is a pebble.Comparer object that implements MVCC-specific
Expand Down Expand Up @@ -936,12 +945,16 @@ func newPebble(ctx context.Context, cfg engineConfig) (p *Pebble, err error) {
if !valueSeparationEnabled.Get(&cfg.settings.SV) {
return pebble.ValueSeparationPolicy{}
}
lowPri := float64(valueSeparationCompactionGarbageThreshold.Get(&cfg.settings.SV)) / 100.0
highPri := float64(valueSeparationCompactionGarbageThresholdHighPriority.Get(&cfg.settings.SV)) / 100.0
highPri = max(highPri, lowPri)
return pebble.ValueSeparationPolicy{
Enabled: true,
MinimumSize: int(valueSeparationMinimumSize.Get(&cfg.settings.SV)),
MaxBlobReferenceDepth: int(valueSeparationMaxReferenceDepth.Get(&cfg.settings.SV)),
RewriteMinimumAge: valueSeparationRewriteMinimumAge.Get(&cfg.settings.SV),
TargetGarbageRatio: float64(valueSeparationCompactionGarbageThreshold.Get(&cfg.settings.SV)) / 100.0,
Enabled: true,
MinimumSize: int(valueSeparationMinimumSize.Get(&cfg.settings.SV)),
MaxBlobReferenceDepth: int(valueSeparationMaxReferenceDepth.Get(&cfg.settings.SV)),
RewriteMinimumAge: valueSeparationRewriteMinimumAge.Get(&cfg.settings.SV),
GarbageRatioLowPriority: lowPri,
GarbageRatioHighPriority: highPri,
}
}
cfg.opts.Experimental.MultiLevelCompactionHeuristic = func() pebble.MultiLevelHeuristic {
Expand Down