diff --git a/consensus/cuckoo/consensus.go b/consensus/cuckoo/consensus.go index 1dcba325fd..51b3d634fd 100644 --- a/consensus/cuckoo/consensus.go +++ b/consensus/cuckoo/consensus.go @@ -883,7 +883,6 @@ func accumulateRewards(config *params.ChainConfig, state *state.StateDB, header, //} if len(uncles) > 0 { - for _, uncle := range uncles { r.Add(uncle.Number, big8) r.Sub(r, header.Number) diff --git a/ctxc/downloader/downloader_test.go b/ctxc/downloader/downloader_test.go index 2b0c8277a7..047b95f9c5 100644 --- a/ctxc/downloader/downloader_test.go +++ b/ctxc/downloader/downloader_test.go @@ -1504,9 +1504,9 @@ func TestDeliverHeadersHang(t *testing.T) { protocol int syncMode SyncMode }{ - {62, FullSync}, - {63, FullSync}, - {63, FastSync}, + //{62, FullSync}, + //{63, FullSync}, + //{63, FastSync}, {64, FullSync}, {64, FastSync}, {65, FullSync}, @@ -1514,7 +1514,7 @@ func TestDeliverHeadersHang(t *testing.T) { } for _, tc := range testCases { t.Run(fmt.Sprintf("protocol %d mode %v", tc.protocol, tc.syncMode), func(t *testing.T) { - t.Parallel() + //t.Parallel() testDeliverHeadersHang(t, tc.protocol, tc.syncMode) }) } diff --git a/go.mod b/go.mod index 83d5b47fa6..6883379453 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1 github.com/CortexFoundation/inference v1.0.2-0.20230307032835-9197d586a4e8 github.com/CortexFoundation/statik v0.0.0-20210315012922-8bb8a7b5dc66 - github.com/CortexFoundation/torrentfs v1.0.43-0.20230424145059-4a78d96b8709 + github.com/CortexFoundation/torrentfs v1.0.43-0.20230430013533-eb5ca1c4750a github.com/VictoriaMetrics/fastcache v1.12.1 github.com/arsham/figurine v1.3.0 github.com/aws/aws-sdk-go-v2 v1.17.7 @@ -17,7 +17,7 @@ require ( github.com/cespare/cp v1.1.1 github.com/charmbracelet/bubbletea v0.23.2 github.com/cloudflare/cloudflare-go v0.57.1 - github.com/cockroachdb/pebble v0.0.0-20230420172751-f87a15b23a26 + github.com/cockroachdb/pebble v0.0.0-20230428220915-dc0efbd4333b github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/deckarep/golang-set/v2 v2.3.0 github.com/docker/docker v23.0.2+incompatible diff --git a/go.sum b/go.sum index 431498fd5c..62c2f94828 100644 --- a/go.sum +++ b/go.sum @@ -67,8 +67,8 @@ github.com/CortexFoundation/statik v0.0.0-20210315012922-8bb8a7b5dc66/go.mod h1: github.com/CortexFoundation/torrentfs v1.0.13-0.20200623060705-ce027f43f2f8/go.mod h1:Ma+tGhPPvz4CEZHaqEJQMOEGOfHeQBiAoNd1zyc/w3Q= github.com/CortexFoundation/torrentfs v1.0.14-0.20200703071639-3fcabcabf274/go.mod h1:qnb3YlIJmuetVBtC6Lsejr0Xru+1DNmDCdTqnwy7lhk= github.com/CortexFoundation/torrentfs v1.0.20-0.20200810031954-d36d26f82fcc/go.mod h1:N5BsicP5ynjXIi/Npl/SRzlJ630n1PJV2sRj0Z0t2HA= -github.com/CortexFoundation/torrentfs v1.0.43-0.20230424145059-4a78d96b8709 h1:FIoIkOouFF/Z76EONdZOwVqPEDYZqErd/C3Xgz80O+4= -github.com/CortexFoundation/torrentfs v1.0.43-0.20230424145059-4a78d96b8709/go.mod h1:KwAVwauTMrKxfWfodK5lqPSlT2Lz/c61AAkEU5kknd4= +github.com/CortexFoundation/torrentfs v1.0.43-0.20230430013533-eb5ca1c4750a h1:vGsEFb94z7Ap5RK7gQp5O0LRn0TfQ97ratluEKxHaO4= +github.com/CortexFoundation/torrentfs v1.0.43-0.20230430013533-eb5ca1c4750a/go.mod h1:eOqycAWG+Zg+ZlkjOj/YLPnnYrry/z9HZRVlqGUGMPY= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= @@ -342,15 +342,15 @@ github.com/cloudflare/cloudflare-go v0.57.1/go.mod h1:cD8AqNMMaL1A0Sj9XKo3Xu9ZVH github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8= github.com/cockroachdb/errors v1.9.1/go.mod h1:2sxOtL2WIc096WSZqZ5h8fa17rdDq9HZOZLBCor4mBk= github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= -github.com/cockroachdb/pebble v0.0.0-20230420172751-f87a15b23a26 h1:B36ZLUlDXPaOUofmJW/usmYqMJMJFH+bRzHX5HbBjeY= -github.com/cockroachdb/pebble v0.0.0-20230420172751-f87a15b23a26/go.mod h1:9lRMC4XN3/BLPtIp6kAKwIaHu369NOf2rMucPzipz50= +github.com/cockroachdb/pebble v0.0.0-20230428220915-dc0efbd4333b h1:s8Bb/AsUb/WuhLTs0ENm9I0dxthCVp4QR79Q1a9ZK8I= +github.com/cockroachdb/pebble v0.0.0-20230428220915-dc0efbd4333b/go.mod h1:TkdVsGYRqtULUppt2RbC+YaKtTHnHoWa2apfFrSKABw= github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= diff --git a/params/protocol_params.go b/params/protocol_params.go index 1fbc194f26..7cd9efcdf7 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -196,7 +196,7 @@ const ( SeedingBlks = 6 // TESTING: for torrent seed spreading MatureBlks = 100 // Blocks between model uploading tx and model ready for use. // For the full node to synchronize the models - BernardMatureBlks = 10 // TESTING: For the full node to synchronize the models, in dolores testnet + BernardMatureBlks = 10 // TESTING: For the full node to synchronize the models, in bernard testnet DoloresMatureBlks = 1 // TESTING: For the full node to synchronize the models, in dolores testnet ExpiredBlks = 1000000000000000000 // TESTING: Model expire blocks. Not effective. 8409600 diff --git a/vendor/github.com/cockroachdb/pebble/db.go b/vendor/github.com/cockroachdb/pebble/db.go index 0e7fbaa7c9..62d2aca5fa 100644 --- a/vendor/github.com/cockroachdb/pebble/db.go +++ b/vendor/github.com/cockroachdb/pebble/db.go @@ -264,7 +264,7 @@ type DB struct { // objProvider is used to access and manage SSTs. objProvider objstorage.Provider - fileLock io.Closer + fileLock *Lock dataDir vfs.File walDir vfs.File @@ -369,9 +369,6 @@ type DB struct { } mem struct { - // Condition variable used to serialize memtable switching. See - // DB.makeRoomForWrite(). - cond sync.Cond // The current mutable memTable. mutable *memTable // Queue of flushables (the mutable memtable is at end). Elements are @@ -2265,7 +2262,6 @@ func (d *DB) recycleWAL() (newLogNum FileNum, prevLogSize uint64) { }) d.mu.Lock() - d.mu.mem.cond.Broadcast() d.mu.versions.metrics.WAL.Files++ @@ -2360,3 +2356,9 @@ func (d *DB) SetCreatorID(creatorID uint64) error { } return d.objProvider.SetCreatorID(objstorage.CreatorID(creatorID)) } + +// ObjProvider returns the objstorage.Provider for this database. Meant to be +// used for internal purposes only. +func (d *DB) ObjProvider() objstorage.Provider { + return d.objProvider +} diff --git a/vendor/github.com/cockroachdb/pebble/ingest.go b/vendor/github.com/cockroachdb/pebble/ingest.go index 1cabaf370b..c57fb59f9a 100644 --- a/vendor/github.com/cockroachdb/pebble/ingest.go +++ b/vendor/github.com/cockroachdb/pebble/ingest.go @@ -846,6 +846,7 @@ func (d *DB) ingest( // approximate ingest-into-L0 stats when using flushable ingests. metaFlushableOverlaps := make([]bool, len(meta)) var mem *flushableEntry + var mut *memTable // asFlushable indicates whether the sstable was ingested as a flushable. var asFlushable bool prepare := func(seqNum uint64) { @@ -890,8 +891,20 @@ func (d *DB) ingest( d.opts.Logger.Infof("ingest error reading flushable for log %s: %s", m.logNum, err) } } + if mem == nil { - // No overlap with any of the queued flushables. + // No overlap with any of the queued flushables, so no need to queue + // after them. + + // New writes with higher sequence numbers may be concurrently + // committed. We must ensure they don't flush before this ingest + // completes. To do that, we ref the mutable memtable as a writer, + // preventing its flushing (and the flushing of all subsequent + // flushables in the queue). Once we've acquired the manifest lock + // to add the ingested sstables to the LSM, we can unref as we're + // guaranteed that the flush won't edit the LSM before this ingest. + mut = d.mu.mem.mutable + mut.writerRef() return } // The ingestion overlaps with some entry in the flushable queue. @@ -903,6 +916,15 @@ func (d *DB) ingest( if mem.flushable == d.mu.mem.mutable { err = d.makeRoomForWrite(nil) } + // New writes with higher sequence numbers may be concurrently + // committed. We must ensure they don't flush before this ingest + // completes. To do that, we ref the mutable memtable as a writer, + // preventing its flushing (and the flushing of all subsequent + // flushables in the queue). Once we've acquired the manifest lock + // to add the ingested sstables to the LSM, we can unref as we're + // guaranteed that the flush won't edit the LSM before this ingest. + mut = d.mu.mem.mutable + mut.writerRef() mem.flushForced = true d.maybeScheduleFlush() return @@ -917,6 +939,9 @@ func (d *DB) ingest( apply := func(seqNum uint64) { if err != nil || asFlushable { // An error occurred during prepare. + if mut != nil { + mut.writerUnref() + } return } @@ -927,6 +952,9 @@ func (d *DB) ingest( if err = ingestUpdateSeqNum( d.cmp, d.opts.Comparer.FormatKey, seqNum, meta, ); err != nil { + if mut != nil { + mut.writerUnref() + } return } @@ -938,7 +966,7 @@ func (d *DB) ingest( // Assign the sstables to the correct level in the LSM and apply the // version edit. - ve, err = d.ingestApply(jobID, meta, targetLevelFunc) + ve, err = d.ingestApply(jobID, meta, targetLevelFunc, mut) } d.commit.AllocateSeqNum(len(meta), prepare, apply) @@ -1021,7 +1049,7 @@ type ingestTargetLevelFunc func( ) (int, error) func (d *DB) ingestApply( - jobID int, meta []*fileMetadata, findTargetLevel ingestTargetLevelFunc, + jobID int, meta []*fileMetadata, findTargetLevel ingestTargetLevelFunc, mut *memTable, ) (*versionEdit, error) { d.mu.Lock() defer d.mu.Unlock() @@ -1038,6 +1066,16 @@ func (d *DB) ingestApply( // logAndApply unconditionally releases the manifest lock, but any earlier // returns must unlock the manifest. d.mu.versions.logLock() + + if mut != nil { + // Unref the mutable memtable to allows its flush to proceed. Now that we've + // acquired the manifest lock, we can be certain that if the mutable + // memtable has received more recent conflicting writes, the flush won't + // beat us to applying to the manifest resulting in sequence number + // inversion. + mut.writerUnref() + } + current := d.mu.versions.currentVersion() baseLevel := d.mu.versions.picker.getBaseLevel() iterOps := IterOptions{logger: d.opts.Logger} diff --git a/vendor/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go b/vendor/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go index cfcee95385..3646a034e8 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go +++ b/vendor/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go @@ -658,14 +658,18 @@ func (s *L0Sublevels) addFileToSublevels(f *FileMetadata, checkInvariant bool) e // Update state in every fileInterval for this file. for i := f.minIntervalIndex; i <= f.maxIntervalIndex; i++ { interval := &s.orderedIntervals[i] - if len(interval.files) > 0 && - subLevel <= interval.files[len(interval.files)-1].SubLevel { + if len(interval.files) > 0 { if checkInvariant && interval.files[len(interval.files)-1].LargestSeqNum > f.LargestSeqNum { // We are sliding this file "underneath" an existing file. Throw away // and start over in NewL0Sublevels. return errInvalidL0SublevelsOpt } - subLevel = interval.files[len(interval.files)-1].SubLevel + 1 + // interval.files is sorted by sublevels, from lowest to highest. + // AddL0Files can only add files at sublevels higher than existing files + // in the same key intervals. + if maxSublevel := interval.files[len(interval.files)-1].SubLevel; subLevel <= maxSublevel { + subLevel = maxSublevel + 1 + } } interval.estimatedBytes += interpolatedBytes if f.minIntervalIndex < interval.filesMinIntervalIndex { diff --git a/vendor/github.com/cockroachdb/pebble/internal/manifest/version.go b/vendor/github.com/cockroachdb/pebble/internal/manifest/version.go index 799e300811..d95631a0b2 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/manifest/version.go +++ b/vendor/github.com/cockroachdb/pebble/internal/manifest/version.go @@ -1058,15 +1058,21 @@ func (v *Version) DebugString(format base.FormatKey) string { return v.string(format, true) } +func describeSublevels(format base.FormatKey, verbose bool, sublevels []LevelSlice) string { + var buf bytes.Buffer + for sublevel := len(sublevels) - 1; sublevel >= 0; sublevel-- { + fmt.Fprintf(&buf, "0.%d:\n", sublevel) + sublevels[sublevel].Each(func(f *FileMetadata) { + fmt.Fprintf(&buf, " %s\n", f.DebugString(format, verbose)) + }) + } + return buf.String() +} + func (v *Version) string(format base.FormatKey, verbose bool) string { var buf bytes.Buffer if len(v.L0SublevelFiles) > 0 { - for sublevel := len(v.L0SublevelFiles) - 1; sublevel >= 0; sublevel-- { - fmt.Fprintf(&buf, "0.%d:\n", sublevel) - v.L0SublevelFiles[sublevel].Each(func(f *FileMetadata) { - fmt.Fprintf(&buf, " %s\n", f.DebugString(format, verbose)) - }) - } + fmt.Fprintf(&buf, "%s", describeSublevels(format, verbose, v.L0SublevelFiles)) } for level := 1; level < NumLevels; level++ { if v.Levels[level].Empty() { diff --git a/vendor/github.com/cockroachdb/pebble/internal/manifest/version_edit.go b/vendor/github.com/cockroachdb/pebble/internal/manifest/version_edit.go index 1470812593..33f70d6a09 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/manifest/version_edit.go +++ b/vendor/github.com/cockroachdb/pebble/internal/manifest/version_edit.go @@ -8,11 +8,14 @@ import ( "bufio" "bytes" "encoding/binary" + "fmt" "io" "sort" + "time" "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/invariants" ) // TODO(peter): describe the MANIFEST file format, independently of the C++ @@ -396,6 +399,48 @@ func (v *VersionEdit) Decode(r io.Reader) error { return nil } +// String implements fmt.Stringer for a VersionEdit. +func (v *VersionEdit) String() string { + var buf bytes.Buffer + if v.ComparerName != "" { + fmt.Fprintf(&buf, " comparer: %s", v.ComparerName) + } + if v.MinUnflushedLogNum != 0 { + fmt.Fprintf(&buf, " log-num: %d\n", v.MinUnflushedLogNum) + } + if v.ObsoletePrevLogNum != 0 { + fmt.Fprintf(&buf, " prev-log-num: %d\n", v.ObsoletePrevLogNum) + } + if v.NextFileNum != 0 { + fmt.Fprintf(&buf, " next-file-num: %d\n", v.NextFileNum) + } + if v.LastSeqNum != 0 { + fmt.Fprintf(&buf, " last-seq-num: %d\n", v.LastSeqNum) + } + entries := make([]DeletedFileEntry, 0, len(v.DeletedFiles)) + for df := range v.DeletedFiles { + entries = append(entries, df) + } + sort.Slice(entries, func(i, j int) bool { + if entries[i].Level != entries[j].Level { + return entries[i].Level < entries[j].Level + } + return entries[i].FileNum < entries[j].FileNum + }) + for _, df := range entries { + fmt.Fprintf(&buf, " deleted: L%d %s\n", df.Level, df.FileNum) + } + for _, nf := range v.NewFiles { + fmt.Fprintf(&buf, " added: L%d %s", nf.Level, nf.Meta.String()) + if nf.Meta.CreationTime != 0 { + fmt.Fprintf(&buf, " (%s)", + time.Unix(nf.Meta.CreationTime, 0).UTC().Format(time.RFC3339)) + } + fmt.Fprintln(&buf) + } + return buf.String() +} + // Encode encodes an edit to the specified writer. // // TODO(bananabrick): Support encoding of virtual sstable state. @@ -922,6 +967,16 @@ func (b *BulkVersionEdit) Apply( v.L0Sublevels, err = curr.L0Sublevels.AddL0Files(addedFiles, flushSplitBytes, &v.Levels[0]) if errors.Is(err, errInvalidL0SublevelsOpt) { err = v.InitL0Sublevels(cmp, formatKey, flushSplitBytes) + } else if invariants.Enabled && err == nil { + copyOfSublevels, err := NewL0Sublevels(&v.Levels[0], cmp, formatKey, flushSplitBytes) + if err != nil { + panic(fmt.Sprintf("error when regenerating sublevels: %s", err)) + } + s1 := describeSublevels(base.DefaultFormatter, false /* verbose */, copyOfSublevels.Levels) + s2 := describeSublevels(base.DefaultFormatter, false /* verbose */, v.L0Sublevels.Levels) + if s1 != s2 { + panic(fmt.Sprintf("incremental L0 sublevel generation produced different output than regeneration: %s != %s", s1, s2)) + } } if err != nil { return nil, errors.Wrap(err, "pebble: internal error") diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_readable.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_readable.go index e5df4a87c0..3b9dc8be19 100644 --- a/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_readable.go +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_readable.go @@ -7,7 +7,6 @@ package objstorageprovider import ( "context" "io" - "sync" "github.com/cockroachdb/pebble/objstorage" "github.com/cockroachdb/pebble/objstorage/shared" @@ -19,36 +18,29 @@ type sharedReadable struct { storage shared.Storage objName string size int64 - - mu struct { - sync.Mutex - // rh is used for direct ReadAt calls without a read handle. - rh sharedReadHandle - } } var _ objstorage.Readable = (*sharedReadable)(nil) func newSharedReadable(storage shared.Storage, objName string, size int64) *sharedReadable { - r := &sharedReadable{ + return &sharedReadable{ storage: storage, objName: objName, size: size, } - r.mu.rh.readable = r - return r } func (r *sharedReadable) ReadAt(ctx context.Context, p []byte, offset int64) error { - r.mu.Lock() - defer r.mu.Unlock() - return r.mu.rh.ReadAt(ctx, p, offset) + rh := sharedReadHandle{ + readable: r, + } + defer func() { _ = rh.Close() }() + return rh.ReadAt(ctx, p, offset) } func (r *sharedReadable) Close() error { - err := r.mu.rh.Close() r.storage = nil - return err + return nil } func (r *sharedReadable) Size() int64 { diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs_readable.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs_readable.go index b0f453618a..cd72da054e 100644 --- a/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs_readable.go +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs_readable.go @@ -15,7 +15,8 @@ import ( "github.com/cockroachdb/pebble/vfs" ) -// fileReadable implements objstorage.Readable on top of vfs.File. +// fileReadable implements objstorage.Readable on top of a vfs.File that +// supports Prefetch and/or on an FS that supports SequentialReadsOption. type fileReadable struct { file vfs.File size int64 @@ -219,5 +220,43 @@ func (r *genericFileReadable) NewReadHandle(_ context.Context) objstorage.ReadHa // TestingCheckMaxReadahead returns true if the ReadHandle has switched to // OS-level read-ahead. func TestingCheckMaxReadahead(rh objstorage.ReadHandle) bool { - return rh.(*vfsReadHandle).sequentialFile != nil + switch rh := rh.(type) { + case *vfsReadHandle: + return rh.sequentialFile != nil + case *PreallocatedReadHandle: + return rh.sequentialFile != nil + default: + panic("unknown ReadHandle type") + } +} + +// PreallocatedReadHandle is used to avoid an allocation in NewReadHandle; see +// UsePreallocatedReadHandle. +type PreallocatedReadHandle struct { + vfsReadHandle +} + +// Close is part of the objstorage.ReadHandle interface. +func (rh *PreallocatedReadHandle) Close() error { + var err error + if rh.sequentialFile != nil { + err = rh.sequentialFile.Close() + } + rh.vfsReadHandle = vfsReadHandle{} + return err +} + +// UsePreallocatedReadHandle is equivalent to calling readable.NewReadHandle() +// but uses the existing storage of a PreallocatedReadHandle when possible +// (currently this happens if we are reading from a local file). +// The returned handle still needs to be closed. +func UsePreallocatedReadHandle( + ctx context.Context, readable objstorage.Readable, rh *PreallocatedReadHandle, +) objstorage.ReadHandle { + if r, ok := readable.(*fileReadable); ok { + // See fileReadable.NewReadHandle. + rh.vfsReadHandle = vfsReadHandle{r: r} + return rh + } + return readable.NewReadHandle(ctx) } diff --git a/vendor/github.com/cockroachdb/pebble/open.go b/vendor/github.com/cockroachdb/pebble/open.go index a4e1cffdc6..7d6852cda8 100644 --- a/vendor/github.com/cockroachdb/pebble/open.go +++ b/vendor/github.com/cockroachdb/pebble/open.go @@ -87,9 +87,22 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { }() // Lock the database directory. - fileLock, err := opts.FS.Lock(base.MakeFilepath(opts.FS, dirname, fileTypeLock, base.FileNum(0).DiskFileNum())) - if err != nil { - return nil, err + var fileLock *Lock + if opts.Lock != nil { + // The caller already acquired the database lock. Ensure that the + // directory matches. + if dirname != opts.Lock.dirname { + return nil, errors.Newf("pebble: opts.Lock acquired in %q not %q", opts.Lock.dirname, dirname) + } + if err := opts.Lock.refForOpen(); err != nil { + return nil, err + } + fileLock = opts.Lock + } else { + fileLock, err = LockDirectory(dirname, opts.FS) + if err != nil { + return nil, err + } } defer func() { if db == nil { @@ -204,7 +217,6 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { if d.mu.mem.nextSize > initialMemTableSize { d.mu.mem.nextSize = initialMemTableSize } - d.mu.mem.cond.L = &d.mu.Mutex d.mu.cleaner.cond.L = &d.mu.Mutex d.mu.compact.cond.L = &d.mu.Mutex d.mu.compact.inProgress = make(map[*compaction]struct{}) @@ -970,6 +982,68 @@ func Peek(dirname string, fs vfs.FS) (*DBDesc, error) { return desc, nil } +// LockDirectory acquires the database directory lock in the named directory, +// preventing another process from opening the database. LockDirectory returns a +// handle to the held lock that may be passed to Open through Options.Lock to +// subsequently open the database, skipping lock acquistion during Open. +// +// LockDirectory may be used to expand the critical section protected by the +// database lock to include setup before the call to Open. +func LockDirectory(dirname string, fs vfs.FS) (*Lock, error) { + fileLock, err := fs.Lock(base.MakeFilepath(fs, dirname, fileTypeLock, base.FileNum(0).DiskFileNum())) + if err != nil { + return nil, err + } + l := &Lock{dirname: dirname, fileLock: fileLock} + l.refs.Store(1) + invariants.SetFinalizer(l, func(obj interface{}) { + if refs := obj.(*Lock).refs.Load(); refs > 0 { + panic(errors.AssertionFailedf("lock for %q finalized with %d refs", dirname, refs)) + } + }) + return l, nil +} + +// Lock represents a file lock on a directory. It may be passed to Open through +// Options.Lock to elide lock aquisition during Open. +type Lock struct { + dirname string + fileLock io.Closer + // refs is a count of the number of handles on the lock. refs must be 0, 1 + // or 2. + // + // When acquired by the client and passed to Open, refs = 1 and the Open + // call increments it to 2. When the database is closed, it's decremented to + // 1. Finally when the original caller, calls Close on the Lock, it's + // drecemented to zero and the underlying file lock is released. + // + // When Open acquires the file lock, refs remains at 1 until the database is + // closed. + refs atomic.Int32 +} + +func (l *Lock) refForOpen() error { + // During Open, when a user passed in a lock, the reference count must be + // exactly 1. If it's zero, the lock is no longer held and is invalid. If + // it's 2, the lock is already in use by another database within the + // process. + if !l.refs.CompareAndSwap(1, 2) { + return errors.Errorf("pebble: unexpected Lock reference count; is the lock already in use?") + } + return nil +} + +// Close releases the lock, permitting another process to lock and open the +// database. Close must not be called until after a database using the Lock has +// been closed. +func (l *Lock) Close() error { + if l.refs.Add(-1) > 0 { + return nil + } + defer func() { l.fileLock = nil }() + return l.fileLock.Close() +} + // ErrDBDoesNotExist is generated when ErrorIfNotExists is set and the database // does not exist. // diff --git a/vendor/github.com/cockroachdb/pebble/options.go b/vendor/github.com/cockroachdb/pebble/options.go index 0a4ef4dab8..4e99437c07 100644 --- a/vendor/github.com/cockroachdb/pebble/options.go +++ b/vendor/github.com/cockroachdb/pebble/options.go @@ -704,6 +704,17 @@ type Options struct { // The default value uses the underlying operating system's file system. FS vfs.FS + // Lock, if set, must be a database lock acquired through LockDirectory for + // the same directory passed to Open. If provided, Open will skip locking + // the directory. Closing the database will not release the lock, and it's + // the responsibility of the caller to release the lock after closing the + // database. + // + // Open will enforce that the Lock passed locks the same directory passed to + // Open. Concurrent calls to Open using the same Lock are detected and + // prohibited. + Lock *Lock + // The count of L0 files necessary to trigger an L0 compaction. L0CompactionFileThreshold int diff --git a/vendor/github.com/cockroachdb/pebble/sstable/format.go b/vendor/github.com/cockroachdb/pebble/sstable/format.go index ad3f01ebab..f719ec0e73 100644 --- a/vendor/github.com/cockroachdb/pebble/sstable/format.go +++ b/vendor/github.com/cockroachdb/pebble/sstable/format.go @@ -59,7 +59,7 @@ func ParseTableFormat(magic []byte, version uint32) (TableFormat, error) { } default: return TableFormatUnspecified, base.CorruptionErrorf( - "pebble/table: invalid table (bad magic number)", + "pebble/table: invalid table (bad magic number: 0x%x)", magic, ) } } diff --git a/vendor/github.com/cockroachdb/pebble/sstable/properties.go b/vendor/github.com/cockroachdb/pebble/sstable/properties.go index 6be7f8dc27..ca417cefb1 100644 --- a/vendor/github.com/cockroachdb/pebble/sstable/properties.go +++ b/vendor/github.com/cockroachdb/pebble/sstable/properties.go @@ -136,6 +136,9 @@ type Properties struct { PropertyCollectorNames string `prop:"rocksdb.property.collectors"` // Total raw key size. RawKeySize uint64 `prop:"rocksdb.raw.key.size"` + // Total raw key size of point deletion tombstones. This value is comparable + // to RawKeySize. + RawPointTombstoneKeySize uint64 `prop:"pebble.raw.point-tombstone.key.size"` // Total raw rangekey key size. RawRangeKeyKeySize uint64 `prop:"pebble.raw.range-key.key.size"` // Total raw rangekey value size. @@ -297,7 +300,7 @@ func (p *Properties) saveString(m map[string][]byte, offset uintptr, value strin m[propOffsetTagMap[offset]] = []byte(value) } -func (p *Properties) save(w *rawBlockWriter) { +func (p *Properties) save(tblFormat TableFormat, w *rawBlockWriter) { m := make(map[string][]byte) for k, v := range p.UserProperties { m[k] = []byte(v) @@ -347,6 +350,13 @@ func (p *Properties) save(w *rawBlockWriter) { p.saveUvarint(m, unsafe.Offsetof(p.NumDeletions), p.NumDeletions) p.saveUvarint(m, unsafe.Offsetof(p.NumMergeOperands), p.NumMergeOperands) p.saveUvarint(m, unsafe.Offsetof(p.NumRangeDeletions), p.NumRangeDeletions) + // NB: We only write out the RawTombstoneKeySize for Pebble formats. This + // isn't strictly necessary because unrecognized properties are interpreted + // as user-defined properties, however writing them prevents byte-for-byte + // equivalence with RocksDB files that some of our testing requires. + if p.RawPointTombstoneKeySize > 0 && tblFormat >= TableFormatPebblev1 { + p.saveUvarint(m, unsafe.Offsetof(p.RawPointTombstoneKeySize), p.RawPointTombstoneKeySize) + } if p.NumRangeKeys() > 0 { p.saveUvarint(m, unsafe.Offsetof(p.NumRangeKeyDels), p.NumRangeKeyDels) p.saveUvarint(m, unsafe.Offsetof(p.NumRangeKeySets), p.NumRangeKeySets) diff --git a/vendor/github.com/cockroachdb/pebble/sstable/reader.go b/vendor/github.com/cockroachdb/pebble/sstable/reader.go index af8872b216..4a79d519ed 100644 --- a/vendor/github.com/cockroachdb/pebble/sstable/reader.go +++ b/vendor/github.com/cockroachdb/pebble/sstable/reader.go @@ -26,6 +26,7 @@ import ( "github.com/cockroachdb/pebble/internal/keyspan" "github.com/cockroachdb/pebble/internal/private" "github.com/cockroachdb/pebble/objstorage" + "github.com/cockroachdb/pebble/objstorage/objstorageprovider" "github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing" ) @@ -197,12 +198,13 @@ type singleLevelIterator struct { bpfs *BlockPropertiesFilterer // Per-block lower/upper bound. Nil if the bound does not apply to the block // because we determined the block lies completely within the bound. - blockLower []byte - blockUpper []byte - reader *Reader - index blockIter - data blockIter - dataRH objstorage.ReadHandle + blockLower []byte + blockUpper []byte + reader *Reader + index blockIter + data blockIter + dataRH objstorage.ReadHandle + dataRHPrealloc objstorageprovider.PreallocatedReadHandle // dataBH refers to the last data block that the iterator considered // loading. It may not actually have loaded the block, due to an error or // because it was considered irrelevant. @@ -210,10 +212,11 @@ type singleLevelIterator struct { vbReader *valueBlockReader // vbRH is the read handle for value blocks, which are in a different // part of the sstable than data blocks. - vbRH objstorage.ReadHandle - err error - closeHook func(i Iterator) error - stats *base.InternalIteratorStats + vbRH objstorage.ReadHandle + vbRHPrealloc objstorageprovider.PreallocatedReadHandle + err error + closeHook func(i Iterator) error + stats *base.InternalIteratorStats // boundsCmp and positionedUsingLatestBounds are for optimizing iteration // that uses multiple adjacent bounds. The seek after setting a new bound @@ -414,7 +417,7 @@ func (i *singleLevelIterator) init( _ = i.index.Close() return err } - i.dataRH = r.readable.NewReadHandle(ctx) + i.dataRH = objstorageprovider.UsePreallocatedReadHandle(ctx, r.readable, &i.dataRHPrealloc) if r.tableFormat == TableFormatPebblev3 { if r.Properties.NumValueBlocks > 0 { // NB: we cannot avoid this ~248 byte allocation, since valueBlockReader @@ -434,7 +437,7 @@ func (i *singleLevelIterator) init( stats: stats, } i.data.lazyValueHandling.vbr = i.vbReader - i.vbRH = r.readable.NewReadHandle(ctx) + i.vbRH = objstorageprovider.UsePreallocatedReadHandle(ctx, r.readable, &i.vbRHPrealloc) } i.data.lazyValueHandling.hasValuePrefix = true } diff --git a/vendor/github.com/cockroachdb/pebble/sstable/table.go b/vendor/github.com/cockroachdb/pebble/sstable/table.go index c08fe77ed3..f5e6a1c1d9 100644 --- a/vendor/github.com/cockroachdb/pebble/sstable/table.go +++ b/vendor/github.com/cockroachdb/pebble/sstable/table.go @@ -362,7 +362,7 @@ func readFooter(f objstorage.Readable) (footer, error) { buf = buf[1:] default: - return footer, base.CorruptionErrorf("pebble/table: invalid table (bad magic number)") + return footer, base.CorruptionErrorf("pebble/table: invalid table (bad magic number: 0x%x)", magic) } { diff --git a/vendor/github.com/cockroachdb/pebble/sstable/value_block.go b/vendor/github.com/cockroachdb/pebble/sstable/value_block.go index e60e3e205e..fc0e7004fc 100644 --- a/vendor/github.com/cockroachdb/pebble/sstable/value_block.go +++ b/vendor/github.com/cockroachdb/pebble/sstable/value_block.go @@ -21,10 +21,76 @@ import ( // Value blocks are supported in TableFormatPebblev3. // +// 1. Motivation and overview +// // Value blocks are a mechanism designed for sstables storing MVCC data, where // there can be many versions of a key that need to be kept, but only the -// latest value is typically read. See the documentation for Comparer.Split -// regarding MVCC keys. +// latest value is typically read (see the documentation for Comparer.Split +// regarding MVCC keys). The goal is faster reads. Unlike Pebble versions, +// which can be eagerly thrown away (except when there are snapshots), MVCC +// versions are long-lived (e.g. default CockroachDB garbage collection +// threshold for older versions is 24 hours) and can significantly slow down +// reads. We have seen CockroachDB production workloads with very slow reads +// due to: +// - 100s of versions for each key in a table. +// +// - Tables with mostly MVCC garbage consisting of 2 versions per key -- a +// real key-value pair, followed by a key-value pair whose value (usually +// with zero byte length) indicates it is an MVCC tombstone. +// +// The value blocks mechanism attempts to improve read throughput in these +// cases when the key size is smaller than the value sizes of older versions. +// This is done by moving the value of an older version to a value block in a +// different part of the sstable. This improves spatial locality of the data +// being read by the workload, which increases caching effectiveness. +// +// Additionally, even when the key size is not smaller than the value of older +// versions (e.g. secondary indexes in CockroachDB), TableFormatPebblev3 +// stores the result of key comparisons done at write time inside the sstable, +// which makes stepping from one key prefix to the next prefix (i.e., skipping +// over older versions of a MVCC key) more efficient by avoiding key +// comparisons and key decoding. See the results in +// https://github.com/cockroachdb/pebble/pull/2149 and more details in the +// comment inside BenchmarkIteratorScanNextPrefix. These improvements are also +// visible in end-to-end CockroachDB tests, as outlined in +// https://github.com/cockroachdb/cockroach/pull/96652. +// +// In TableFormatPebblev3, each SET has a one byte value prefix that tells us +// whether the value is in-place or in a value block. This 1 byte prefix +// encodes additional information: +// +// - ShortAttribute: This is an attribute of the value. Currently, CockroachDB +// uses it to represent whether the value is a tombstone or not. This avoids +// the need to fetch a value from the value block if the caller only wants +// to figure out whether it is an MVCC tombstone. The length of the value is +// another attribute that the caller can be interested in, and it is also +// accessible without reading the value in the value block (see the value +// handle in the details section). +// +// - SET-same-prefix: this enables the aforementioned optimization when +// stepping from one key prefix to the next key prefix. +// +// We further optimize this iteration over prefixes by using the restart +// points in a block to encode whether the SET at a restart point has the same +// prefix since the last restart point. This allows us to skip over restart +// points within the same block. See the comment in blockWriter, and how both +// SET-same-prefix and the restart point information is used in +// blockIter.nextPrefixV3. +// +// This flexibility of values that are in-place or in value blocks requires +// flexibility in the iterator interface. The InternalIterator interface +// returns a LazyValue instead of a byte slice. Additionally, pebble.Iterator +// allows the caller to ask for a LazyValue. See lazy_value.go for details, +// including the memory lifetime management. +// +// For historical discussions about this feature, see the issue +// https://github.com/cockroachdb/pebble/issues/1170 and the prototype in +// https://github.com/cockroachdb/pebble/pull/1443. +// +// The code in this file mainly covers value block and related encodings. We +// discuss these in the next section. +// +// 2. Details // // Note that the notion of the latest value is local to the sstable. It is // possible that that latest value has been deleted by a sstable in a higher @@ -35,7 +101,7 @@ import ( // read them. The code in this file is agnostic to the policy regarding what // should be stored in value blocks -- it allows even the latest MVCC version // to be stored in a value block. The policy decision in made in the -// sstable.Writer. See Writer.makeAddPointDecision. +// sstable.Writer. See Writer.makeAddPointDecisionV3. // // Data blocks contain two kinds of SET keys: those with in-place values and // those with a value handle. To distinguish these two cases we use a single diff --git a/vendor/github.com/cockroachdb/pebble/sstable/writer.go b/vendor/github.com/cockroachdb/pebble/sstable/writer.go index ab5f5a8ff5..aaeda43ea1 100644 --- a/vendor/github.com/cockroachdb/pebble/sstable/writer.go +++ b/vendor/github.com/cockroachdb/pebble/sstable/writer.go @@ -932,6 +932,7 @@ func (w *Writer) addPoint(key InternalKey, value []byte) error { switch key.Kind() { case InternalKeyKindDelete, InternalKeyKindSingleDelete: w.props.NumDeletions++ + w.props.RawPointTombstoneKeySize += uint64(key.Size()) case InternalKeyKindMerge: w.props.NumMergeOperands++ } @@ -1948,7 +1949,7 @@ func (w *Writer) Close() (err error) { // reduces table size without a significant impact on performance. raw.restartInterval = propertiesBlockRestartInterval w.props.CompressionOptions = rocksDBCompressionOptions - w.props.save(&raw) + w.props.save(w.tableFormat, &raw) bh, err := w.writeBlock(raw.finish(), NoCompression, &w.blockBuf) if err != nil { return err diff --git a/vendor/github.com/cockroachdb/pebble/table_stats.go b/vendor/github.com/cockroachdb/pebble/table_stats.go index 3c5ce1ffef..1394fb635a 100644 --- a/vendor/github.com/cockroachdb/pebble/table_stats.go +++ b/vendor/github.com/cockroachdb/pebble/table_stats.go @@ -304,12 +304,12 @@ func (d *DB) loadTablePointKeyStats( // We could write the ranges of 'clusters' of point tombstones to // a sstable property and call averageValueSizeBeneath for each of // these narrower ranges to improve the estimate. - avgKeySize, avgValSize, err := d.averageEntrySizeBeneath(v, level, meta) + avgValSize, err := d.averagePhysicalValueSizeBeneath(v, level, meta) if err != nil { return err } stats.PointDeletionsBytesEstimate = - pointDeletionsBytesEstimate(&r.Properties, avgKeySize, avgValSize) + pointDeletionsBytesEstimate(meta.Size, &r.Properties, avgValSize) return nil } @@ -420,9 +420,9 @@ func (d *DB) loadTableRangeDelStats( return compactionHints, err } -func (d *DB) averageEntrySizeBeneath( +func (d *DB) averagePhysicalValueSizeBeneath( v *version, level int, meta physicalMeta, -) (avgKeySize, avgValueSize uint64, err error) { +) (avgValueSize uint64, err error) { // Find all files in lower levels that overlap with meta, // summing their value sizes and entry counts. var fileSum, keySum, valSum, entryCount uint64 @@ -449,31 +449,28 @@ func (d *DB) averageEntrySizeBeneath( } if err != nil { - return 0, 0, err + return 0, err } } } if entryCount == 0 { - return 0, 0, nil + return 0, nil } - // RawKeySize and RawValueSize are uncompressed totals. Scale them + // RawKeySize and RawValueSize are uncompressed totals. Scale the value sum // according to the data size to account for compression, index blocks and // metadata overhead. Eg: // - // Compression rate × Average uncompressed key size + // Compression rate × Average uncompressed value size // // ↓ // - // FileSize RawKeySize - // ----------------------- × ---------- + // FileSize RawValueSize + // ----------------------- × ------------ // RawKeySize+RawValueSize NumEntries - // - // We refactor the calculation to avoid error from rounding/truncation. - totalSizePerEntry := fileSum / entryCount - uncompressedSum := keySum + valSum - avgKeySize = keySum * totalSizePerEntry / uncompressedSum - avgValueSize = valSum * totalSizePerEntry / uncompressedSum - return avgKeySize, avgValueSize, err + uncompressedSum := float64(keySum + valSum) + compressionRatio := float64(fileSum) / uncompressedSum + avgCompressedValueSize := (float64(valSum) / float64(entryCount)) * compressionRatio + return uint64(avgCompressedValueSize), nil } func (d *DB) estimateReclaimedSizeBeneath( @@ -600,8 +597,8 @@ func maybeSetStatsFromProperties(meta physicalMeta, props *sstable.Properties) b // doesn't require any additional IO and since the number of point // deletions in the file is low, the error introduced by this crude // estimate is expected to be small. - avgKeySize, avgValSize := estimateEntrySizes(meta.Size, props) - pointEstimate = pointDeletionsBytesEstimate(props, avgKeySize, avgValSize) + avgValSize := estimateValuePhysicalSize(meta.Size, props) + pointEstimate = pointDeletionsBytesEstimate(meta.Size, props, avgValSize) } meta.Stats.NumEntries = props.NumEntries @@ -614,47 +611,76 @@ func maybeSetStatsFromProperties(meta physicalMeta, props *sstable.Properties) b return true } -func pointDeletionsBytesEstimate(props *sstable.Properties, avgKeySize, avgValSize uint64) uint64 { +func pointDeletionsBytesEstimate( + fileSize uint64, props *sstable.Properties, avgValPhysicalSize uint64, +) (estimate uint64) { if props.NumEntries == 0 { return 0 } - // Estimate the potential space to reclaim using the table's own - // properties. There may or may not be keys covered by any individual - // point tombstone. If not, compacting the point tombstone into L6 will at - // least allow us to drop the point deletion key and will reclaim the key - // bytes. If there are covered key(s), we also get to drop key and value - // bytes for each covered key. + numPointDels := props.NumPointDeletions() + if numPointDels == 0 { + return 0 + } + // Estimate the potential space to reclaim using the table's own properties. + // There may or may not be keys covered by any individual point tombstone. + // If not, compacting the point tombstone into L6 will at least allow us to + // drop the point deletion key and will reclaim the tombstone's key bytes. + // If there are covered key(s), we also get to drop key and value bytes for + // each covered key. // // We estimate assuming that each point tombstone on average covers 1 key. // This is almost certainly an overestimate, but that's probably okay // because point tombstones can slow range iterations even when they don't // cover a key. It may be beneficial in the future to more accurately // estimate which tombstones cover keys and which do not. - numPointDels := props.NumPointDeletions() - return numPointDels*avgKeySize + numPointDels*(avgKeySize+avgValSize) + + // Calculate the contribution of the key sizes: 1x for the point tombstone + // itself and 1x for the key that's expected to exist lower in the LSM. + var keysLogicalSize uint64 + if props.RawPointTombstoneKeySize > 0 { + // This table has a RawPointTombstoneKeySize property, so we know the + // exact size of the logical, uncompressed bytes of the point tombstone + // keys. + keysLogicalSize = 2 * props.RawPointTombstoneKeySize + } else { + // This sstable predates the existence of the RawPointTombstoneKeySize + // property. We can use the average key size within the file itself and + // the count of point deletions to estimate the size. + keysLogicalSize = 2 * numPointDels * props.RawKeySize / props.NumEntries + } + // Scale the logical key size by the logical:physical ratio of the file to + // account for compression, metadata overhead, etc. + // + // Physical FileSize + // ----------- = ----------------------- + // Logical RawKeySize+RawValueSize + // + estimate += (keysLogicalSize * fileSize) / (props.RawKeySize + props.RawValueSize) + + // Calculate the contribution of the deleted values. The caller has already + // computed an average physical size (possibly comptued across many + // sstables), so there's no need to scale it. + estimate += numPointDels * avgValPhysicalSize + return estimate } -func estimateEntrySizes( - fileSize uint64, props *sstable.Properties, -) (avgKeySize, avgValSize uint64) { - // RawKeySize and RawValueSize are uncompressed totals. Scale them - // according to the data size to account for compression, index blocks and - // metadata overhead. Eg: +func estimateValuePhysicalSize(fileSize uint64, props *sstable.Properties) (avgValSize uint64) { + // RawKeySize and RawValueSize are uncompressed totals. Scale according to + // the data size to account for compression, index blocks and metadata + // overhead. Eg: // - // Compression rate × Average uncompressed key size + // Compression rate × Average uncompressed value size // // ↓ // - // FileSize RawKeySize + // FileSize RawValSize // ----------------------- × ---------- // RawKeySize+RawValueSize NumEntries // - // We refactor the calculation to avoid error from rounding/truncation. - fileSizePerEntry := fileSize / props.NumEntries uncompressedSum := props.RawKeySize + props.RawValueSize - avgKeySize = props.RawKeySize * fileSizePerEntry / uncompressedSum - avgValSize = props.RawValueSize * fileSizePerEntry / uncompressedSum - return avgKeySize, avgValSize + compressionRatio := float64(fileSize) / float64(uncompressedSum) + avgCompressedValueSize := (float64(props.RawValueSize) / float64(props.NumEntries)) * compressionRatio + return uint64(avgCompressedValueSize) } // newCombinedDeletionKeyspanIter returns a keyspan.FragmentIterator that diff --git a/vendor/github.com/cockroachdb/pebble/vfs/disk_health.go b/vendor/github.com/cockroachdb/pebble/vfs/disk_health.go index 33ce3d2e88..1c0b9c5976 100644 --- a/vendor/github.com/cockroachdb/pebble/vfs/disk_health.go +++ b/vendor/github.com/cockroachdb/pebble/vfs/disk_health.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "os" + "path/filepath" "sync" "sync/atomic" "time" @@ -375,7 +376,8 @@ func (i DiskSlowInfo) String() string { // SafeFormat implements redact.SafeFormatter. func (i DiskSlowInfo) SafeFormat(w redact.SafePrinter, _ rune) { w.Printf("disk slowness detected: %s on file %s (%d bytes) has been ongoing for %0.1fs", - redact.Safe(i.OpType.String()), i.Path, i.WriteSize, redact.Safe(i.Duration.Seconds())) + redact.Safe(i.OpType.String()), redact.Safe(filepath.Base(i.Path)), + redact.Safe(i.WriteSize), redact.Safe(i.Duration.Seconds())) } // diskHealthCheckingFS adds disk-health checking facilities to a VFS. diff --git a/vendor/modules.txt b/vendor/modules.txt index ec5e626247..16ab072c6c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -45,7 +45,7 @@ github.com/CortexFoundation/merkletree # github.com/CortexFoundation/statik v0.0.0-20210315012922-8bb8a7b5dc66 ## explicit; go 1.16 github.com/CortexFoundation/statik -# github.com/CortexFoundation/torrentfs v1.0.43-0.20230424145059-4a78d96b8709 +# github.com/CortexFoundation/torrentfs v1.0.43-0.20230430013533-eb5ca1c4750a ## explicit; go 1.20 github.com/CortexFoundation/torrentfs github.com/CortexFoundation/torrentfs/backend @@ -329,7 +329,7 @@ github.com/cockroachdb/errors/withstack # github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b ## explicit; go 1.16 github.com/cockroachdb/logtags -# github.com/cockroachdb/pebble v0.0.0-20230420172751-f87a15b23a26 +# github.com/cockroachdb/pebble v0.0.0-20230428220915-dc0efbd4333b ## explicit; go 1.19 github.com/cockroachdb/pebble github.com/cockroachdb/pebble/bloom