-
Notifications
You must be signed in to change notification settings - Fork 543
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Store Gateway: Add experimental configuration to use MAP_POPULATE for…
… indexheader reads. (#2019) Introduces a new experimental configuration option (`-blocks-storage.bucket-store.index-header.map-populate-enabled`). This enables the use of the `MAP_POPULATE` flag when `mmap`-ing index-header files in the store-gateway. What this flag does is advise the kernel to (synchronously) pre-fault all pages in the memory region, loading them into the file system cache. Why is this a good idea? - The initial read process of the index-header files has shown to cause hangups in the store-gateway. - By using this option, I/O is done in the mmap() syscall, which the Go scheduler can cope with. - We reduce the likelyhood of Goroutines getting stalled in major page faults. - The initial read process walks the entire file anyway, so we are not doing any more I/O. - It's a very low risk change compared to re-writing the BinaryReader (work in progress). Why is this not perfect? - The Kernel does not guarantee the pages will stay in memory, so we are only reducing the probability of major page faults. Rationale about the implementation: - I have copied the mmap utilities from Prometheus as a temporary measure, for the sake of evaluating this change.
- Loading branch information
Showing
18 changed files
with
189 additions
and
27 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| // SPDX-License-Identifier: AGPL-3.0-only | ||
| // Provenance-includes-location: https://github.com/prometheus/prometheus/blob/main/tsdb/fileutil/mmap.go | ||
| // Provenance-includes-license: Apache-2.0 | ||
| // Provenance-includes-copyright: The Prometheus Authors. | ||
|
|
||
| package fileutil | ||
|
|
||
| import ( | ||
| "os" | ||
|
|
||
| "github.com/pkg/errors" | ||
| ) | ||
|
|
||
| type MmapFile struct { | ||
| f *os.File | ||
| b []byte | ||
| } | ||
|
|
||
| func OpenMmapFile(path string, populate bool) (*MmapFile, error) { | ||
| return OpenMmapFileWithSize(path, 0, populate) | ||
| } | ||
|
|
||
| func OpenMmapFileWithSize(path string, size int, populate bool) (mf *MmapFile, retErr error) { | ||
| f, err := os.Open(path) | ||
| if err != nil { | ||
| return nil, errors.Wrap(err, "try lock file") | ||
| } | ||
| defer func() { | ||
| if retErr != nil { | ||
| f.Close() | ||
| } | ||
| }() | ||
| if size <= 0 { | ||
| info, err := f.Stat() | ||
| if err != nil { | ||
| return nil, errors.Wrap(err, "stat") | ||
| } | ||
| size = int(info.Size()) | ||
| } | ||
|
|
||
| b, err := mmap(f, size, populate) | ||
| if err != nil { | ||
| return nil, errors.Wrapf(err, "mmap, size %d", size) | ||
| } | ||
|
|
||
| return &MmapFile{f: f, b: b}, nil | ||
| } | ||
|
|
||
| func (f *MmapFile) Close() error { | ||
| err0 := munmap(f.b) | ||
| err1 := f.f.Close() | ||
|
|
||
| if err0 != nil { | ||
| return err0 | ||
| } | ||
| return err1 | ||
| } | ||
|
|
||
| func (f *MmapFile) File() *os.File { | ||
| return f.f | ||
| } | ||
|
|
||
| func (f *MmapFile) Bytes() []byte { | ||
| return f.b | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| // SPDX-License-Identifier: AGPL-3.0-only | ||
| // Provenance-includes-location: https://github.com/prometheus/prometheus/blob/main/tsdb/fileutil/mmap_unix.go | ||
| // Provenance-includes-license: Apache-2.0 | ||
| // Provenance-includes-copyright: The Prometheus Authors. | ||
|
|
||
| //go:build !windows && !plan9 | ||
| // +build !windows,!plan9 | ||
|
|
||
| package fileutil | ||
|
|
||
| import ( | ||
| "os" | ||
|
|
||
| "golang.org/x/sys/unix" | ||
| ) | ||
|
|
||
| func mmap(f *os.File, length int, populate bool) ([]byte, error) { | ||
| flags := unix.MAP_SHARED | ||
| if populate { | ||
| flags |= unix.MAP_POPULATE | ||
| } | ||
| return unix.Mmap(int(f.Fd()), 0, length, unix.PROT_READ, flags) | ||
| } | ||
|
|
||
| func munmap(b []byte) (err error) { | ||
| return unix.Munmap(b) | ||
| } |
Oops, something went wrong.