From e8b05c5ff29b9d3a71199c91efdfe07bce52cec9 Mon Sep 17 00:00:00 2001 From: Timo Beckers Date: Tue, 17 Dec 2024 21:21:43 +0100 Subject: [PATCH] prog: add ProgramOptions.LogSizeStart to obtain full log after verifier bug When the verifier hits a bug, it returns EFAULT while clobbering other retcodes. This defeats the buffer growth algorithm, so the user needs a break-glass procedure for obtaining the full verifier log. Also, reinstate maxVerifierLogSize since the maxu32 >> 2 limit is still in effect in the Linux kernel. This can be removed if the limit is removed one day and someone has a verifier log that's actually this big. EFAULT can't be reliably triggered in CI, and I'm currently out of ideas for testing this. Signed-off-by: Timo Beckers --- prog.go | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/prog.go b/prog.go index 1c97ae3de..3767f2873 100644 --- a/prog.go +++ b/prog.go @@ -51,6 +51,11 @@ const ( // verifier log. const minVerifierLogSize = 64 * 1024 +// maxVerifierLogSize is the maximum size of verifier log buffer the kernel +// will accept before returning EINVAL. May be increased to MaxUint32 in the +// future, but avoid the unnecessary EINVAL for now. +const maxVerifierLogSize = math.MaxUint32 >> 2 + // ProgramOptions control loading a program into the kernel. type ProgramOptions struct { // Bitmap controlling the detail emitted by the kernel's eBPF verifier log. @@ -70,6 +75,11 @@ type ProgramOptions struct { // attempt at loading the program. LogLevel LogLevel + // Starting size of the verifier log buffer. If the verifier log is larger + // than this size, the buffer will be grown to fit the entire log. Leave at + // its default value unless troubleshooting. + LogSizeStart uint32 + // Disables the verifier log completely, regardless of other options. LogDisabled bool @@ -401,9 +411,10 @@ func newProgramWithOptions(spec *ProgramSpec, opts ProgramOptions) (*Program, er // The caller requested a specific verifier log level. Set up the log buffer // so that there is a chance of loading the program in a single shot. + logSize := internal.Between(opts.LogSizeStart, minVerifierLogSize, maxVerifierLogSize) var logBuf []byte if !opts.LogDisabled && opts.LogLevel != 0 { - logBuf = make([]byte, minVerifierLogSize) + logBuf = make([]byte, logSize) attr.LogLevel = opts.LogLevel attr.LogSize = uint32(len(logBuf)) attr.LogBuf = sys.NewSlicePointer(logBuf) @@ -437,12 +448,11 @@ func newProgramWithOptions(spec *ProgramSpec, opts ProgramOptions) (*Program, er attr.LogLevel = LogLevelBranch } - // Make an educated guess how large the buffer should be. Start - // at minVerifierLogSize and then double the size. - logSize := uint32(max(len(logBuf)*2, minVerifierLogSize)) - if int(logSize) < len(logBuf) { - return nil, errors.New("overflow while probing log buffer size") - } + // Make an educated guess how large the buffer should be by multiplying. + // Ensure the size doesn't overflow. + const factor = 2 + logSize := internal.Between(logSize, minVerifierLogSize, maxVerifierLogSize/factor) + logSize *= factor if attr.LogTrueSize != 0 { // The kernel has given us a hint how large the log buffer has to be. @@ -468,6 +478,12 @@ func newProgramWithOptions(spec *ProgramSpec, opts ProgramOptions) (*Program, er return nil, fmt.Errorf("load program: %w (MEMLOCK may be too low, consider rlimit.RemoveMemlock)", err) } + case errors.Is(err, unix.EFAULT): + // EFAULT is returned when the kernel hits a verifier bug, and always + // overrides ENOSPC, defeating the buffer growth strategy. Warn the user + // that they may need to increase the buffer size manually. + return nil, fmt.Errorf("load program: %w (hit verifier bug, increase LogSizeStart to fit the log and check dmesg)", err) + case errors.Is(err, unix.EINVAL): if bytes.Contains(tail, coreBadCall) { err = errBadRelocation