Skip to content

Commit

Permalink
fix: allow promtail docker_sd_config to handle long log lines (#6256)
Browse files Browse the repository at this point in the history
In promtail's docker_sd_config, parsing of a container's
logs would fail if any log line was over 64k bytes in length.
This is because bufio.Scanner uses a 64k buffer by default.
Since there doesn't appear to be a way to elegantly catch
`bufio.ErrTooLong` and carry on, the parsing approach was changed
to bufio.Reader which can handle lines of arbitrary length.
  • Loading branch information
srstsavage authored Jun 3, 2022
1 parent 69eabfc commit 135f672
Show file tree
Hide file tree
Showing 3 changed files with 29 additions and 10 deletions.
36 changes: 27 additions & 9 deletions clients/pkg/promtail/targets/docker/target.go
Original file line number Diff line number Diff line change
Expand Up @@ -148,14 +148,38 @@ func extractTs(line string) (time.Time, string, error) {
return ts, pair[1], nil
}

// https://devmarkpro.com/working-big-files-golang
func readLine(r *bufio.Reader) (string, error) {
var (
isPrefix = true
err error
line, ln []byte
)

for isPrefix && err == nil {
line, isPrefix, err = r.ReadLine()
ln = append(ln, line...)
}

return string(ln), err
}

func (t *Target) process(r io.Reader, logStream string) {
defer func() {
t.wg.Done()
}()

scanner := bufio.NewScanner(r)
for scanner.Scan() {
line := scanner.Text()
reader := bufio.NewReader(r)
for {
line, err := readLine(reader)
if err != nil {
if err == io.EOF {
break
}
level.Error(t.logger).Log("msg", "error reading docker log line, skipping line", "err", err)
t.metrics.dockerErrors.Inc()
}

ts, line, err := extractTs(line)
if err != nil {
level.Error(t.logger).Log("msg", "could not extract timestamp, skipping line", "err", err)
Expand Down Expand Up @@ -189,12 +213,6 @@ func (t *Target) process(r io.Reader, logStream string) {
t.metrics.dockerEntries.Inc()
t.positions.Put(positions.CursorKey(t.containerName), ts.Unix())
}

err := scanner.Err()
if err != nil {
level.Warn(t.logger).Log("msg", "finished scanning logs lines with an error", "err", err)
}

}

// startIfNotRunning starts processing container logs. The operation is idempotent , i.e. the processing cannot be started twice.
Expand Down
3 changes: 2 additions & 1 deletion clients/pkg/promtail/targets/docker/targetmanager_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ func Test_TargetManager(t *testing.T) {
require.True(t, ta.Ready())

require.Eventually(t, func() bool {
return len(entryHandler.Received()) >= 5
return len(entryHandler.Received()) >= 6
}, 20*time.Second, 100*time.Millisecond)

received := entryHandler.Received()
Expand All @@ -109,4 +109,5 @@ func Test_TargetManager(t *testing.T) {
actualLines = append(actualLines, entry.Line)
}
require.ElementsMatch(t, actualLines, expectedLines)
require.Equal(t, 99969, len(received[5].Line))
}
Binary file modified clients/pkg/promtail/targets/docker/testdata/flog.log
Binary file not shown.

0 comments on commit 135f672

Please sign in to comment.