Skip to content

pprof: support limit the max number for dumping goroutines. #50771

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions api/next.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
pkg runtime/pprof, func SetMaxDumpGoroutineNum(int)
32 changes: 29 additions & 3 deletions src/runtime/mprof.go
Original file line number Diff line number Diff line change
Expand Up @@ -755,6 +755,17 @@ func runtime_goroutineProfileWithLabels(p []StackRecord, labels []unsafe.Pointer

// labels may be nil. If labels is non-nil, it must have the same length as p.
func goroutineProfileWithLabels(p []StackRecord, labels []unsafe.Pointer) (n int, ok bool) {
return goroutineProfileWithLabelsAndLimit(p, labels, 0)
}

//go:linkname runtime_goroutineProfileWithLabelsAndLimit runtime/pprof.runtime_goroutineProfileWithLabelsAndLimit
func runtime_goroutineProfileWithLabelsAndLimit(p []StackRecord, labels []unsafe.Pointer, max int) (n int, ok bool) {
return goroutineProfileWithLabelsAndLimit(p, labels, max)
}

// limit the max goroutine, sample them when return there are more than max goroutines.
// 0 means not limitation.
func goroutineProfileWithLabelsAndLimit(p []StackRecord, labels []unsafe.Pointer, max int) (m int, ok bool) {
if labels != nil && len(labels) != len(p) {
labels = nil
}
Expand All @@ -769,14 +780,19 @@ func goroutineProfileWithLabels(p []StackRecord, labels []unsafe.Pointer) (n int
stopTheWorld("profile")

// World is stopped, no locking required.
n = 1
n := 1
forEachGRace(func(gp1 *g) {
if isOK(gp1) {
n++
}
})

if n <= len(p) {
m = n
if max > 0 && n > max {
m = max
}

if m <= len(p) {
ok = true
r, lbl := p, labels

Expand All @@ -794,12 +810,22 @@ func goroutineProfileWithLabels(p []StackRecord, labels []unsafe.Pointer) (n int
lbl = lbl[1:]
}

got, i := 1, 0
rand := fastrand()

// Save other goroutines.
forEachGRace(func(gp1 *g) {
if !isOK(gp1) {
return
}

i++
// make sure we will have m goroutines.
if m < n && int(rand%uint32(n-i)) >= (m-got) {
return
}
got++

if len(r) == 0 {
// Should be impossible, but better to return a
// truncated profile than to crash the entire process.
Expand All @@ -819,7 +845,7 @@ func goroutineProfileWithLabels(p []StackRecord, labels []unsafe.Pointer) (n int
}

startTheWorld()
return n, ok
return m, ok
}

// GoroutineProfile returns n, the number of records in the active goroutine stack profile.
Expand Down
23 changes: 16 additions & 7 deletions src/runtime/pprof/pprof.go
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,14 @@ func Lookup(name string) *Profile {
return profiles.m[name]
}

// no limit by default
var maxDumpGoroutineNum = 0

// SetMaxDumpGoroutineNum set the max number for dumping goroutine
func SetMaxDumpGoroutineNum(limit int) {
maxDumpGoroutineNum = limit
}

// Profiles returns a slice of all the known profiles, sorted by name.
func Profiles() []*Profile {
lockProfiles()
Expand Down Expand Up @@ -664,7 +672,7 @@ func writeThreadCreate(w io.Writer, debug int) error {
// Until https://golang.org/issues/6104 is addressed, wrap
// ThreadCreateProfile because there's no point in tracking labels when we
// don't get any stack-traces.
return writeRuntimeProfile(w, debug, "threadcreate", func(p []runtime.StackRecord, _ []unsafe.Pointer) (n int, ok bool) {
return writeRuntimeProfile(w, debug, "threadcreate", func(p []runtime.StackRecord, _ []unsafe.Pointer, _ int) (n int, ok bool) {
return runtime.ThreadCreateProfile(p)
})
}
Expand All @@ -674,15 +682,15 @@ func countGoroutine() int {
return runtime.NumGoroutine()
}

// runtime_goroutineProfileWithLabels is defined in runtime/mprof.go
func runtime_goroutineProfileWithLabels(p []runtime.StackRecord, labels []unsafe.Pointer) (n int, ok bool)
// runtime_goroutineProfileWithLabelsAndLimit is defined in runtime/mprof.go
func runtime_goroutineProfileWithLabelsAndLimit(p []runtime.StackRecord, labels []unsafe.Pointer, max int) (n int, ok bool)

// writeGoroutine writes the current runtime GoroutineProfile to w.
func writeGoroutine(w io.Writer, debug int) error {
if debug >= 2 {
return writeGoroutineStacks(w)
}
return writeRuntimeProfile(w, debug, "goroutine", runtime_goroutineProfileWithLabels)
return writeRuntimeProfile(w, debug, "goroutine", runtime_goroutineProfileWithLabelsAndLimit)
}

func writeGoroutineStacks(w io.Writer) error {
Expand All @@ -706,7 +714,7 @@ func writeGoroutineStacks(w io.Writer) error {
return err
}

func writeRuntimeProfile(w io.Writer, debug int, name string, fetch func([]runtime.StackRecord, []unsafe.Pointer) (int, bool)) error {
func writeRuntimeProfile(w io.Writer, debug int, name string, fetch func([]runtime.StackRecord, []unsafe.Pointer, int) (int, bool)) error {
// Find out how many records there are (fetch(nil)),
// allocate that many records, and get the data.
// There's a race—more records might be added between
Expand All @@ -715,14 +723,15 @@ func writeRuntimeProfile(w io.Writer, debug int, name string, fetch func([]runti
// The loop should only execute one iteration in the common case.
var p []runtime.StackRecord
var labels []unsafe.Pointer
n, ok := fetch(nil, nil)
max := maxDumpGoroutineNum
n, ok := fetch(nil, nil, max)
for {
// Allocate room for a slightly bigger profile,
// in case a few more entries have been added
// since the call to ThreadProfile.
p = make([]runtime.StackRecord, n+10)
labels = make([]unsafe.Pointer, n+10)
n, ok = fetch(p, labels)
n, ok = fetch(p, labels, max)
if ok {
p = p[0:n]
break
Expand Down