Memory leak in daemon

[bluntik]

Hi,
we are using client send in daemon and we have memory leak. During debug I found next
Example of code

package main

import (
	"context"
	cloudevents "github.com/cloudevents/sdk-go/v2"
	"log"
	"net/http"
	_ "net/http/pprof"
	"sync"
)

func main() {
	wg := sync.WaitGroup{}

	wg.Add(1)
	go func() {
		defer wg.Done()
		log.Println(http.ListenAndServe("localhost:6060", nil))
	}()

	c, err := cloudevents.NewClientHTTP()
	if err != nil {
		log.Fatalf("failed to create client, %v", err)
	}

	queue := make(chan struct{}, 100)

	for i := 0; i < 500; i++ {
		queue <- struct{}{}

		go func() {
			// Create an Event.
			event :=  cloudevents.NewEvent()
			event.SetSource("example/uri")
			event.SetType("example.type")
			event.SetData(cloudevents.ApplicationJSON, map[string]string{"hello": "world"})

			// Set a target.
			ctx := cloudevents.ContextWithTarget(context.Background(), "http://localhost:14046/health-check")

			// Send that Event.
			if result := c.Send(ctx, event); cloudevents.IsUndelivered(result) {
				log.Fatalf("failed to send, %v", result)
			}
			<-queue
		}()
	}

	wg.Wait()
}

If I sending 500 request in example in debug goruitine we will see 1000 not close gorutine 500 for read 500 for write
http://localhost:6060/debug/pprof/goroutine?debug=1

goroutine profile: total 1005
500 @ 0x103b8a5 0x104c7d7 0x132aae7 0x1071c61
#	0x132aae6	net/http.(*persistConn).readLoop+0x966	/usr/local/Cellar/go/1.16.4/libexec/src/net/http/transport.go:2203

500 @ 0x103b8a5 0x104c7d7 0x132bef7 0x1071c61
#	0x132bef6	net/http.(*persistConn).writeLoop+0xf6	/usr/local/Cellar/go/1.16.4/libexec/src/net/http/transport.go:2382

1 @ 0x103b8a5 0x103421b 0x106c155 0x10c8f25 0x10c9b55 0x10c9b37 0x12289cf 0x1234571 0x130a418 0x1071c61
#	0x106c154	internal/poll.runtime_pollWait+0x54		/usr/local/Cellar/go/1.16.4/libexec/src/runtime/netpoll.go:222
#	0x10c8f24	internal/poll.(*pollDesc).wait+0x44		/usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_poll_runtime.go:87
#	0x10c9b54	internal/poll.(*pollDesc).waitRead+0x1d4	/usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_poll_runtime.go:92
#	0x10c9b36	internal/poll.(*FD).Read+0x1b6			/usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_unix.go:166
#	0x12289ce	net.(*netFD).Read+0x4e				/usr/local/Cellar/go/1.16.4/libexec/src/net/fd_posix.go:55
#	0x1234570	net.(*conn).Read+0x90				/usr/local/Cellar/go/1.16.4/libexec/src/net/net.go:183
#	0x130a417	net/http.(*connReader).backgroundRead+0x57	/usr/local/Cellar/go/1.16.4/libexec/src/net/http/server.go:692

1 @ 0x103b8a5 0x103421b 0x106c155 0x10c8f25 0x10c9b55 0x10c9b37 0x12289cf 0x1234571 0x130a999 0x1101988 0x110267d 0x11028d4 0x12a4436 0x1305eaa 0x1305eab 0x130be3d 0x1310a45 0x1071c61
#	0x106c154	internal/poll.runtime_pollWait+0x54		/usr/local/Cellar/go/1.16.4/libexec/src/runtime/netpoll.go:222
#	0x10c8f24	internal/poll.(*pollDesc).wait+0x44		/usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_poll_runtime.go:87
#	0x10c9b54	internal/poll.(*pollDesc).waitRead+0x1d4	/usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_poll_runtime.go:92
#	0x10c9b36	internal/poll.(*FD).Read+0x1b6			/usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_unix.go:166
#	0x12289ce	net.(*netFD).Read+0x4e				/usr/local/Cellar/go/1.16.4/libexec/src/net/fd_posix.go:55
#	0x1234570	net.(*conn).Read+0x90				/usr/local/Cellar/go/1.16.4/libexec/src/net/net.go:183
#	0x130a998	net/http.(*connReader).Read+0x1b8		/usr/local/Cellar/go/1.16.4/libexec/src/net/http/server.go:800
#	0x1101987	bufio.(*Reader).fill+0x107			/usr/local/Cellar/go/1.16.4/libexec/src/bufio/bufio.go:101
#	0x110267c	bufio.(*Reader).ReadSlice+0x3c			/usr/local/Cellar/go/1.16.4/libexec/src/bufio/bufio.go:360
#	0x11028d3	bufio.(*Reader).ReadLine+0x33			/usr/local/Cellar/go/1.16.4/libexec/src/bufio/bufio.go:389
#	0x12a4435	net/textproto.(*Reader).readLineSlice+0xd5	/usr/local/Cellar/go/1.16.4/libexec/src/net/textproto/reader.go:57
#	0x1305ea9	net/textproto.(*Reader).ReadLine+0xa9		/usr/local/Cellar/go/1.16.4/libexec/src/net/textproto/reader.go:38
#	0x1305eaa	net/http.readRequest+0xaa			/usr/local/Cellar/go/1.16.4/libexec/src/net/http/request.go:1027
#	0x130be3c	net/http.(*conn).readRequest+0x19c		/usr/local/Cellar/go/1.16.4/libexec/src/net/http/server.go:986
#	0x1310a44	net/http.(*conn).serve+0x704			/usr/local/Cellar/go/1.16.4/libexec/src/net/http/server.go:1878

1 @ 0x103b8a5 0x103421b 0x106c155 0x10c8f25 0x10ca632 0x10ca614 0x1229885 0x123b832 0x123a885 0x13157c5 0x13154da 0x13a97a5 0x13a97a6 0x1071c61
#	0x106c154	internal/poll.runtime_pollWait+0x54		/usr/local/Cellar/go/1.16.4/libexec/src/runtime/netpoll.go:222
#	0x10c8f24	internal/poll.(*pollDesc).wait+0x44		/usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_poll_runtime.go:87
#	0x10ca631	internal/poll.(*pollDesc).waitRead+0x211	/usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_poll_runtime.go:92
#	0x10ca613	internal/poll.(*FD).Accept+0x1f3		/usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_unix.go:401
#	0x1229884	net.(*netFD).accept+0x44			/usr/local/Cellar/go/1.16.4/libexec/src/net/fd_unix.go:172
#	0x123b831	net.(*TCPListener).accept+0x31			/usr/local/Cellar/go/1.16.4/libexec/src/net/tcpsock_posix.go:139
#	0x123a884	net.(*TCPListener).Accept+0x64			/usr/local/Cellar/go/1.16.4/libexec/src/net/tcpsock.go:261
#	0x13157c4	net/http.(*Server).Serve+0x284			/usr/local/Cellar/go/1.16.4/libexec/src/net/http/server.go:2981
#	0x13154d9	net/http.(*Server).ListenAndServe+0xb9		/usr/local/Cellar/go/1.16.4/libexec/src/net/http/server.go:2910
#	0x13a97a4	net/http.ListenAndServe+0x84			/usr/local/Cellar/go/1.16.4/libexec/src/net/http/server.go:3164
#	0x13a97a5	main.main.func1+0x85				/Users/a.ryazanov/go/src/test/sdk-go/sdk-go.go:18

1 @ 0x103b8a5 0x104d2a5 0x104d28e 0x106db85 0x107da25 0x13a9705 0x103b476 0x1071c61
#	0x106db84	sync.runtime_Semacquire+0x44	/usr/local/Cellar/go/1.16.4/libexec/src/runtime/sema.go:56
#	0x107da24	sync.(*WaitGroup).Wait+0x64	/usr/local/Cellar/go/1.16.4/libexec/src/sync/waitgroup.go:130
#	0x13a9704	main.main+0x1a4			/Users/a.ryazanov/go/src/test/sdk-go/sdk-go.go:49
#	0x103b475	runtime.main+0x255		/usr/local/Cellar/go/1.16.4/libexec/src/runtime/proc.go:225

1 @ 0x106bd9d 0x1398cee 0x1398ac5 0x1395652 0x13a6f25 0x13a8777 0x1311ec4 0x1313d4d 0x13153e3 0x1310c0d 0x1071c61
#	0x106bd9c	runtime/pprof.runtime_goroutineProfileWithLabels+0x5c	/usr/local/Cellar/go/1.16.4/libexec/src/runtime/mprof.go:716
#	0x1398ced	runtime/pprof.writeRuntimeProfile+0xcd			/usr/local/Cellar/go/1.16.4/libexec/src/runtime/pprof/pprof.go:724
#	0x1398ac4	runtime/pprof.writeGoroutine+0xa4			/usr/local/Cellar/go/1.16.4/libexec/src/runtime/pprof/pprof.go:684
#	0x1395651	runtime/pprof.(*Profile).WriteTo+0x3f1			/usr/local/Cellar/go/1.16.4/libexec/src/runtime/pprof/pprof.go:331
#	0x13a6f24	net/http/pprof.handler.ServeHTTP+0x384			/usr/local/Cellar/go/1.16.4/libexec/src/net/http/pprof/pprof.go:253
#	0x13a8776	net/http/pprof.Index+0x8d6				/usr/local/Cellar/go/1.16.4/libexec/src/net/http/pprof/pprof.go:371
#	0x1311ec3	net/http.HandlerFunc.ServeHTTP+0x43			/usr/local/Cellar/go/1.16.4/libexec/src/net/http/server.go:2069
#	0x1313d4c	net/http.(*ServeMux).ServeHTTP+0x1ac			/usr/local/Cellar/go/1.16.4/libexec/src/net/http/server.go:2448
#	0x13153e2	net/http.serverHandler.ServeHTTP+0xa2			/usr/local/Cellar/go/1.16.4/libexec/src/net/http/server.go:2887
#	0x1310c0c	net/http.(*conn).serve+0x8cc				/usr/local/Cellar/go/1.16.4/libexec/src/net/http/server.go:1952

for fix this problem need to close Body.Close() https://github.com/cloudevents/sdk-go/blob/main/v2/protocol/http/protocol_retry.go#L36

example

resp, err := p.Client.Do(req)

if resp != nil {
  defer resp.Body.Close()
}

[n3wscott]

thanks! I will look into this and the linked PR

[n3wscott]

I think this was fixed by #689, but we have not had a chance to make a release yet.

Could you try your test again with what is at HEAD?

[bluntik]

It looks like it really fixing it. I just tested on snipped code from this issue. I will test tomorrow on my project daemon code.

[n3wscott]

Ok awesome, I will wait for you to confirm and then run a release tomorrow. Thanks for the report!!

[bluntik]

All fine. The code from the latest main branch works as expected. All read/write goroutines are closed. No memory leak or overhead memory limit using. Thanks for fast answer, we need it release for our production using.

[n3wscott]

awesome, will make a release then, thanks!