concurrent map writes to cm.records

[alexcb]

I came across a concurrent map writes failure when writing to
https://github.com/moby/buildkit/blob/master/cache/manager.go#L542

Here's a very large collection of stacktraces if it helps; the one caveat is it's running out of our fork in https://github.com/earthly/buildkit but I don't think we've touched the cache package in any way.

fatal error: concurrent map writes
--
  |  
  | goroutine 130608 [running]:
  | runtime.throw(0x135b6ab, 0x15)
  | /usr/local/go/src/runtime/panic.go:774 +0x72 fp=0xc003e33080 sp=0xc003e33050 pc=0x42f892
  | runtime.mapassign_faststr(0x1159620, 0xc00010ac30, 0xc00a0fc041, 0x19, 0x1)
  | /usr/local/go/src/runtime/map_faststr.go:211 +0x417 fp=0xc003e330e8 sp=0xc003e33080 pc=0x413787
  | github.com/moby/buildkit/cache.(*cacheManager).New(0xc000194280, 0x15353c0, 0xc00a0bf410, 0x1550740, 0xc005628000, 0x150b720, 0xc00347b560, 0xc0047b00c0, 0x1, 0x1, ...)
  | /src/cache/manager.go:542 +0x733 fp=0xc003e33358 sp=0xc003e330e8 pc=0xc5d843
  | github.com/moby/buildkit/solver/llbsolver/ops.(*execOp).Exec.func1(0xc005bbeb40, 0x1550740, 0xc005628000, 0x7f1ad2165f00, 0xc005628000, 0x440a21, 0xc003752608)
  | /src/solver/llbsolver/ops/exec.go:241 +0x264 fp=0xc003e33440 sp=0xc003e33358 pc=0xf29c84
  | github.com/moby/buildkit/frontend/gateway.PrepareMounts(0x15353c0, 0xc00a0bf410, 0xc004152900, 0x154a100, 0xc000194280, 0x150b720, 0xc00347b560, 0xc005030fe0, 0x3, 0x4, ...)
  | /src/frontend/gateway/container.go:172 +0x972 fp=0xc003e33668 sp=0xc003e33440 pc=0xcbc7f2
  | github.com/moby/buildkit/solver/llbsolver/ops.(*execOp).Exec(0xc003ff7f20, 0x15353c0, 0xc00a0bf410, 0x150b720, 0xc00347b560, 0xc0037a81b0, 0x1, 0x1, 0x0, 0x0, ...)
  | /src/solver/llbsolver/ops/exec.go:239 +0x2f8 fp=0xc003e33c90 sp=0xc003e33668 pc=0xf21028
  | github.com/moby/buildkit/solver.(*sharedOp).Exec.func2(0x15353c0, 0xc00a0bf410, 0x0, 0x0, 0x0, 0x0)
  | /src/solver/jobs.go:778 +0x37b fp=0xc003e33e18 sp=0xc003e33c90 pc=0xc0507b
  | github.com/moby/buildkit/util/flightcontrol.(*call).run(0xc007c473b0)
  | /src/util/flightcontrol/flightcontrol.go:121 +0xee fp=0xc003e33ef8 sp=0xc003e33e18 pc=0xa6789e
  | github.com/moby/buildkit/util/flightcontrol.(*call).run-fm()
  | /src/util/flightcontrol/flightcontrol.go:117 +0x2a fp=0xc003e33f10 sp=0xc003e33ef8 pc=0xa695fa
  | sync.(*Once).doSlow(0xc007c47410, 0xc0037a8370)
  | /usr/local/go/src/sync/once.go:66 +0xe3 fp=0xc003e33fb0 sp=0xc003e33f10 pc=0x46e263
  | sync.(*Once).Do(0xc007c47410, 0xc0037a8370)
  | /usr/local/go/src/sync/once.go:57 +0x45 fp=0xc003e33fd0 sp=0xc003e33fb0 pc=0x46e175
  | runtime.goexit()
  | /usr/local/go/src/runtime/asm_amd64.s:1357 +0x1 fp=0xc003e33fd8 sp=0xc003e33fd0 pc=0x45f2e1
  | created by github.com/moby/buildkit/util/flightcontrol.(*call).wait
  | /src/util/flightcontrol/flightcontrol.go:156 +0x3a9
  |  
  | goroutine 1 [select, 3 minutes]:
  | main.main.func3(0xc0000e31e0, 0x0, 0x0)
  | /src/cmd/buildkitd/main.go:270 +0xad1
  | github.com/urfave/cli.HandleAction(0x112d260, 0x13a3cb0, 0xc0000e31e0, 0xc0000e31e0, 0x0)
  | /src/vendor/github.com/urfave/cli/app.go:523 +0xbe
  | github.com/urfave/cli.(*App).Run(0xc00056c700, 0xc0000ba000, 0x2, 0x2, 0x0, 0x0)
  | /src/vendor/github.com/urfave/cli/app.go:285 +0x5ec
  | main.main()
  | /src/cmd/buildkitd/main.go:297 +0xd59
  |  
  | goroutine 20 [syscall, 3 minutes]:
  | os/signal.signal_recv(0x0)
  | /usr/local/go/src/runtime/sigqueue.go:147 +0x9c
  | os/signal.loop()
  | /usr/local/go/src/os/signal/signal_unix.go:23 +0x22
  | created by os/signal.init.0
  | /usr/local/go/src/os/signal/signal_unix.go:29 +0x41
  |  
  | goroutine 36 [chan receive, 3 minutes]:
  | github.com/moby/buildkit/util/appcontext.Context.func1.1(0xc00024e6c0, 0xc000312000, 0xc000242000)
  | /src/util/appcontext/appcontext.go:30 +0x38
  | created by github.com/moby/buildkit/util/appcontext.Context.func1
  | /src/util/appcontext/appcontext.go:28 +0xfe
  |  
  | goroutine 130755 [sync.Cond.Wait]:
  | runtime.goparkunlock(...)
  | /usr/local/go/src/runtime/proc.go:310
  | sync.runtime_notifyListWait(0xc0086f3750, 0xc000000000)
  | /usr/local/go/src/runtime/sema.go:510 +0xf8
  | sync.(*Cond).Wait(0xc0086f3740)
  | /usr/local/go/src/sync/cond.go:56 +0x9d
  | github.com/moby/buildkit/util/progress.(*progressReader).Read(0xc00a0f58c0, 0x1535340, 0xc0000b0080, 0x0, 0x0, 0x0, 0x0, 0x0)
  | /src/util/progress/progress.go:132 +0x12e
  | github.com/moby/buildkit/util/flightcontrol.(*progressState).run(0xc00a0f5830, 0x150bb20, 0xc00a0f58c0)
  | /src/util/flightcontrol/flightcontrol.go:288 +0x16d
  | created by github.com/moby/buildkit/util/flightcontrol.newCall
  | /src/util/flightcontrol/flightcontrol.go:112 +0x1d8
  |  
  | goroutine 130607 [chan receive]:
  | github.com/moby/buildkit/util/flightcontrol.(*Group).do.func1(0xc007c473b0, 0xc003bb58d0, 0x1347cb0, 0x4)
  | /src/util/flightcontrol/flightcontrol.go:71 +0x3c
  | created by github.com/moby/buildkit/util/flightcontrol.(*Group).do
  | /src/util/flightcontrol/flightcontrol.go:69 +0x1b4
  |  
  | goroutine 99178 [syscall]:
  | syscall.Syscall6(0xf7, 0x1, 0x237c, 0xc003d1e5e8, 0x1000004, 0x0, 0x0, 0xc003d1e728, 0xa65858, 0xc00407fc80)
  | /usr/local/go/src/syscall/asm_linux_amd64.s:44 +0x5
  | os.(*Process).blockUntilWaitable(0xc002b78d80, 0xc0035805a0, 0x0, 0x0)
  | /usr/local/go/src/os/wait_waitid.go:31 +0x98
  | os.(*Process).wait(0xc002b78d80, 0xc0007c5a90, 0xc006aab900, 0xc004f9dae0)
  | /usr/local/go/src/os/exec_unix.go:22 +0x39
  | os.(*Process).Wait(...)
  | /usr/local/go/src/os/exec.go:125
  | os/exec.(*Cmd).Wait(0xc004495080, 0x13a46a8, 0x437a56)
  | /usr/local/go/src/os/exec/exec.go:506 +0x60
  | github.com/containerd/go-runc.(*defaultMonitor).Start.func1(0xc004495080, 0xc004adc5a0)
  | /src/vendor/github.com/containerd/go-runc/monitor.go:55 +0x2f
  | created by github.com/containerd/go-runc.(*defaultMonitor).Start
  | /src/vendor/github.com/containerd/go-runc/monitor.go:53 +0xa5
  |  
  | goroutine 9 [semacquire, 3 minutes]:
  | sync.runtime_Semacquire(0xc000a04850)
  | /usr/local/go/src/runtime/sema.go:56 +0x42
  | sync.(*WaitGroup).Wait(0xc000a04848)
  | /usr/local/go/src/sync/waitgroup.go:130 +0x64
  | golang.org/x/sync/errgroup.(*Group).Wait(0xc000a04840, 0xc0004167e0, 0xc0004c1fb0)
  | /src/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31
  | main.serveGRPC.func2(0xc001389920, 0xc000a04840)
  | /src/cmd/buildkitd/main.go:339 +0x2b
  | created by main.serveGRPC
  | /src/cmd/buildkitd/main.go:338 +0x39e
  |  
  | goroutine 8 [IO wait, 3 minutes]:
  | internal/poll.runtime_pollWait(0x7f1ade40b6f0, 0x72, 0x0)
  | /usr/local/go/src/runtime/netpoll.go:184 +0x55
  | internal/poll.(*pollDesc).wait(0xc000112498, 0x72, 0x0, 0x0, 0x134af2e)
  | /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
  | internal/poll.(*pollDesc).waitRead(...)
  | /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
  | internal/poll.(*FD).Accept(0xc000112480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
  | /usr/local/go/src/internal/poll/fd_unix.go:384 +0x1f8
  | net.(*netFD).accept(0xc000112480, 0xc00000e008, 0x0, 0x0)
  | /usr/local/go/src/net/fd_unix.go:238 +0x42
  | net.(*UnixListener).accept(0xc000a04930, 0xc001543d30, 0xc001543d38, 0x18)
  | /usr/local/go/src/net/unixsock_posix.go:162 +0x32
  | net.(*UnixListener).Accept(0xc000a04930, 0x13a3840, 0xc00017c540, 0x1547460, 0xc00000e008)
  | /usr/local/go/src/net/unixsock.go:260 +0x47
  | google.golang.org/grpc.(*Server).Serve(0xc00017c540, 0x152ef40, 0xc000a04930, 0x0, 0x0)
  | /src/vendor/google.golang.org/grpc/server.go:695 +0x22e
  | main.serveGRPC.func1.1(0x0, 0x0)
  | /src/cmd/buildkitd/main.go:334 +0x142
  | golang.org/x/sync/errgroup.(*Group).Go.func1(0xc000a04840, 0xc00140d8a0)
  | /src/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x64
  | created by golang.org/x/sync/errgroup.(*Group).Go
  | /src/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66
  |  
  | goroutine 59 [chan receive, 3 minutes]:
  | github.com/moby/buildkit/solver.(*scheduler).loop.func2(0xc0002cec40)
  | /src/solver/scheduler.go:76 +0x3c
  | created by github.com/moby/buildkit/solver.(*scheduler).loop
  | /src/solver/scheduler.go:75 +0x77
  |  
  | goroutine 7 [semacquire]:
  | sync.runtime_Semacquire(0xc00991af10)
  | /usr/local/go/src/runtime/sema.go:56 +0x42
  | sync.(*WaitGroup).Wait(0xc00991af08)
  | /usr/local/go/src/sync/waitgroup.go:130 +0x64
  | golang.org/x/sync/errgroup.(*Group).Wait(0xc00991af00, 0xc004ae0190, 0x2)
  | /src/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31
  | github.com/moby/buildkit/solver.(*combinedCacheManager).Query(0xc004ae00a0, 0xc004ae0190, 0x2, 0x2, 0x1, 0xc003338ff0, 0x47, 0x0, 0x203000, 0x8, ...)
  | /src/solver/combinedcache.go:59 +0x1f2
  | github.com/moby/buildkit/solver.(*edge).processUpdate(0xc005740f00, 0x1535b00, 0xc002ae86e0, 0x4461ee)
  | /src/solver/edge.go:511 +0xad8
  | github.com/moby/buildkit/solver.(*edge).unpark(0xc005740f00, 0xc0045cc910, 0x1, 0x1, 0xc0045cc930, 0x1, 0x1, 0xc0045cc920, 0x1, 0x1, ...)
  | /src/solver/edge.go:326 +0x81
  | github.com/moby/buildkit/solver.(*scheduler).dispatch(0xc0002cec40, 0xc005740f00)
  | /src/solver/scheduler.go:136 +0x430
  | github.com/moby/buildkit/solver.(*scheduler).loop(0xc0002cec40)
  | /src/solver/scheduler.go:104 +0x179
  | created by github.com/moby/buildkit/solver.newScheduler
  | /src/solver/scheduler.go:35 +0x1ab
  |  
  | goroutine 131290 [syscall]:
  | syscall.Syscall6(0xf7, 0x1, 0x52ee, 0xc00467cde8, 0x1000004, 0x0, 0x0, 0xc00467cf28, 0xa65858, 0xc003fdb920)
  | /usr/local/go/src/syscall/asm_linux_amd64.s:44 +0x5
  | os.(*Process).blockUntilWaitable(0xc0067eff80, 0x45bf60, 0x0, 0xc00342c000)
  | /usr/local/go/src/os/wait_waitid.go:31 +0x98
  | os.(*Process).wait(0xc0067eff80, 0x0, 0x8, 0xc00467cf80)
  | /usr/local/go/src/os/exec_unix.go:22 +0x39
  | os.(*Process).Wait(...)
  | /usr/local/go/src/os/exec.go:125
  | os/exec.(*Cmd).Wait(0xc00074e160, 0xc00235e701, 0x437a56)
  | /usr/local/go/src/os/exec/exec.go:506 +0x60
  | github.com/containerd/go-runc.(*defaultMonitor).Start.func1(0xc00074e160, 0xc00cadab40)
  | /src/vendor/github.com/containerd/go-runc/monitor.go:55 +0x2f
  | created by github.com/containerd/go-runc.(*defaultMonitor).Start
  | /src/vendor/github.com/containerd/go-runc/monitor.go:53 +0xa5
  |  
  | goroutine 130668 [select]:
  | github.com/moby/buildkit/util/flightcontrol.(*call).wait(0xc00bc30090, 0x1535300, 0xc008644a40, 0x0, 0x0, 0x0, 0x0)
  | /src/util/flightcontrol/flightcontrol.go:158 +0x447
  | github.com/moby/buildkit/util/flightcontrol.(*Group).do(0xc003b220d0, 0x1535300, 0xc008644600, 0x1347cb0, 0x4, 0xc0086447c0, 0x10, 0x11c7700, 0x47, 0xc002fdacf0)
  | /src/util/flightcontrol/flightcontrol.go:78 +0x1eb
  | github.com/moby/buildkit/util/flightcontrol.(*Group).Do(0xc003b220d0, 0x1535300, 0xc008644600, 0x1347cb0, 0x4, 0xc0086447c0, 0x40a5b5, 0x11c7700, 0x1217b80, 0x0)
  | /src/util/flightcontrol/flightcontrol.go:37 +0x7a
  | github.com/moby/buildkit/solver.(*sharedOp).Exec(0xc003b220c0, 0x1535300, 0xc008644600, 0xc003935280, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, ...)
  | /src/solver/jobs.go:762 +0x1c5
  | github.com/moby/buildkit/solver.(*edge).execOp(0xc003b01a40, 0x1535300, 0xc008644600, 0xc00a0f3140, 0xc00a0f3140, 0x0, 0xc004f11160)
  | /src/solver/edge.go:892 +0xb3
  | github.com/moby/buildkit/solver/internal/pipe.NewWithFunction.func2()
  | /src/solver/internal/pipe/pipe.go:82 +0x44
  | created by github.com/moby/buildkit/solver.(*scheduler).newRequestWithFunc
  | /src/solver/scheduler.go:289 +0x142
  |  

the stack trace is too large to fully post above; the entire stack traces can be found under https://gist.github.com/alexcb/6868ee55604932cf6d5198e8e916b16e

[tonistiigi]

Can you run with the race detector to see where the other write is.