ctr-remote image rpull error

[yangxiaolaing]

[root@nydus nydus]# go version
go version go1.17.12 linux/amd64
[root@nydus nydus]#

[root@nydus nydus]# curl -XGET http://localhost:5000/v2/ubuntu-nydus/tags/list
{"name":"ubuntu-nydus","tags":["latest"]}
[root@nydus nydus]#

[root@nydus nydus]# ctr-remote image rpull --plain-http localhost:5000/ubuntu-nydus:latest
fatal error: unexpected signal during runtime execution
[signal SIGSEGV: segmentation violation code=0x1 addr=0x63 pc=0x7f106b5f4bd0]

runtime stack:
runtime.throw({0x1038ef0, 0x7f108198e880})
/opt/hostedtoolcache/go/1.17.12/x64/src/runtime/panic.go:1198 +0x71
runtime.sigpanic()
/opt/hostedtoolcache/go/1.17.12/x64/src/runtime/signal_unix.go:719 +0x396

goroutine 13 [syscall]:
runtime.cgocall(0xd55940, 0xc000061590)
/opt/hostedtoolcache/go/1.17.12/x64/src/runtime/cgocall.go:156 +0x5c fp=0xc000061568 sp=0xc000061530 pc=0x40537c
net._C2func_getaddrinfo(0xc0003be8c0, 0x0, 0xc000482db0, 0xc0000106f8)
_cgo_gotypes.go:91 +0x56 fp=0xc000061590 sp=0xc000061568 pc=0x60f616
net.cgoLookupIPCNAME.func1({0xc0003be8c0, 0xc000061648, 0x870ba5}, 0xc0003be780, 0xc000061650)
/opt/hostedtoolcache/go/1.17.12/x64/src/net/cgo_unix.go:163 +0x9f fp=0xc0000615e8 sp=0xc000061590 pc=0x61135f
net.cgoLookupIPCNAME({0x100a086, 0x3}, {0xc0003be780, 0x4})
/opt/hostedtoolcache/go/1.17.12/x64/src/net/cgo_unix.go:163 +0x16d fp=0xc000061738 sp=0xc0000615e8 pc=0x610bad
net.cgoIPLookup(0x18a23d0, {0x100a086, 0xc0003be790}, {0xc0003be780, 0xc0003c09a0})
/opt/hostedtoolcache/go/1.17.12/x64/src/net/cgo_unix.go:220 +0x3b fp=0xc0000617a8 sp=0xc000061738 pc=0x61141b
net.cgoLookupIP·dwrap·25()
/opt/hostedtoolcache/go/1.17.12/x64/src/net/cgo_unix.go:230 +0x36 fp=0xc0000617e0 sp=0xc0000617a8 pc=0x611896
runtime.goexit()
/opt/hostedtoolcache/go/1.17.12/x64/src/runtime/asm_amd64.s:1581 +0x1 fp=0xc0000617e8 sp=0xc0000617e0 pc=0x467b61
created by net.cgoLookupIP
/opt/hostedtoolcache/go/1.17.12/x64/src/net/cgo_unix.go:230 +0x125

goroutine 1 [select]:
net/http.(*Transport).getConn(0xc0003cec80, 0xc000337840, {{}, 0x0, {0xc000267d40, 0x4}, {0xc0003be780, 0xe}, 0x0})
/opt/hostedtoolcache/go/1.17.12/x64/src/net/http/transport.go:1372 +0x5d2
net/http.(*Transport).roundTrip(0xc0003cec80, 0xc0001bf700)
/opt/hostedtoolcache/go/1.17.12/x64/src/net/http/transport.go:581 +0x774
net/http.(*Transport).RoundTrip(0x30, 0x114d2e0)
/opt/hostedtoolcache/go/1.17.12/x64/src/net/http/roundtrip.go:18 +0x19
net/http.send(0xc0001bf700, {0x114d2e0, 0xc0003cec80}, {0xff3480, 0x1, 0x0})
/opt/hostedtoolcache/go/1.17.12/x64/src/net/http/client.go:252 +0x5d8
net/http.(*Client).send(0xc0004828a0, 0xc0001bf700, {0x203000, 0x1, 0x0})
/opt/hostedtoolcache/go/1.17.12/x64/src/net/http/client.go:176 +0x9b
net/http.(*Client).do(0xc0004828a0, 0xc0001bf700)
/opt/hostedtoolcache/go/1.17.12/x64/src/net/http/client.go:725 +0x908
net/http.(*Client).Do(...)
/opt/hostedtoolcache/go/1.17.12/x64/src/net/http/client.go:593
golang.org/x/net/context/ctxhttp.Do({0x1169d88, 0xc000482810}, 0xc000482810, 0xc0001bf600)
/home/runner/go/pkg/mod/golang.org/x/net@v0.0.0-20211216030914-fe4d6282115f/context/ctxhttp/ctxhttp.go:27 +0x206
github.com/containerd/containerd/remotes/docker.(*request).do(0xc0001accf0, {0x1169d88, 0xc0004826f0})
/home/runner/go/pkg/mod/github.com/containerd/containerd@v1.6.6/remotes/docker/resolver.go:569 +0x5a6
github.com/containerd/containerd/remotes/docker.(*request).doWithRetries(0x1169d88, {0x1169d88, 0xc0004826f0}, {0x0, 0x0, 0x0})
/home/runner/go/pkg/mod/github.com/containerd/containerd@v1.6.6/remotes/docker/resolver.go:578 +0x4a
github.com/containerd/containerd/remotes/docker.(*dockerResolver).Resolve(0xc000482090, {0x1169d88, 0xc0003b7f80}, {0x7ffe39adc6b0, 0x22})
/home/runner/go/pkg/mod/github.com/containerd/containerd@v1.6.6/remotes/docker/resolver.go:279 +0xc33
github.com/containerd/containerd.(*Client).fetch(0xc0003be730, {0x1169d88, 0xc0003b7f80}, 0xc0004869c0, {0x7ffe39adc6b0, 0x22}, 0x1)
/home/runner/go/pkg/mod/github.com/containerd/containerd@v1.6.6/pull.go:131 +0x10b
github.com/containerd/containerd.(*Client).Pull(0xc0003b02a0, {0x1169d88, 0xc0003b7f80}, {0x7ffe39adc6b0, 0x22}, {0xc0002de6f8, 0x7, 0xc0002de680})
/home/runner/go/pkg/mod/github.com/containerd/containerd@v1.6.6/pull.go:93 +0x794
github.com/dragonflyoss/image-service/contrib/ctr-remote/commands.pull({0x1169d88, 0xc0003b7f80}, 0xc0000b9e40, {0x7ffe39adc6b0, 0x22}, 0xc0002de8b0)
/home/runner/work/image-service/image-service/contrib/ctr-remote/commands/rpull.go:90 +0x385
github.com/dragonflyoss/image-service/contrib/ctr-remote/commands.glob..func1(0xc0000b9e40)
/home/runner/work/image-service/image-service/contrib/ctr-remote/commands/rpull.go:71 +0x1a9
github.com/urfave/cli.HandleAction({0xea50e0, 0x1062738}, 0x5)
/home/runner/go/pkg/mod/github.com/urfave/cli@v1.22.5/app.go:524 +0xa8
github.com/urfave/cli.Command.Run({{0x100b77c, 0x5}, {0x0, 0x0}, {0x0, 0x0, 0x0}, {0x104c6d5, 0x3a}, {0x0, ...}, ...}, ...)
/home/runner/go/pkg/mod/github.com/urfave/cli@v1.22.5/command.go:173 +0x652
github.com/urfave/cli.(*App).RunAsSubcommand(0xc0000f7a40, 0xc0000b9b80)
/home/runner/go/pkg/mod/github.com/urfave/cli@v1.22.5/app.go:405 +0x9ec
github.com/urfave/cli.Command.startApp({{0x100c7fc, 0x6}, {0x0, 0x0}, {0x180fa00, 0x2, 0x2}, {0x101585b, 0xd}, {0x0, ...}, ...}, ...)
/home/runner/go/pkg/mod/github.com/urfave/cli@v1.22.5/command.go:372 +0x6e9
github.com/urfave/cli.Command.Run({{0x100c7fc, 0x6}, {0x0, 0x0}, {0x180fa00, 0x2, 0x2}, {0x101585b, 0xd}, {0x0, ...}, ...}, ...)
/home/runner/go/pkg/mod/github.com/urfave/cli@v1.22.5/command.go:102 +0x808
github.com/urfave/cli.(*App).Run(0xc0000f7880, {0xc0000320a0, 0x5, 0x5})
/home/runner/go/pkg/mod/github.com/urfave/cli@v1.22.5/app.go:277 +0x705
main.main()
/home/runner/work/image-service/image-service/contrib/ctr-remote/cmd/main.go:61 +0x1be

goroutine 6 [select]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000325680)
/home/runner/go/pkg/mod/google.golang.org/grpc@v1.43.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
/home/runner/go/pkg/mod/google.golang.org/grpc@v1.43.0/balancer_conn_wrappers.go:67 +0x246

goroutine 9 [IO wait]:
internal/poll.runtime_pollWait(0x7f108219b9d0, 0x72)
/opt/hostedtoolcache/go/1.17.12/x64/src/runtime/netpoll.go:303 +0x85
internal/poll.(*pollDesc).wait(0xc0003b4280, 0xc0003e0000, 0x0)
/opt/hostedtoolcache/go/1.17.12/x64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/opt/hostedtoolcache/go/1.17.12/x64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0003b4280, {0xc0003e0000, 0x8000, 0x8000})
/opt/hostedtoolcache/go/1.17.12/x64/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0003b4280, {0xc0003e0000, 0x1060100000000, 0x8})
/opt/hostedtoolcache/go/1.17.12/x64/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000010668, {0xc0003e0000, 0x7f10a9783f18, 0x18})
/opt/hostedtoolcache/go/1.17.12/x64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00008b860, {0xc0003b03c0, 0x9, 0x18})
/opt/hostedtoolcache/go/1.17.12/x64/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x114bac0, 0xc00008b860}, {0xc0003b03c0, 0x9, 0x9}, 0x9)
/opt/hostedtoolcache/go/1.17.12/x64/src/io/io.go:328 +0x9a
io.ReadFull(...)
/opt/hostedtoolcache/go/1.17.12/x64/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0003b03c0, 0x9, 0x86bba7}, {0x114bac0, 0xc00008b860})
/home/runner/go/pkg/mod/golang.org/x/net@v0.0.0-20211216030914-fe4d6282115f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0003b0380)
/home/runner/go/pkg/mod/golang.org/x/net@v0.0.0-20211216030914-fe4d6282115f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00000c3c0)
/home/runner/go/pkg/mod/google.golang.org/grpc@v1.43.0/internal/transport/http2_client.go:1499 +0x41f
created by google.golang.org/grpc/internal/transport.newHTTP2Client
/home/runner/go/pkg/mod/google.golang.org/grpc@v1.43.0/internal/transport/http2_client.go:364 +0x192f

goroutine 10 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0003258b0, 0x1)
/home/runner/go/pkg/mod/google.golang.org/grpc@v1.43.0/internal/transport/controlbuf.go:407 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00008b920)
/home/runner/go/pkg/mod/google.golang.org/grpc@v1.43.0/internal/transport/controlbuf.go:534 +0x85
google.golang.org/grpc/internal/transport.newHTTP2Client.func3()
/home/runner/go/pkg/mod/google.golang.org/grpc@v1.43.0/internal/transport/http2_client.go:414 +0x65
created by google.golang.org/grpc/internal/transport.newHTTP2Client
/home/runner/go/pkg/mod/google.golang.org/grpc@v1.43.0/internal/transport/http2_client.go:412 +0x1f85

goroutine 11 [select]:
net.(*Resolver).lookupIPAddr(0x18a23c0, {0x1169d50, 0xc00008be60}, {0x100a086, 0xc000325b80}, {0xc0003be780, 0x9})
/opt/hostedtoolcache/go/1.17.12/x64/src/net/lookup.go:302 +0x5c7
net.(*Resolver).internetAddrList(0x1169d50, {0x1169d50, 0xc00008be60}, {0x100a086, 0x3}, {0xc0003be780, 0xe})
/opt/hostedtoolcache/go/1.17.12/x64/src/net/ipsock.go:288 +0x67a
net.(*Resolver).resolveAddrList(0x18db030, {0x1169d50, 0xc00008be60}, {0x100a630, 0x4}, {0x100a086, 0x0}, {0xc0003be780, 0xe}, {0x0, ...})
/opt/hostedtoolcache/go/1.17.12/x64/src/net/dial.go:221 +0x41b
net.(*Dialer).DialContext(0xc00008bb60, {0x1169d88, 0xc000482810}, {0x100a086, 0x18db030}, {0xc0003be780, 0xc000072a10})
/opt/hostedtoolcache/go/1.17.12/x64/src/net/dial.go:406 +0x448
net/http.(*Transport).dial(0x0, {0x1169d88, 0xc000482810}, {0x100a086, 0x0}, {0xc0003be780, 0x0})
/opt/hostedtoolcache/go/1.17.12/x64/src/net/http/transport.go:1166 +0xda
net/http.(*Transport).dialConn(0xc0003cec80, {0x1169d88, 0xc000482810}, {{}, 0x0, {0xc000267d40, 0x4}, {0xc0003be780, 0xe}, 0x0})
/opt/hostedtoolcache/go/1.17.12/x64/src/net/http/transport.go:1604 +0x845
net/http.(*Transport).dialConnFor(0xc00000c3c0, 0xc0003c09a0)
/opt/hostedtoolcache/go/1.17.12/x64/src/net/http/transport.go:1446 +0xb0
created by net/http.(*Transport).queueForDial
/opt/hostedtoolcache/go/1.17.12/x64/src/net/http/transport.go:1415 +0x3d7

goroutine 12 [select]:
net.cgoLookupIP({0x1169ce0, 0xc0003378c0}, {0x100a086, 0x9}, {0xc0003be780, 0x0})
/opt/hostedtoolcache/go/1.17.12/x64/src/net/cgo_unix.go:231 +0x1b7
net.(*Resolver).lookupIP(0x18a23c0, {0x1169ce0, 0xc0003378c0}, {0x100a086, 0x3}, {0xc0003be780, 0x9})
/opt/hostedtoolcache/go/1.17.12/x64/src/net/lookup_unix.go:97 +0x128
net.glob..func1({0x1169ce0, 0xc0003378c0}, 0xc0003be550, {0x100a086, 0x58}, {0xc0003be780, 0x8510d0})
/opt/hostedtoolcache/go/1.17.12/x64/src/net/hook.go:23 +0x3d
net.(*Resolver).lookupIPAddr.func1()
/opt/hostedtoolcache/go/1.17.12/x64/src/net/lookup.go:296 +0x9f
internal/singleflight.(*Group).doCall(0x18a23d0, 0xc000325bd0, {0xc0003be790, 0xd}, 0xc0003c09a0)
/opt/hostedtoolcache/go/1.17.12/x64/src/internal/singleflight/singleflight.go:95 +0x3b
created by internal/singleflight.(*Group).DoChan
/opt/hostedtoolcache/go/1.17.12/x64/src/internal/singleflight/singleflight.go:88 +0x2f1
[root@nydus nydus]#

[hsiangkao]

That is strange. Do you have more clues about it?

[imeoer]

Could you share the containerd version connected by ctr-remote? And does ctr-remote pull <oci-image> work fine?

[yangxiaolaing]

If you directly unzip the targz(nydus-static-v2.1.0-alpha.6-linux-amd64.tgz) package, you will get an error. If you compile the source code yourself, you will not

[imeoer]

@yangxiaolaing Thanks for sharing the key point, will take a look.