net: panic: runtime error: invalid memory address or nil pointer dereference

[zhangriyueming]

What version of Go are you using (go version)?

$ go version
go version go1.12.5 linux/amd64

Does this issue reproduce with the latest release?

yes

What operating system and processor architecture are you using (go env)?

go env Output

$ go env
GOARCH="amd64"
GOBIN=""
GOCACHE="/root/.cache/go-build"
GOEXE=""
GOFLAGS=""
GOHOSTARCH="amd64"
GOHOSTOS="linux"
GOOS="linux"
GOPATH="/go"
GOPROXY=""
GORACE=""
GOROOT="/usr/local/go"
GOTMPDIR=""
GOTOOLDIR="/usr/local/go/pkg/tool/linux_amd64"
GCCGO="gccgo"
CC="gcc"
CXX="g++"
CGO_ENABLED="1"
GOMOD=""
CGO_CFLAGS="-g -O2"
CGO_CPPFLAGS=""
CGO_CXXFLAGS="-g -O2"
CGO_FFLAGS="-g -O2"
CGO_LDFLAGS="-g -O2"
PKG_CONFIG="pkg-config"
GOGCCFLAGS="-fPIC -m64 -pthread -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build000458693=/tmp/go-build -gno-record-gcc-switches"

What did you do?

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"strings"
	"time"
)

type Transport struct {
	upstream  http.RoundTripper
	jar       http.CookieJar
	delay     int
	userAgent string
}

func NewTransport(upstream http.RoundTripper) (*Transport, error) {
	jar, err := cookiejar.New(nil)
	if err != nil {
		return nil, err
	}
	userAgent := "Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0"
	return &Transport{
		upstream:  upstream,
		jar:       jar,
		delay:     8,
		userAgent: userAgent}, nil
}

func (t *Transport) RoundTrip(r *http.Request) (*http.Response, error) {
	resp, err := t.upstream.RoundTrip(r)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode == 503 &&
		strings.HasPrefix(resp.Header.Get("Server"), "cloudflare") {

		req, err := http.NewRequest("GET", "https://www.google.co.jp/", nil)
		if err != nil {
			return nil, err
		}

		client := http.Client{
			Transport: t,
			Jar:       t.jar,
			CheckRedirect: func(req *http.Request, via []*http.Request) error {
				return http.ErrUseLastResponse
			},
		}

		resp, err = client.Do(req)
		if err != nil {
			return nil, err
		}

		redirectUrl := resp.Header.Get("Location")
		redirectLocation, err := url.Parse(redirectUrl)
		if err != nil {
			return nil, err
		}
		if redirectLocation.Host == "" {
			redirectUrl = fmt.Sprintf("%s://%s%s",
				resp.Request.URL.Scheme,
				resp.Request.URL.Host,
				redirectUrl)
		}

		req, err = http.NewRequest("GET", redirectUrl, nil)
		if err != nil {
			return nil, err
		}

		client = http.Client{
			Transport: t,
			Jar:       t.jar,
		}
		resp, err = client.Do(req)
		return resp, err
	}

	return resp, err
}

type Job struct {
	baseUrl             string
	transport           *Transport
	client              *http.Client
	clientWithTransport *http.Client
}

type Response struct {
	Key1 string `json:key1`
	Key2 string `json:key2`
	Key3 string `json:key3`
}

func (job *Job) syncPart1(v interface{}) error {
	req, err := http.NewRequest("GET", job.baseUrl+"part1", nil)
	resp, err := job.client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	decErr := json.NewDecoder(resp.Body).Decode(v)
	if decErr == io.EOF {
		decErr = nil // ignore EOF errors caused by empty response body
	}
	if decErr != nil {
		err = decErr
	}

	return err
}

func (job *Job) syncPart2() ([]byte, error) {
	req, err := http.NewRequest("GET", job.baseUrl+"part2", nil)
	if err != nil {
		return nil, err
	}
	resp, err := job.clientWithTransport.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	return body, nil
}

func (job *Job) Sync() {
	var r Response
	job.syncPart1(&r)
	job.syncPart2()
}

func NewJob() *Job {
	http.DefaultTransport.(*http.Transport).Proxy = nil
	transport, _ := NewTransport(http.DefaultTransport)
	return &Job{
		baseUrl:             "https://www.google.com/",
		transport:           transport,
		client:              &http.Client{},
		clientWithTransport: &http.Client{Transport: transport},
	}
}

func handler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Greeting! %s", r.URL.Path[1:])
}

func main() {

	job := NewJob()

	go func() {
		for {
			job.Sync()
			time.Sleep(20 * time.Second)
		}
	}()

	http.HandleFunc("/", handler)
	http.ListenAndServe(":8080", nil)
}

What did you expect to see?

Works without panics.

What did you see instead?

api_1  | 14:44:05 app         | panic: runtime error: invalid memory address or nil pointer dereference
api_1  | [signal SIGSEGV: segmentation violation code=0x1 addr=0x49 pc=0x5c596a]
api_1  | 
api_1  | goroutine 102 [running]:
api_1  | panic(0xa16e40, 0x10071d0)
api_1  | 	/usr/local/go/src/runtime/panic.go:565 +0x2c5 fp=0xc000633b48 sp=0xc000633ab8 pc=0x42bca5
api_1  | runtime.panicmem(...)
api_1  | 	/usr/local/go/src/runtime/panic.go:82
api_1  | runtime.sigpanic()
api_1  | 	/usr/local/go/src/runtime/signal_unix.go:390 +0x411 fp=0xc000633b78 sp=0xc000633b48 pc=0x4404e1
api_1  | net.(*conf).hostLookupOrder(0x1009720, 0x101ea40, 0xc000550040, 0x11, 0x0)
api_1  | 	/usr/local/go/src/net/conf.go:133 +0x6a fp=0xc000633c50 sp=0xc000633b78 pc=0x5c596a
api_1  | net.(*Resolver).lookupIP(0x101ea40, 0xbd3120, 0xc0002d2080, 0xab6ad4, 0x3, 0xc000550040, 0x11, 0xc00040be98, 0xc00040be38, 0xc00006b200, ...)
api_1  | 	/usr/local/go/src/net/lookup_unix.go:94 +0xa3 fp=0xc000633db8 sp=0xc000633c50 pc=0x5e5023
api_1  | net.(*Resolver).lookupIP-fm(0xbd3120, 0xc0002d2080, 0xab6ad4, 0x3, 0xc000550040, 0x11, 0x42addf
api_1  | 14:44:05 app         | , 0x8, 0xc00016c100, 0xbf2e7e748010f248, ...)
api_1  | 	/usr/local/go/src/net/lookup_unix.go:90 +0x76 fp=0xc000633e28 sp=0xc000633db8 pc=0x5fef86
api_1  | net.glob..func1(0xbd3120, 0xc0002d2080, 0xc00049d420, 0xab6ad4, 0x3, 0xc000550040, 0x11, 0x24, 0x0, 0x0, ...)
api_1  | 	/usr/local/go/src/net/hook.go:23 +0x72 fp=0xc000633e90 sp=0xc000633e28 pc=0x5f8e82
api_1  | net.(*Resolver).lookupIPAddr.func1(0x0, 0x0, 0x0, 0x0)
api_1  | 	/usr/local/go/src/net/lookup.go:269 +0x116 fp=0xc000633f48 sp=0xc000633e90 pc=0x5fa006
api_1  | internal/singleflight.(*Group).doCall(0x101ea50, 0xc0005a8280, 0xc0005500a0, 0x15, 0xc0002d2140)
api_1  | 	/usr/local/go/src/internal/singleflight/singleflight.go:95 +0x2e fp=0xc000633fb8 sp=0xc000633f48 pc=0x5c39de
api_1  | runtime.goexit()
api_1  | 	/usr/local/go/src/runtime/asm_amd64.s:1337 +0x1 fp=0xc000633fc0 sp=0xc000633fb8 pc=0x457bf1
api_1  | created by internal/singleflight.(*Group).DoChan
api_1  | 	/usr/local/go/src/internal/singleflight/singleflight.go:88 +0x29d

This panic is very rarely, tbh it was first time after hundreds hours running. And it maybe linked to #31986 .

[bradfitz]

Like your other bug, I don't see how this can happen.

It seems that c.resolv is nil in c.resolv.unknownOpt on line 133 of Go 1.12's src/net/conf.go, but I see no path where resolv can be nil.

Can you run with the race detector?

/cc @ianlancetaylor

[ianlancetaylor]

If the program ran for 100 hours and then crashed, that can only mean that confVal.resolv was initialized, and then somehow became nil. I don't see how that could happen. You say that this panic is very rare; does it look the same every time?

[zhangriyueming]

@bradfitz @ianlancetaylor, I have ran it in my local machine with race detector enabled, unfortunately it crashed my computer, but after restarted my computer, the terminal output messages are restored, and there is no race detected so far.

So, is it possible for data races to cause these panics?

[ianlancetaylor]

Yes, it is possible for a data race to corrupt memory and thus to cause an error like this.

It's not especially likely but it is possible.

[zhangriyueming]

@ianlancetaylor , I can't say it is unfortunate or fortunate. But after I made a build with cgo enabled and race detector enabled. It have ran more than 50 hours without any data race or panic……

I'm going on with race detector enabled running, wait till the first data race coming out or the first panic occurred.

[ALTree]

This has been in waiting for info for more than 2 years, and it was reported against a version of Go that is no longer supported, so let's archive this. If it comes out again on a recent Go version, we can open a new issue.