feat(netxlite): add HTTP/HTTPS proxy dialer #1162
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,222 @@ | ||
| package netxlite | ||
|
|
||
| import ( | ||
| "context" | ||
| "errors" | ||
| "fmt" | ||
| "github.com/ooni/probe-cli/v3/internal/model" | ||
| "golang.org/x/net/proxy" | ||
| "net" | ||
| "net/http" | ||
| "strconv" | ||
| "strings" | ||
| "time" | ||
| ) | ||
|
|
||
| // A HttpDialer holds HTTP-specific options | ||
| // Specifically for HTTP proxy, we build an HTTP tunnel | ||
| type HttpDialer struct { | ||
Murphy-OrangeMud marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| proxy.Dialer | ||
| proxyNetwork string | ||
| proxyAddress string | ||
| timeout time.Duration | ||
| ProxyDial func(context.Context, string, string) (net.Conn, error) | ||
| } | ||
|
|
||
| func (d *HttpDialer) Dial(network, address string) (net.Conn, error) { | ||
Murphy-OrangeMud marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| if err := d.validateTarget(network, address); err != nil { | ||
| proxy, dst, _ := d.pathAddrs(address) | ||
| return nil, &net.OpError{Op: http.MethodConnect, Net: network, Source: proxy, Addr: dst, Err: err} | ||
| } | ||
| var err error | ||
| var c net.Conn | ||
|
|
||
| if d.ProxyDial != nil { | ||
| c, err = d.ProxyDial(context.Background(), d.proxyNetwork, d.proxyAddress) | ||
| } else { | ||
| nd := &net.Dialer{Timeout: d.timeout} | ||
| c, err = nd.DialContext(context.Background(), d.proxyNetwork, d.proxyAddress) | ||
| } | ||
|
|
||
| if err != nil { | ||
| proxy, dst, _ := d.pathAddrs(address) | ||
| return nil, &net.OpError{Op: http.MethodConnect, Net: network, Source: proxy, Addr: dst, Err: err} | ||
| } | ||
| if err := d.DialWithConn(context.Background(), c, network, address); err != nil { | ||
| c.Close() | ||
| return nil, err | ||
| } | ||
| return c, nil | ||
| } | ||
|
|
||
| func (d *HttpDialer) validateTarget(network, address string) error { | ||
Murphy-OrangeMud marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| switch network { | ||
| case "tcp", "tcp6", "tcp4": | ||
| default: | ||
| return errors.New("network not implemented") | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| type Addr struct { | ||
Murphy-OrangeMud marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| network string | ||
| Name string // fully-qualified domain name | ||
| IP net.IP | ||
| Port int | ||
| } | ||
|
|
||
| func (a *Addr) Network() string { return a.network } | ||
|
|
||
| func (a *Addr) String() string { | ||
| if a == nil { | ||
| return "<nil>" | ||
| } | ||
| port := strconv.Itoa(a.Port) | ||
| if a.IP == nil { | ||
| return net.JoinHostPort(a.Name, port) | ||
| } | ||
| return net.JoinHostPort(a.IP.String(), port) | ||
| } | ||
|
|
||
| func splitHostPort(address string) (string, int, error) { | ||
| host, port, err := net.SplitHostPort(address) | ||
| if err != nil { | ||
| return "", 0, err | ||
| } | ||
| portnum, err := strconv.Atoi(port) | ||
| if err != nil { | ||
| return "", 0, err | ||
| } | ||
| if 1 > portnum || portnum > 0xffff { | ||
| return "", 0, errors.New("port number out of range " + port) | ||
| } | ||
| return host, portnum, nil | ||
| } | ||
|
|
||
| func (d *HttpDialer) pathAddrs(address string) (proxy, dst net.Addr, err error) { | ||
| for i, s := range []string{d.proxyAddress, address} { | ||
| host, port, err := splitHostPort(s) | ||
| if err != nil { | ||
| return nil, nil, err | ||
| } | ||
| a := &Addr{Port: port} | ||
| a.IP = net.ParseIP(host) | ||
| if a.IP == nil { | ||
| a.Name = host | ||
| } | ||
| if i == 0 { | ||
| proxy = a | ||
| } else { | ||
| dst = a | ||
| } | ||
| } | ||
| return | ||
| } | ||
|
|
||
| func (d *HttpDialer) DialWithConn(ctx context.Context, c net.Conn, network, address string) error { | ||
Murphy-OrangeMud marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| if err := d.validateTarget(network, address); err != nil { | ||
| proxy, dst, _ := d.pathAddrs(address) | ||
| return &net.OpError{Op: http.MethodConnect, Net: network, Source: proxy, Addr: dst, Err: err} | ||
| } | ||
| if ctx == nil { | ||
| proxy, dst, _ := d.pathAddrs(address) | ||
| return &net.OpError{Op: http.MethodConnect, Net: network, Source: proxy, Addr: dst, Err: errors.New("nil context")} | ||
| } | ||
|
|
||
| connectReq := fmt.Sprintf("%v %v HTTP/1.1\r\n"+ | ||
| "Host: %v\r\n"+ | ||
| "Proxy-Connection: keep-alive\r\n"+ | ||
| "User-Agent: %v\r\n\r\n", http.MethodConnect, address, address, model.HTTPHeaderUserAgent) | ||
|
|
||
| b := []byte(connectReq) | ||
|
|
||
| n, err := c.Write(b) | ||
| if err != nil { | ||
| proxy, dst, _ := d.pathAddrs(address) | ||
| return &net.OpError{Op: http.MethodConnect, Net: network, Source: proxy, Addr: dst, Err: err} | ||
| } | ||
| if n != len(b) { | ||
| proxy, dst, _ := d.pathAddrs(address) | ||
| return &net.OpError{Op: http.MethodConnect, Net: network, Source: proxy, Addr: dst, Err: errors.New("not write enough bytes")} | ||
| } | ||
|
|
||
| c.Read(b) | ||
| if err != nil { | ||
| proxy, dst, _ := d.pathAddrs(address) | ||
| return &net.OpError{Op: http.MethodConnect, Net: network, Source: proxy, Addr: dst, Err: err} | ||
| } | ||
|
|
||
| str := string(b) | ||
| if strings.Split(str, " ")[1] != "200" { | ||
| proxy, dst, _ := d.pathAddrs(address) | ||
| return &net.OpError{Op: http.MethodConnect, Net: network, Source: proxy, Addr: dst, Err: errors.New("cannot establish connection")} | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func (d *HttpDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { | ||
Murphy-OrangeMud marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| if err := d.validateTarget(network, address); err != nil { | ||
Murphy-OrangeMud marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| proxy, dst, _ := d.pathAddrs(address) | ||
| return nil, &net.OpError{Op: http.MethodConnect, Net: network, Source: proxy, Addr: dst, Err: err} | ||
Murphy-OrangeMud marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| } | ||
| if ctx == nil { | ||
| proxy, dst, _ := d.pathAddrs(address) | ||
| return nil, &net.OpError{Op: http.MethodConnect, Net: network, Source: proxy, Addr: dst, Err: errors.New("nil context")} | ||
| } | ||
Murphy-OrangeMud marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
|
||
| // proxy dial | ||
| var err error | ||
| var c net.Conn | ||
Murphy-OrangeMud marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
|
||
| if d.ProxyDial != nil { | ||
| c, err = d.ProxyDial(ctx, d.proxyNetwork, d.proxyAddress) | ||
| } else { | ||
| nd := &net.Dialer{Timeout: d.timeout} | ||
| c, err = nd.DialContext(ctx, d.proxyNetwork, d.proxyAddress) | ||
| } | ||
| if err != nil { | ||
| proxy, dst, _ := d.pathAddrs(address) | ||
| return nil, &net.OpError{Op: http.MethodConnect, Net: network, Source: proxy, Addr: dst, Err: err} | ||
| } | ||
|
|
||
| connectReq := fmt.Sprintf("%v %v HTTP/1.1\r\n"+ | ||
| "Host: %v\r\n"+ | ||
| "Proxy-Connection: keep-alive\r\n"+ | ||
| "User-Agent: %v\r\n\r\n", http.MethodConnect, address, address, model.HTTPHeaderUserAgent) | ||
|
|
||
| b := []byte(connectReq) | ||
Murphy-OrangeMud marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
|
||
| n, err := c.Write(b) | ||
Murphy-OrangeMud marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| if err != nil { | ||
| proxy, dst, _ := d.pathAddrs(address) | ||
| return nil, &net.OpError{Op: http.MethodConnect, Net: network, Source: proxy, Addr: dst, Err: err} | ||
| } | ||
| if n != len(b) { | ||
Murphy-OrangeMud marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| proxy, dst, _ := d.pathAddrs(address) | ||
| return nil, &net.OpError{Op: http.MethodConnect, Net: network, Source: proxy, Addr: dst, Err: errors.New("not write enough bytes")} | ||
| } | ||
|
|
||
| c.Read(b) | ||
|
There was a problem hiding this comment. This seems wrong since you're not checking for an error or the amount of bytes read here. |
||
| if err != nil { | ||
| proxy, dst, _ := d.pathAddrs(address) | ||
| return nil, &net.OpError{Op: http.MethodConnect, Net: network, Source: proxy, Addr: dst, Err: err} | ||
| } | ||
|
|
||
| str := string(b) | ||
| if strings.Split(str, " ")[1] != "200" { | ||
| proxy, dst, _ := d.pathAddrs(address) | ||
| return nil, &net.OpError{Op: http.MethodConnect, Net: network, Source: proxy, Addr: dst, Err: errors.New("cannot establish connection")} | ||
| } | ||
Murphy-OrangeMud marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
|
||
| return c, nil | ||
|
|
||
| } | ||
|
|
||
| func NewHTTPDialer(network, address string) *HttpDialer { | ||
| return &HttpDialer{ | ||
| proxyNetwork: network, | ||
| proxyAddress: address, | ||
| timeout: dialerDefaultTimeout, | ||
| } | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,102 @@ | ||
| package netxlite | ||
|
|
||
| import ( | ||
| "context" | ||
| "errors" | ||
| "fmt" | ||
| "net" | ||
| "net/http" | ||
| "testing" | ||
| ) | ||
|
|
||
| func TestHTTPProxyDialer(t *testing.T) { | ||
| // REMINDER: This test need a http proxy running locally | ||
| dialer := NewHTTPDialer("tcp", "localhost:7890") | ||
|
There was a problem hiding this comment. We recently added support for proxies in the |
||
| t.Run("DialContextSuccess", func(t *testing.T) { | ||
| conn, err := dialer.DialContext(context.Background(), "tcp", "google.com:443") | ||
| if err != nil { | ||
| t.Fatal(fmt.Sprintf("unexpected error: %v", err)) | ||
| } | ||
| if conn == nil { | ||
| t.Fatal("unexpected nil connection") | ||
| } | ||
| }) | ||
|
|
||
| t.Run("DialSuccess", func(t *testing.T) { | ||
| conn, err := dialer.Dial("tcp", "google.com:443") | ||
| if err != nil { | ||
| t.Fatal(fmt.Sprintf("unexpected error: %v", err)) | ||
| } | ||
| if conn == nil { | ||
| t.Fatal("unexpected nil connection") | ||
| } | ||
| }) | ||
|
|
||
| t.Run("DialContextInvalidNetwork", func(t *testing.T) { | ||
| expected := errors.New("network not implemented") | ||
| conn, err := dialer.DialContext(context.Background(), "udp", "google.com:443") | ||
| if conn != nil { | ||
| t.Fatal("unexpected connection") | ||
| } | ||
| if err.(*net.OpError).Err.Error() != expected.Error() { | ||
| t.Fatal(fmt.Sprintf("unexpected error: %v", err)) | ||
| } | ||
| }) | ||
|
|
||
| t.Run("DialInvalidNetwork", func(t *testing.T) { | ||
| expected := errors.New("network not implemented") | ||
| conn, err := dialer.Dial("udp", "google.com:443") | ||
| if conn != nil { | ||
| t.Fatal("unexpected connection") | ||
| } | ||
| if err.(*net.OpError).Err.Error() != expected.Error() { | ||
| t.Fatal(fmt.Sprintf("unexpected error: %v", err)) | ||
| } | ||
| }) | ||
| } | ||
|
|
||
| func TestHTTPProxyDialerFailure(t *testing.T) { | ||
| dialer := NewHTTPDialer("tcp", "localhost:8888") | ||
| go func() { | ||
| listener, err := net.Listen("tcp", "localhost:8888") | ||
| defer listener.Close() | ||
| if err != nil { | ||
| t.Error(fmt.Sprintf("error: listen failed%v", err)) | ||
| return | ||
| } | ||
| err = http.Serve(listener, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | ||
| if r.Method == http.MethodConnect { | ||
| w.WriteHeader(500) | ||
| return | ||
| } else { | ||
| t.Error("error: unexpected request") | ||
| return | ||
| } | ||
| })) | ||
| }() | ||
| t.Run("DialContextFailure", func(t *testing.T) { | ||
| expected := errors.New("cannot establish connection") | ||
|
|
||
| conn, err := dialer.DialContext(context.Background(), "tcp", "google.com:443") | ||
| if conn != nil { | ||
| t.Fatal("unexpected connection") | ||
| } | ||
| if err.(*net.OpError).Err.Error() != expected.Error() { | ||
| t.Fatal("unexpected error") | ||
| } | ||
| }) | ||
|
|
||
| t.Run("DialFailure", func(t *testing.T) { | ||
| expected := errors.New("cannot establish connection") | ||
|
|
||
| conn, err := dialer.Dial("tcp", "google.com:443") | ||
| if conn != nil { | ||
| t.Fatal("unexpected connection") | ||
| } | ||
| if err.(*net.OpError).Err.Error() != expected.Error() { | ||
| t.Fatal(fmt.Sprintf("unexpected error: %v", err)) | ||
| } | ||
| }) | ||
|
|
||
| return | ||
Murphy-OrangeMud marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This code is very fragile and assumes that we're creating a specific chain of wrappers. I'd rather try to rewrite the pull request such that we avoid using this code entirely. My understanding is that you don't want to wrap the connection more than once. Wrapping multiple times should be fine, but probably it's also possible to avoid wrapping by using a
model.UnderlyingNetwork, which provides a dialer that, in the common case, is the standard library dialer.