Merge pull request #6096 from ipfs/feat/gateway-subdomains

feat: gateway subdomains + http proxy mode

cmd/ipfs/daemon.go

@@ -12,6 +12,8 @@ import (
 	"sort"
 	"sync"
 
+	multierror "github.com/hashicorp/go-multierror"
+
 	version "github.com/ipfs/go-ipfs"
 	config "github.com/ipfs/go-ipfs-config"
 	cserial "github.com/ipfs/go-ipfs-config/serialize"
@@ -27,7 +29,6 @@ import (
 	migrate "github.com/ipfs/go-ipfs/repo/fsrepo/migrations"
 	sockets "github.com/libp2p/go-socket-activation"
 
-	"github.com/hashicorp/go-multierror"
 	cmds "github.com/ipfs/go-ipfs-cmds"
 	mprome "github.com/ipfs/go-metrics-prometheus"
 	goprocess "github.com/jbenet/goprocess"
@@ -298,9 +299,9 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
 
 	// Start assembling node config
 	ncfg := &core.BuildCfg{
-		Repo:                        repo,
-		Permanent:                   true, // It is temporary way to signify that node is permanent
-		Online:                      !offline,
+		Repo:      repo,
+		Permanent: true, // It is temporary way to signify that node is permanent
+		Online:    !offline,
 		DisableEncryptedConnections: unencrypted,
 		ExtraOpts: map[string]bool{
 			"pubsub": pubsub,
@@ -636,15 +637,15 @@ func serveHTTPGateway(req *cmds.Request, cctx *oldcmds.Context) (<-chan error, e
 
 	var opts = []corehttp.ServeOption{
 		corehttp.MetricsCollectionOption("gateway"),
-		corehttp.IPNSHostnameOption(),
+		corehttp.HostnameOption(),
 		corehttp.GatewayOption(writable, "/ipfs", "/ipns"),
 		corehttp.VersionOption(),
 		corehttp.CheckVersionOption(),
 		corehttp.CommandsROOption(cmdctx),
 	}
 
 	if cfg.Experimental.P2pHttpProxy {
-		opts = append(opts, corehttp.ProxyOption())
+		opts = append(opts, corehttp.P2PProxyOption())
 	}
 
 	if len(cfg.Gateway.RootRedirect) > 0 {

core/corehttp/corehttp.go

@@ -43,7 +43,17 @@ func makeHandler(n *core.IpfsNode, l net.Listener, options ...ServeOption) (http
 			return nil, err
 		}
 	}
-	return topMux, nil
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// ServeMux does not support requests with CONNECT method,
+		// so we need to handle them separately
+		// https://golang.org/src/net/http/request.go#L111
+		if r.Method == http.MethodConnect {
+			w.WriteHeader(http.StatusOK)
+			return
+		}
+		topMux.ServeHTTP(w, r)
+	})
+	return handler, nil
 }
 
 // ListenAndServe runs an HTTP server listening at |listeningMultiAddr| with
@@ -70,6 +80,8 @@ func ListenAndServe(n *core.IpfsNode, listeningMultiAddr string, options ...Serv
 	return Serve(n, manet.NetListener(list), options...)
 }
 
+// Serve accepts incoming HTTP connections on the listener and pass them
+// to ServeOption handlers.
 func Serve(node *core.IpfsNode, lis net.Listener, options ...ServeOption) error {
 	// make sure we close this no matter what.
 	defer lis.Close()

core/corehttp/gateway_handler.go

@@ -14,17 +14,16 @@ import (
 	"strings"
 	"time"
 
-	"github.com/dustin/go-humanize"
+	humanize "github.com/dustin/go-humanize"
 	"github.com/ipfs/go-cid"
 	files "github.com/ipfs/go-ipfs-files"
 	dag "github.com/ipfs/go-merkledag"
-	"github.com/ipfs/go-mfs"
-	"github.com/ipfs/go-path"
+	mfs "github.com/ipfs/go-mfs"
+	path "github.com/ipfs/go-path"
 	"github.com/ipfs/go-path/resolver"
 	coreiface "github.com/ipfs/interface-go-ipfs-core"
 	ipath "github.com/ipfs/interface-go-ipfs-core/path"
 	routing "github.com/libp2p/go-libp2p-core/routing"
-	"github.com/multiformats/go-multibase"
 )
 
 const (
@@ -39,6 +38,25 @@ type gatewayHandler struct {
 	api    coreiface.CoreAPI
 }
 
+// StatusResponseWriter enables us to override HTTP Status Code passed to
+// WriteHeader function inside of http.ServeContent.  Decision is based on
+// presence of HTTP Headers such as Location.
+type statusResponseWriter struct {
+	http.ResponseWriter
+}
+
+func (sw *statusResponseWriter) WriteHeader(code int) {
+	// Check if we need to adjust Status Code to account for scheduled redirect
+	// This enables us to return payload along with HTTP 301
+	// for subdomain redirect in web browsers while also returning body for cli
+	// tools which do not follow redirects by default (curl, wget).
+	redirect := sw.ResponseWriter.Header().Get("Location")
+	if redirect != "" && code == http.StatusOK {
+		code = http.StatusMovedPermanently
+	}
+	sw.ResponseWriter.WriteHeader(code)
+}
+
 func newGatewayHandler(c GatewayConfig, api coreiface.CoreAPI) *gatewayHandler {
 	i := &gatewayHandler{
 		config: c,
@@ -143,17 +161,17 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
 		}
 	}
 
-	// IPNSHostnameOption might have constructed an IPNS path using the Host header.
+	// HostnameOption might have constructed an IPNS/IPFS path using the Host header.
 	// In this case, we need the original path for constructing redirects
 	// and links that match the requested URL.
 	// For example, http://example.net would become /ipns/example.net, and
 	// the redirects and links would end up as http://example.net/ipns/example.net
-	originalUrlPath := prefix + urlPath
-	ipnsHostname := false
-	if hdr := r.Header.Get("X-Ipns-Original-Path"); len(hdr) > 0 {
-		originalUrlPath = prefix + hdr
-		ipnsHostname = true
+	requestURI, err := url.ParseRequestURI(r.RequestURI)
+	if err != nil {
+		webError(w, "failed to parse request path", err, http.StatusInternalServerError)
+		return
 	}
+	originalUrlPath := prefix + requestURI.Path
 
 	// Service Worker registration request
 	if r.Header.Get("Service-Worker") == "script" {
@@ -206,39 +224,6 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
 	w.Header().Set("X-IPFS-Path", urlPath)
 	w.Header().Set("Etag", etag)
 
-	// Suborigin header, sandboxes apps from each other in the browser (even
-	// though they are served from the same gateway domain).
-	//
-	// Omitted if the path was treated by IPNSHostnameOption(), for example
-	// a request for http://example.net/ would be changed to /ipns/example.net/,
-	// which would turn into an incorrect Suborigin header.
-	// In this case the correct thing to do is omit the header because it is already
-	// handled correctly without a Suborigin.
-	//
-	// NOTE: This is not yet widely supported by browsers.
-	if !ipnsHostname {
-		// e.g.: 1="ipfs", 2="QmYuNaKwY...", ...
-		pathComponents := strings.SplitN(urlPath, "/", 4)
-
-		var suboriginRaw []byte
-		cidDecoded, err := cid.Decode(pathComponents[2])
-		if err != nil {
-			// component 2 doesn't decode with cid, so it must be a hostname
-			suboriginRaw = []byte(strings.ToLower(pathComponents[2]))
-		} else {
-			suboriginRaw = cidDecoded.Bytes()
-		}
-
-		base32Encoded, err := multibase.Encode(multibase.Base32, suboriginRaw)
-		if err != nil {
-			internalWebError(w, err)
-			return
-		}
-
-		suborigin := pathComponents[1] + "000" + strings.ToLower(base32Encoded)
-		w.Header().Set("Suborigin", suborigin)
-	}
-
 	// set these headers _after_ the error, for we may just not have it
 	// and dont want the client to cache a 500 response...
 	// and only if it's /ipfs!
@@ -322,10 +307,10 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
 
 	// construct the correct back link
 	// https://github.com/ipfs/go-ipfs/issues/1365
-	var backLink string = prefix + urlPath
+	var backLink string = originalUrlPath
 
 	// don't go further up than /ipfs/$hash/
-	pathSplit := path.SplitList(backLink)
+	pathSplit := path.SplitList(urlPath)
 	switch {
 	// keep backlink
 	case len(pathSplit) == 3: // url: /ipfs/$hash
@@ -342,18 +327,8 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
 		}
 	}
 
-	// strip /ipfs/$hash from backlink if IPNSHostnameOption touched the path.
-	if ipnsHostname {
-		backLink = prefix + "/"
-		if len(pathSplit) > 5 {
-			// also strip the trailing segment, because it's a backlink
-			backLinkParts := pathSplit[3 : len(pathSplit)-2]
-			backLink += path.Join(backLinkParts) + "/"
-		}
-	}
-
 	var hash string
-	if !strings.HasPrefix(originalUrlPath, ipfsPathPrefix) {
+	if !strings.HasPrefix(urlPath, ipfsPathPrefix) {
 		hash = resolvedPath.Cid().String()
 	}
 
@@ -410,6 +385,7 @@ func (i *gatewayHandler) serveFile(w http.ResponseWriter, req *http.Request, nam
 	}
 	w.Header().Set("Content-Type", ctype)
 
+	w = &statusResponseWriter{w}
 	http.ServeContent(w, req, name, modtime, content)
 }
 

core/corehttp/gateway_test.go

@@ -138,7 +138,7 @@ func newTestServerAndNode(t *testing.T, ns mockNamesys) (*httptest.Server, iface
 
 	dh.Handler, err = makeHandler(n,
 		ts.Listener,
-		IPNSHostnameOption(),
+		HostnameOption(),
 		GatewayOption(false, "/ipfs", "/ipns"),
 		VersionOption(),
 	)
@@ -184,12 +184,12 @@ func TestGatewayGet(t *testing.T) {
 		status int
 		text   string
 	}{
-		{"localhost:5001", "/", http.StatusNotFound, "404 page not found\n"},
-		{"localhost:5001", "/" + k.Cid().String(), http.StatusNotFound, "404 page not found\n"},
-		{"localhost:5001", k.String(), http.StatusOK, "fnord"},
-		{"localhost:5001", "/ipns/nxdomain.example.com", http.StatusNotFound, "ipfs resolve -r /ipns/nxdomain.example.com: " + namesys.ErrResolveFailed.Error() + "\n"},
-		{"localhost:5001", "/ipns/%0D%0A%0D%0Ahello", http.StatusNotFound, "ipfs resolve -r /ipns/%0D%0A%0D%0Ahello: " + namesys.ErrResolveFailed.Error() + "\n"},
-		{"localhost:5001", "/ipns/example.com", http.StatusOK, "fnord"},
+		{"127.0.0.1:8080", "/", http.StatusNotFound, "404 page not found\n"},
+		{"127.0.0.1:8080", "/" + k.Cid().String(), http.StatusNotFound, "404 page not found\n"},
+		{"127.0.0.1:8080", k.String(), http.StatusOK, "fnord"},
+		{"127.0.0.1:8080", "/ipns/nxdomain.example.com", http.StatusNotFound, "ipfs resolve -r /ipns/nxdomain.example.com: " + namesys.ErrResolveFailed.Error() + "\n"},
+		{"127.0.0.1:8080", "/ipns/%0D%0A%0D%0Ahello", http.StatusNotFound, "ipfs resolve -r /ipns/%0D%0A%0D%0Ahello: " + namesys.ErrResolveFailed.Error() + "\n"},
+		{"127.0.0.1:8080", "/ipns/example.com", http.StatusOK, "fnord"},
 		{"example.com", "/", http.StatusOK, "fnord"},
 
 		{"working.example.com", "/", http.StatusOK, "fnord"},
@@ -381,7 +381,7 @@ func TestIPNSHostnameBacklinks(t *testing.T) {
 	if !strings.Contains(s, "Index of /foo? #<'/") {
 		t.Fatalf("expected a path in directory listing")
 	}
-	if !strings.Contains(s, "<a href=\"/\">") {
+	if !strings.Contains(s, "<a href=\"/foo%3F%20%23%3C%27/./..\">") {
 		t.Fatalf("expected backlink in directory listing")
 	}
 	if !strings.Contains(s, "<a href=\"/foo%3F%20%23%3C%27/file.txt\">") {
@@ -447,7 +447,7 @@ func TestIPNSHostnameBacklinks(t *testing.T) {
 	if !strings.Contains(s, "Index of /foo? #&lt;&#39;/bar/") {
 		t.Fatalf("expected a path in directory listing")
 	}
-	if !strings.Contains(s, "<a href=\"/foo%3F%20%23%3C%27/\">") {
+	if !strings.Contains(s, "<a href=\"/foo%3F%20%23%3C%27/bar/./..\">") {
 		t.Fatalf("expected backlink in directory listing")
 	}
 	if !strings.Contains(s, "<a href=\"/foo%3F%20%23%3C%27/bar/file.txt\">") {