From 11c229bb14726e2bdd99d706ded2383a4a41adb4 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 14 Mar 2019 16:07:01 -0700 Subject: [PATCH 1/6] rename ProxyOption to P2PProxyOption (we're implementing an _actual_ proxy) License: MIT Signed-off-by: Steven Allen --- cmd/ipfs/daemon.go | 2 +- core/corehttp/{proxy.go => p2p_proxy.go} | 4 ++-- core/corehttp/{proxy_test.go => p2p_proxy_test.go} | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename core/corehttp/{proxy.go => p2p_proxy.go} (94%) rename core/corehttp/{proxy_test.go => p2p_proxy_test.go} (100%) diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index d4647d2f54e..dec47a18dcb 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -644,7 +644,7 @@ func serveHTTPGateway(req *cmds.Request, cctx *oldcmds.Context) (<-chan error, e } if cfg.Experimental.P2pHttpProxy { - opts = append(opts, corehttp.ProxyOption()) + opts = append(opts, corehttp.P2PProxyOption()) } if len(cfg.Gateway.RootRedirect) > 0 { diff --git a/core/corehttp/proxy.go b/core/corehttp/p2p_proxy.go similarity index 94% rename from core/corehttp/proxy.go rename to core/corehttp/p2p_proxy.go index 17cb0052824..0a615c33a2a 100644 --- a/core/corehttp/proxy.go +++ b/core/corehttp/p2p_proxy.go @@ -14,8 +14,8 @@ import ( p2phttp "github.com/libp2p/go-libp2p-http" ) -// ProxyOption is an endpoint for proxying a HTTP request to another ipfs peer -func ProxyOption() ServeOption { +// P2PProxyOption is an endpoint for proxying a HTTP request to another ipfs peer +func P2PProxyOption() ServeOption { return func(ipfsNode *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { mux.HandleFunc("/p2p/", func(w http.ResponseWriter, request *http.Request) { // parse request diff --git a/core/corehttp/proxy_test.go b/core/corehttp/p2p_proxy_test.go similarity index 100% rename from core/corehttp/proxy_test.go rename to core/corehttp/p2p_proxy_test.go From 72490f7ed04494b5d96bae06476faff472bc37a9 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 14 Mar 2019 16:09:00 -0700 Subject: [PATCH 2/6] gateway: simplify/improve dnslink rewrite handling Instead of adding a new fake header (that could be spoofed by the client...), just read the original request URI from the request object. This also removes support for suborigins. They have never been implemented in browsers and it looks like efforts have stalled. We can add support back if we need it but, well, maintaining support was going to be more trouble than it was worth. License: MIT Signed-off-by: Steven Allen --- core/corehttp/gateway_handler.go | 59 ++++---------------------------- core/corehttp/ipns_hostname.go | 1 - 2 files changed, 7 insertions(+), 53 deletions(-) diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index de8038f5306..5b1382a8655 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -24,7 +24,6 @@ import ( coreiface "github.com/ipfs/interface-go-ipfs-core" ipath "github.com/ipfs/interface-go-ipfs-core/path" routing "github.com/libp2p/go-libp2p-core/routing" - "github.com/multiformats/go-multibase" ) const ( @@ -148,12 +147,11 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request // and links that match the requested URL. // For example, http://example.net would become /ipns/example.net, and // the redirects and links would end up as http://example.net/ipns/example.net - originalUrlPath := prefix + urlPath - ipnsHostname := false - if hdr := r.Header.Get("X-Ipns-Original-Path"); len(hdr) > 0 { - originalUrlPath = prefix + hdr - ipnsHostname = true + requestURI, err := url.ParseRequestURI(r.RequestURI) + if err != nil { + webError(w, "failed to parse request path", err, http.StatusInternalServerError) } + originalUrlPath := prefix + requestURI.Path // Service Worker registration request if r.Header.Get("Service-Worker") == "script" { @@ -206,39 +204,6 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request w.Header().Set("X-IPFS-Path", urlPath) w.Header().Set("Etag", etag) - // Suborigin header, sandboxes apps from each other in the browser (even - // though they are served from the same gateway domain). - // - // Omitted if the path was treated by IPNSHostnameOption(), for example - // a request for http://example.net/ would be changed to /ipns/example.net/, - // which would turn into an incorrect Suborigin header. - // In this case the correct thing to do is omit the header because it is already - // handled correctly without a Suborigin. - // - // NOTE: This is not yet widely supported by browsers. - if !ipnsHostname { - // e.g.: 1="ipfs", 2="QmYuNaKwY...", ... - pathComponents := strings.SplitN(urlPath, "/", 4) - - var suboriginRaw []byte - cidDecoded, err := cid.Decode(pathComponents[2]) - if err != nil { - // component 2 doesn't decode with cid, so it must be a hostname - suboriginRaw = []byte(strings.ToLower(pathComponents[2])) - } else { - suboriginRaw = cidDecoded.Bytes() - } - - base32Encoded, err := multibase.Encode(multibase.Base32, suboriginRaw) - if err != nil { - internalWebError(w, err) - return - } - - suborigin := pathComponents[1] + "000" + strings.ToLower(base32Encoded) - w.Header().Set("Suborigin", suborigin) - } - // set these headers _after_ the error, for we may just not have it // and dont want the client to cache a 500 response... // and only if it's /ipfs! @@ -322,10 +287,10 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request // construct the correct back link // https://github.com/ipfs/go-ipfs/issues/1365 - var backLink string = prefix + urlPath + var backLink string = originalUrlPath // don't go further up than /ipfs/$hash/ - pathSplit := path.SplitList(backLink) + pathSplit := path.SplitList(urlPath) switch { // keep backlink case len(pathSplit) == 3: // url: /ipfs/$hash @@ -342,18 +307,8 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request } } - // strip /ipfs/$hash from backlink if IPNSHostnameOption touched the path. - if ipnsHostname { - backLink = prefix + "/" - if len(pathSplit) > 5 { - // also strip the trailing segment, because it's a backlink - backLinkParts := pathSplit[3 : len(pathSplit)-2] - backLink += path.Join(backLinkParts) + "/" - } - } - var hash string - if !strings.HasPrefix(originalUrlPath, ipfsPathPrefix) { + if !strings.HasPrefix(urlPath, ipfsPathPrefix) { hash = resolvedPath.Cid().String() } diff --git a/core/corehttp/ipns_hostname.go b/core/corehttp/ipns_hostname.go index d5512779b35..ddcd58c6129 100644 --- a/core/corehttp/ipns_hostname.go +++ b/core/corehttp/ipns_hostname.go @@ -28,7 +28,6 @@ func IPNSHostnameOption() ServeOption { name := "/ipns/" + host _, err := n.Namesys.Resolve(ctx, name, nsopts.Depth(1)) if err == nil || err == namesys.ErrResolveRecursion { - r.Header.Set("X-Ipns-Original-Path", r.URL.Path) r.URL.Path = name + r.URL.Path } } From 848d4c7f18649b4c994af01dd6fb6b4764cb1127 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Sat, 7 Mar 2020 01:56:48 +0100 Subject: [PATCH 3/6] feat: IPFS_NS_MAP Allows static DNSLink mappings with IPFS_NS_MAP. License: MIT Signed-off-by: Marcin Rataj --- docs/environment-variables.md | 14 ++++++++++++++ namesys/cache.go | 8 ++++++++ namesys/namesys.go | 25 +++++++++++++++++++++++-- 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 82eb5c3126c..fdc92f461c3 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -82,6 +82,20 @@ the `--migrate` flag). Default: https://ipfs.io/ipfs/$something (depends on the IPFS version) +## `IPFS_NS_MAP` + +Prewarms namesys cache with static records for deteministic tests and debugging. +Useful for testing things like DNSLink without real DNS lookup. + +Example: + +```console +$ IPFS_NS_MAP="dnslink-test1.example.com:/ipfs/bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am,dnslink-test2.example.com:/ipns/dnslink-test1.example.com" ipfs daemon +... +$ ipfs resolve -r /ipns/dnslink-test2.example.com +/ipfs/bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am +``` + ## `LIBP2P_MUX_PREFS` Tells go-ipfs which multiplexers to use in which order. diff --git a/namesys/cache.go b/namesys/cache.go index 4a5cb5113ae..a0029829d33 100644 --- a/namesys/cache.go +++ b/namesys/cache.go @@ -7,6 +7,14 @@ import ( ) func (ns *mpns) cacheGet(name string) (path.Path, bool) { + // existence of optional mapping defined via IPFS_NS_MAP is checked first + if ns.staticMap != nil { + val, ok := ns.staticMap[name] + if ok { + return val, true + } + } + if ns.cache == nil { return "", false } diff --git a/namesys/namesys.go b/namesys/namesys.go index 079eecccc03..11f4646f174 100644 --- a/namesys/namesys.go +++ b/namesys/namesys.go @@ -2,6 +2,7 @@ package namesys import ( "context" + "os" "strings" "time" @@ -29,25 +30,45 @@ type mpns struct { dnsResolver, proquintResolver, ipnsResolver resolver ipnsPublisher Publisher - cache *lru.Cache + staticMap map[string]path.Path + cache *lru.Cache } // NewNameSystem will construct the IPFS naming system based on Routing func NewNameSystem(r routing.ValueStore, ds ds.Datastore, cachesize int) NameSystem { - var cache *lru.Cache + var ( + cache *lru.Cache + staticMap map[string]path.Path + ) if cachesize > 0 { cache, _ = lru.New(cachesize) } + // Prewarm namesys cache with static records for deteministic tests and debugging. + // Useful for testing things like DNSLink without real DNS lookup. + // Example: + // IPFS_NS_MAP="dnslink-test.example.com:/ipfs/bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am" + if list := os.Getenv("IPFS_NS_MAP"); list != "" { + staticMap = make(map[string]path.Path) + for _, pair := range strings.Split(list, ",") { + mapping := strings.SplitN(pair, ":", 2) + key := mapping[0] + value := path.FromString(mapping[1]) + staticMap[key] = value + } + } + return &mpns{ dnsResolver: NewDNSResolver(), proquintResolver: new(ProquintResolver), ipnsResolver: NewIpnsResolver(r), ipnsPublisher: NewIpnsPublisher(r, ds), + staticMap: staticMap, cache: cache, } } +// DefaultResolverCacheTTL defines max ttl of a record placed in namesys cache. const DefaultResolverCacheTTL = time.Minute // Resolve implements Resolver. From 3ecccd6e1dff567db8da8a53cebd8226ecf2f446 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 14 Mar 2019 17:21:38 -0700 Subject: [PATCH 4/6] feat(gateway): subdomain and proxy gateway License: MIT Signed-off-by: Marcin Rataj --- cmd/ipfs/daemon.go | 11 +- core/corehttp/corehttp.go | 14 +- core/corehttp/gateway_handler.go | 9 +- core/corehttp/gateway_test.go | 18 +- core/corehttp/hostname.go | 358 ++++++++++++ core/corehttp/hostname_test.go | 152 +++++ core/corehttp/ipns_hostname.go | 38 -- docs/config.md | 151 ++++- docs/environment-variables.md | 2 +- go.mod | 2 +- go.sum | 8 +- namesys/namesys.go | 23 +- namesys/namesys_test.go | 14 +- namesys/routing.go | 1 + test/sharness/t0111-gateway-writeable.sh | 20 +- test/sharness/t0114-gateway-subdomains.sh | 641 +++++++++++++++++++++ test/sharness/t0160-resolve.sh | 9 + test/sharness/t0184-http-proxy-over-p2p.sh | 35 +- 18 files changed, 1421 insertions(+), 85 deletions(-) create mode 100644 core/corehttp/hostname.go create mode 100644 core/corehttp/hostname_test.go delete mode 100644 core/corehttp/ipns_hostname.go create mode 100755 test/sharness/t0114-gateway-subdomains.sh diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index dec47a18dcb..b8486f8aba1 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -12,6 +12,8 @@ import ( "sort" "sync" + multierror "github.com/hashicorp/go-multierror" + version "github.com/ipfs/go-ipfs" config "github.com/ipfs/go-ipfs-config" cserial "github.com/ipfs/go-ipfs-config/serialize" @@ -27,7 +29,6 @@ import ( migrate "github.com/ipfs/go-ipfs/repo/fsrepo/migrations" sockets "github.com/libp2p/go-socket-activation" - "github.com/hashicorp/go-multierror" cmds "github.com/ipfs/go-ipfs-cmds" mprome "github.com/ipfs/go-metrics-prometheus" goprocess "github.com/jbenet/goprocess" @@ -298,9 +299,9 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment // Start assembling node config ncfg := &core.BuildCfg{ - Repo: repo, - Permanent: true, // It is temporary way to signify that node is permanent - Online: !offline, + Repo: repo, + Permanent: true, // It is temporary way to signify that node is permanent + Online: !offline, DisableEncryptedConnections: unencrypted, ExtraOpts: map[string]bool{ "pubsub": pubsub, @@ -636,7 +637,7 @@ func serveHTTPGateway(req *cmds.Request, cctx *oldcmds.Context) (<-chan error, e var opts = []corehttp.ServeOption{ corehttp.MetricsCollectionOption("gateway"), - corehttp.IPNSHostnameOption(), + corehttp.HostnameOption(), corehttp.GatewayOption(writable, "/ipfs", "/ipns"), corehttp.VersionOption(), corehttp.CheckVersionOption(), diff --git a/core/corehttp/corehttp.go b/core/corehttp/corehttp.go index c52bea8f5c7..d99a0769119 100644 --- a/core/corehttp/corehttp.go +++ b/core/corehttp/corehttp.go @@ -43,7 +43,17 @@ func makeHandler(n *core.IpfsNode, l net.Listener, options ...ServeOption) (http return nil, err } } - return topMux, nil + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // ServeMux does not support requests with CONNECT method, + // so we need to handle them separately + // https://golang.org/src/net/http/request.go#L111 + if r.Method == http.MethodConnect { + w.WriteHeader(http.StatusOK) + return + } + topMux.ServeHTTP(w, r) + }) + return handler, nil } // ListenAndServe runs an HTTP server listening at |listeningMultiAddr| with @@ -70,6 +80,8 @@ func ListenAndServe(n *core.IpfsNode, listeningMultiAddr string, options ...Serv return Serve(n, manet.NetListener(list), options...) } +// Serve accepts incoming HTTP connections on the listener and pass them +// to ServeOption handlers. func Serve(node *core.IpfsNode, lis net.Listener, options ...ServeOption) error { // make sure we close this no matter what. defer lis.Close() diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index 5b1382a8655..cf74202434a 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -14,12 +14,12 @@ import ( "strings" "time" - "github.com/dustin/go-humanize" + humanize "github.com/dustin/go-humanize" "github.com/ipfs/go-cid" files "github.com/ipfs/go-ipfs-files" dag "github.com/ipfs/go-merkledag" - "github.com/ipfs/go-mfs" - "github.com/ipfs/go-path" + mfs "github.com/ipfs/go-mfs" + path "github.com/ipfs/go-path" "github.com/ipfs/go-path/resolver" coreiface "github.com/ipfs/interface-go-ipfs-core" ipath "github.com/ipfs/interface-go-ipfs-core/path" @@ -142,7 +142,7 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request } } - // IPNSHostnameOption might have constructed an IPNS path using the Host header. + // HostnameOption might have constructed an IPNS/IPFS path using the Host header. // In this case, we need the original path for constructing redirects // and links that match the requested URL. // For example, http://example.net would become /ipns/example.net, and @@ -150,6 +150,7 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request requestURI, err := url.ParseRequestURI(r.RequestURI) if err != nil { webError(w, "failed to parse request path", err, http.StatusInternalServerError) + return } originalUrlPath := prefix + requestURI.Path diff --git a/core/corehttp/gateway_test.go b/core/corehttp/gateway_test.go index 9128aa0175d..daf1af07c6c 100644 --- a/core/corehttp/gateway_test.go +++ b/core/corehttp/gateway_test.go @@ -138,7 +138,7 @@ func newTestServerAndNode(t *testing.T, ns mockNamesys) (*httptest.Server, iface dh.Handler, err = makeHandler(n, ts.Listener, - IPNSHostnameOption(), + HostnameOption(), GatewayOption(false, "/ipfs", "/ipns"), VersionOption(), ) @@ -184,12 +184,12 @@ func TestGatewayGet(t *testing.T) { status int text string }{ - {"localhost:5001", "/", http.StatusNotFound, "404 page not found\n"}, - {"localhost:5001", "/" + k.Cid().String(), http.StatusNotFound, "404 page not found\n"}, - {"localhost:5001", k.String(), http.StatusOK, "fnord"}, - {"localhost:5001", "/ipns/nxdomain.example.com", http.StatusNotFound, "ipfs resolve -r /ipns/nxdomain.example.com: " + namesys.ErrResolveFailed.Error() + "\n"}, - {"localhost:5001", "/ipns/%0D%0A%0D%0Ahello", http.StatusNotFound, "ipfs resolve -r /ipns/%0D%0A%0D%0Ahello: " + namesys.ErrResolveFailed.Error() + "\n"}, - {"localhost:5001", "/ipns/example.com", http.StatusOK, "fnord"}, + {"127.0.0.1:8080", "/", http.StatusNotFound, "404 page not found\n"}, + {"127.0.0.1:8080", "/" + k.Cid().String(), http.StatusNotFound, "404 page not found\n"}, + {"127.0.0.1:8080", k.String(), http.StatusOK, "fnord"}, + {"127.0.0.1:8080", "/ipns/nxdomain.example.com", http.StatusNotFound, "ipfs resolve -r /ipns/nxdomain.example.com: " + namesys.ErrResolveFailed.Error() + "\n"}, + {"127.0.0.1:8080", "/ipns/%0D%0A%0D%0Ahello", http.StatusNotFound, "ipfs resolve -r /ipns/%0D%0A%0D%0Ahello: " + namesys.ErrResolveFailed.Error() + "\n"}, + {"127.0.0.1:8080", "/ipns/example.com", http.StatusOK, "fnord"}, {"example.com", "/", http.StatusOK, "fnord"}, {"working.example.com", "/", http.StatusOK, "fnord"}, @@ -381,7 +381,7 @@ func TestIPNSHostnameBacklinks(t *testing.T) { if !strings.Contains(s, "Index of /foo? #<'/") { t.Fatalf("expected a path in directory listing") } - if !strings.Contains(s, "") { + if !strings.Contains(s, "") { t.Fatalf("expected backlink in directory listing") } if !strings.Contains(s, "") { @@ -447,7 +447,7 @@ func TestIPNSHostnameBacklinks(t *testing.T) { if !strings.Contains(s, "Index of /foo? #<'/bar/") { t.Fatalf("expected a path in directory listing") } - if !strings.Contains(s, "") { + if !strings.Contains(s, "") { t.Fatalf("expected backlink in directory listing") } if !strings.Contains(s, "") { diff --git a/core/corehttp/hostname.go b/core/corehttp/hostname.go new file mode 100644 index 00000000000..910ba5bc87d --- /dev/null +++ b/core/corehttp/hostname.go @@ -0,0 +1,358 @@ +package corehttp + +import ( + "context" + "fmt" + "net" + "net/http" + "net/url" + "strings" + + cid "github.com/ipfs/go-cid" + core "github.com/ipfs/go-ipfs/core" + coreapi "github.com/ipfs/go-ipfs/core/coreapi" + namesys "github.com/ipfs/go-ipfs/namesys" + isd "github.com/jbenet/go-is-domain" + "github.com/libp2p/go-libp2p-core/peer" + mbase "github.com/multiformats/go-multibase" + + config "github.com/ipfs/go-ipfs-config" + iface "github.com/ipfs/interface-go-ipfs-core" + options "github.com/ipfs/interface-go-ipfs-core/options" + nsopts "github.com/ipfs/interface-go-ipfs-core/options/namesys" +) + +var defaultPaths = []string{"/ipfs/", "/ipns/", "/api/", "/p2p/", "/version"} + +var pathGatewaySpec = config.GatewaySpec{ + Paths: defaultPaths, + UseSubdomains: false, +} + +var subdomainGatewaySpec = config.GatewaySpec{ + Paths: defaultPaths, + UseSubdomains: true, +} + +var defaultKnownGateways = map[string]config.GatewaySpec{ + "localhost": subdomainGatewaySpec, + "ipfs.io": pathGatewaySpec, + "gateway.ipfs.io": pathGatewaySpec, + "dweb.link": subdomainGatewaySpec, +} + +// HostnameOption rewrites an incoming request based on the Host header. +func HostnameOption() ServeOption { + return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { + childMux := http.NewServeMux() + + coreApi, err := coreapi.NewCoreAPI(n) + if err != nil { + return nil, err + } + + cfg, err := n.Repo.Config() + if err != nil { + return nil, err + } + knownGateways := make( + map[string]config.GatewaySpec, + len(defaultKnownGateways)+len(cfg.Gateway.PublicGateways), + ) + for hostname, gw := range defaultKnownGateways { + knownGateways[hostname] = gw + } + for hostname, gw := range cfg.Gateway.PublicGateways { + if gw == nil { + // Allows the user to remove gateways but _also_ + // allows us to continuously update the list. + delete(knownGateways, hostname) + } else { + knownGateways[hostname] = *gw + } + } + + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + // Unfortunately, many (well, ipfs.io) gateways use + // DNSLink so if we blindly rewrite with DNSLink, we'll + // break /ipfs links. + // + // We fix this by maintaining a list of known gateways + // and the paths that they serve "gateway" content on. + // That way, we can use DNSLink for everything else. + + // HTTP Host & Path check: is this one of our "known gateways"? + if gw, ok := isKnownHostname(r.Host, knownGateways); ok { + // This is a known gateway but request is not using + // the subdomain feature. + + // Does this gateway _handle_ this path? + if hasPrefix(r.URL.Path, gw.Paths...) { + // It does. + + // Should this gateway use subdomains instead of paths? + if gw.UseSubdomains { + // Yes, redirect if applicable + // Example: dweb.link/ipfs/{cid} → {cid}.ipfs.dweb.link + if newURL, ok := toSubdomainURL(r.Host, r.URL.Path, r); ok { + http.Redirect(w, r, newURL, http.StatusMovedPermanently) + return + } + } + + // Not a subdomain resource, continue with path processing + // Example: 127.0.0.1:8080/ipfs/{CID}, ipfs.io/ipfs/{CID} etc + childMux.ServeHTTP(w, r) + return + } + // Not a whitelisted path + + // Try DNSLink, if it was not explicitly disabled for the hostname + if !gw.NoDNSLink && isDNSLinkRequest(n.Context(), coreApi, r) { + // rewrite path and handle as DNSLink + r.URL.Path = "/ipns/" + stripPort(r.Host) + r.URL.Path + childMux.ServeHTTP(w, r) + return + } + + // If not, resource does not exist on the hostname, return 404 + http.NotFound(w, r) + return + } + + // HTTP Host check: is this one of our subdomain-based "known gateways"? + // Example: {cid}.ipfs.localhost, {cid}.ipfs.dweb.link + if gw, hostname, ns, rootID, ok := knownSubdomainDetails(r.Host, knownGateways); ok { + // Looks like we're using known subdomain gateway. + + // Assemble original path prefix. + pathPrefix := "/" + ns + "/" + rootID + + // Does this gateway _handle_ this path? + if !(gw.UseSubdomains && hasPrefix(pathPrefix, gw.Paths...)) { + // If not, resource does not exist, return 404 + http.NotFound(w, r) + return + } + + // Do we need to fix multicodec in PeerID represented as CIDv1? + if isPeerIDNamespace(ns) { + keyCid, err := cid.Decode(rootID) + if err == nil && keyCid.Type() != cid.Libp2pKey { + if newURL, ok := toSubdomainURL(hostname, pathPrefix+r.URL.Path, r); ok { + // Redirect to CID fixed inside of toSubdomainURL() + http.Redirect(w, r, newURL, http.StatusMovedPermanently) + return + } + } + } + + // Rewrite the path to not use subdomains + r.URL.Path = pathPrefix + r.URL.Path + + // Serve path request + childMux.ServeHTTP(w, r) + return + } + // We don't have a known gateway. Fallback on DNSLink lookup + + // Wildcard HTTP Host check: + // 1. is wildcard DNSLink enabled (Gateway.NoDNSLink=false)? + // 2. does Host header include a fully qualified domain name (FQDN)? + // 3. does DNSLink record exist in DNS? + if !cfg.Gateway.NoDNSLink && isDNSLinkRequest(n.Context(), coreApi, r) { + // rewrite path and handle as DNSLink + r.URL.Path = "/ipns/" + stripPort(r.Host) + r.URL.Path + childMux.ServeHTTP(w, r) + return + } + + // else, treat it as an old school gateway, I guess. + childMux.ServeHTTP(w, r) + }) + return childMux, nil + } +} + +// isKnownHostname checks Gateway.PublicGateways and returns matching +// GatewaySpec with gracefull fallback to version without port +func isKnownHostname(hostname string, knownGateways map[string]config.GatewaySpec) (gw config.GatewaySpec, ok bool) { + // Try hostname (host+optional port - value from Host header as-is) + if gw, ok := knownGateways[hostname]; ok { + return gw, ok + } + // Fallback to hostname without port + gw, ok = knownGateways[stripPort(hostname)] + return gw, ok +} + +// Parses Host header and looks for a known subdomain gateway host. +// If found, returns GatewaySpec and subdomain components. +// Note: hostname is host + optional port +func knownSubdomainDetails(hostname string, knownGateways map[string]config.GatewaySpec) (gw config.GatewaySpec, knownHostname, ns, rootID string, ok bool) { + labels := strings.Split(hostname, ".") + // Look for FQDN of a known gateway hostname. + // Example: given "dist.ipfs.io.ipns.dweb.link": + // 1. Lookup "link" TLD in knownGateways: negative + // 2. Lookup "dweb.link" in knownGateways: positive + // + // Stops when we have 2 or fewer labels left as we need at least a + // rootId and a namespace. + for i := len(labels) - 1; i >= 2; i-- { + fqdn := strings.Join(labels[i:], ".") + gw, ok := isKnownHostname(fqdn, knownGateways) + if !ok { + continue + } + + ns := labels[i-1] + if !isSubdomainNamespace(ns) { + break + } + + // Merge remaining labels (could be a FQDN with DNSLink) + rootID := strings.Join(labels[:i-1], ".") + return gw, fqdn, ns, rootID, true + } + // not a known subdomain gateway + return gw, "", "", "", false +} + +// isDNSLinkRequest returns bool that indicates if request +// should return data from content path listed in DNSLink record (if exists) +func isDNSLinkRequest(ctx context.Context, ipfs iface.CoreAPI, r *http.Request) bool { + fqdn := stripPort(r.Host) + if len(fqdn) == 0 && !isd.IsDomain(fqdn) { + return false + } + name := "/ipns/" + fqdn + // check if DNSLink exists + depth := options.Name.ResolveOption(nsopts.Depth(1)) + _, err := ipfs.Name().Resolve(ctx, name, depth) + return err == nil || err == namesys.ErrResolveRecursion +} + +func isSubdomainNamespace(ns string) bool { + switch ns { + case "ipfs", "ipns", "p2p", "ipld": + return true + default: + return false + } +} + +func isPeerIDNamespace(ns string) bool { + switch ns { + case "ipns", "p2p": + return true + default: + return false + } +} + +// Converts a hostname/path to a subdomain-based URL, if applicable. +func toSubdomainURL(hostname, path string, r *http.Request) (redirURL string, ok bool) { + var scheme, ns, rootID, rest string + + query := r.URL.RawQuery + parts := strings.SplitN(path, "/", 4) + safeRedirectURL := func(in string) (out string, ok bool) { + safeURI, err := url.ParseRequestURI(in) + if err != nil { + return "", false + } + return safeURI.String(), true + } + + // Support X-Forwarded-Proto if added by a reverse proxy + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-Proto + xproto := r.Header.Get("X-Forwarded-Proto") + if xproto == "https" { + scheme = "https:" + } else { + scheme = "http:" + } + + switch len(parts) { + case 4: + rest = parts[3] + fallthrough + case 3: + ns = parts[1] + rootID = parts[2] + default: + return "", false + } + + if !isSubdomainNamespace(ns) { + return "", false + } + + // add prefix if query is present + if query != "" { + query = "?" + query + } + + // Normalize problematic PeerIDs (eg. ed25519+identity) to CID representation + if isPeerIDNamespace(ns) && !isd.IsDomain(rootID) { + peerID, err := peer.Decode(rootID) + // Note: PeerID CIDv1 with protobuf multicodec will fail, but we fix it + // in the next block + if err == nil { + rootID = peer.ToCid(peerID).String() + } + } + + // If rootID is a CID, ensure it uses DNS-friendly text representation + if rootCid, err := cid.Decode(rootID); err == nil { + multicodec := rootCid.Type() + + // PeerIDs represented as CIDv1 are expected to have libp2p-key + // multicodec (https://github.com/libp2p/specs/pull/209). + // We ease the transition by fixing multicodec on the fly: + // https://github.com/ipfs/go-ipfs/issues/5287#issuecomment-492163929 + if isPeerIDNamespace(ns) && multicodec != cid.Libp2pKey { + multicodec = cid.Libp2pKey + } + + // if object turns out to be a valid CID, + // ensure text representation used in subdomain is CIDv1 in Base32 + // https://github.com/ipfs/in-web-browsers/issues/89 + rootID, err = cid.NewCidV1(multicodec, rootCid.Hash()).StringOfBase(mbase.Base32) + if err != nil { + // should not error, but if it does, its clealy not possible to + // produce a subdomain URL + return "", false + } + } + + return safeRedirectURL(fmt.Sprintf( + "%s//%s.%s.%s/%s%s", + scheme, + rootID, + ns, + hostname, + rest, + query, + )) +} + +func hasPrefix(path string, prefixes ...string) bool { + for _, prefix := range prefixes { + // Assume people are creative with trailing slashes in Gateway config + p := strings.TrimSuffix(prefix, "/") + // Support for both /version and /ipfs/$cid + if p == path || strings.HasPrefix(path, p+"/") { + return true + } + } + return false +} + +func stripPort(hostname string) string { + host, _, err := net.SplitHostPort(hostname) + if err == nil { + return host + } + return hostname +} diff --git a/core/corehttp/hostname_test.go b/core/corehttp/hostname_test.go new file mode 100644 index 00000000000..9a297464891 --- /dev/null +++ b/core/corehttp/hostname_test.go @@ -0,0 +1,152 @@ +package corehttp + +import ( + "net/http/httptest" + "testing" + + config "github.com/ipfs/go-ipfs-config" +) + +func TestToSubdomainURL(t *testing.T) { + r := httptest.NewRequest("GET", "http://request-stub.example.com", nil) + for _, test := range []struct { + // in: + hostname string + path string + // out: + url string + ok bool + }{ + // DNSLink + {"localhost", "/ipns/dnslink.io", "http://dnslink.io.ipns.localhost/", true}, + // Hostname with port + {"localhost:8080", "/ipns/dnslink.io", "http://dnslink.io.ipns.localhost:8080/", true}, + // CIDv0 → CIDv1base32 + {"localhost", "/ipfs/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", "http://bafybeif7a7gdklt6hodwdrmwmxnhksctcuav6lfxlcyfz4khzl3qfmvcgu.ipfs.localhost/", true}, + // PeerID as CIDv1 needs to have libp2p-key multicodec + {"localhost", "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", "http://bafzbeieqhtl2l3mrszjnhv6hf2iloiitsx7mexiolcnywnbcrzkqxwslja.ipns.localhost/", true}, + {"localhost", "/ipns/bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", "http://bafzbeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm.ipns.localhost/", true}, + // PeerID: ed25519+identity multihash + {"localhost", "/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", "http://bafzaajaiaejcat4yhiwnr2qz73mtu6vrnj2krxlpfoa3wo2pllfi37quorgwh2jw.ipns.localhost/", true}, + } { + url, ok := toSubdomainURL(test.hostname, test.path, r) + if ok != test.ok || url != test.url { + t.Errorf("(%s, %s) returned (%s, %t), expected (%s, %t)", test.hostname, test.path, url, ok, test.url, ok) + } + } +} + +func TestHasPrefix(t *testing.T) { + for _, test := range []struct { + prefixes []string + path string + out bool + }{ + {[]string{"/ipfs"}, "/ipfs/cid", true}, + {[]string{"/ipfs/"}, "/ipfs/cid", true}, + {[]string{"/version/"}, "/version", true}, + {[]string{"/version"}, "/version", true}, + } { + out := hasPrefix(test.path, test.prefixes...) + if out != test.out { + t.Errorf("(%+v, %s) returned '%t', expected '%t'", test.prefixes, test.path, out, test.out) + } + } +} + +func TestPortStripping(t *testing.T) { + for _, test := range []struct { + in string + out string + }{ + {"localhost:8080", "localhost"}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.localhost:8080", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.localhost"}, + {"example.com:443", "example.com"}, + {"example.com", "example.com"}, + {"foo-dweb.ipfs.pvt.k12.ma.us:8080", "foo-dweb.ipfs.pvt.k12.ma.us"}, + {"localhost", "localhost"}, + {"[::1]:8080", "::1"}, + } { + out := stripPort(test.in) + if out != test.out { + t.Errorf("(%s): returned '%s', expected '%s'", test.in, out, test.out) + } + } + +} + +func TestKnownSubdomainDetails(t *testing.T) { + gwSpec := config.GatewaySpec{ + UseSubdomains: true, + } + knownGateways := map[string]config.GatewaySpec{ + "localhost": gwSpec, + "dweb.link": gwSpec, + "dweb.ipfs.pvt.k12.ma.us": gwSpec, // note the sneaky ".ipfs." ;-) + } + + for _, test := range []struct { + // in: + hostHeader string + // out: + hostname string + ns string + rootID string + ok bool + }{ + // no subdomain + {"127.0.0.1:8080", "", "", "", false}, + {"[::1]:8080", "", "", "", false}, + {"hey.look.example.com", "", "", "", false}, + {"dweb.link", "", "", "", false}, + // malformed Host header + {".....dweb.link", "", "", "", false}, + {"link", "", "", "", false}, + {"8080:dweb.link", "", "", "", false}, + {" ", "", "", "", false}, + {"", "", "", "", false}, + // unknown gateway host + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.unknown.example.com", "", "", "", false}, + // cid in subdomain, known gateway + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.localhost:8080", "localhost:8080", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.link", "dweb.link", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + // capture everything before .ipfs. + {"foo.bar.boo-buzz.ipfs.dweb.link", "dweb.link", "ipfs", "foo.bar.boo-buzz", true}, + // ipns + {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.localhost:8080", "localhost:8080", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, + {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.link", "dweb.link", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, + // edge case check: public gateway under long TLD (see: https://publicsuffix.org) + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.ipfs.pvt.k12.ma.us", "dweb.ipfs.pvt.k12.ma.us", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.ipfs.pvt.k12.ma.us", "dweb.ipfs.pvt.k12.ma.us", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, + // dnslink in subdomain + {"en.wikipedia-on-ipfs.org.ipns.localhost:8080", "localhost:8080", "ipns", "en.wikipedia-on-ipfs.org", true}, + {"en.wikipedia-on-ipfs.org.ipns.localhost", "localhost", "ipns", "en.wikipedia-on-ipfs.org", true}, + {"dist.ipfs.io.ipns.localhost:8080", "localhost:8080", "ipns", "dist.ipfs.io", true}, + {"en.wikipedia-on-ipfs.org.ipns.dweb.link", "dweb.link", "ipns", "en.wikipedia-on-ipfs.org", true}, + // edge case check: public gateway under long TLD (see: https://publicsuffix.org) + {"foo.dweb.ipfs.pvt.k12.ma.us", "", "", "", false}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.ipfs.pvt.k12.ma.us", "dweb.ipfs.pvt.k12.ma.us", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.ipfs.pvt.k12.ma.us", "dweb.ipfs.pvt.k12.ma.us", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, + // other namespaces + {"api.localhost", "", "", "", false}, + {"peerid.p2p.localhost", "localhost", "p2p", "peerid", true}, + } { + gw, hostname, ns, rootID, ok := knownSubdomainDetails(test.hostHeader, knownGateways) + if ok != test.ok { + t.Errorf("knownSubdomainDetails(%s): ok is %t, expected %t", test.hostHeader, ok, test.ok) + } + if rootID != test.rootID { + t.Errorf("knownSubdomainDetails(%s): rootID is '%s', expected '%s'", test.hostHeader, rootID, test.rootID) + } + if ns != test.ns { + t.Errorf("knownSubdomainDetails(%s): ns is '%s', expected '%s'", test.hostHeader, ns, test.ns) + } + if hostname != test.hostname { + t.Errorf("knownSubdomainDetails(%s): hostname is '%s', expected '%s'", test.hostHeader, hostname, test.hostname) + } + if ok && gw.UseSubdomains != gwSpec.UseSubdomains { + t.Errorf("knownSubdomainDetails(%s): gw is %+v, expected %+v", test.hostHeader, gw, gwSpec) + } + } + +} diff --git a/core/corehttp/ipns_hostname.go b/core/corehttp/ipns_hostname.go deleted file mode 100644 index ddcd58c6129..00000000000 --- a/core/corehttp/ipns_hostname.go +++ /dev/null @@ -1,38 +0,0 @@ -package corehttp - -import ( - "context" - "net" - "net/http" - "strings" - - core "github.com/ipfs/go-ipfs/core" - namesys "github.com/ipfs/go-ipfs/namesys" - - nsopts "github.com/ipfs/interface-go-ipfs-core/options/namesys" - isd "github.com/jbenet/go-is-domain" -) - -// IPNSHostnameOption rewrites an incoming request if its Host: header contains -// an IPNS name. -// The rewritten request points at the resolved name on the gateway handler. -func IPNSHostnameOption() ServeOption { - return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { - childMux := http.NewServeMux() - mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - ctx, cancel := context.WithCancel(n.Context()) - defer cancel() - - host := strings.SplitN(r.Host, ":", 2)[0] - if len(host) > 0 && isd.IsDomain(host) { - name := "/ipns/" + host - _, err := n.Namesys.Resolve(ctx, name, nsopts.Depth(1)) - if err == nil || err == namesys.ErrResolveRecursion { - r.URL.Path = name + r.URL.Path - } - } - childMux.ServeHTTP(w, r) - }) - return childMux, nil - } -} diff --git a/docs/config.md b/docs/config.md index 5a3446b9dd9..9efe56c5359 100644 --- a/docs/config.md +++ b/docs/config.md @@ -83,10 +83,13 @@ Available profiles: - [`Routing.Type`](#routingtype) - [`Gateway`](#gateway) - [`Gateway.NoFetch`](#gatewaynofetch) + - [`Gateway.NoDNSLink`](#gatewaynodnslink) - [`Gateway.HTTPHeaders`](#gatewayhttpheaders) - [`Gateway.RootRedirect`](#gatewayrootredirect) - [`Gateway.Writable`](#gatewaywritable) - [`Gateway.PathPrefixes`](#gatewaypathprefixes) + - [`Gateway.PublicGateways`](#gatewaypublicgateways) + - [`Gateway` recipes](#gateway-recipes) - [`Identity`](#identity) - [`Identity.PeerID`](#identitypeerid) - [`Identity.PrivKey`](#identityprivkey) @@ -348,6 +351,14 @@ and will not fetch files from the network. Default: `false` +### `Gateway.NoDNSLink` + +A boolean to configure whether DNSLink lookup for value in `Host` HTTP header +should be performed. If DNSLink is present, content path stored in the DNS TXT +record becomes the `/` and respective payload is returned to the client. + +Default: `false` + ### `Gateway.HTTPHeaders` Headers to set on gateway responses. @@ -379,7 +390,6 @@ A boolean to configure whether the gateway is writeable or not. Default: `false` - ### `Gateway.PathPrefixes` Array of acceptable url paths that a client can specify in X-Ipfs-Path-Prefix @@ -409,6 +419,145 @@ location /blog/ { Default: `[]` + +### `Gateway.PublicGateways` + +`PublicGateways` is a dictionary for defining gateway behavior on specified hostnames. + +#### `Gateway.PublicGateways: Paths` + +Array of paths that should be exposed on the hostname. + +Example: +```json +{ + "Gateway": { + "PublicGateways": { + "example.com": { + "Paths": ["/ipfs", "/ipns"], +``` + +Above enables `http://example.com/ipfs/*` and `http://example.com/ipns/*` but not `http://example.com/api/*` + +Default: `[]` + +#### `Gateway.PublicGateways: UseSubdomains` + +A boolean to configure whether the gateway at the hostname provides [Origin isolation](https://developer.mozilla.org/en-US/docs/Web/Security/Same-origin_policy) +between content roots. + +- `true` - enables [subdomain gateway](#https://docs-beta.ipfs.io/how-to/address-ipfs-on-web/#subdomain-gateway) at `http://*.{hostname}/` + - **Requires whitelist:** make sure respective `Paths` are set. + For example, `Paths: ["/ipfs", "/ipns"]` are required for `http://{cid}.ipfs.{hostname}` and `http://{foo}.ipns.{hostname}` to work: + ```json + { + "Gateway": { + "PublicGateways": { + "dweb.link": { + "UseSubdomains": true, + "Paths": ["/ipfs", "/ipns"], + ``` + - **Backward-compatible:** requests for content paths such as `http://{hostname}/ipfs/{cid}` produce redirect to `http://{cid}.ipfs.{hostname}` + - **API:** if `/api` is on the `Paths` whitelist, `http://{hostname}/api/{cmd}` produces redirect to `http://api.{hostname}/api/{cmd}` + +- `false` - enables [path gateway](https://docs-beta.ipfs.io/how-to/address-ipfs-on-web/#path-gateway) at `http://{hostname}/*` + - Example: + ```json + { + "Gateway": { + "PublicGateways": { + "ipfs.io": { + "UseSubdomains": false, + "Paths": ["/ipfs", "/ipns", "/api"], + ``` + + +Default: `false` + + +#### `Gateway.PublicGateways: NoDNSLink` + +A boolean to configure whether DNSLink for hostname present in `Host` +HTTP header should be resolved. Overrides global setting. +If `Paths` are defined, they take priority over DNSLink. + +Default: `false` (DNSLink lookup enabled by default for every defined hostname) + +#### Implicit defaults of `Gateway.PublicGateways` + +Default entries for `localhost` hostname and loopback IPs are always present. +If additional config is provided for those hostnames, it will be merged on top of implicit values: +```json +{ + "Gateway": { + "PublicGateways": { + "localhost": { + "Paths": ["/ipfs", "/ipns"], + "UseSubdomains": true + } + } + } +} +``` + +It is also possible to remove a default by setting it to `null`. +For example, to disable subdomain gateway on `localhost` +and make that hostname act the same as `127.0.0.1`: + +```console +$ ipfs config --json Gateway.PublicGateways '{"localhost": null }' +``` + +### `Gateway` recipes + +Below is a list of the most common public gateway setups. + +* Public [subdomain gateway](https://docs-beta.ipfs.io/how-to/address-ipfs-on-web/#subdomain-gateway) at `http://{cid}.ipfs.dweb.link` (each content root gets its own Origin) + ```console + $ ipfs config --json Gateway.PublicGateways '{ + "dweb.link": { + "UseSubdomains": true, + "Paths": ["/ipfs", "/ipns"] + } + }' + ``` + **Note:** this enables automatic redirects from content paths to subdomains + `http://dweb.link/ipfs/{cid}` → `http://{cid}.ipfs.dweb.link` + +* Public [path gateway](https://docs-beta.ipfs.io/how-to/address-ipfs-on-web/#path-gateway) at `http://ipfs.io/ipfs/{cid}` (no Origin separation) + ```console + $ ipfs config --json Gateway.PublicGateways '{ + "ipfs.io": { + "UseSubdomains": false, + "Paths": ["/ipfs", "/ipns", "/api"] + } + }' + ``` + +* Public [DNSLink](https://dnslink.io/) gateway resolving every hostname passed in `Host` header. + ```console + $ ipfs config --json Gateway.NoDNSLink true + ``` + * Note that `NoDNSLink: false` is the default (it works out of the box unless set to `true` manually) + +* Hardened, site-specific [DNSLink gateway](https://docs-beta.ipfs.io/how-to/address-ipfs-on-web/#dnslink-gateway). + Disable fetching of remote data (`NoFetch: true`) + and resolving DNSLink at unknown hostnames (`NoDNSLink: true`). + Then, enable DNSLink gateway only for the specific hostname (for which data + is already present on the node), without exposing any content-addressing `Paths`: + "NoFetch": true, + "NoDNSLink": true, + ```console + $ ipfs config --json Gateway.NoFetch true + $ ipfs config --json Gateway.NoDNSLink true + $ ipfs config --json Gateway.PublicGateways '{ + "en.wikipedia-on-ipfs.org": { + "NoDNSLink": false, + "Paths": [] + } + }' + ``` + ## `Identity` ### `Identity.PeerID` diff --git a/docs/environment-variables.md b/docs/environment-variables.md index fdc92f461c3..b64a4e2481d 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -84,7 +84,7 @@ Default: https://ipfs.io/ipfs/$something (depends on the IPFS version) ## `IPFS_NS_MAP` -Prewarms namesys cache with static records for deteministic tests and debugging. +Adds static namesys records for deteministic tests and debugging. Useful for testing things like DNSLink without real DNS lookup. Example: diff --git a/go.mod b/go.mod index 53ccb013976..c1823843bd9 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/ipfs/go-ipfs-blockstore v0.1.4 github.com/ipfs/go-ipfs-chunker v0.0.4 github.com/ipfs/go-ipfs-cmds v0.1.2 - github.com/ipfs/go-ipfs-config v0.2.1 + github.com/ipfs/go-ipfs-config v0.3.0 github.com/ipfs/go-ipfs-ds-help v0.1.1 github.com/ipfs/go-ipfs-exchange-interface v0.0.1 github.com/ipfs/go-ipfs-exchange-offline v0.0.1 diff --git a/go.sum b/go.sum index 88d51c67606..9a8b551dc0b 100644 --- a/go.sum +++ b/go.sum @@ -268,14 +268,10 @@ github.com/ipfs/go-ipfs-chunker v0.0.1 h1:cHUUxKFQ99pozdahi+uSC/3Y6HeRpi9oTeUHbE github.com/ipfs/go-ipfs-chunker v0.0.1/go.mod h1:tWewYK0we3+rMbOh7pPFGDyypCtvGcBFymgY4rSDLAw= github.com/ipfs/go-ipfs-chunker v0.0.4 h1:nb2ZIgtOk0TxJ5KDBEk+sv6iqJTF/PHg6owN2xCrUjE= github.com/ipfs/go-ipfs-chunker v0.0.4/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8= -github.com/ipfs/go-ipfs-cmds v0.1.1 h1:H9/BLf5rcsULHMj/x8gC0e5o+raYhqk1OQsfzbGMNM4= -github.com/ipfs/go-ipfs-cmds v0.1.1/go.mod h1:k1zMXcOLtljA9iAnZHddbH69yVm5+weRL0snmMD/rK0= -github.com/ipfs/go-ipfs-cmds v0.1.2-0.20200316211807-0c2a21b0dacc h1:HIG2l6XUnov+M6UwcUKKrwGc8Q+n9AYGbiGM4pK21SM= -github.com/ipfs/go-ipfs-cmds v0.1.2-0.20200316211807-0c2a21b0dacc/go.mod h1:a9LyFOtQCnVc3BvbAgW+GrMXEuN29aLCNi3Wk0IM8wo= github.com/ipfs/go-ipfs-cmds v0.1.2 h1:02FLzTA9jYRle/xdMWYwGwxu3gzC3GhPUaz35dH+FrY= github.com/ipfs/go-ipfs-cmds v0.1.2/go.mod h1:a9LyFOtQCnVc3BvbAgW+GrMXEuN29aLCNi3Wk0IM8wo= -github.com/ipfs/go-ipfs-config v0.2.1 h1:Mpyvdf9Zc8k3jg+sRe8e9iylYXHYXqFMuePUjAZQvsE= -github.com/ipfs/go-ipfs-config v0.2.1/go.mod h1:zCKH1uf1XIvf67589BnQ5IAv/Pld2J3gQoQYvG8TK8w= +github.com/ipfs/go-ipfs-config v0.3.0 h1:fGs3JBqB9ia/Joi8up47uiKn150EOEqqVFwv8HZqXao= +github.com/ipfs/go-ipfs-config v0.3.0/go.mod h1:nSLCFtlaL+2rbl3F+9D4gQZQbT1LjRKx7TJg/IHz6oM= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= diff --git a/namesys/namesys.go b/namesys/namesys.go index 11f4646f174..a486b83b8a3 100644 --- a/namesys/namesys.go +++ b/namesys/namesys.go @@ -2,11 +2,13 @@ package namesys import ( "context" + "fmt" "os" "strings" "time" lru "github.com/hashicorp/golang-lru" + cid "github.com/ipfs/go-cid" ds "github.com/ipfs/go-datastore" path "github.com/ipfs/go-path" opts "github.com/ipfs/interface-go-ipfs-core/options/namesys" @@ -14,7 +16,6 @@ import ( ci "github.com/libp2p/go-libp2p-core/crypto" peer "github.com/libp2p/go-libp2p-core/peer" routing "github.com/libp2p/go-libp2p-core/routing" - mh "github.com/multiformats/go-multihash" ) // mpns (a multi-protocol NameSystem) implements generic IPFS naming. @@ -133,12 +134,28 @@ func (ns *mpns) resolveOnceAsync(ctx context.Context, name string, options opts. } // Resolver selection: - // 1. if it is a multihash resolve through "ipns". + // 1. if it is a PeerID/CID/multihash resolve through "ipns". // 2. if it is a domain name, resolve through "dns" // 3. otherwise resolve through the "proquint" resolver var res resolver - if _, err := mh.FromB58String(key); err == nil { + _, err := peer.Decode(key) + + // CIDs in IPNS are expected to have libp2p-key multicodec + // We ease the transition by returning a more meaningful error with a valid CID + if err != nil && err.Error() == "can't convert CID of type protobuf to a peer ID" { + ipnsCid, cidErr := cid.Decode(key) + if cidErr == nil && ipnsCid.Version() == 1 && ipnsCid.Type() != cid.Libp2pKey { + fixedCid := cid.NewCidV1(cid.Libp2pKey, ipnsCid.Hash()).String() + codecErr := fmt.Errorf("peer ID represented as CIDv1 require libp2p-key multicodec: retry with /ipns/%s", fixedCid) + log.Debugf("RoutingResolver: could not convert public key hash %s to peer ID: %s\n", key, codecErr) + out <- onceResult{err: codecErr} + close(out) + return out + } + } + + if err == nil { res = ns.ipnsResolver } else if isd.IsDomain(key) { res = ns.dnsResolver diff --git a/namesys/namesys_test.go b/namesys/namesys_test.go index a0ffbc50d80..b3e963c9e9e 100644 --- a/namesys/namesys_test.go +++ b/namesys/namesys_test.go @@ -11,7 +11,7 @@ import ( offroute "github.com/ipfs/go-ipfs-routing/offline" ipns "github.com/ipfs/go-ipns" path "github.com/ipfs/go-path" - "github.com/ipfs/go-unixfs" + unixfs "github.com/ipfs/go-unixfs" opts "github.com/ipfs/interface-go-ipfs-core/options/namesys" ci "github.com/libp2p/go-libp2p-core/crypto" peer "github.com/libp2p/go-libp2p-core/peer" @@ -49,10 +49,12 @@ func (r *mockResolver) resolveOnceAsync(ctx context.Context, name string, option func mockResolverOne() *mockResolver { return &mockResolver{ entries: map[string]string{ - "QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy": "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", - "QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n": "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", - "QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD": "/ipns/ipfs.io", - "QmQ4QZh8nrsczdUEwTyfBope4THUhqxqc1fx6qYhhzZQei": "/ipfs/QmP3ouCnU8NNLsW6261pAx2pNLV2E4dQoisB1sgda12Act", + "QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy": "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", + "QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n": "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", + "QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD": "/ipns/ipfs.io", + "QmQ4QZh8nrsczdUEwTyfBope4THUhqxqc1fx6qYhhzZQei": "/ipfs/QmP3ouCnU8NNLsW6261pAx2pNLV2E4dQoisB1sgda12Act", + "12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5": "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", // ed25519+identity multihash + "bafzbeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm": "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", // cidv1 in base32 with libp2p-key multicodec }, } } @@ -82,6 +84,8 @@ func TestNamesysResolution(t *testing.T) { testResolution(t, r, "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", 1, "/ipns/ipfs.io", ErrResolveRecursion) testResolution(t, r, "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", 2, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", ErrResolveRecursion) testResolution(t, r, "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", 3, "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", ErrResolveRecursion) + testResolution(t, r, "/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", 1, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", ErrResolveRecursion) + testResolution(t, r, "/ipns/bafzbeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", 1, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", ErrResolveRecursion) } func TestPublishWithCache0(t *testing.T) { diff --git a/namesys/routing.go b/namesys/routing.go index c2d0d0252d4..60928fbca69 100644 --- a/namesys/routing.go +++ b/namesys/routing.go @@ -59,6 +59,7 @@ func (r *IpnsResolver) resolveOnceAsync(ctx context.Context, name string, option } name = strings.TrimPrefix(name, "/ipns/") + pid, err := peer.Decode(name) if err != nil { log.Debugf("RoutingResolver: could not convert public key hash %s to peer ID: %s\n", name, err) diff --git a/test/sharness/t0111-gateway-writeable.sh b/test/sharness/t0111-gateway-writeable.sh index f63b49bcf4b..6708b91a6db 100755 --- a/test/sharness/t0111-gateway-writeable.sh +++ b/test/sharness/t0111-gateway-writeable.sh @@ -41,7 +41,7 @@ test_expect_success "HTTP gateway gives access to sample file" ' test_expect_success "HTTP POST file gives Hash" ' echo "$RANDOM" >infile && - URL="http://localhost:$port/ipfs/" && + URL="http://127.0.0.1:$port/ipfs/" && curl -svX POST --data-binary @infile "$URL" 2>curl_post.out && grep "HTTP/1.1 201 Created" curl_post.out && LOCATION=$(grep Location curl_post.out) && @@ -49,7 +49,7 @@ test_expect_success "HTTP POST file gives Hash" ' ' test_expect_success "We can HTTP GET file just created" ' - URL="http://localhost:${port}${HASH}" && + URL="http://127.0.0.1:${port}${HASH}" && curl -so outfile "$URL" && test_cmp infile outfile ' @@ -60,7 +60,7 @@ test_expect_success "We got the correct hash" ' ' test_expect_success "HTTP GET empty directory" ' - URL="http://localhost:$port/ipfs/$HASH_EMPTY_DIR/" && + URL="http://127.0.0.1:$port/ipfs/$HASH_EMPTY_DIR/" && echo "GET $URL" && curl -so outfile "$URL" 2>curl_getEmpty.out && grep "Index of /ipfs/$HASH_EMPTY_DIR/" outfile @@ -68,7 +68,7 @@ test_expect_success "HTTP GET empty directory" ' test_expect_success "HTTP PUT file to construct a hierarchy" ' echo "$RANDOM" >infile && - URL="http://localhost:$port/ipfs/$HASH_EMPTY_DIR/test.txt" && + URL="http://127.0.0.1:$port/ipfs/$HASH_EMPTY_DIR/test.txt" && echo "PUT $URL" && curl -svX PUT --data-binary @infile "$URL" 2>curl_put.out && grep "HTTP/1.1 201 Created" curl_put.out && @@ -77,7 +77,7 @@ test_expect_success "HTTP PUT file to construct a hierarchy" ' ' test_expect_success "We can HTTP GET file just created" ' - URL="http://localhost:$port/ipfs/$HASH/test.txt" && + URL="http://127.0.0.1:$port/ipfs/$HASH/test.txt" && echo "GET $URL" && curl -so outfile "$URL" && test_cmp infile outfile @@ -85,7 +85,7 @@ test_expect_success "We can HTTP GET file just created" ' test_expect_success "HTTP PUT file to append to existing hierarchy" ' echo "$RANDOM" >infile2 && - URL="http://localhost:$port/ipfs/$HASH/test/test.txt" && + URL="http://127.0.0.1:$port/ipfs/$HASH/test/test.txt" && echo "PUT $URL" && curl -svX PUT --data-binary @infile2 "$URL" 2>curl_putAgain.out && grep "HTTP/1.1 201 Created" curl_putAgain.out && @@ -95,7 +95,7 @@ test_expect_success "HTTP PUT file to append to existing hierarchy" ' test_expect_success "We can HTTP GET file just updated" ' - URL="http://localhost:$port/ipfs/$HASH/test/test.txt" && + URL="http://127.0.0.1:$port/ipfs/$HASH/test/test.txt" && echo "GET $URL" && curl -svo outfile2 "$URL" 2>curl_getAgain.out && test_cmp infile2 outfile2 @@ -103,7 +103,7 @@ test_expect_success "We can HTTP GET file just updated" ' test_expect_success "HTTP PUT to replace a directory" ' echo "$RANDOM" >infile3 && - URL="http://localhost:$port/ipfs/$HASH/test" && + URL="http://127.0.0.1:$port/ipfs/$HASH/test" && echo "PUT $URL" && curl -svX PUT --data-binary @infile3 "$URL" 2>curl_putOverDirectory.out && grep "HTTP/1.1 201 Created" curl_putOverDirectory.out && @@ -112,7 +112,7 @@ test_expect_success "HTTP PUT to replace a directory" ' ' test_expect_success "We can HTTP GET file just put over a directory" ' - URL="http://localhost:$port/ipfs/$HASH/test" && + URL="http://127.0.0.1:$port/ipfs/$HASH/test" && echo "GET $URL" && curl -svo outfile3 "$URL" 2>curl_getOverDirectory.out && test_cmp infile3 outfile3 @@ -120,7 +120,7 @@ test_expect_success "We can HTTP GET file just put over a directory" ' test_expect_success "HTTP PUT to /ipns fails" ' PEERID=`ipfs id --format=""` && - URL="http://localhost:$port/ipns/$PEERID/test.txt" && + URL="http://127.0.0.1:$port/ipns/$PEERID/test.txt" && echo "PUT $URL" && curl -svX PUT --data-binary @infile1 "$URL" 2>curl_putIpns.out && grep "HTTP/1.1 400 Bad Request" curl_putIpns.out diff --git a/test/sharness/t0114-gateway-subdomains.sh b/test/sharness/t0114-gateway-subdomains.sh new file mode 100755 index 00000000000..38aa4cc9710 --- /dev/null +++ b/test/sharness/t0114-gateway-subdomains.sh @@ -0,0 +1,641 @@ +#!/usr/bin/env bash +# +# Copyright (c) Protocol Labs + +test_description="Test subdomain support on the HTTP gateway" + + +. lib/test-lib.sh + +## ============================================================================ +## Helpers specific to subdomain tests +## ============================================================================ + +# Helper that tests gateway response over direct HTTP +# and in all supported HTTP proxy modes +test_localhost_gateway_response_should_contain() { + local label="$1" + local expected="$3" + + # explicit "Host: $hostname" header to match browser behavior + # and also make tests independent from DNS + local host=$(echo $2 | cut -d'/' -f3 | cut -d':' -f1) + local hostname=$(echo $2 | cut -d'/' -f3 | cut -d':' -f1,2) + + # Proxy is the same as HTTP Gateway, we use raw IP and port to be sure + local proxy="http://127.0.0.1:$GWAY_PORT" + + # Create a raw URL version with IP to ensure hostname from Host header is used + # (removes false-positives, Host header is used for passing hostname already) + local url="$2" + local rawurl=$(echo "$url" | sed "s/$hostname/127.0.0.1:$GWAY_PORT/") + + #echo "hostname: $hostname" + #echo "url before: $url" + #echo "url after: $rawurl" + + # regular HTTP request + # (hostname in Host header, raw IP in URL) + test_expect_success "$label (direct HTTP)" " + curl -H \"Host: $hostname\" -sD - \"$rawurl\" > response && + test_should_contain \"$expected\" response + " + + # HTTP proxy + # (hostname is passed via URL) + # Note: proxy client should not care, but curl does DNS lookup + # for some reason anyway, so we pass static DNS mapping + test_expect_success "$label (HTTP proxy)" " + curl -x $proxy --resolve $hostname:127.0.0.1 -sD - \"$url\" > response && + test_should_contain \"$expected\" response + " + + # HTTP proxy 1.0 + # (repeating proxy test with older spec, just to be sure) + test_expect_success "$label (HTTP proxy 1.0)" " + curl --proxy1.0 $proxy --resolve $hostname:127.0.0.1 -sD - \"$url\" > response && + test_should_contain \"$expected\" response + " + + # HTTP proxy tunneling (CONNECT) + # https://tools.ietf.org/html/rfc7231#section-4.3.6 + # In HTTP/1.x, the pseudo-method CONNECT + # can be used to convert an HTTP connection into a tunnel to a remote host + test_expect_success "$label (HTTP proxy tunneling)" " + curl --proxytunnel -x $proxy -H \"Host: $hostname\" -sD - \"$rawurl\" > response && + test_should_contain \"$expected\" response + " +} + +# Helper that checks gateway resonse for specific hostname in Host header +test_hostname_gateway_response_should_contain() { + local label="$1" + local hostname="$2" + local url="$3" + local rawurl=$(echo "$url" | sed "s/$hostname/127.0.0.1:$GWAY_PORT/") + local expected="$4" + test_expect_success "$label" " + curl -H \"Host: $hostname\" -sD - \"$rawurl\" > response && + test_should_contain \"$expected\" response + " +} + +## ============================================================================ +## Start IPFS Node and prepare test CIDs +## ============================================================================ + +test_init_ipfs +test_launch_ipfs_daemon --offline + +# CIDv0to1 is necessary because raw-leaves are enabled by default during +# "ipfs add" with CIDv1 and disabled with CIDv0 +test_expect_success "Add test text file" ' + CID_VAL="hello" + CIDv1=$(echo $CID_VAL | ipfs add --cid-version 1 -Q) + CIDv0=$(echo $CID_VAL | ipfs add --cid-version 0 -Q) + CIDv0to1=$(echo "$CIDv0" | ipfs cid base32) +' + +test_expect_success "Add the test directory" ' + mkdir -p testdirlisting/subdir1/subdir2 && + echo "hello" > testdirlisting/hello && + echo "subdir2-bar" > testdirlisting/subdir1/subdir2/bar && + mkdir -p testdirlisting/api && + mkdir -p testdirlisting/ipfs && + echo "I am a txt file" > testdirlisting/api/file.txt && + echo "I am a txt file" > testdirlisting/ipfs/file.txt && + DIR_CID=$(ipfs add -Qr --cid-version 1 testdirlisting) +' + +test_expect_success "Publish test text file to IPNS" ' + PEERID=$(ipfs id --format="") + IPNS_IDv0=$(echo "$PEERID" | ipfs cid format -v 0) + IPNS_IDv1=$(echo "$PEERID" | ipfs cid format -v 1 --codec libp2p-key -b base32) + IPNS_IDv1_DAGPB=$(echo "$IPNS_IDv0" | ipfs cid format -v 1 -b base32) + test_check_peerid "${PEERID}" && + ipfs name publish --allow-offline -Q "/ipfs/$CIDv1" > name_publish_out && + ipfs name resolve "$PEERID" > output && + printf "/ipfs/%s\n" "$CIDv1" > expected2 && + test_cmp expected2 output +' + + +# ensure we start with empty Gateway.PublicGateways +test_expect_success 'start daemon with empty config for Gateway.PublicGateways' ' + test_kill_ipfs_daemon && + ipfs config --json Gateway.PublicGateways "{}" && + test_launch_ipfs_daemon --offline +' + +## ============================================================================ +## Test path-based requests to a local gateway with default config +## (forced redirects to http://*.localhost) +## ============================================================================ + +# /ipfs/ + +# IP remains old school path-based gateway + +test_localhost_gateway_response_should_contain \ + "request for 127.0.0.1/ipfs/{CID} stays on path" \ + "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1" \ + "$CID_VAL" + +# 'localhost' hostname is used for subdomains, and should not return +# payload directly, but redirect to URL with proper origin isolation + +test_localhost_gateway_response_should_contain \ + "request for localhost/ipfs/{CIDv1} redirects to subdomain" \ + "http://localhost:$GWAY_PORT/ipfs/$CIDv1" \ + "Location: http://$CIDv1.ipfs.localhost:$GWAY_PORT/" + +test_localhost_gateway_response_should_contain \ + "request for localhost/ipfs/{CIDv0} redirects to CIDv1 representation in subdomain" \ + "http://localhost:$GWAY_PORT/ipfs/$CIDv0" \ + "Location: http://${CIDv0to1}.ipfs.localhost:$GWAY_PORT/" + +# /ipns/ + +test_localhost_gateway_response_should_contain \ + "request for localhost/ipns/{CIDv0} redirects to CIDv1 with libp2p-key multicodec in subdomain" \ + "http://localhost:$GWAY_PORT/ipns/$IPNS_IDv0" \ + "Location: http://${IPNS_IDv1}.ipns.localhost:$GWAY_PORT/" + +# /ipns/ + +test_localhost_gateway_response_should_contain \ + "request for localhost/ipns/{fqdn} redirects to DNSLink in subdomain" \ + "http://localhost:$GWAY_PORT/ipns/en.wikipedia-on-ipfs.org/wiki" \ + "Location: http://en.wikipedia-on-ipfs.org.ipns.localhost:$GWAY_PORT/wiki" + +# API on localhost subdomain gateway + +# /api/v0 present on the root hostname +test_localhost_gateway_response_should_contain \ + "request for localhost/api" \ + "http://localhost:$GWAY_PORT/api/v0/refs?arg=${DIR_CID}&r=true" \ + "Ref" + +# /api/v0 not mounted on content root subdomains +test_localhost_gateway_response_should_contain \ + "request for {cid}.ipfs.localhost/api returns data if present on the content root" \ + "http://${DIR_CID}.ipfs.localhost:$GWAY_PORT/api/file.txt" \ + "I am a txt file" + +test_localhost_gateway_response_should_contain \ + "request for {cid}.ipfs.localhost/api/v0/refs returns 404" \ + "http://${DIR_CID}.ipfs.localhost:$GWAY_PORT/api/v0/refs?arg=${DIR_CID}&r=true" \ + "404 Not Found" + +## ============================================================================ +## Test subdomain-based requests to a local gateway with default config +## (origin per content root at http://*.localhost) +## ============================================================================ + +# {CID}.ipfs.localhost + +test_localhost_gateway_response_should_contain \ + "request for {CID}.ipfs.localhost should return expected payload" \ + "http://${CIDv1}.ipfs.localhost:$GWAY_PORT" \ + "$CID_VAL" + +# ensure /ipfs/ namespace is not mounted on subdomain +test_localhost_gateway_response_should_contain \ + "request for {CID}.ipfs.localhost/ipfs/{CID} should return HTTP 404" \ + "http://${CIDv1}.ipfs.localhost:$GWAY_PORT/ipfs/$CIDv1" \ + "404 Not Found" + +# ensure requests to /ipfs/* are not blocked, if content root has such subdirectory +test_localhost_gateway_response_should_contain \ + "request for {CID}.ipfs.localhost/ipfs/file.txt should return data from a file in CID content root" \ + "http://${DIR_CID}.ipfs.localhost:$GWAY_PORT/ipfs/file.txt" \ + "I am a txt file" + +# {CID}.ipfs.localhost/sub/dir (Directory Listing) +DIR_HOSTNAME="${DIR_CID}.ipfs.localhost:$GWAY_PORT" + +test_expect_success "valid file and subdirectory paths in directory listing at {cid}.ipfs.localhost" ' + curl -s --resolve $DIR_HOSTNAME:127.0.0.1 "http://$DIR_HOSTNAME" > list_response && + test_should_contain "hello" list_response && + test_should_contain "subdir1" list_response +' + +test_expect_success "valid parent directory path in directory listing at {cid}.ipfs.localhost/sub/dir" ' + curl -s --resolve $DIR_HOSTNAME:127.0.0.1 "http://$DIR_HOSTNAME/subdir1/subdir2/" > list_response && + test_should_contain ".." list_response && + test_should_contain "bar" list_response +' + +test_expect_success "request for deep path resource at {cid}.ipfs.localhost/sub/dir/file" ' + curl -s --resolve $DIR_HOSTNAME:127.0.0.1 "http://$DIR_HOSTNAME/subdir1/subdir2/bar" > list_response && + test_should_contain "subdir2-bar" list_response +' + +# *.ipns.localhost + +# .ipns.localhost + +test_localhost_gateway_response_should_contain \ + "request for {CIDv1-libp2p-key}.ipns.localhost returns expected payload" \ + "http://${IPNS_IDv1}.ipns.localhost:$GWAY_PORT" \ + "$CID_VAL" + +test_localhost_gateway_response_should_contain \ + "request for {CIDv1-dag-pb}.ipns.localhost redirects to CID with libp2p-key multicodec" \ + "http://${IPNS_IDv1_DAGPB}.ipns.localhost:$GWAY_PORT" \ + "Location: http://${IPNS_IDv1}.ipns.localhost:$GWAY_PORT/" + +# .ipns.localhost + +# DNSLink test requires a daemon in online mode with precached /ipns/ mapping +test_kill_ipfs_daemon +DNSLINK_FQDN="dnslink-test.example.com" +export IPFS_NS_MAP="$DNSLINK_FQDN:/ipfs/$CIDv1" +test_launch_ipfs_daemon + +test_localhost_gateway_response_should_contain \ + "request for {dnslink}.ipns.localhost returns expected payload" \ + "http://$DNSLINK_FQDN.ipns.localhost:$GWAY_PORT" \ + "$CID_VAL" + +# api.localhost/api + +# Note: we use DIR_CID so refs -r returns some CIDs for child nodes +test_localhost_gateway_response_should_contain \ + "request for api.localhost returns API response" \ + "http://api.localhost:$GWAY_PORT/api/v0/refs?arg=$DIR_CID&r=true" \ + "Ref" + +## ============================================================================ +## Test subdomain-based requests with a custom hostname config +## (origin per content root at http://*.example.com) +## ============================================================================ + +# set explicit subdomain gateway config for the hostname +ipfs config --json Gateway.PublicGateways '{ + "example.com": { + "UseSubdomains": true, + "Paths": ["/ipfs", "/ipns", "/api"] + } +}' || exit 1 +# restart daemon to apply config changes +test_kill_ipfs_daemon +test_launch_ipfs_daemon --offline + + +# example.com/ip(f|n)s/* +# ============================================================================= + +# path requests to the root hostname should redirect +# to a subdomain URL with proper origin isolation + +test_hostname_gateway_response_should_contain \ + "request for example.com/ipfs/{CIDv1} produces redirect to {CIDv1}.ipfs.example.com" \ + "example.com" \ + "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1" \ + "Location: http://$CIDv1.ipfs.example.com/" + +# error message should include original CID +# (and it should be case-sensitive, as we can't assume everyone uses base32) +test_hostname_gateway_response_should_contain \ + "request for example.com/ipfs/{InvalidCID} produces useful error before redirect" \ + "example.com" \ + "http://127.0.0.1:$GWAY_PORT/ipfs/QmInvalidCID" \ + 'invalid path \"/ipfs/QmInvalidCID\"' + +test_hostname_gateway_response_should_contain \ + "request for example.com/ipfs/{CIDv0} produces redirect to {CIDv1}.ipfs.example.com" \ + "example.com" \ + "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv0" \ + "Location: http://${CIDv0to1}.ipfs.example.com/" + +# Support X-Forwarded-Proto +test_expect_success "request for http://example.com/ipfs/{CID} with X-Forwarded-Proto: https produces redirect to HTTPS URL" " + curl -H \"X-Forwarded-Proto: https\" -H \"Host: example.com\" -sD - \"http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1\" > response && + test_should_contain \"Location: https://$CIDv1.ipfs.example.com/\" response +" + + + +# example.com/ipns/ + +test_hostname_gateway_response_should_contain \ + "request for example.com/ipns/{CIDv0} redirects to CIDv1 with libp2p-key multicodec in subdomain" \ + "example.com" \ + "http://127.0.0.1:$GWAY_PORT/ipns/$IPNS_IDv0" \ + "Location: http://${IPNS_IDv1}.ipns.example.com/" + +# example.com/ipns/ + +test_hostname_gateway_response_should_contain \ + "request for example.com/ipns/{fqdn} redirects to DNSLink in subdomain" \ + "example.com" \ + "http://127.0.0.1:$GWAY_PORT/ipns/en.wikipedia-on-ipfs.org/wiki" \ + "Location: http://en.wikipedia-on-ipfs.org.ipns.example.com/wiki" + +# *.ipfs.example.com: subdomain requests made with custom FQDN in Host header + +test_hostname_gateway_response_should_contain \ + "request for {CID}.ipfs.example.com should return expected payload" \ + "${CIDv1}.ipfs.example.com" \ + "http://127.0.0.1:$GWAY_PORT/" \ + "$CID_VAL" + +test_hostname_gateway_response_should_contain \ + "request for {CID}.ipfs.example.com/ipfs/{CID} should return HTTP 404" \ + "${CIDv1}.ipfs.example.com" \ + "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1" \ + "404 Not Found" + +# {CID}.ipfs.example.com/sub/dir (Directory Listing) +DIR_FQDN="${DIR_CID}.ipfs.example.com" + +test_expect_success "valid file and directory paths in directory listing at {cid}.ipfs.example.com" ' + curl -s -H "Host: $DIR_FQDN" http://127.0.0.1:$GWAY_PORT > list_response && + test_should_contain "hello" list_response && + test_should_contain "subdir1" list_response +' + +test_expect_success "valid parent directory path in directory listing at {cid}.ipfs.example.com/sub/dir" ' + curl -s -H "Host: $DIR_FQDN" http://127.0.0.1:$GWAY_PORT/subdir1/subdir2/ > list_response && + test_should_contain ".." list_response && + test_should_contain "bar" list_response +' + +test_expect_success "request for deep path resource {cid}.ipfs.example.com/sub/dir/file" ' + curl -s -H "Host: $DIR_FQDN" http://127.0.0.1:$GWAY_PORT/subdir1/subdir2/bar > list_response && + test_should_contain "subdir2-bar" list_response +' + +# *.ipns.example.com +# ============================================================================ + +# .ipns.example.com + +test_hostname_gateway_response_should_contain \ + "request for {CIDv1-libp2p-key}.ipns.example.com returns expected payload" \ + "${IPNS_IDv1}.ipns.example.com" \ + "http://127.0.0.1:$GWAY_PORT" \ + "$CID_VAL" + +test_hostname_gateway_response_should_contain \ + "request for {CIDv1-dag-pb}.ipns.localhost redirects to CID with libp2p-key multicodec" \ + "${IPNS_IDv1_DAGPB}.ipns.example.com" \ + "http://127.0.0.1:$GWAY_PORT" \ + "Location: http://${IPNS_IDv1}.ipns.example.com/" + +# API on subdomain gateway example.com +# ============================================================================ + +# present at the root domain +test_hostname_gateway_response_should_contain \ + "request for example.com/api/v0/refs returns expected payload when /api is on Paths whitelist" \ + "example.com" \ + "http://127.0.0.1:$GWAY_PORT/api/v0/refs?arg=${DIR_CID}&r=true" \ + "Ref" + +# not mounted on content root subdomains +test_hostname_gateway_response_should_contain \ + "request for {cid}.ipfs.example.com/api returns data if present on the content root" \ + "$DIR_CID.ipfs.example.com" \ + "http://127.0.0.1:$GWAY_PORT/api/file.txt" \ + "I am a txt file" + +test_hostname_gateway_response_should_contain \ + "request for {cid}.ipfs.example.com/api/v0/refs returns 404" \ + "$CIDv1.ipfs.example.com" \ + "http://127.0.0.1:$GWAY_PORT/api/v0/refs?arg=${DIR_CID}&r=true" \ + "404 Not Found" + +# disable /api on example.com +ipfs config --json Gateway.PublicGateways '{ + "example.com": { + "UseSubdomains": true, + "Paths": ["/ipfs", "/ipns"] + } +}' || exit 1 +# restart daemon to apply config changes +test_kill_ipfs_daemon +test_launch_ipfs_daemon --offline + +# not mounted at the root domain +test_hostname_gateway_response_should_contain \ + "request for example.com/api/v0/refs returns 404 if /api not on Paths whitelist" \ + "example.com" \ + "http://127.0.0.1:$GWAY_PORT/api/v0/refs?arg=${DIR_CID}&r=true" \ + "404 Not Found" + +# not mounted on content root subdomains +test_hostname_gateway_response_should_contain \ + "request for {cid}.ipfs.example.com/api returns data if present on the content root" \ + "$DIR_CID.ipfs.example.com" \ + "http://127.0.0.1:$GWAY_PORT/api/file.txt" \ + "I am a txt file" + +# DNSLink: .ipns.example.com +# (not really useful outside of localhost, as setting TLS for more than one +# level of wildcard is a pain, but we support it if someone really wants it) +# ============================================================================ + +# DNSLink test requires a daemon in online mode with precached /ipns/ mapping +test_kill_ipfs_daemon +DNSLINK_FQDN="dnslink-subdomain-gw-test.example.org" +export IPFS_NS_MAP="$DNSLINK_FQDN:/ipfs/$CIDv1" +test_launch_ipfs_daemon + +test_hostname_gateway_response_should_contain \ + "request for {dnslink}.ipns.example.com returns expected payload" \ + "$DNSLINK_FQDN.ipns.example.com" \ + "http://127.0.0.1:$GWAY_PORT" \ + "$CID_VAL" + +# Disable selected Paths for the subdomain gateway hostname +# ============================================================================= + +# disable /ipns for the hostname by not whitelisting it +ipfs config --json Gateway.PublicGateways '{ + "example.com": { + "UseSubdomains": true, + "Paths": ["/ipfs"] + } +}' || exit 1 +# restart daemon to apply config changes +test_kill_ipfs_daemon +test_launch_ipfs_daemon --offline + +# refuse requests to Paths that were not explicitly whitelisted for the hostname +test_hostname_gateway_response_should_contain \ + "request for *.ipns.example.com returns HTTP 404 Not Found when /ipns is not on Paths whitelist" \ + "${IPNS_IDv1}.ipns.example.com" \ + "http://127.0.0.1:$GWAY_PORT" \ + "404 Not Found" + + +## ============================================================================ +## Test path-based requests with a custom hostname config +## ============================================================================ + +# set explicit subdomain gateway config for the hostname +ipfs config --json Gateway.PublicGateways '{ + "example.com": { + "UseSubdomains": false, + "Paths": ["/ipfs"] + } +}' || exit 1 + +# restart daemon to apply config changes +test_kill_ipfs_daemon +test_launch_ipfs_daemon --offline + +# example.com/ip(f|n)s/* smoke-tests +# ============================================================================= + +# confirm path gateway works for /ipfs +test_hostname_gateway_response_should_contain \ + "request for example.com/ipfs/{CIDv1} returns expected payload" \ + "example.com" \ + "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1" \ + "$CID_VAL" + +# refuse subdomain requests on path gateway +# (we don't want false sense of security) +test_hostname_gateway_response_should_contain \ + "request for {CID}.ipfs.example.com/ipfs/{CID} should return HTTP 404 Not Found" \ + "${CIDv1}.ipfs.example.com" \ + "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1" \ + "404 Not Found" + +# refuse requests to Paths that were not explicitly whitelisted for the hostname +test_hostname_gateway_response_should_contain \ + "request for example.com/ipns/ returns HTTP 404 Not Found when /ipns is not on Paths whitelist" \ + "example.com" \ + "http://127.0.0.1:$GWAY_PORT/ipns/$IPNS_IDv1" \ + "404 Not Found" + +## ============================================================================ +## Test DNSLink requests with a custom PublicGateway (hostname config) +## (DNSLink site at http://dnslink-test.example.com) +## ============================================================================ + +test_kill_ipfs_daemon + +# disable wildcard DNSLink gateway +# and enable it on specific NSLink hostname +ipfs config --json Gateway.NoDNSLink true && \ +ipfs config --json Gateway.PublicGateways '{ + "dnslink-enabled-on-fqdn.example.org": { + "NoDNSLink": false, + "UseSubdomains": false, + "Paths": ["/ipfs"] + }, + "only-dnslink-enabled-on-fqdn.example.org": { + "NoDNSLink": false, + "UseSubdomains": false, + "Paths": [] + }, + "dnslink-disabled-on-fqdn.example.com": { + "NoDNSLink": true, + "UseSubdomains": false, + "Paths": [] + } +}' || exit 1 + +# DNSLink test requires a daemon in online mode with precached /ipns/ mapping +DNSLINK_FQDN="dnslink-enabled-on-fqdn.example.org" +ONLY_DNSLINK_FQDN="only-dnslink-enabled-on-fqdn.example.org" +NO_DNSLINK_FQDN="dnslink-disabled-on-fqdn.example.com" +export IPFS_NS_MAP="$DNSLINK_FQDN:/ipfs/$CIDv1,$ONLY_DNSLINK_FQDN:/ipfs/$DIR_CID" + +# restart daemon to apply config changes +test_launch_ipfs_daemon + +# make sure test setup is valid (fail if CoreAPI is unable to resolve) +test_expect_success "spoofed DNSLink record resolves in cli" " + ipfs resolve /ipns/$DNSLINK_FQDN > result && + test_should_contain \"$CIDv1\" result && + ipfs cat /ipns/$DNSLINK_FQDN > result && + test_should_contain \"$CID_VAL\" result +" + +# DNSLink enabled + +test_hostname_gateway_response_should_contain \ + "request for http://{dnslink-fqdn}/ PublicGateway returns expected payload" \ + "$DNSLINK_FQDN" \ + "http://127.0.0.1:$GWAY_PORT/" \ + "$CID_VAL" + +test_hostname_gateway_response_should_contain \ + "request for {dnslink-fqdn}/ipfs/{cid} returns expected payload when /ipfs is on Paths whitelist" \ + "$DNSLINK_FQDN" \ + "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1" \ + "$CID_VAL" + +# Test for a fun edge case: DNSLink-only gateway without /ipfs/ namespace +# mounted, and with subdirectory named "ipfs" ¯\_(ツ)_/¯ +test_hostname_gateway_response_should_contain \ + "request for {dnslink-fqdn}/ipfs/file.txt returns data from content root when /ipfs in not on Paths whitelist" \ + "$ONLY_DNSLINK_FQDN" \ + "http://127.0.0.1:$GWAY_PORT/ipfs/file.txt" \ + "I am a txt file" + +test_hostname_gateway_response_should_contain \ + "request for {dnslink-fqdn}/ipns/{peerid} returns 404 when path is not whitelisted" \ + "$DNSLINK_FQDN" \ + "http://127.0.0.1:$GWAY_PORT/ipns/$IPNS_IDv0" \ + "404 Not Found" + +# DNSLink disabled + +test_hostname_gateway_response_should_contain \ + "request for http://{dnslink-fqdn}/ returns 404 when NoDNSLink=true" \ + "$NO_DNSLINK_FQDN" \ + "http://127.0.0.1:$GWAY_PORT/" \ + "404 Not Found" + +test_hostname_gateway_response_should_contain \ + "request for {dnslink-fqdn}/ipfs/{cid} returns 404 when path is not whitelisted" \ + "$NO_DNSLINK_FQDN" \ + "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv0" \ + "404 Not Found" + + +## ============================================================================ +## Test wildcard DNSLink (any hostname, with default config) +## ============================================================================ + +test_kill_ipfs_daemon + +# enable wildcard DNSLink gateway (any value in Host header) +# and remove custom PublicGateways +ipfs config --json Gateway.NoDNSLink false && \ +ipfs config --json Gateway.PublicGateways '{}' || exit 1 + +# DNSLink test requires a daemon in online mode with precached /ipns/ mapping +DNSLINK_FQDN="wildcard-dnslink-not-in-config.example.com" +export IPFS_NS_MAP="$DNSLINK_FQDN:/ipfs/$CIDv1" + +# restart daemon to apply config changes +test_launch_ipfs_daemon + +# make sure test setup is valid (fail if CoreAPI is unable to resolve) +test_expect_success "spoofed DNSLink record resolves in cli" " + ipfs resolve /ipns/$DNSLINK_FQDN > result && + test_should_contain \"$CIDv1\" result && + ipfs cat /ipns/$DNSLINK_FQDN > result && + test_should_contain \"$CID_VAL\" result +" + +# gateway test +test_hostname_gateway_response_should_contain \ + "request for http://{dnslink-fqdn}/ (wildcard) returns expected payload" \ + "$DNSLINK_FQDN" \ + "http://127.0.0.1:$GWAY_PORT/" \ + "$CID_VAL" + +# ============================================================================= +# ensure we end with empty Gateway.PublicGateways +ipfs config --json Gateway.PublicGateways '{}' +test_kill_ipfs_daemon + +test_done diff --git a/test/sharness/t0160-resolve.sh b/test/sharness/t0160-resolve.sh index 87740685644..62a6f5b3bfa 100755 --- a/test/sharness/t0160-resolve.sh +++ b/test/sharness/t0160-resolve.sh @@ -116,6 +116,15 @@ test_resolve_cmd_b32() { test_resolve_setup_name "self" "/ipfs/$c_hash_b32" test_resolve "/ipns/$self_hash" "/ipfs/$c_hash_b32" --cid-base=base32 + + # peer ID represented as CIDv1 require libp2p-key multicodec + # https://github.com/libp2p/specs/blob/master/RFC/0001-text-peerid-cid.md + local self_hash_b32protobuf=$(echo $self_hash | ipfs cid format -v 1 -b b --codec protobuf) + local self_hash_b32libp2pkey=$(echo $self_hash | ipfs cid format -v 1 -b b --codec libp2p-key) + test_expect_success "resolve of /ipns/{cidv1} with multicodec other than libp2p-key returns a meaningful error" ' + test_expect_code 1 ipfs resolve /ipns/$self_hash_b32protobuf 2>cidcodec_error && + grep "Error: peer ID represented as CIDv1 require libp2p-key multicodec: retry with /ipns/$self_hash_b32libp2pkey" cidcodec_error + ' } diff --git a/test/sharness/t0184-http-proxy-over-p2p.sh b/test/sharness/t0184-http-proxy-over-p2p.sh index 578a3f424df..06a3b9ccb18 100755 --- a/test/sharness/t0184-http-proxy-over-p2p.sh +++ b/test/sharness/t0184-http-proxy-over-p2p.sh @@ -144,10 +144,19 @@ test_expect_success 'configure nodes' ' iptb testbed create -type localipfs -count 2 -force -init && ipfsi 0 config --json Experimental.Libp2pStreamMounting true && ipfsi 1 config --json Experimental.Libp2pStreamMounting true && - ipfsi 0 config --json Experimental.P2pHttpProxy true + ipfsi 0 config --json Experimental.P2pHttpProxy true && ipfsi 0 config --json Addresses.Gateway "[\"/ip4/127.0.0.1/tcp/$IPFS_GATEWAY_PORT\"]" ' +test_expect_success 'configure a subdomain gateway with /p2p/ path whitelisted' " + ipfsi 0 config --json Gateway.PublicGateways '{ + \"example.com\": { + \"UseSubdomains\": true, + \"Paths\": [\"/p2p/\"] + } + }' +" + test_expect_success 'start and connect nodes' ' iptb start -wait && iptb connect 0 1 ' @@ -206,6 +215,30 @@ test_expect_success 'handle multipart/form-data http request' ' curl_send_multipart_form_request 200 ' +# subdomain gateway at *.p2p.example.com requires PeerdID in base32 +RECEIVER_ID_CIDv1=$( ipfs cid format -v 1 -b b --codec libp2p-key -- $RECEIVER_ID) + +# OK: $peerid.p2p.example.com/http/index.txt +test_expect_success "handle http request to a subdomain gateway" ' + serve_content "SUBDOMAIN PROVIDES ORIGIN ISOLATION PER RECEIVER_ID" && + curl -H "Host: $RECEIVER_ID_CIDv1.p2p.example.com" -sD - $SENDER_GATEWAY/http/index.txt > p2p_response && + test_should_contain "SUBDOMAIN PROVIDES ORIGIN ISOLATION PER RECEIVER_ID" p2p_response +' + +# FAIL: $peerid.p2p.example.com/p2p/$peerid/http/index.txt +test_expect_success "handle invalid http request to a subdomain gateway" ' + serve_content "SUBDOMAIN DOES NOT SUPPORT FULL /p2p/ PATH" && + curl -H "Host: $RECEIVER_ID_CIDv1.p2p.example.com" -sD - $SENDER_GATEWAY/p2p/$RECEIVER_ID/http/index.txt > p2p_response && + test_should_contain "400 Bad Request" p2p_response +' + +# REDIRECT: example.com/p2p/$peerid/http/index.txt → $peerid.p2p.example.com/http/index.txt +test_expect_success "redirect http path request to subdomain gateway" ' + serve_content "SUBDOMAIN ROOT REDIRECTS /p2p/ PATH TO SUBDOMAIN" && + curl -H "Host: example.com" -sD - $SENDER_GATEWAY/p2p/$RECEIVER_ID/http/index.txt > p2p_response && + test_should_contain "Location: http://$RECEIVER_ID_CIDv1.p2p.example.com/http/index.txt" p2p_response +' + test_expect_success 'stop http server' ' teardown_remote_server ' From f9567a0a0fb12b2d27735708e85b9e16a84f528c Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 11 Mar 2020 18:07:28 +0100 Subject: [PATCH 5/6] fix(gateway): curl without redirect on localhost When request is sent to http://localhost:8080/ipfs/$cid response has HTTP 301 status code and "Location" header with redirect destination at $cid.ipfs.localhost:8080 Redirect is followed by browsersi, but not by commandline tools. Status 301 is ignored by curl in default mode: it will print response and won't follow redirect, user needs to add -L for that. To fix curl, we return correct payload in body of HTTP 301 response, but set Clear-Site-Data header to ensure Origin sandbox can't be abused. This requires a surgical workaround: If Location header is present in ResponseWriter's Header map, we ensure http.ServeContent() returns HTTP 301 Context: https://github.com/ipfs/go-ipfs/pull/6982 License: MIT Signed-off-by: Marcin Rataj --- core/corehttp/gateway_handler.go | 20 ++++++++++++++++++++ core/corehttp/hostname.go | 20 ++++++++++++++++++-- test/sharness/t0114-gateway-subdomains.sh | 23 ++++++++++++++++++++++- 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index cf74202434a..d3c4d26392d 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -38,6 +38,25 @@ type gatewayHandler struct { api coreiface.CoreAPI } +// StatusResponseWriter enables us to override HTTP Status Code passed to +// WriteHeader function inside of http.ServeContent. Decision is based on +// presence of HTTP Headers such as Location. +type statusResponseWriter struct { + http.ResponseWriter +} + +func (sw *statusResponseWriter) WriteHeader(code int) { + // Check if we need to adjust Status Code to account for scheduled redirect + // This enables us to return payload along with HTTP 301 + // for subdomain redirect in web browsers while also returning body for cli + // tools which do not follow redirects by default (curl, wget). + redirect := sw.ResponseWriter.Header().Get("Location") + if redirect != "" && code == http.StatusOK { + code = http.StatusMovedPermanently + } + sw.ResponseWriter.WriteHeader(code) +} + func newGatewayHandler(c GatewayConfig, api coreiface.CoreAPI) *gatewayHandler { i := &gatewayHandler{ config: c, @@ -366,6 +385,7 @@ func (i *gatewayHandler) serveFile(w http.ResponseWriter, req *http.Request, nam } w.Header().Set("Content-Type", ctype) + w = &statusResponseWriter{w} http.ServeContent(w, req, name, modtime, content) } diff --git a/core/corehttp/hostname.go b/core/corehttp/hostname.go index 910ba5bc87d..143435106e5 100644 --- a/core/corehttp/hostname.go +++ b/core/corehttp/hostname.go @@ -95,8 +95,24 @@ func HostnameOption() ServeOption { // Yes, redirect if applicable // Example: dweb.link/ipfs/{cid} → {cid}.ipfs.dweb.link if newURL, ok := toSubdomainURL(r.Host, r.URL.Path, r); ok { - http.Redirect(w, r, newURL, http.StatusMovedPermanently) - return + // Just to be sure single Origin can't be abused in + // web browsers that ignored the redirect for some + // reason, Clear-Site-Data header clears browsing + // data (cookies, storage etc) associated with + // hostname's root Origin + // Note: we can't use "*" due to bug in Chromium: + // https://bugs.chromium.org/p/chromium/issues/detail?id=898503 + w.Header().Set("Clear-Site-Data", "\"cookies\", \"storage\"") + + // Set "Location" header with redirect destination. + // It is ignored by curl in default mode, but will + // be respected by user agents that follow + // redirects by default, namely web browsers + w.Header().Set("Location", newURL) + + // Note: we continue regular gateway processing: + // HTTP Status Code http.StatusMovedPermanently + // will be set later, in statusResponseWriter } } diff --git a/test/sharness/t0114-gateway-subdomains.sh b/test/sharness/t0114-gateway-subdomains.sh index 38aa4cc9710..d079f56e6c7 100755 --- a/test/sharness/t0114-gateway-subdomains.sh +++ b/test/sharness/t0114-gateway-subdomains.sh @@ -145,10 +145,31 @@ test_localhost_gateway_response_should_contain \ # payload directly, but redirect to URL with proper origin isolation test_localhost_gateway_response_should_contain \ - "request for localhost/ipfs/{CIDv1} redirects to subdomain" \ + "request for localhost/ipfs/{CIDv1} returns status code HTTP 301" \ + "http://localhost:$GWAY_PORT/ipfs/$CIDv1" \ + "301 Moved Permanently" + +test_localhost_gateway_response_should_contain \ + "request for localhost/ipfs/{CIDv1} returns Location HTTP header for subdomain redirect in browsers" \ "http://localhost:$GWAY_PORT/ipfs/$CIDv1" \ "Location: http://$CIDv1.ipfs.localhost:$GWAY_PORT/" +# Responses to the root domain of subdomain gateway hostname should Clear-Site-Data +# https://github.com/ipfs/go-ipfs/issues/6975#issuecomment-597472477 +test_localhost_gateway_response_should_contain \ + "request for localhost/ipfs/{CIDv1} returns Clear-Site-Data header to purge Origin cookies and storage" \ + "http://localhost:$GWAY_PORT/ipfs/$CIDv1" \ + 'Clear-Site-Data: \"cookies\", \"storage\"' + +# We return body with HTTP 301 so existing cli scripts that use path-based +# gateway do not break (curl doesn't auto-redirect without passing -L; wget +# does not span across hostnames by default) +# Context: https://github.com/ipfs/go-ipfs/issues/6975 +test_localhost_gateway_response_should_contain \ + "request for localhost/ipfs/{CIDv1} includes valid payload in body for CLI tools like curl" \ + "http://localhost:$GWAY_PORT/ipfs/$CIDv1" \ + "$CID_VAL" + test_localhost_gateway_response_should_contain \ "request for localhost/ipfs/{CIDv0} redirects to CIDv1 representation in subdomain" \ "http://localhost:$GWAY_PORT/ipfs/$CIDv0" \ From 05fe3086a5823a642622d24d0621f8a46527ce68 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 17 Mar 2020 20:10:52 -0700 Subject: [PATCH 6/6] fix(sharness): fix sharness tests to take removed bootstrap peers into account --- test/sharness/t0120-bootstrap.sh | 26 +------------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/test/sharness/t0120-bootstrap.sh b/test/sharness/t0120-bootstrap.sh index 883e29a8be6..538c5bde0ef 100755 --- a/test/sharness/t0120-bootstrap.sh +++ b/test/sharness/t0120-bootstrap.sh @@ -10,14 +10,6 @@ BP2="/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19 BP3="/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb" BP4="/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt" BP5="/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" -BP6="/ip4/104.236.179.241/tcp/4001/p2p/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM" -BP7="/ip4/104.236.76.40/tcp/4001/p2p/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64" -BP8="/ip4/128.199.219.111/tcp/4001/p2p/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu" -BP9="/ip4/178.62.158.247/tcp/4001/p2p/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd" -BP10="/ip6/2400:6180:0:d0::151:6001/tcp/4001/p2p/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu" -BP11="/ip6/2604:a880:1:20::203:d001/tcp/4001/p2p/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM" -BP12="/ip6/2604:a880:800:10::4a:5001/tcp/4001/p2p/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64" -BP13="/ip6/2a03:b0c0:0:1010::23:1001/tcp/4001/p2p/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd" test_description="Test ipfs repo operations" @@ -99,18 +91,10 @@ test_bootstrap_cmd() { echo "added $BP3" >>add2_expected && echo "added $BP4" >>add2_expected && echo "added $BP5" >>add2_expected && - echo "added $BP6" >>add2_expected && - echo "added $BP7" >>add2_expected && - echo "added $BP8" >>add2_expected && - echo "added $BP9" >>add2_expected && - echo "added $BP10" >>add2_expected && - echo "added $BP11" >>add2_expected && - echo "added $BP12" >>add2_expected && - echo "added $BP13" >>add2_expected && test_cmp add2_expected add2_actual ' - test_bootstrap_list_cmd $BP1 $BP2 $BP3 $BP4 $BP5 $BP6 $BP7 $BP8 $BP9 $BP10 $BP11 $BP12 $BP13 $BP14 $BP15 $BP16 $BP17 + test_bootstrap_list_cmd $BP1 $BP2 $BP3 $BP4 $BP5 test_expect_success "'ipfs bootstrap rm --all' succeeds" ' ipfs bootstrap rm --all >rm2_actual @@ -122,14 +106,6 @@ test_bootstrap_cmd() { echo "removed $BP3" >>rm2_expected && echo "removed $BP4" >>rm2_expected && echo "removed $BP5" >>rm2_expected && - echo "removed $BP6" >>rm2_expected && - echo "removed $BP7" >>rm2_expected && - echo "removed $BP8" >>rm2_expected && - echo "removed $BP9" >>rm2_expected && - echo "removed $BP10" >>rm2_expected && - echo "removed $BP11" >>rm2_expected && - echo "removed $BP12" >>rm2_expected && - echo "removed $BP13" >>rm2_expected && test_cmp rm2_expected rm2_actual '