feat: built-in content blocking based on IPIP-383 (#10161)

Fixes #8492

This introduces "nopfs" as a preloaded plugin into Kubo
with support for denylists from ipfs/specs#383

It automatically makes Kubo watch *.deny files found in:

- /etc/ipfs/denylists
- $XDG_CONFIG_HOME/ipfs/denylists
- $IPFS_PATH/denylists

* test: Gateway.NoFetch and GatewayOverLibp2p

adds missing tests for "no fetch" gateways one can expose,
in both cases the offline mode is done by passing custom
blockservice/exchange into path resolver, which means
global path resolver that has nopfs intercept is not used,
and the content blocking does not happen on these gateways.

* fix: use offline path resolvers where appropriate

this fixes the problem described in
#10161 (comment)
by adding explicit offline path resolvers that are backed
by offline exchange, and using them in NoFetch gateways
instead of the default online ones

---------

Co-authored-by: Henrique Dias <hacdias@gmail.com>
Co-authored-by: Marcin Rataj <lidel@lidel.org>

README.md

@@ -30,6 +30,7 @@ Featureset
 - [HTTP Kubo RPC API](https://docs.ipfs.tech/reference/kubo/rpc/) (`/api/v0`) to access and control the daemon
 - [Command Line Interface](https://docs.ipfs.tech/reference/kubo/cli/) based on (`/api/v0`) RPC API
 - [WebUI](https://github.com/ipfs/ipfs-webui/#readme) to manage the Kubo node
+- [Content blocking](/docs/content-blocking.md) support for operators of public nodes
 
 ### Other implementations
 

core/commands/dag/export.go

@@ -14,26 +14,32 @@ import (
  cid "github.com/ipfs/go-cid"
  ipld "github.com/ipfs/go-ipld-format"
  "github.com/ipfs/kubo/core/commands/cmdenv"
+ "github.com/ipfs/kubo/core/commands/cmdutils"
 
  cmds "github.com/ipfs/go-ipfs-cmds"
  gocar "github.com/ipld/go-car"
  selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse"
 )
 
 func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
- c, err := cid.Decode(req.Arguments[0])
+ // Accept CID or a content path
+ p, err := cmdutils.PathOrCidPath(req.Arguments[0])
  if err != nil {
- return fmt.Errorf(
- "unable to parse root specification (currently only bare CIDs are supported): %s",
- err,
- )
+ return err
  }
 
  api, err := cmdenv.GetApi(env, req)
  if err != nil {
  return err
  }
 
+ // Resolve path and confirm the root block is available, fail fast if not
+ b, err := api.Block().Stat(req.Context, p)
+ if err != nil {
+ return err
+ }
+ c := b.Path().RootCid()
+
  pipeR, pipeW := io.Pipe()
 
  errCh := make(chan error, 2) // we only report the 1st error

core/core.go

@@ -76,35 +76,39 @@ type IpfsNode struct {
  PNetFingerprint libp2p.PNetFingerprint `optional:"true"` // fingerprint of private network
 
  // Services
- Peerstore pstore.Peerstore `optional:"true"` // storage for other Peer instances
- Blockstore bstore.GCBlockstore // the block store (lower level)
- Filestore *filestore.Filestore `optional:"true"` // the filestore blockstore
- BaseBlocks node.BaseBlocks // the raw blockstore, no filestore wrapping
- GCLocker bstore.GCLocker // the locker used to protect the blockstore during gc
- Blocks bserv.BlockService // the block service, get/add blocks.
- DAG ipld.DAGService // the merkle dag service, get/add objects.
- IPLDFetcherFactory fetcher.Factory `name:"ipldFetcher"` // fetcher that paths over the IPLD data model
- UnixFSFetcherFactory fetcher.Factory `name:"unixfsFetcher"` // fetcher that interprets UnixFS data
- Reporter *metrics.BandwidthCounter `optional:"true"`
- Discovery mdns.Service `optional:"true"`
- FilesRoot *mfs.Root
- RecordValidator record.Validator
+ Peerstore pstore.Peerstore `optional:"true"` // storage for other Peer instances
+ Blockstore bstore.GCBlockstore // the block store (lower level)
+ Filestore *filestore.Filestore `optional:"true"` // the filestore blockstore
+ BaseBlocks node.BaseBlocks // the raw blockstore, no filestore wrapping
+ GCLocker bstore.GCLocker // the locker used to protect the blockstore during gc
+ Blocks bserv.BlockService // the block service, get/add blocks.
+ DAG ipld.DAGService // the merkle dag service, get/add objects.
+ IPLDFetcherFactory fetcher.Factory `name:"ipldFetcher"` // fetcher that paths over the IPLD data model
+ UnixFSFetcherFactory fetcher.Factory `name:"unixfsFetcher"` // fetcher that interprets UnixFS data
+ OfflineIPLDFetcherFactory fetcher.Factory `name:"offlineIpldFetcher"` // fetcher that paths over the IPLD data model without fetching new blocks
+ OfflineUnixFSFetcherFactory fetcher.Factory `name:"offlineUnixfsFetcher"` // fetcher that interprets UnixFS data without fetching new blocks
+ Reporter *metrics.BandwidthCounter `optional:"true"`
+ Discovery mdns.Service `optional:"true"`
+ FilesRoot *mfs.Root
+ RecordValidator record.Validator
 
  // Online
- PeerHost p2phost.Host `optional:"true"` // the network host (server+client)
- Peering *peering.PeeringService `optional:"true"`
- Filters *ma.Filters `optional:"true"`
- Bootstrapper io.Closer `optional:"true"` // the periodic bootstrapper
- Routing irouting.ProvideManyRouter `optional:"true"` // the routing system. recommend ipfs-dht
- DNSResolver *madns.Resolver // the DNS resolver
- IPLDPathResolver pathresolver.Resolver `name:"ipldPathResolver"` // The IPLD path resolver
- UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"` // The UnixFS path resolver
- Exchange exchange.Interface // the block exchange + strategy (bitswap)
- Namesys namesys.NameSystem // the name system, resolves paths to hashes
- Provider provider.System // the value provider system
- IpnsRepub *ipnsrp.Republisher `optional:"true"`
- GraphExchange graphsync.GraphExchange `optional:"true"`
- ResourceManager network.ResourceManager `optional:"true"`
+ PeerHost p2phost.Host `optional:"true"` // the network host (server+client)
+ Peering *peering.PeeringService `optional:"true"`
+ Filters *ma.Filters `optional:"true"`
+ Bootstrapper io.Closer `optional:"true"` // the periodic bootstrapper
+ Routing irouting.ProvideManyRouter `optional:"true"` // the routing system. recommend ipfs-dht
+ DNSResolver *madns.Resolver // the DNS resolver
+ IPLDPathResolver pathresolver.Resolver `name:"ipldPathResolver"` // The IPLD path resolver
+ UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"` // The UnixFS path resolver
+ OfflineIPLDPathResolver pathresolver.Resolver `name:"offlineIpldPathResolver"` // The IPLD path resolver that uses only locally available blocks
+ OfflineUnixFSPathResolver pathresolver.Resolver `name:"offlineUnixFSPathResolver"` // The UnixFS path resolver that uses only locally available blocks
+ Exchange exchange.Interface // the block exchange + strategy (bitswap)
+ Namesys namesys.NameSystem // the name system, resolves paths to hashes
+ Provider provider.System // the value provider system
+ IpnsRepub *ipnsrp.Republisher `optional:"true"`
+ GraphExchange graphsync.GraphExchange `optional:"true"`
+ ResourceManager network.ResourceManager `optional:"true"`
 
  PubSub *pubsub.PubSub `optional:"true"`
  PSRouter *psrouter.PubsubValueStore `optional:"true"`

core/corehttp/gateway.go

@@ -81,7 +81,11 @@ func Libp2pGatewayOption() ServeOption {
  return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
  bserv := blockservice.New(n.Blocks.Blockstore(), offline.Exchange(n.Blocks.Blockstore()))
 
- backend, err := gateway.NewBlocksBackend(bserv)
+ backend, err := gateway.NewBlocksBackend(bserv,
+ // GatewayOverLibp2p only returns things that are in local blockstore
+ // (same as Gateway.NoFetch=true), we have to pass offline path resolver
+ gateway.WithResolver(n.OfflineUnixFSPathResolver),
+ )
  if err != nil {
  return nil, err
  }
@@ -111,6 +115,8 @@ func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) {
  bserv := n.Blocks
  var vsRouting routing.ValueStore = n.Routing
  nsys := n.Namesys
+ pathResolver := n.UnixFSPathResolver
+
  if cfg.Gateway.NoFetch {
  bserv = blockservice.New(bserv.Blockstore(), offline.Exchange(bserv.Blockstore()))
 
@@ -130,9 +136,17 @@ func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) {
  if err != nil {
  return nil, fmt.Errorf("error constructing namesys: %w", err)
  }
+
+ // Gateway.NoFetch=true requires offline path resolver
+ // to avoid fetching missing blocks during path traversal
+ pathResolver = n.OfflineUnixFSPathResolver
  }
 
- backend, err := gateway.NewBlocksBackend(bserv, gateway.WithValueStore(vsRouting), gateway.WithNameSystem(nsys))
+ backend, err := gateway.NewBlocksBackend(bserv,
+ gateway.WithValueStore(vsRouting),
+ gateway.WithNameSystem(nsys),
+ gateway.WithResolver(pathResolver),
+ )
  if err != nil {
  return nil, err
  }

core/node/core.go

@@ -7,6 +7,7 @@ import (
  "github.com/ipfs/boxo/blockservice"
  blockstore "github.com/ipfs/boxo/blockstore"
  exchange "github.com/ipfs/boxo/exchange"
+ offline "github.com/ipfs/boxo/exchange/offline"
  "github.com/ipfs/boxo/fetcher"
  bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice"
  "github.com/ipfs/boxo/filestore"
@@ -21,9 +22,6 @@ import (
  format "github.com/ipfs/go-ipld-format"
  "github.com/ipfs/go-unixfsnode"
  dagpb "github.com/ipld/go-codec-dagpb"
- "github.com/ipld/go-ipld-prime"
- basicnode "github.com/ipld/go-ipld-prime/node/basic"
- "github.com/ipld/go-ipld-prime/schema"
  "go.uber.org/fx"
 
  "github.com/ipfs/kubo/core/node/helpers"
@@ -87,43 +85,58 @@ func (s *syncDagService) Session(ctx context.Context) format.NodeGetter {
 // FetchersOut allows injection of fetchers.
 type FetchersOut struct {
  fx.Out
- IPLDFetcher fetcher.Factory `name:"ipldFetcher"`
- UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"`
+ IPLDFetcher fetcher.Factory `name:"ipldFetcher"`
+ UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"`
+ OfflineIPLDFetcher fetcher.Factory `name:"offlineIpldFetcher"`
+ OfflineUnixfsFetcher fetcher.Factory `name:"offlineUnixfsFetcher"`
 }
 
 // FetchersIn allows using fetchers for other dependencies.
 type FetchersIn struct {
  fx.In
- IPLDFetcher fetcher.Factory `name:"ipldFetcher"`
- UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"`
+ IPLDFetcher fetcher.Factory `name:"ipldFetcher"`
+ UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"`
+ OfflineIPLDFetcher fetcher.Factory `name:"offlineIpldFetcher"`
+ OfflineUnixfsFetcher fetcher.Factory `name:"offlineUnixfsFetcher"`
 }
 
 // FetcherConfig returns a fetcher config that can build new fetcher instances
 func FetcherConfig(bs blockservice.BlockService) FetchersOut {
  ipldFetcher := bsfetcher.NewFetcherConfig(bs)
- ipldFetcher.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) {
- if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok {
- return tlnkNd.LinkTargetNodePrototype(), nil
- }
- return basicnode.Prototype.Any, nil
- })
-
+ ipldFetcher.PrototypeChooser = dagpb.AddSupportToChooser(bsfetcher.DefaultPrototypeChooser)
  unixFSFetcher := ipldFetcher.WithReifier(unixfsnode.Reify)
- return FetchersOut{IPLDFetcher: ipldFetcher, UnixfsFetcher: unixFSFetcher}
+
+ // Construct offline versions which we can safely use in contexts where
+ // path resolution should not fetch new blocks via exchange.
+ offlineBs := blockservice.New(bs.Blockstore(), offline.Exchange(bs.Blockstore()))
+ offlineIpldFetcher := bsfetcher.NewFetcherConfig(offlineBs)
+ offlineIpldFetcher.PrototypeChooser = dagpb.AddSupportToChooser(bsfetcher.DefaultPrototypeChooser)
+ offlineUnixFSFetcher := offlineIpldFetcher.WithReifier(unixfsnode.Reify)
+
+ return FetchersOut{
+ IPLDFetcher: ipldFetcher,
+ UnixfsFetcher: unixFSFetcher,
+ OfflineIPLDFetcher: offlineIpldFetcher,
+ OfflineUnixfsFetcher: offlineUnixFSFetcher,
+ }
 }
 
 // PathResolversOut allows injection of path resolvers
 type PathResolversOut struct {
  fx.Out
- IPLDPathResolver pathresolver.Resolver `name:"ipldPathResolver"`
- UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"`
+ IPLDPathResolver pathresolver.Resolver `name:"ipldPathResolver"`
+ UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"`
+ OfflineIPLDPathResolver pathresolver.Resolver `name:"offlineIpldPathResolver"`
+ OfflineUnixFSPathResolver pathresolver.Resolver `name:"offlineUnixFSPathResolver"`
 }
 
 // PathResolverConfig creates path resolvers with the given fetchers.
 func PathResolverConfig(fetchers FetchersIn) PathResolversOut {
  return PathResolversOut{
- IPLDPathResolver: pathresolver.NewBasicResolver(fetchers.IPLDFetcher),
- UnixFSPathResolver: pathresolver.NewBasicResolver(fetchers.UnixfsFetcher),
+ IPLDPathResolver: pathresolver.NewBasicResolver(fetchers.IPLDFetcher),
+ UnixFSPathResolver: pathresolver.NewBasicResolver(fetchers.UnixfsFetcher),
+ OfflineIPLDPathResolver: pathresolver.NewBasicResolver(fetchers.OfflineIPLDFetcher),
+ OfflineUnixFSPathResolver: pathresolver.NewBasicResolver(fetchers.OfflineUnixfsFetcher),
  }
 }
 

docs/changelogs/v0.24.md

@@ -6,6 +6,7 @@
 
 - [Overview](#overview)
 - [🔦 Highlights](#-highlights)
+ - [Support for content blocking](#support-for-content-blocking)
  - [Gateway: the root of the CARs are no longer meaningful](#gateway-the-root-of-the-cars-are-no-longer-meaningful)
  - [IPNS: improved publishing defaults](#ipns-improved-publishing-defaults)
  - [IPNS: record TTL is used for caching](#ipns-record-ttl-is-used-for-caching)
@@ -16,6 +17,14 @@
 
 ### 🔦 Highlights
 
+#### Support for content blocking
+
+This Kubo release ships with built-in content-blocking subsystem [announced earlier this year](https://blog.ipfs.tech/2023-content-blocking-for-the-ipfs-stack/).
+Content blocking is an opt-in decision made by the operator of `ipfs daemon`.
+The official build does not ship with any denylists.
+
+Learn more at [`/docs/content-blocking.md`](https://github.com/ipfs/kubo/blob/master/docs/content-blocking.md)
+
 #### Gateway: the root of the CARs are no longer meaningful
 
 When requesting a CAR from the gateway, the root of the CAR might no longer be

docs/content-blocking.md

@@ -0,0 +1,73 @@
+<h1 align="center">
+ <br>
+ <a href="#readme"><img src="https://github.com/ipfs-shipyard/nopfs/blob/41484a818e6542314f784da852fc41b76f2d48a6/logo.png?raw=true" alt="content blocking logo" title="content blocking in Kubo" width="200"></a>
+ <br>
+ Content Blocking in Kubo
+ <br>
+</h1>
+
+Kubo ships with built-in support for denylist format from [IPIP-383](https://github.com/ipfs/specs/pull/383).
+
+## Default behavior
+
+Official Kubo build does not ship with any denylists enabled by default.
+
+Content blocking is an opt-in decision made by the operator of `ipfs daemon`.
+
+## How to enable blocking
+
+Place a `*.deny` file in one of directories:
+
+- `$IPFS_PATH/denylists/` (`$HOME/.ipfs/denylists/` if `IPFS_PATH` is not set)
+- `$XDG_CONFIG_HOME/ipfs/denylists/` (`$HOME/.config/ipfs/denylists/` if `XDG_CONFIG_HOME` is not set)
+- `/etc/ipfs/denylists/` (global)
+
+Files need to be present before starting the `ipfs daemon` in order to be watched for updates.
+
+If a new denylist file is added, `ipfs daemon` needs to be restarted.
+
+CLI and Gateway users will receive errors in response to request impacted by a blocklist:
+
+```
+Error: /ipfs/QmQvjk82hPkSaZsyJ8vNER5cmzKW7HyGX5XVusK7EAenCN is blocked and cannot be provided
+```
+
+End user is not informed about the exact reason, see [How to
+debug](#how-to-debug) if you need to find out which line of which denylist
+caused the request to be blocked.
+
+## Denylist file format
+
+[NOpfs](https://github.com/ipfs-shipyard/nopfs) supports the format from [IPIP-383](https://github.com/ipfs/specs/pull/383).
+
+Clear-text rules are simple: just put content paths to block, one per line.
+Paths with unicode and whitespace need to be percend-encoded:
+
+```
+/ipfs/QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR
+/ipfs/bafybeihfg3d7rdltd43u3tfvncx7n5loqofbsobojcadtmokrljfthuc7y/927%20-%20Standards/927%20-%20Standards.png
+```
+
+Sensitive content paths can be double-hashed to block without revealing them.
+Double-hashed list example: https://badbits.dwebops.pub/badbits.deny
+
+See [IPIP-383](https://github.com/ipfs/specs/pull/383) for detailed format specification and more examples.
+
+## How to suspend blocking without removing denylists
+
+Set `IPFS_CONTENT_BLOCKING_DISABLE` environment variable to `true` and restart the daemon.
+
+
+## How to debug
+
+Debug logging of `nopfs` subsystem can be enabled with `GOLOG_LOG_LEVEL="nopfs=debug"`
+
+All block events are logged as warnings on a separate level named `nopfs-blocks`.
+
+To only log requests for blocked content set `GOLOG_LOG_LEVEL="nopfs-blocks=warn"`:
+
+```
+WARN (...) QmRFniDxwxoG2n4AcnGhRdjqDjCM5YeUcBE75K8WXmioH3: blocked (test.deny:9)
+```
+
+

docs/environment-variables.md

@@ -131,6 +131,12 @@ The above will replace implicit HTTP routers with single one, allowing for
 inspection/debug of HTTP requests sent by Kubo via `while true ; do nc -l 7423; done`
 or more advanced tools like [mitmproxy](https://docs.mitmproxy.org/stable/#mitmproxy).
 
+
+## `IPFS_CONTENT_BLOCKING_DISABLE`
+
+Disables the content-blocking subsystem. No denylists will be watched and no
+content will be blocked.
+
 ## `LIBP2P_TCP_REUSEPORT`
 
 Kubo tries to reuse the same source port for all connections to improve NAT

docs/examples/kubo-as-a-library/go.mod

@@ -7,7 +7,7 @@ go 1.20
 replace github.com/ipfs/kubo => ./../../..
 
 require (
- github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd
+ github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b
  github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
  github.com/libp2p/go-libp2p v0.31.0
  github.com/multiformats/go-multiaddr v0.11.0
@@ -40,6 +40,7 @@ require (
  github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 // indirect
  github.com/flynn/noise v1.0.0 // indirect
  github.com/francoispqt/gojay v1.2.13 // indirect
+ github.com/fsnotify/fsnotify v1.6.0 // indirect
  github.com/gabriel-vasile/mimetype v1.4.1 // indirect
  github.com/go-logr/logr v1.2.4 // indirect
  github.com/go-logr/stdr v1.2.2 // indirect
@@ -60,6 +61,8 @@ require (
  github.com/hashicorp/golang-lru v0.5.4 // indirect
  github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
  github.com/huin/goupnp v1.2.0 // indirect
+ github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c // indirect
+ github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c // indirect
  github.com/ipfs/bbloom v0.0.4 // indirect
  github.com/ipfs/go-bitfield v1.1.0 // indirect
  github.com/ipfs/go-block-format v0.2.0 // indirect
@@ -194,5 +197,6 @@ require (
  google.golang.org/grpc v1.55.0 // indirect
  google.golang.org/protobuf v1.31.0 // indirect
  gopkg.in/square/go-jose.v2 v2.5.1 // indirect
+ gopkg.in/yaml.v3 v3.0.1 // indirect
  lukechampine.com/blake3 v1.2.1 // indirect
 )