feat(cmds): extend block size check for dag|block put (#8751)

* feat(cmds): extend block size check for dag|block put
* feat(cmds): block size check for dag import
* style: dag-pb → UnixFS, 1MB → 1MiB

Co-authored-by: Marcin Rataj <lidel@lidel.org>

Author: schomatis

core/commands/block.go

@@ -10,6 +10,7 @@ import (
 
 	util "github.com/ipfs/go-ipfs/blocks/blockstoreutil"
 	cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv"
+	"github.com/ipfs/go-ipfs/core/commands/cmdutils"
 
 	cmds "github.com/ipfs/go-ipfs-cmds"
 	options "github.com/ipfs/interface-go-ipfs-core/options"
@@ -138,6 +139,7 @@ other than 'sha2-256' or format to anything other than 'v0' will result in CIDv1
 		cmds.StringOption(mhtypeOptionName, "multihash hash function").WithDefault("sha2-256"),
 		cmds.IntOption(mhlenOptionName, "multihash hash length").WithDefault(-1),
 		cmds.BoolOption(pinOptionName, "pin added blocks recursively").WithDefault(false),
+		cmdutils.AllowBigBlockOption,
 	},
 	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
 		api, err := cmdenv.GetApi(env, req)
@@ -182,6 +184,10 @@ other than 'sha2-256' or format to anything other than 'v0' will result in CIDv1
 				return err
 			}
 
+			if err := cmdutils.CheckBlockSize(req, uint64(p.Size())); err != nil {
+				return err
+			}
+
 			err = res.Emit(&BlockStat{
 				Key:  p.Path().Cid().String(),
 				Size: p.Size(),

core/commands/cmdutils/utils.go

@@ -0,0 +1,51 @@
+package cmdutils
+
+import (
+	"fmt"
+
+	cmds "github.com/ipfs/go-ipfs-cmds"
+
+	"github.com/ipfs/go-cid"
+	coreiface "github.com/ipfs/interface-go-ipfs-core"
+)
+
+const (
+	AllowBigBlockOptionName = "allow-big-block"
+	SoftBlockLimit          = 1024 * 1024 // https://github.com/ipfs/go-ipfs/issues/7421#issuecomment-910833499
+)
+
+var AllowBigBlockOption cmds.Option
+
+func init() {
+	AllowBigBlockOption = cmds.BoolOption(AllowBigBlockOptionName, "Disable block size check and allow creation of blocks bigger than 1MiB. WARNING: such blocks won't be transferable over the standard bitswap.").WithDefault(false)
+}
+
+func CheckCIDSize(req *cmds.Request, c cid.Cid, dagAPI coreiface.APIDagService) error {
+	n, err := dagAPI.Get(req.Context, c)
+	if err != nil {
+		return fmt.Errorf("CheckCIDSize: getting dag: %w", err)
+	}
+
+	nodeSize, err := n.Size()
+	if err != nil {
+		return fmt.Errorf("CheckCIDSize: getting node size: %w", err)
+	}
+
+	return CheckBlockSize(req, nodeSize)
+}
+
+func CheckBlockSize(req *cmds.Request, size uint64) error {
+	allowAnyBlockSize, _ := req.Options[AllowBigBlockOptionName].(bool)
+	if allowAnyBlockSize {
+		return nil
+	}
+
+	// We do not allow producing blocks bigger than 1 MiB to avoid errors
+	// when transmitting them over BitSwap. The 1 MiB constant is an
+	// unenforced and undeclared rule of thumb hard-coded here.
+	if size > SoftBlockLimit {
+		return fmt.Errorf("produced block is over 1MiB: big blocks can't be exchanged with other peers. consider using UnixFS for automatic chunking of bigger files, or pass --allow-big-block to override")
+	}
+	return nil
+
+}

core/commands/dag/dag.go

@@ -5,6 +5,7 @@ import (
 	"io"
 
 	"github.com/ipfs/go-ipfs/core/commands/cmdenv"
+	"github.com/ipfs/go-ipfs/core/commands/cmdutils"
 
 	cid "github.com/ipfs/go-cid"
 	cidenc "github.com/ipfs/go-cidutil/cidenc"
@@ -88,6 +89,7 @@ into an object of the specified format.
 		cmds.StringOption("input-codec", "Codec that the input object is encoded in").WithDefault("dag-json"),
 		cmds.BoolOption("pin", "Pin this object when adding."),
 		cmds.StringOption("hash", "Hash function to use").WithDefault("sha2-256"),
+		cmdutils.AllowBigBlockOption,
 	},
 	Run:  dagPut,
 	Type: OutputObject{},
@@ -205,6 +207,7 @@ Maximum supported CAR version: 1
 		cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing.").WithDefault(true),
 		cmds.BoolOption(silentOptionName, "No output."),
 		cmds.BoolOption(statsOptionName, "Output stats."),
+		cmdutils.AllowBigBlockOption,
 	},
 	Type: CarImportOutput{},
 	Run:  dagImport,

core/commands/dag/import.go

@@ -8,6 +8,7 @@ import (
 	cid "github.com/ipfs/go-cid"
 	files "github.com/ipfs/go-ipfs-files"
 	"github.com/ipfs/go-ipfs/core/commands/cmdenv"
+	"github.com/ipfs/go-ipfs/core/commands/cmdutils"
 	ipld "github.com/ipfs/go-ipld-format"
 	iface "github.com/ipfs/interface-go-ipfs-core"
 	"github.com/ipfs/interface-go-ipfs-core/options"
@@ -180,6 +181,9 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI,
 				} else if block == nil {
 					break
 				}
+				if err := cmdutils.CheckBlockSize(req, uint64(len(block.RawData()))); err != nil {
+					return err
+				}
 
 				// the double-decode is suboptimal, but we need it for batching
 				nd, err := ipld.Decode(block)

core/commands/dag/put.go

@@ -7,6 +7,7 @@ import (
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-ipfs/core/commands/cmdenv"
+	"github.com/ipfs/go-ipfs/core/commands/cmdutils"
 	ipldlegacy "github.com/ipfs/go-ipld-legacy"
 	"github.com/ipld/go-ipld-prime/multicodec"
 	basicnode "github.com/ipld/go-ipld-prime/node/basic"
@@ -102,6 +103,10 @@ func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e
 			Node:  n,
 		}
 
+		if err := cmdutils.CheckBlockSize(req, uint64(bd.Len())); err != nil {
+			return err
+		}
+
 		if err := b.Add(req.Context, &ln); err != nil {
 			return err
 		}

core/commands/get.go

@@ -257,7 +257,7 @@ func getCompressOptions(req *cmds.Request) (int, error) {
 	return cmplvl, nil
 }
 
-// DefaultBufSize is the buffer size for gets. for now, 1MB, which is ~4 blocks.
+// DefaultBufSize is the buffer size for gets. for now, 1MiB, which is ~4 blocks.
 // TODO: does this need to be configurable?
 var DefaultBufSize = 1048576
 

core/commands/object/patch.go

@@ -4,20 +4,14 @@ import (
 	"fmt"
 	"io"
 
-	"github.com/ipfs/go-cid"
 	cmds "github.com/ipfs/go-ipfs-cmds"
 	"github.com/ipfs/go-ipfs/core/commands/cmdenv"
-	coreiface "github.com/ipfs/interface-go-ipfs-core"
+	"github.com/ipfs/go-ipfs/core/commands/cmdutils"
 
 	"github.com/ipfs/interface-go-ipfs-core/options"
 	"github.com/ipfs/interface-go-ipfs-core/path"
 )
 
-const (
-	softBlockLimit = 1024 * 1024 // https://github.com/ipfs/go-ipfs/issues/7421#issuecomment-910833499
-	allowBigBlock  = "allow-big-block"
-)
-
 var ObjectPatchCmd = &cmds.Command{
 	Helptext: cmds.HelpText{
 		Tagline: "Deprecated way to create a new merkledag object based on an existing one. Use MFS with 'files cp|rm' instead.",
@@ -49,7 +43,7 @@ For modern use cases, use MFS with 'files' commands: 'ipfs files --help'.
 		"set-data":    patchSetDataCmd,
 	},
 	Options: []cmds.Option{
-		cmds.BoolOption(allowBigBlock, "Disable block size check and allow creation of blocks bigger than 1MB. WARNING: such blocks won't be transferable over the standard bitswap.").WithDefault(false),
+		cmdutils.AllowBigBlockOption,
 	},
 }
 
@@ -64,7 +58,7 @@ Example:
 	$ echo "hello" | ipfs object patch $HASH append-data
 
 NOTE: This does not append data to a file - it modifies the actual raw
-data within a dag-pb object. Blocks have a max size of 1MB and objects larger than
+data within a dag-pb object. Blocks have a max size of 1MiB and objects larger than
 the limit will not be respected by the network.
 
 DEPRECATED and provided for legacy reasons. Use 'ipfs add' or 'ipfs files' instead.
@@ -92,7 +86,7 @@ DEPRECATED and provided for legacy reasons. Use 'ipfs add' or 'ipfs files' inste
 			return err
 		}
 
-		if err := checkBlockSize(req, p.Cid(), api.Dag()); err != nil {
+		if err := cmdutils.CheckCIDSize(req, p.Cid(), api.Dag()); err != nil {
 			return err
 		}
 
@@ -142,7 +136,7 @@ DEPRECATED and provided for legacy reasons. Use 'files cp' and 'dag put' instead
 			return err
 		}
 
-		if err := checkBlockSize(req, p.Cid(), api.Dag()); err != nil {
+		if err := cmdutils.CheckCIDSize(req, p.Cid(), api.Dag()); err != nil {
 			return err
 		}
 
@@ -184,7 +178,7 @@ DEPRECATED and provided for legacy reasons. Use 'files rm' instead.
 			return err
 		}
 
-		if err := checkBlockSize(req, p.Cid(), api.Dag()); err != nil {
+		if err := cmdutils.CheckCIDSize(req, p.Cid(), api.Dag()); err != nil {
 			return err
 		}
 
@@ -254,7 +248,7 @@ Use MFS and 'files' commands instead:
 			return err
 		}
 
-		if err := checkBlockSize(req, p.Cid(), api.Dag()); err != nil {
+		if err := cmdutils.CheckCIDSize(req, p.Cid(), api.Dag()); err != nil {
 			return err
 		}
 
@@ -268,26 +262,3 @@ Use MFS and 'files' commands instead:
 		}),
 	},
 }
-
-func checkBlockSize(req *cmds.Request, c cid.Cid, dagAPI coreiface.APIDagService) error {
-	allowAnyBlockSize, _ := req.Options[allowBigBlock].(bool)
-	if allowAnyBlockSize {
-		return nil
-	}
-
-	// We do not allow producing blocks bigger than 1 MiB to avoid errors
-	// when transmitting them over BitSwap. The 1 MiB constant is an
-	// unenforced and undeclared rule of thumb hard-coded here.
-	modifiedNode, err := dagAPI.Get(req.Context, c)
-	if err != nil {
-		return err
-	}
-	modifiedNodeSize, err := modifiedNode.Size()
-	if err != nil {
-		return err
-	}
-	if modifiedNodeSize > softBlockLimit {
-		return fmt.Errorf("produced block is over 1MB, object API is deprecated and does not support HAMT-sharding: to create big directories, please use the files API (MFS)")
-	}
-	return nil
-}

test/sharness/t0050-block.sh

@@ -248,4 +248,18 @@ test_expect_success "put with sha3 and cidv0 fails" '
   echo "foooo" | test_must_fail ipfs block put --mhtype=sha3 --mhlen=20 --format=v0
 '
 
+test_expect_success "'ipfs block put' check block size" '
+    dd if=/dev/zero bs=2MB count=1 > 2-MB-file &&
+    test_expect_code 1 ipfs block put 2-MB-file >block_put_out 2>&1
+  '
+
+  test_expect_success "ipfs block put output has the correct error" '
+    grep "produced block is over 1MiB" block_put_out
+  '
+
+  test_expect_success "ipfs block put --allow-big-block=true works" '
+    test_expect_code 0 ipfs block put 2-MB-file --allow-big-block=true &&
+    rm 2-MB-file
+  '
+
 test_done

test/sharness/t0051-object.sh

@@ -229,12 +229,12 @@ test_object_cmd() {
     do
        DIR=$(ipfs object patch "$DIR" add-link "$DIR.jpg" "$DIR")
     done
-    # Fail when new block goes over the BS limit of 1MB, but allow manual override
+    # Fail when new block goes over the BS limit of 1MiB, but allow manual override
     test_expect_code 1 ipfs object patch "$DIR" add-link "$DIR.jpg" "$DIR"  >patch_out 2>&1
   '
 
   test_expect_success "ipfs object patch add-link output has the correct error" '
-    grep "produced block is over 1MB, object API is deprecated and does not support HAMT-sharding: to create big directories, please use the files API (MFS)" patch_out
+    grep "produced block is over 1MiB" patch_out
   '
 
   test_expect_success "ipfs object patch --allow-big-block=true add-link works" '

test/sharness/t0053-dag.sh

@@ -44,6 +44,20 @@ test_dag_cmd() {
     test $EXPHASH = $IPLDHASH
   '
 
+test_expect_success "'ipfs dag put' check block size" '
+    dd if=/dev/zero bs=2MB count=1 > 2-MB-file &&
+    test_expect_code 1 ipfs dag put --input-codec=raw --store-codec=raw 2-MB-file >dag_put_out 2>&1
+  '
+
+  test_expect_success "ipfs dag put output has the correct error" '
+    grep "produced block is over 1MiB" dag_put_out
+  '
+
+  test_expect_success "ipfs dag put --allow-big-block=true works" '
+    test_expect_code 0 ipfs dag put --input-codec=raw --store-codec=raw 2-MB-file --allow-big-block=true &&
+    rm 2-MB-file
+  '
+
   test_expect_success "can add an ipld object using dag-json to dag-json" '
     IPLDHASH=$(cat ipld_object | ipfs dag put --input-codec dag-json --store-codec dag-json)
   '

test/sharness/t0054-dag-car-import-export.sh

@@ -233,4 +233,17 @@ test_expect_success "naked root import expected output" '
    test_cmp_sorted naked_root_import_json_expected naked_root_import_json_actual
 '
 
+test_expect_success "'ipfs dag import' check block size" '
+    BIG_CID=$(dd if=/dev/zero bs=2MB count=1 | ipfs dag put --input-codec=raw --store-codec=raw --allow-big-block) &&
+    ipfs dag export $BIG_CID > 2-MB-block.car &&
+    test_expect_code 1 ipfs dag import 2-MB-block.car >dag_import_out 2>&1
+'
+test_expect_success "ipfs dag import output has the correct error" '
+    grep "block is over 1MiB" dag_import_out
+'
+
+test_expect_success "ipfs dag import --allow-big-block works" '
+    test_expect_code 0 ipfs dag import --allow-big-block 2-MB-block.car
+'
+
 test_done