feat: dag import --stats (#8237)

* feat: report block count on `dag import`
* fix: clean-up dag import message format
* Only print stats when --stats flag is passed

This applies to both text and json output encoding.

- Stats data is now contained within a Stats datastructure
- Stats are printed after root so that first line of output is the same as previously, even when stats are output using --stats

* fix sharness test

* Add PayloadBytesCount to stats

* Attempt to stabilize flaky tests

* Rename PayloadBytesCount to BlockBytesCount

* Correctly calculate size or imported dag

* Use RawSize of original block for import bytes calc

* test: dag import without --stats

basic regression tests for the default output (text and json)

Co-authored-by: gammazero <gammazero@users.noreply.github.com>
Co-authored-by: Marcin Rataj <lidel@lidel.org>

core/commands/dag/dag.go

@@ -16,9 +16,10 @@ import (
 )
 
 const (
+	pinRootsOptionName = "pin-roots"
 	progressOptionName = "progress"
 	silentOptionName   = "silent"
-	pinRootsOptionName = "pin-roots"
+	statsOptionName    = "stats"
 )
 
 // DagCmd provides a subset of commands for interacting with ipld dag objects
@@ -53,9 +54,15 @@ type ResolveOutput struct {
 	RemPath string
 }
 
+type CarImportStats struct {
+	BlockCount      uint64
+	BlockBytesCount uint64
+}
+
 // CarImportOutput is the output type of the 'dag import' commands
 type CarImportOutput struct {
-	Root RootMeta
+	Root  *RootMeta       `json:",omitempty"`
+	Stats *CarImportStats `json:",omitempty"`
 }
 
 // RootMeta is the metadata for a root pinning response
@@ -160,8 +167,10 @@ var DagResolveCmd = &cmds.Command{
 }
 
 type importResult struct {
-	roots map[cid.Cid]struct{}
-	err   error
+	blockCount      uint64
+	blockBytesCount uint64
+	roots           map[cid.Cid]struct{}
+	err             error
 }
 
 // DagImportCmd is a command for importing a car to ipfs
@@ -193,8 +202,9 @@ Maximum supported CAR version: 1
 		cmds.FileArg("path", true, true, "The path of a .car file.").EnableStdin(),
 	},
 	Options: []cmds.Option{
-		cmds.BoolOption(silentOptionName, "No output."),
 		cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing.").WithDefault(true),
+		cmds.BoolOption(silentOptionName, "No output."),
+		cmds.BoolOption(statsOptionName, "Output stats."),
 	},
 	Type: CarImportOutput{},
 	Run:  dagImport,
@@ -206,6 +216,22 @@ Maximum supported CAR version: 1
 				return nil
 			}
 
+			// event should have only one of `Root` or `Stats` set, not both
+			if event.Root == nil {
+				if event.Stats == nil {
+					return fmt.Errorf("Unexpected message from DAG import")
+				}
+				stats, _ := req.Options[statsOptionName].(bool)
+				if stats {
+					fmt.Fprintf(w, "Imported %d blocks (%d bytes)\n", event.Stats.BlockCount, event.Stats.BlockBytesCount)
+				}
+				return nil
+			}
+
+			if event.Stats != nil {
+				return fmt.Errorf("Unexpected message from DAG import")
+			}
+
 			enc, err := cmdenv.GetLowLevelCidEncoder(req)
 			if err != nil {
 				return err

core/commands/dag/import.go

@@ -101,7 +101,7 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
 				failedPins++
 			}
 
-			if err := res.Emit(&CarImportOutput{Root: ret}); err != nil {
+			if err := res.Emit(&CarImportOutput{Root: &ret}); err != nil {
 				return err
 			}
 		}
@@ -115,6 +115,19 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
 		}
 	}
 
+	stats, _ := req.Options[statsOptionName].(bool)
+	if stats {
+		err = res.Emit(&CarImportOutput{
+			Stats: &CarImportStats{
+				BlockCount:      done.blockCount,
+				BlockBytesCount: done.blockBytesCount,
+			},
+		})
+		if err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -126,6 +139,7 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI,
 	batch := ipld.NewBatch(req.Context, api.Dag())
 
 	roots := make(map[cid.Cid]struct{})
+	var blockCount, blockBytesCount uint64
 
 	it := req.Files.Entries()
 	for it.Next() {
@@ -176,6 +190,8 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI,
 				if err := batch.Add(req.Context, nd); err != nil {
 					return err
 				}
+				blockCount++
+				blockBytesCount += uint64(len(block.RawData()))
 			}
 
 			return nil
@@ -197,5 +213,8 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI,
 		return
 	}
 
-	ret <- importResult{roots: roots}
+	ret <- importResult{
+		blockCount:      blockCount,
+		blockBytesCount: blockBytesCount,
+		roots:           roots}
 }

test/sharness/lib/test-lib.sh

@@ -269,7 +269,7 @@ test_launch_ipfs_daemon() {
 
   # wait for api file to show up
   test_expect_success "api file shows up" '
-    test_wait_for_file 50 100ms "$IPFS_PATH/api"
+    test_wait_for_file 50 200ms "$IPFS_PATH/api"
   '
 
   test_set_address_vars actual_daemon

test/sharness/t0041-ping.sh

@@ -43,7 +43,7 @@ test_expect_success "test ping 0" '
 '
 
 test_expect_success "test ping offline" '
-  iptb stop 1 &&
+  iptb stop 1 && sleep 2 &&
   ! ipfsi 0 ping -n2 -- "$PEERID_1"
 '
 

test/sharness/t0054-dag-car-import-export.sh

@@ -55,18 +55,25 @@ run_online_imp_exp_tests() {
   reset_blockstore 0
   reset_blockstore 1
 
-  cat > basic_import_expected <<EOE
+  cat > basic_import_stats_expected <<EOE
+Imported 1198 blocks (468513 bytes)
 Pinned root${tab}bafkqaaa${tab}success
 Pinned root${tab}bafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u${tab}success
 Pinned root${tab}bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy${tab}success
 EOE
+  # output without the --stats line at the top
+  tail -n +2 basic_import_stats_expected > basic_import_expected
 
+  # Explainer:
+  # naked_root_import_json_expected output is produced by dag import of combined_naked_roots_genesis_and_128.car
+  # executed when roots are already present in the repo - thus the BlockCount=0
+  # (if blocks were not present in the repo, blockstore: block not found would be returned)
   cat >naked_root_import_json_expected <<EOE
 {"Root":{"Cid":{"/":"bafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u"},"PinErrorMsg":""}}
 {"Root":{"Cid":{"/":"bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy"},"PinErrorMsg":""}}
+{"Stats":{"BlockCount":0,"BlockBytesCount":0}}
 EOE
 
-
   test_expect_success "basic import" '
     do_import 0 \
       ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
@@ -79,6 +86,18 @@ EOE
     test_cmp_sorted basic_import_expected basic_import_actual
   '
 
+  test_expect_success "basic import with --stats" '
+    do_import 0 --stats \
+      ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
+      ../t0054-dag-car-import-export-data/lotus_testnet_export_128_shuffled_nulroot.car \
+      ../t0054-dag-car-import-export-data/lotus_devnet_genesis_shuffled_nulroot.car \
+    > basic_import_actual
+  '
+
+  test_expect_success "basic import output with --stats as expected" '
+    test_cmp_sorted basic_import_stats_expected basic_import_actual
+  '
+
   test_expect_success "basic fetch+export 1" '
     ipfsi 1 dag export bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy > reexported_testnet_128.car
   '
@@ -98,7 +117,7 @@ EOE
   '
 
   test_expect_success "import/pin naked roots only, relying on local blockstore having all the data" '
-    ipfsi 1 dag import --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
+    ipfsi 1 dag import --stats --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
       > naked_import_result_json_actual
   '
 
@@ -117,7 +136,7 @@ EOE
         cat ../t0054-dag-car-import-export-data/lotus_testnet_export_128_shuffled_nulroot.car > pipe_testnet &
         cat ../t0054-dag-car-import-export-data/lotus_devnet_genesis_shuffled_nulroot.car > pipe_devnet &
 
-        do_import 0 \
+        do_import 0 --stats \
           pipe_testnet \
           pipe_devnet \
           ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
@@ -134,7 +153,7 @@ EOE
   '
 
   test_expect_success "fifo-import output as expected" '
-    test_cmp_sorted basic_import_expected basic_fifo_import_actual
+    test_cmp_sorted basic_import_stats_expected basic_fifo_import_actual
   '
 }
 
@@ -168,33 +187,46 @@ test_expect_success "correct error" '
   test_cmp_sorted offline_fetch_error_expected offline_fetch_error_actual
 '
 
-
-cat >multiroot_import_json_expected <<EOE
+cat >multiroot_import_json_stats_expected <<EOE
 {"Root":{"Cid":{"/":"bafy2bzaceb55n7uxyfaelplulk3ev2xz7gnq6crncf3ahnvu46hqqmpucizcw"},"PinErrorMsg":""}}
 {"Root":{"Cid":{"/":"bafy2bzacebedrc4n2ac6cqdkhs7lmj5e4xiif3gu7nmoborihajxn3fav3vdq"},"PinErrorMsg":""}}
 {"Root":{"Cid":{"/":"bafy2bzacede2hsme6hparlbr4g2x6pylj43olp4uihwjq3plqdjyrdhrv7cp4"},"PinErrorMsg":""}}
+{"Stats":{"BlockCount":2825,"BlockBytesCount":1339709}}
 EOE
-test_expect_success "multiroot import works" '
+# output without --stats line
+head -3 multiroot_import_json_stats_expected > multiroot_import_json_expected
+
+test_expect_success "multiroot import works (--enc=json)" '
   ipfs dag import --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual
 '
 test_expect_success "multiroot import expected output" '
   test_cmp_sorted multiroot_import_json_expected multiroot_import_json_actual
 '
 
+test_expect_success "multiroot import works with --stats" '
+  ipfs dag import --stats --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual
+'
+test_expect_success "multiroot import expected output" '
+  test_cmp_sorted multiroot_import_json_stats_expected multiroot_import_json_actual
+'
+
 
+cat >pin_import_expected << EOE
+{"Stats":{"BlockCount":1198,"BlockBytesCount":468513}}
+EOE
 test_expect_success "pin-less import works" '
-  ipfs dag import --enc=json --pin-roots=false \
+  ipfs dag import --stats --enc=json --pin-roots=false \
   ../t0054-dag-car-import-export-data/lotus_devnet_genesis.car \
   ../t0054-dag-car-import-export-data/lotus_testnet_export_128.car \
     > no-pin_import_actual
 '
-test_expect_success "expected silence on --pin-roots=false" '
-  test_cmp /dev/null no-pin_import_actual
+test_expect_success "expected no pins on --pin-roots=false" '
+  test_cmp pin_import_expected no-pin_import_actual
 '
 
 
 test_expect_success "naked root import works" '
-  ipfs dag import --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
+  ipfs dag import --stats --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
     > naked_root_import_json_actual
 '
 test_expect_success "naked root import expected output" '