feat: dag import --stats (#8237)

* feat: report block count on `dag import`
* fix: clean-up dag import message format
* Only print stats when --stats flag is passed

This applies to both text and json output encoding.

- Stats data is now contained within a Stats datastructure
- Stats are printed after root so that first line of output is the same as previously, even when stats are output using --stats

* fix sharness test

* Add PayloadBytesCount to stats

* Attempt to stabilize flaky tests

* Rename PayloadBytesCount to BlockBytesCount

* Correctly calculate size or imported dag

* Use RawSize of original block for import bytes calc

* test: dag import without --stats

basic regression tests for the default output (text and json)

Co-authored-by: gammazero <gammazero@users.noreply.github.com>
Co-authored-by: Marcin Rataj <lidel@lidel.org>
(cherry picked from commit 0057199)

core/commands/dag/dag.go

@@ -16,9 +16,10 @@ import (
 )
 
 const (
+ pinRootsOptionName = "pin-roots"
  progressOptionName = "progress"
  silentOptionName = "silent"
- pinRootsOptionName = "pin-roots"
+ statsOptionName = "stats"
 )
 
 // DagCmd provides a subset of commands for interacting with ipld dag objects
@@ -53,9 +54,15 @@ type ResolveOutput struct {
  RemPath string
 }
 
+type CarImportStats struct {
+ BlockCount uint64
+ BlockBytesCount uint64
+}
+
 // CarImportOutput is the output type of the 'dag import' commands
 type CarImportOutput struct {
- Root RootMeta
+ Root *RootMeta `json:",omitempty"`
+ Stats *CarImportStats `json:",omitempty"`
 }
 
 // RootMeta is the metadata for a root pinning response
@@ -160,8 +167,10 @@ var DagResolveCmd = &cmds.Command{
 }
 
 type importResult struct {
- roots map[cid.Cid]struct{}
- err error
+ blockCount uint64
+ blockBytesCount uint64
+ roots map[cid.Cid]struct{}
+ err error
 }
 
 // DagImportCmd is a command for importing a car to ipfs
@@ -193,8 +202,9 @@ Maximum supported CAR version: 1
  cmds.FileArg("path", true, true, "The path of a .car file.").EnableStdin(),
  },
  Options: []cmds.Option{
- cmds.BoolOption(silentOptionName, "No output."),
  cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing.").WithDefault(true),
+ cmds.BoolOption(silentOptionName, "No output."),
+ cmds.BoolOption(statsOptionName, "Output stats."),
  },
  Type: CarImportOutput{},
  Run: dagImport,
@@ -206,6 +216,22 @@ Maximum supported CAR version: 1
  return nil
  }
 
+ // event should have only one of `Root` or `Stats` set, not both
+ if event.Root == nil {
+ if event.Stats == nil {
+ return fmt.Errorf("Unexpected message from DAG import")
+ }
+ stats, _ := req.Options[statsOptionName].(bool)
+ if stats {
+ fmt.Fprintf(w, "Imported %d blocks (%d bytes)\n", event.Stats.BlockCount, event.Stats.BlockBytesCount)
+ }
+ return nil
+ }
+
+ if event.Stats != nil {
+ return fmt.Errorf("Unexpected message from DAG import")
+ }
+
  enc, err := cmdenv.GetLowLevelCidEncoder(req)
  if err != nil {
  return err

core/commands/dag/import.go

@@ -101,7 +101,7 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
  failedPins++
  }
 
- if err := res.Emit(&CarImportOutput{Root: ret}); err != nil {
+ if err := res.Emit(&CarImportOutput{Root: &ret}); err != nil {
  return err
  }
  }
@@ -115,6 +115,19 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
  }
  }
 
+ stats, _ := req.Options[statsOptionName].(bool)
+ if stats {
+ err = res.Emit(&CarImportOutput{
+ Stats: &CarImportStats{
+ BlockCount: done.blockCount,
+ BlockBytesCount: done.blockBytesCount,
+ },
+ })
+ if err != nil {
+ return err
+ }
+ }
+
  return nil
 }
 
@@ -126,6 +139,7 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI,
  batch := ipld.NewBatch(req.Context, api.Dag())
 
  roots := make(map[cid.Cid]struct{})
+ var blockCount, blockBytesCount uint64
 
  it := req.Files.Entries()
  for it.Next() {
@@ -176,6 +190,8 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI,
  if err := batch.Add(req.Context, nd); err != nil {
  return err
  }
+ blockCount++
+ blockBytesCount += uint64(len(block.RawData()))
  }
 
  return nil
@@ -197,5 +213,8 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI,
  return
  }
 
- ret <- importResult{roots: roots}
+ ret <- importResult{
+ blockCount: blockCount,
+ blockBytesCount: blockBytesCount,
+ roots: roots}
 }

test/sharness/lib/test-lib.sh

@@ -269,7 +269,7 @@ test_launch_ipfs_daemon() {
 
  # wait for api file to show up
  test_expect_success "api file shows up" '
- test_wait_for_file 50 100ms "$IPFS_PATH/api"
+ test_wait_for_file 50 200ms "$IPFS_PATH/api"
  '
 
  test_set_address_vars actual_daemon

test/sharness/t0041-ping.sh

@@ -43,7 +43,7 @@ test_expect_success "test ping 0" '
 '
 
 test_expect_success "test ping offline" '
- iptb stop 1 &&
+ iptb stop 1 && sleep 2 &&
  ! ipfsi 0 ping -n2 -- "$PEERID_1"
 '
 

test/sharness/t0054-dag-car-import-export.sh

@@ -55,18 +55,25 @@ run_online_imp_exp_tests() {
  reset_blockstore 0
  reset_blockstore 1
 
- cat > basic_import_expected <<EOE
+ cat > basic_import_stats_expected <<EOE
+Imported 1198 blocks (468513 bytes)
 Pinned root${tab}bafkqaaa${tab}success
 Pinned root${tab}bafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u${tab}success
 Pinned root${tab}bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy${tab}success
 EOE
+ # output without the --stats line at the top
+ tail -n +2 basic_import_stats_expected > basic_import_expected
 
+ # Explainer:
+ # naked_root_import_json_expected output is produced by dag import of combined_naked_roots_genesis_and_128.car
+ # executed when roots are already present in the repo - thus the BlockCount=0
+ # (if blocks were not present in the repo, blockstore: block not found would be returned)
  cat >naked_root_import_json_expected <<EOE
 {"Root":{"Cid":{"/":"bafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u"},"PinErrorMsg":""}}
 {"Root":{"Cid":{"/":"bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy"},"PinErrorMsg":""}}
+{"Stats":{"BlockCount":0,"BlockBytesCount":0}}
 EOE
 
-
  test_expect_success "basic import" '
  do_import 0 \
  ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
@@ -79,6 +86,18 @@ EOE
  test_cmp_sorted basic_import_expected basic_import_actual
  '
 
+ test_expect_success "basic import with --stats" '
+ do_import 0 --stats \
+ ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
+ ../t0054-dag-car-import-export-data/lotus_testnet_export_128_shuffled_nulroot.car \
+ ../t0054-dag-car-import-export-data/lotus_devnet_genesis_shuffled_nulroot.car \
+ > basic_import_actual
+ '
+
+ test_expect_success "basic import output with --stats as expected" '
+ test_cmp_sorted basic_import_stats_expected basic_import_actual
+ '
+
  test_expect_success "basic fetch+export 1" '
  ipfsi 1 dag export bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy > reexported_testnet_128.car
  '
@@ -98,7 +117,7 @@ EOE
  '
 
  test_expect_success "import/pin naked roots only, relying on local blockstore having all the data" '
- ipfsi 1 dag import --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
+ ipfsi 1 dag import --stats --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
  > naked_import_result_json_actual
  '
 
@@ -117,7 +136,7 @@ EOE
  cat ../t0054-dag-car-import-export-data/lotus_testnet_export_128_shuffled_nulroot.car > pipe_testnet &
  cat ../t0054-dag-car-import-export-data/lotus_devnet_genesis_shuffled_nulroot.car > pipe_devnet &
 
- do_import 0 \
+ do_import 0 --stats \
  pipe_testnet \
  pipe_devnet \
  ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
@@ -134,7 +153,7 @@ EOE
  '
 
  test_expect_success "fifo-import output as expected" '
- test_cmp_sorted basic_import_expected basic_fifo_import_actual
+ test_cmp_sorted basic_import_stats_expected basic_fifo_import_actual
  '
 }
 
@@ -168,33 +187,46 @@ test_expect_success "correct error" '
  test_cmp_sorted offline_fetch_error_expected offline_fetch_error_actual
 '
 
-
-cat >multiroot_import_json_expected <<EOE
+cat >multiroot_import_json_stats_expected <<EOE
 {"Root":{"Cid":{"/":"bafy2bzaceb55n7uxyfaelplulk3ev2xz7gnq6crncf3ahnvu46hqqmpucizcw"},"PinErrorMsg":""}}
 {"Root":{"Cid":{"/":"bafy2bzacebedrc4n2ac6cqdkhs7lmj5e4xiif3gu7nmoborihajxn3fav3vdq"},"PinErrorMsg":""}}
 {"Root":{"Cid":{"/":"bafy2bzacede2hsme6hparlbr4g2x6pylj43olp4uihwjq3plqdjyrdhrv7cp4"},"PinErrorMsg":""}}
+{"Stats":{"BlockCount":2825,"BlockBytesCount":1339709}}
 EOE
-test_expect_success "multiroot import works" '
+# output without --stats line
+head -3 multiroot_import_json_stats_expected > multiroot_import_json_expected
+
+test_expect_success "multiroot import works (--enc=json)" '
  ipfs dag import --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual
 '
 test_expect_success "multiroot import expected output" '
  test_cmp_sorted multiroot_import_json_expected multiroot_import_json_actual
 '
 
+test_expect_success "multiroot import works with --stats" '
+ ipfs dag import --stats --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual
+'
+test_expect_success "multiroot import expected output" '
+ test_cmp_sorted multiroot_import_json_stats_expected multiroot_import_json_actual
+'
+
 
+cat >pin_import_expected << EOE
+{"Stats":{"BlockCount":1198,"BlockBytesCount":468513}}
+EOE
 test_expect_success "pin-less import works" '
- ipfs dag import --enc=json --pin-roots=false \
+ ipfs dag import --stats --enc=json --pin-roots=false \
  ../t0054-dag-car-import-export-data/lotus_devnet_genesis.car \
  ../t0054-dag-car-import-export-data/lotus_testnet_export_128.car \
  > no-pin_import_actual
 '
-test_expect_success "expected silence on --pin-roots=false" '
- test_cmp /dev/null no-pin_import_actual
+test_expect_success "expected no pins on --pin-roots=false" '
+ test_cmp pin_import_expected no-pin_import_actual
 '
 
 
 test_expect_success "naked root import works" '
- ipfs dag import --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
+ ipfs dag import --stats --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
  > naked_root_import_json_actual
 '
 test_expect_success "naked root import expected output" '