diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 8cd9864a6b6..8fabe628948 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -16,9 +16,10 @@ import ( ) const ( + pinRootsOptionName = "pin-roots" progressOptionName = "progress" silentOptionName = "silent" - pinRootsOptionName = "pin-roots" + statsOptionName = "stats" ) // DagCmd provides a subset of commands for interacting with ipld dag objects @@ -53,9 +54,15 @@ type ResolveOutput struct { RemPath string } +type CarImportStats struct { + BlockCount uint64 + BlockBytesCount uint64 +} + // CarImportOutput is the output type of the 'dag import' commands type CarImportOutput struct { - Root RootMeta + Root *RootMeta `json:",omitempty"` + Stats *CarImportStats `json:",omitempty"` } // RootMeta is the metadata for a root pinning response @@ -160,8 +167,10 @@ var DagResolveCmd = &cmds.Command{ } type importResult struct { - roots map[cid.Cid]struct{} - err error + blockCount uint64 + blockBytesCount uint64 + roots map[cid.Cid]struct{} + err error } // DagImportCmd is a command for importing a car to ipfs @@ -193,8 +202,9 @@ Maximum supported CAR version: 1 cmds.FileArg("path", true, true, "The path of a .car file.").EnableStdin(), }, Options: []cmds.Option{ - cmds.BoolOption(silentOptionName, "No output."), cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing.").WithDefault(true), + cmds.BoolOption(silentOptionName, "No output."), + cmds.BoolOption(statsOptionName, "Output stats."), }, Type: CarImportOutput{}, Run: dagImport, @@ -206,6 +216,22 @@ Maximum supported CAR version: 1 return nil } + // event should have only one of `Root` or `Stats` set, not both + if event.Root == nil { + if event.Stats == nil { + return fmt.Errorf("Unexpected message from DAG import") + } + stats, _ := req.Options[statsOptionName].(bool) + if stats { + fmt.Fprintf(w, "Imported %d blocks (%d bytes)\n", event.Stats.BlockCount, event.Stats.BlockBytesCount) + } + return nil + } + + if event.Stats != nil { + return fmt.Errorf("Unexpected message from DAG import") + } + enc, err := cmdenv.GetLowLevelCidEncoder(req) if err != nil { return err diff --git a/core/commands/dag/import.go b/core/commands/dag/import.go index b83af8b911d..244260601bf 100644 --- a/core/commands/dag/import.go +++ b/core/commands/dag/import.go @@ -101,7 +101,7 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment failedPins++ } - if err := res.Emit(&CarImportOutput{Root: ret}); err != nil { + if err := res.Emit(&CarImportOutput{Root: &ret}); err != nil { return err } } @@ -115,6 +115,19 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment } } + stats, _ := req.Options[statsOptionName].(bool) + if stats { + err = res.Emit(&CarImportOutput{ + Stats: &CarImportStats{ + BlockCount: done.blockCount, + BlockBytesCount: done.blockBytesCount, + }, + }) + if err != nil { + return err + } + } + return nil } @@ -126,6 +139,7 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI, batch := ipld.NewBatch(req.Context, api.Dag()) roots := make(map[cid.Cid]struct{}) + var blockCount, blockBytesCount uint64 it := req.Files.Entries() for it.Next() { @@ -176,6 +190,8 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI, if err := batch.Add(req.Context, nd); err != nil { return err } + blockCount++ + blockBytesCount += uint64(len(block.RawData())) } return nil @@ -197,5 +213,8 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI, return } - ret <- importResult{roots: roots} + ret <- importResult{ + blockCount: blockCount, + blockBytesCount: blockBytesCount, + roots: roots} } diff --git a/test/sharness/lib/test-lib.sh b/test/sharness/lib/test-lib.sh index f4b3c68db16..a20668f50e1 100644 --- a/test/sharness/lib/test-lib.sh +++ b/test/sharness/lib/test-lib.sh @@ -269,7 +269,7 @@ test_launch_ipfs_daemon() { # wait for api file to show up test_expect_success "api file shows up" ' - test_wait_for_file 50 100ms "$IPFS_PATH/api" + test_wait_for_file 50 200ms "$IPFS_PATH/api" ' test_set_address_vars actual_daemon diff --git a/test/sharness/t0041-ping.sh b/test/sharness/t0041-ping.sh index 276cd4802d1..c4665b9ba14 100755 --- a/test/sharness/t0041-ping.sh +++ b/test/sharness/t0041-ping.sh @@ -43,7 +43,7 @@ test_expect_success "test ping 0" ' ' test_expect_success "test ping offline" ' - iptb stop 1 && + iptb stop 1 && sleep 2 && ! ipfsi 0 ping -n2 -- "$PEERID_1" ' diff --git a/test/sharness/t0054-dag-car-import-export.sh b/test/sharness/t0054-dag-car-import-export.sh index 450591ff792..311833748f0 100755 --- a/test/sharness/t0054-dag-car-import-export.sh +++ b/test/sharness/t0054-dag-car-import-export.sh @@ -55,18 +55,25 @@ run_online_imp_exp_tests() { reset_blockstore 0 reset_blockstore 1 - cat > basic_import_expected < basic_import_stats_expected < basic_import_expected + # Explainer: + # naked_root_import_json_expected output is produced by dag import of combined_naked_roots_genesis_and_128.car + # executed when roots are already present in the repo - thus the BlockCount=0 + # (if blocks were not present in the repo, blockstore: block not found would be returned) cat >naked_root_import_json_expected < basic_import_actual + ' + + test_expect_success "basic import output with --stats as expected" ' + test_cmp_sorted basic_import_stats_expected basic_import_actual + ' + test_expect_success "basic fetch+export 1" ' ipfsi 1 dag export bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy > reexported_testnet_128.car ' @@ -98,7 +117,7 @@ EOE ' test_expect_success "import/pin naked roots only, relying on local blockstore having all the data" ' - ipfsi 1 dag import --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \ + ipfsi 1 dag import --stats --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \ > naked_import_result_json_actual ' @@ -117,7 +136,7 @@ EOE cat ../t0054-dag-car-import-export-data/lotus_testnet_export_128_shuffled_nulroot.car > pipe_testnet & cat ../t0054-dag-car-import-export-data/lotus_devnet_genesis_shuffled_nulroot.car > pipe_devnet & - do_import 0 \ + do_import 0 --stats \ pipe_testnet \ pipe_devnet \ ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \ @@ -134,7 +153,7 @@ EOE ' test_expect_success "fifo-import output as expected" ' - test_cmp_sorted basic_import_expected basic_fifo_import_actual + test_cmp_sorted basic_import_stats_expected basic_fifo_import_actual ' } @@ -168,33 +187,46 @@ test_expect_success "correct error" ' test_cmp_sorted offline_fetch_error_expected offline_fetch_error_actual ' - -cat >multiroot_import_json_expected <multiroot_import_json_stats_expected < multiroot_import_json_expected + +test_expect_success "multiroot import works (--enc=json)" ' ipfs dag import --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual ' test_expect_success "multiroot import expected output" ' test_cmp_sorted multiroot_import_json_expected multiroot_import_json_actual ' +test_expect_success "multiroot import works with --stats" ' + ipfs dag import --stats --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual +' +test_expect_success "multiroot import expected output" ' + test_cmp_sorted multiroot_import_json_stats_expected multiroot_import_json_actual +' + +cat >pin_import_expected << EOE +{"Stats":{"BlockCount":1198,"BlockBytesCount":468513}} +EOE test_expect_success "pin-less import works" ' - ipfs dag import --enc=json --pin-roots=false \ + ipfs dag import --stats --enc=json --pin-roots=false \ ../t0054-dag-car-import-export-data/lotus_devnet_genesis.car \ ../t0054-dag-car-import-export-data/lotus_testnet_export_128.car \ > no-pin_import_actual ' -test_expect_success "expected silence on --pin-roots=false" ' - test_cmp /dev/null no-pin_import_actual +test_expect_success "expected no pins on --pin-roots=false" ' + test_cmp pin_import_expected no-pin_import_actual ' test_expect_success "naked root import works" ' - ipfs dag import --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \ + ipfs dag import --stats --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \ > naked_root_import_json_actual ' test_expect_success "naked root import expected output" '