Add global --cid-base option and enable it for ipfs add and tar comma…

…nds.

This also adds a global --output-cidv1 option.

License: MIT
Signed-off-by: Kevin Atkinson <k@kevina.org>

core/commands/add.go

@@ -225,6 +225,11 @@ You can now check what blocks have been created by:
 			outChan := make(chan interface{})
 			req := res.Request()
 
+			_, err := cmdenv.ProcCidBaseClientSide(req)
+			if err != nil {
+				return err
+			}
+
 			sizeFile, ok := req.Files.(files.SizeFile)
 			if ok {
 				// Could be slow.
@@ -279,8 +284,9 @@ You can now check what blocks have been created by:
 							break LOOP
 						}
 						output := out.(*coreiface.AddEvent)
-						if len(output.Hash) > 0 {
-							lastHash = output.Hash
+						hash := output.Hash.String()
+						if len(hash) > 0 {
+							lastHash = hash
 							if quieter {
 								continue
 							}
@@ -290,9 +296,9 @@ You can now check what blocks have been created by:
 								fmt.Fprintf(os.Stderr, "\033[2K\r")
 							}
 							if quiet {
-								fmt.Fprintf(os.Stdout, "%s\n", output.Hash)
+								fmt.Fprintf(os.Stdout, "%s\n", hash)
 							} else {
-								fmt.Fprintf(os.Stdout, "added %s %s\n", output.Hash, output.Name)
+								fmt.Fprintf(os.Stdout, "added %s %s\n", hash, output.Name)
 							}
 
 						} else {

core/commands/cmdenv/cidbase.go

@@ -0,0 +1,82 @@
+package cmdenv
+
+import (
+	cidenc "gx/ipfs/QmWf8NwKFLbTBvAvZst3bYF7WEEetzxWyMhvQ885cj9MM8/go-cidutil/cidenc"
+	cmds "gx/ipfs/Qma6uuSyjkecGhMFFLfzyJDPyoDtNJSHJNweDccZhaWkgU/go-ipfs-cmds"
+	cmdkit "gx/ipfs/Qmde5VP1qUkyQXKCfmEUA7bP64V2HAptbJ7phuPp7jXWwg/go-ipfs-cmdkit"
+	mbase "gx/ipfs/QmekxXDhCxCJRNuzmHreuaT3BsuJcsjcXWNrtV9C8DRHtd/go-multibase"
+)
+
+var OptionCidBase = cmdkit.StringOption("cid-base", "mbase", "Multi-base encoding used for version 1 CIDs in output.")
+var OptionOutputCidV1 = cmdkit.BoolOption("output-cidv1", "Upgrade CID version 0 to version 1 in output.")
+
+// ProcCidBase processes the `cid-base` and `output-cidv1` options.
+// In the future it may also be used to process relevant config
+// settings.
+func ProcCidBase(req *cmds.Request, opts ...ProcCidBaseOption) (*CidBaseInfo, error) {
+	h := &CidBaseInfo{}
+	h.base, _ = req.Options["cid-base"].(string)
+	h.upgrade, h.upgradeDefined = req.Options["output-cidv1"].(bool)
+
+	for _, opt := range opts {
+		if err := opt(h); err != nil {
+			return nil, err
+		}
+	}
+
+	e := cidenc.Default
+
+	if h.base != "" {
+		var err error
+		e.Base, err = mbase.EncoderByName(h.base)
+		if err != nil {
+			return h, err
+		}
+		if !h.upgradeDefined {
+			e.Upgrade = true
+		}
+	}
+
+	if h.upgradeDefined {
+		e.Upgrade = h.upgrade
+	}
+
+	if h.enc == nil {
+		h.enc = &cidenc.Encoder{}
+	}
+	*h.enc = e
+	return h, nil
+}
+
+func ProcCidBaseClientSide(req *cmds.Request, opts ...ProcCidBaseOption) (*CidBaseInfo, error) {
+	return ProcCidBase(req, append([]ProcCidBaseOption{UseGlobal(true)}, opts...)...)
+}
+
+type ProcCidBaseOption func(*CidBaseInfo) error
+
+// UseGlobal enables the use of the global default.  This is somewhat
+// of a hack and should be used with care.  In particular it should
+// only be used on the client side and not the server side.
+func UseGlobal(yes bool) ProcCidBaseOption {
+	return func(h *CidBaseInfo) error {
+		if yes {
+			h.enc = &cidenc.Default
+		} else {
+			h.enc = nil
+		}
+		return nil
+	}
+}
+
+// CidBaseInfo is a returned by ProcCidBase
+type CidBaseInfo struct {
+	base           string
+	upgrade        bool
+	upgradeDefined bool
+	enc            *cidenc.Encoder
+}
+
+// Encoder returns a copy of the underlying Encoder
+func (h *CidBaseInfo) Encoder() cidenc.Encoder {
+	return *h.enc
+}

core/commands/root.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 
 	lgc "github.com/ipfs/go-ipfs/commands/legacy"
+	cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv"
 	dag "github.com/ipfs/go-ipfs/core/commands/dag"
 	name "github.com/ipfs/go-ipfs/core/commands/name"
 	ocmd "github.com/ipfs/go-ipfs/core/commands/object"
@@ -97,6 +98,9 @@ The CLI will exit with one of the following values:
 		cmdkit.StringOption(ApiOption, "Use a specific API instance (defaults to /ip4/127.0.0.1/tcp/5001)"),
 
 		// global options, added to every command
+		cmdenv.OptionCidBase,
+		cmdenv.OptionOutputCidV1,
+
 		cmds.OptionEncodingType,
 		cmds.OptionStreamChannels,
 		cmds.OptionTimeout,

core/commands/tar.go

@@ -10,6 +10,7 @@ import (
 	tar "github.com/ipfs/go-ipfs/tar"
 
 	"gx/ipfs/QmRG3XuGwT7GYuAqgWDJBKTzdaHMwAnc1x7J2KHEXNHxzG/go-path"
+	apicid "gx/ipfs/QmWf8NwKFLbTBvAvZst3bYF7WEEetzxWyMhvQ885cj9MM8/go-cidutil/apicid"
 	cmds "gx/ipfs/Qma6uuSyjkecGhMFFLfzyJDPyoDtNJSHJNweDccZhaWkgU/go-ipfs-cmds"
 	dag "gx/ipfs/QmaDBne4KeY3UepeqSVKYpSmQGa3q9zP6x3LfVF2UjF3Hc/go-merkledag"
 	cmdkit "gx/ipfs/Qmde5VP1qUkyQXKCfmEUA7bP64V2HAptbJ7phuPp7jXWwg/go-ipfs-cmdkit"
@@ -59,12 +60,17 @@ represent it.
 		fi.FileName()
 		return cmds.EmitOnce(res, &coreiface.AddEvent{
 			Name: fi.FileName(),
-			Hash: c.String(),
+			Hash: apicid.FromCid(c),
 		})
 	},
 	Type: coreiface.AddEvent{},
 	Encoders: cmds.EncoderMap{
 		cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *coreiface.AddEvent) error {
+			_, err := cmdenv.ProcCidBaseClientSide(req)
+			if err != nil {
+				return err
+			}
+
 			fmt.Fprintln(w, out.Hash)
 			return nil
 		}),

core/coreapi/interface/unixfs.go

@@ -6,16 +6,17 @@ import (
 
 	options "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
 
+	apicid "gx/ipfs/QmWf8NwKFLbTBvAvZst3bYF7WEEetzxWyMhvQ885cj9MM8/go-cidutil/apicid"
 	files "gx/ipfs/QmZMWMvWMVKCbHetJ4RgndbuEF1io2UpUxwQwtNjtYPzSC/go-ipfs-files"
 	ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format"
 )
 
 // TODO: ideas on making this more coreapi-ish without breaking the http API?
 type AddEvent struct {
 	Name  string
-	Hash  string `json:",omitempty"`
-	Bytes int64  `json:",omitempty"`
-	Size  string `json:",omitempty"`
+	Hash  apicid.Hash `json:",omitempty"`
+	Bytes int64       `json:",omitempty"`
+	Size  string      `json:",omitempty"`
 }
 
 type UnixfsFile interface {

core/coreapi/unixfs_test.go

@@ -24,8 +24,10 @@ import (
 	"github.com/ipfs/go-ipfs/repo"
 
 	ci "gx/ipfs/QmNiJiXwWE3kRhZrC5ej3kSjWHm337pYfhjLGSCDNKJP2s/go-libp2p-crypto"
+	cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid"
 	cbor "gx/ipfs/QmRoARq3nkUb13HSKZGepCZSWe5GrVPwx7xURJGZ7KWv9V/go-ipld-cbor"
 	pstore "gx/ipfs/QmUymf8fJtideyv3z727BcZUifGBjMZMpCJqu3Gxk5aRUk/go-libp2p-peerstore"
+	apicid "gx/ipfs/QmWf8NwKFLbTBvAvZst3bYF7WEEetzxWyMhvQ885cj9MM8/go-cidutil/apicid"
 	unixfs "gx/ipfs/QmXLCwhHh7bxRsBnCKNE9BAN87V44aSxXLquZYTtjr6fZ3/go-unixfs"
 	mocknet "gx/ipfs/QmXnpYYg2onGLXVxM4Q5PEFcx29k8zeJQkPeLAk9h9naxg/go-libp2p/p2p/net/mock"
 	files "gx/ipfs/QmZMWMvWMVKCbHetJ4RgndbuEF1io2UpUxwQwtNjtYPzSC/go-ipfs-files"
@@ -165,6 +167,14 @@ func wrapped(f files.File) files.File {
 	})
 }
 
+func apiCid(hash string) apicid.Hash {
+	c, err := cid.Decode(hash)
+	if err != nil {
+		panic(err)
+	}
+	return apicid.FromCid(c)
+}
+
 func TestAdd(t *testing.T) {
 	ctx := context.Background()
 	_, api, err := makeAPI(ctx)
@@ -384,7 +394,7 @@ func TestAdd(t *testing.T) {
 			data: strFile(helloStr),
 			path: "/ipfs/zb2rhdhmJjJZs9qkhQCpCQ7VREFkqWw3h1r8utjVvQugwHPFd",
 			events: []coreiface.AddEvent{
-				{Name: "zb2rhdhmJjJZs9qkhQCpCQ7VREFkqWw3h1r8utjVvQugwHPFd", Hash: "zb2rhdhmJjJZs9qkhQCpCQ7VREFkqWw3h1r8utjVvQugwHPFd", Size: strconv.Itoa(len(helloStr))},
+				{Name: "zb2rhdhmJjJZs9qkhQCpCQ7VREFkqWw3h1r8utjVvQugwHPFd", Hash: apiCid("zb2rhdhmJjJZs9qkhQCpCQ7VREFkqWw3h1r8utjVvQugwHPFd"), Size: strconv.Itoa(len(helloStr))},
 			},
 			opts: []options.UnixfsAddOption{options.Unixfs.RawLeaves(true)},
 		},
@@ -393,8 +403,8 @@ func TestAdd(t *testing.T) {
 			data: twoLevelDir(),
 			path: "/ipfs/QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr",
 			events: []coreiface.AddEvent{
-				{Name: "t/abc", Hash: "QmU7nuGs2djqK99UNsNgEPGh6GV4662p6WtsgccBNGTDxt", Size: "62"},
-				{Name: "t", Hash: "QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr", Size: "229"},
+				{Name: "t/abc", Hash: apiCid("QmU7nuGs2djqK99UNsNgEPGh6GV4662p6WtsgccBNGTDxt"), Size: "62"},
+				{Name: "t", Hash: apiCid("QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr"), Size: "229"},
 			},
 			recursive: true,
 			opts:      []options.UnixfsAddOption{options.Unixfs.Silent(true)},
@@ -404,11 +414,11 @@ func TestAdd(t *testing.T) {
 			data: twoLevelDir(),
 			path: "/ipfs/QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr",
 			events: []coreiface.AddEvent{
-				{Name: "t/abc/def", Hash: "QmNyJpQkU1cEkBwMDhDNFstr42q55mqG5GE5Mgwug4xyGk", Size: "13"},
-				{Name: "t/bar", Hash: "QmS21GuXiRMvJKHos4ZkEmQDmRBqRaF5tQS2CQCu2ne9sY", Size: "14"},
-				{Name: "t/foo", Hash: "QmfAjGiVpTN56TXi6SBQtstit5BEw3sijKj1Qkxn6EXKzJ", Size: "14"},
-				{Name: "t/abc", Hash: "QmU7nuGs2djqK99UNsNgEPGh6GV4662p6WtsgccBNGTDxt", Size: "62"},
-				{Name: "t", Hash: "QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr", Size: "229"},
+				{Name: "t/abc/def", Hash: apiCid("QmNyJpQkU1cEkBwMDhDNFstr42q55mqG5GE5Mgwug4xyGk"), Size: "13"},
+				{Name: "t/bar", Hash: apiCid("QmS21GuXiRMvJKHos4ZkEmQDmRBqRaF5tQS2CQCu2ne9sY"), Size: "14"},
+				{Name: "t/foo", Hash: apiCid("QmfAjGiVpTN56TXi6SBQtstit5BEw3sijKj1Qkxn6EXKzJ"), Size: "14"},
+				{Name: "t/abc", Hash: apiCid("QmU7nuGs2djqK99UNsNgEPGh6GV4662p6WtsgccBNGTDxt"), Size: "62"},
+				{Name: "t", Hash: apiCid("QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr"), Size: "229"},
 			},
 			recursive: true,
 		},
@@ -424,7 +434,7 @@ func TestAdd(t *testing.T) {
 				{Name: "", Bytes: 524288},
 				{Name: "", Bytes: 786432},
 				{Name: "", Bytes: 1000000},
-				{Name: "QmXXNNbwe4zzpdMg62ZXvnX1oU7MwSrQ3vAEtuwFKCm1oD", Hash: "QmXXNNbwe4zzpdMg62ZXvnX1oU7MwSrQ3vAEtuwFKCm1oD", Size: "1000256"},
+				{Name: "QmXXNNbwe4zzpdMg62ZXvnX1oU7MwSrQ3vAEtuwFKCm1oD", Hash: apiCid("QmXXNNbwe4zzpdMg62ZXvnX1oU7MwSrQ3vAEtuwFKCm1oD"), Size: "1000256"},
 			},
 			recursive: true,
 			opts:      []options.UnixfsAddOption{options.Unixfs.Progress(true)},

core/coreunix/add.go

@@ -18,6 +18,7 @@ import (
 	posinfo "gx/ipfs/QmR6YMs8EkXQLXNwQKxLnQp2VBZSepoEJ8KCZAyanJHhJu/go-ipfs-posinfo"
 	cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid"
 	bstore "gx/ipfs/QmSNLNnL3kq3A1NGdQA9AtgxM9CWKiiSEup3W435jCkRQS/go-ipfs-blockstore"
+	apicid "gx/ipfs/QmWf8NwKFLbTBvAvZst3bYF7WEEetzxWyMhvQ885cj9MM8/go-cidutil/apicid"
 	unixfs "gx/ipfs/QmXLCwhHh7bxRsBnCKNE9BAN87V44aSxXLquZYTtjr6fZ3/go-unixfs"
 	balanced "gx/ipfs/QmXLCwhHh7bxRsBnCKNE9BAN87V44aSxXLquZYTtjr6fZ3/go-unixfs/importer/balanced"
 	ihelper "gx/ipfs/QmXLCwhHh7bxRsBnCKNE9BAN87V44aSxXLquZYTtjr6fZ3/go-unixfs/importer/helpers"
@@ -37,12 +38,13 @@ const progressReaderIncrement = 1024 * 256
 var liveCacheSize = uint64(256 << 10)
 
 type Link struct {
-	Name, Hash string
-	Size       uint64
+	Name string
+	Hash apicid.Hash
+	Size uint64
 }
 
 type Object struct {
-	Hash  string
+	Hash  apicid.Hash
 	Links []Link
 	Size  string
 }
@@ -599,7 +601,7 @@ func getOutput(dagnode ipld.Node) (*Object, error) {
 	}
 
 	output := &Object{
-		Hash:  c.String(),
+		Hash:  apicid.FromCid(c),
 		Size:  strconv.FormatUint(s, 10),
 		Links: make([]Link, len(dagnode.Links())),
 	}

core/coreunix/add_test.go

@@ -94,10 +94,11 @@ func TestAddGCLive(t *testing.T) {
 
 	}()
 
-	addedHashes := make(map[string]struct{})
+	addedHashes := make(map[cid.Cid]struct{})
 	select {
 	case o := <-out:
-		addedHashes[o.(*coreiface.AddEvent).Hash] = struct{}{}
+		c, _ := o.(*coreiface.AddEvent).Hash.Cid()
+		addedHashes[c] = struct{}{}
 	case <-addDone:
 		t.Fatal("add shouldnt complete yet")
 	}
@@ -125,23 +126,24 @@ func TestAddGCLive(t *testing.T) {
 
 	// receive next object from adder
 	o := <-out
-	addedHashes[o.(*coreiface.AddEvent).Hash] = struct{}{}
+	c, _ := o.(*coreiface.AddEvent).Hash.Cid()
+	addedHashes[c] = struct{}{}
 
 	<-gcstarted
 
 	for r := range gcout {
 		if r.Error != nil {
 			t.Fatal(err)
 		}
-		if _, ok := addedHashes[r.KeyRemoved.String()]; ok {
+		if _, ok := addedHashes[r.KeyRemoved]; ok {
 			t.Fatal("gc'ed a hash we just added")
 		}
 	}
 
 	var last cid.Cid
 	for a := range out {
 		// wait for it to finish
-		c, err := cid.Decode(a.(*coreiface.AddEvent).Hash)
+		c, err := a.(*coreiface.AddEvent).Hash.Cid()
 		if err != nil {
 			t.Fatal(err)
 		}

test/sharness/t0040-add-and-cat.sh

@@ -272,6 +272,36 @@ test_add_cat_file() {
     echo "added QmZQWnfcqJ6hNkkPvrY9Q5X39GP3jUnUbAV4AbmbbR3Cb1 test_current_dir" > expected
     test_cmp expected actual
   '
+
+  # --cid-base=base32
+
+  test_expect_success "ipfs add --cid-base=base32 succeeds" '
+    echo "Hello Worlds!" >mountdir/hello.txt &&
+    ipfs add --cid-base=base32 mountdir/hello.txt >actual
+  '
+
+  test_expect_success "ipfs add output looks good" '
+    HASH="bafybeidpq7lcjx4w5c6yr4vuthzvlav54hgxsremwk73to5ferdc2rxhai" &&
+    echo "added $HASH hello.txt" >expected &&
+    test_cmp expected actual
+  '
+
+  test_expect_success "ipfs add --cid-base=base32 --only-hash succeeds" '
+    ipfs add --cid-base=base32 --only-hash mountdir/hello.txt > oh_actual
+  '
+
+  test_expect_success "ipfs add --only-hash output looks good" '
+    test_cmp expected oh_actual
+  '
+
+  test_expect_success "ipfs cat succeeds" '
+    ipfs cat "$HASH" >actual
+  '
+
+  test_expect_success "ipfs cat output looks good" '
+    echo "Hello Worlds!" >expected &&
+    test_cmp expected actual
+  '
 }
 
 test_add_cat_5MB() {
@@ -312,6 +342,25 @@ test_add_cat_5MB() {
   test_expect_success FUSE "cat ipfs/bigfile looks good" '
     test_cmp mountdir/bigfile actual
   '
+
+  test_expect_success "get base32 version of CID" '
+    ipfs cid base32 $EXP_HASH > base32_cid &&
+    BASE32_HASH=`cat base32_cid`
+  '
+
+  test_expect_success "ipfs add --cid-base=base32 bigfile' succeeds" '
+    ipfs add $ADD_FLAGS --cid-base=base32 mountdir/bigfile >actual ||
+    test_fsh cat daemon_err
+  '
+
+  test_expect_success "'ipfs add bigfile --cid-base=base32' output looks good" '
+    echo "added $BASE32_HASH bigfile" >expected &&
+    test_cmp expected actual
+  '
+
+  test_expect_success "'ipfs cat $BASE32_HASH' succeeds" '
+    ipfs cat "$BASE32_HASH" >actual
+  '
 }
 
 test_add_cat_raw() {