Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add initial support for identity hashes. #4910

Closed
wants to merge 5 commits into from
Closed

Add initial support for identity hashes. #4910

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
f7b6e67
Add initial support for identity hashes.
kevina Apr 3, 2018
6a6fd90
Enforce a length limit on identity hashes.
kevina Apr 5, 2018
42ee69d
Use "CidOpts" instead of just "Prefix" for additional flexibility.
kevina Apr 12, 2018
bb4a520
Add "id-hash-limit" option to "ipfs add" command.
kevina Apr 12, 2018
f0a4f29
Add "--inline" option to add, Use 64 as the default id-hash limit.
kevina Apr 16, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions core/builder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
pin "github.com/ipfs/go-ipfs/pin"
repo "github.com/ipfs/go-ipfs/repo"
cfg "github.com/ipfs/go-ipfs/repo/config"
"github.com/ipfs/go-ipfs/thirdparty/idstore"
"github.com/ipfs/go-ipfs/thirdparty/verifbs"
uio "github.com/ipfs/go-ipfs/unixfs/io"

Expand Down Expand Up @@ -189,14 +190,16 @@ func setupNode(ctx context.Context, n *IpfsNode, cfg *BuildCfg) error {
opts.HasBloomFilterSize = 0
}

cbs, err := bstore.CachedBlockstore(ctx, bs, opts)
wbs, err := bstore.CachedBlockstore(ctx, bs, opts)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What does w in wbs stand for?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

"wrapped"

if err != nil {
return err
}

n.BaseBlocks = cbs
wbs = idstore.IdStore(wbs)

n.BaseBlocks = wbs
n.GCLocker = bstore.NewGCLocker()
n.Blockstore = bstore.NewGCBlockstore(cbs, n.GCLocker)
n.Blockstore = bstore.NewGCBlockstore(wbs, n.GCLocker)

if conf.Experimental.FilestoreEnabled {
// hash security
Expand Down
19 changes: 17 additions & 2 deletions core/commands/add.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
dag "github.com/ipfs/go-ipfs/merkledag"
dagtest "github.com/ipfs/go-ipfs/merkledag/test"
mfs "github.com/ipfs/go-ipfs/mfs"
cide "github.com/ipfs/go-ipfs/thirdparty/cidextra"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I assume this is temporary.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

More or less, I would like to get something working first, merge this p.r. and then worry about finding a better place for the two new packages in thirdparty.

ft "github.com/ipfs/go-ipfs/unixfs"

mh "gx/ipfs/QmZyZDi491cCNTLfAhwcaDii2Kg4pwKRkhqQzURGDvY6ua/go-multihash"
Expand Down Expand Up @@ -43,6 +44,8 @@ const (
fstoreCacheOptionName = "fscache"
cidVersionOptionName = "cid-version"
hashOptionName = "hash"
idHashLimitOptionName = "id-hash-limit"
inlineOptionName = "inline"
)

const adderOutChanSize = 8
Expand Down Expand Up @@ -119,6 +122,8 @@ You can now check what blocks have been created by:
cmdkit.BoolOption(fstoreCacheOptionName, "Check the filestore for pre-existing blocks. (experimental)"),
cmdkit.IntOption(cidVersionOptionName, "CID version. Defaults to 0 unless an option that depends on CIDv1 is passed. (experimental)"),
cmdkit.StringOption(hashOptionName, "Hash function to use. Implies CIDv1 if not sha2-256. (experimental)").WithDefault("sha2-256"),
cmdkit.BoolOption(inlineOptionName, "Inline small objects using identity hash. (experimental)"),
cmdkit.IntOption(idHashLimitOptionName, "Identity hash maxium size. (experimental)").WithDefault(64),
},
PreRun: func(req *cmds.Request, env cmds.Environment) error {
quiet, _ := req.Options[quietOptionName].(bool)
Expand Down Expand Up @@ -172,6 +177,8 @@ You can now check what blocks have been created by:
fscache, _ := req.Options[fstoreCacheOptionName].(bool)
cidVer, cidVerSet := req.Options[cidVersionOptionName].(int)
hashFunStr, _ := req.Options[hashOptionName].(string)
inline, _ := req.Options[inlineOptionName].(bool)
idHashLimit, _ := req.Options[idHashLimitOptionName].(int)

// The arguments are subject to the following constraints.
//
Expand Down Expand Up @@ -279,13 +286,21 @@ You can now check what blocks have been created by:
fileAdder.Silent = silent
fileAdder.RawLeaves = rawblks
fileAdder.NoCopy = nocopy
fileAdder.Prefix = &prefix
fileAdder.CidOpts = &cide.Opts{Prefix: prefix}

if inline {
err = fileAdder.CidOpts.SetIdHashLimit(idHashLimit)
if err != nil {
res.SetError(err, cmdkit.ErrNormal)
return
}
}

if hash {
md := dagtest.Mock()
emptyDirNode := ft.EmptyDirNode()
// Use the same prefix for the "empty" MFS root as for the file adder.
emptyDirNode.Prefix = *fileAdder.Prefix
emptyDirNode.CidOpts = *fileAdder.CidOpts
mr, err := mfs.NewRoot(req.Context, md, emptyDirNode, nil)
if err != nil {
res.SetError(err, cmdkit.ErrNormal)
Expand Down
27 changes: 16 additions & 11 deletions core/commands/files.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
mfs "github.com/ipfs/go-ipfs/mfs"
path "github.com/ipfs/go-ipfs/path"
resolver "github.com/ipfs/go-ipfs/path/resolver"
cide "github.com/ipfs/go-ipfs/thirdparty/cidextra"
ft "github.com/ipfs/go-ipfs/unixfs"
uio "github.com/ipfs/go-ipfs/unixfs/io"

Expand Down Expand Up @@ -834,7 +835,7 @@ Examples:

flush, _, _ := req.Option("flush").Bool()

prefix, err := getPrefix(req)
opts, err := getCidOpts(req)
if err != nil {
res.SetError(err, cmdkit.ErrNormal)
return
Expand All @@ -844,7 +845,7 @@ Examples:
err = mfs.Mkdir(root, dirtomake, mfs.MkdirOpts{
Mkparents: dashp,
Flush: flush,
Prefix: prefix,
CidOpts: opts,
})
if err != nil {
res.SetError(err, cmdkit.ErrNormal)
Expand Down Expand Up @@ -916,13 +917,13 @@ Change the cid version or hash function of the root node of a given path.

flush, _, _ := req.Option("flush").Bool()

prefix, err := getPrefix(req)
opts, err := getCidOpts(req)
if err != nil {
res.SetError(err, cmdkit.ErrNormal)
return
}

err = updatePath(nd.FilesRoot, path, prefix, flush)
err = updatePath(nd.FilesRoot, path, opts, flush)
if err != nil {
res.SetError(err, cmdkit.ErrNormal)
return
Expand All @@ -932,8 +933,8 @@ Change the cid version or hash function of the root node of a given path.
},
}

func updatePath(rt *mfs.Root, pth string, prefix *cid.Prefix, flush bool) error {
if prefix == nil {
func updatePath(rt *mfs.Root, pth string, opts *cide.Opts, flush bool) error {
if opts == nil {
return nil
}

Expand All @@ -944,7 +945,7 @@ func updatePath(rt *mfs.Root, pth string, prefix *cid.Prefix, flush bool) error

switch n := nd.(type) {
case *mfs.Directory:
n.SetPrefix(prefix)
n.SetCidOpts(opts)
default:
return fmt.Errorf("can only update directories")
}
Expand Down Expand Up @@ -1091,7 +1092,7 @@ func getPrefixNew(req *cmds.Request) (*cid.Prefix, error) {
return &prefix, nil
}

func getPrefix(req oldcmds.Request) (*cid.Prefix, error) {
func getCidOpts(req oldcmds.Request) (*cide.Opts, error) {
cidVer, cidVerSet, _ := req.Option("cid-version").Int()
hashFunStr, hashFunSet, _ := req.Option("hash").String()

Expand All @@ -1117,7 +1118,7 @@ func getPrefix(req oldcmds.Request) (*cid.Prefix, error) {
prefix.MhLength = -1
}

return &prefix, nil
return &cide.Opts{Prefix: prefix}, nil
}

func getFileHandle(r *mfs.Root, path string, create bool, prefix *cid.Prefix) (*mfs.File, error) {
Expand Down Expand Up @@ -1146,12 +1147,16 @@ func getFileHandle(r *mfs.Root, path string, create bool, prefix *cid.Prefix) (*
if !ok {
return nil, fmt.Errorf("%s was not a directory", dirname)
}

var opts *cide.Opts
if prefix == nil {
prefix = pdir.GetPrefix()
opts = pdir.GetCidOpts()
} else {
opts = &cide.Opts{Prefix: *prefix}
}

nd := dag.NodeWithData(ft.FilePBData(nil, 0))
nd.SetPrefix(prefix)
nd.SetCidOpts(opts)
err = pdir.AddChild(fname, nd)
if err != nil {
return nil, err
Expand Down
15 changes: 8 additions & 7 deletions core/coredag/dagpb.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"math"

"github.com/ipfs/go-ipfs/merkledag"
cide "github.com/ipfs/go-ipfs/thirdparty/cidextra"

mh "gx/ipfs/QmZyZDi491cCNTLfAhwcaDii2Kg4pwKRkhqQzURGDvY6ua/go-multihash"
cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid"
Expand All @@ -25,7 +26,7 @@ func dagpbJSONParser(r io.Reader, mhType uint64, mhLen int) ([]ipld.Node, error)
return nil, err
}

nd.SetPrefix(cidPrefix(mhType, mhLen))
nd.SetCidOpts(cidOpts(mhType, mhLen))

return []ipld.Node{nd}, nil
}
Expand All @@ -41,26 +42,26 @@ func dagpbRawParser(r io.Reader, mhType uint64, mhLen int) ([]ipld.Node, error)
return nil, err
}

nd.SetPrefix(cidPrefix(mhType, mhLen))
nd.SetCidOpts(cidOpts(mhType, mhLen))

return []ipld.Node{nd}, nil
}

func cidPrefix(mhType uint64, mhLen int) *cid.Prefix {
func cidOpts(mhType uint64, mhLen int) *cide.Opts {
if mhType == math.MaxUint64 {
mhType = mh.SHA2_256
}

prefix := &cid.Prefix{
opts := &cide.Opts{Prefix: cid.Prefix{
MhType: mhType,
MhLength: mhLen,
Version: 1,
Codec: cid.DagProtobuf,
}
}}

if mhType == mh.SHA2_256 {
prefix.Version = 0
opts.Version = 0
}

return prefix
return opts
}
13 changes: 7 additions & 6 deletions core/coreunix/add.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
dag "github.com/ipfs/go-ipfs/merkledag"
mfs "github.com/ipfs/go-ipfs/mfs"
"github.com/ipfs/go-ipfs/pin"
cide "github.com/ipfs/go-ipfs/thirdparty/cidextra"
unixfs "github.com/ipfs/go-ipfs/unixfs"

logging "gx/ipfs/QmRb5jh8z2E8hMGN2tkvs1yHynUanqnZ3UeKwgN1i9P1F8/go-log"
Expand Down Expand Up @@ -108,7 +109,7 @@ type Adder struct {
mroot *mfs.Root
unlocker bstore.Unlocker
tempRoot *cid.Cid
Prefix *cid.Prefix
CidOpts *cide.Opts
liveNodes uint64
}

Expand All @@ -117,7 +118,7 @@ func (adder *Adder) mfsRoot() (*mfs.Root, error) {
return adder.mroot, nil
}
rnode := unixfs.EmptyDirNode()
rnode.SetPrefix(adder.Prefix)
rnode.SetCidOpts(adder.CidOpts)
mr, err := mfs.NewRoot(adder.ctx, adder.dagService, rnode, nil)
if err != nil {
return nil, err
Expand All @@ -143,7 +144,7 @@ func (adder *Adder) add(reader io.Reader) (ipld.Node, error) {
RawLeaves: adder.RawLeaves,
Maxlinks: ihelper.DefaultLinksPerBlock,
NoCopy: adder.NoCopy,
Prefix: adder.Prefix,
CidOpts: adder.CidOpts,
}

if adder.Trickle {
Expand Down Expand Up @@ -407,7 +408,7 @@ func (adder *Adder) addNode(node ipld.Node, path string) error {
opts := mfs.MkdirOpts{
Mkparents: true,
Flush: false,
Prefix: adder.Prefix,
CidOpts: adder.CidOpts,
}
if err := mfs.Mkdir(mr, dir, opts); err != nil {
return err
Expand Down Expand Up @@ -470,7 +471,7 @@ func (adder *Adder) addFile(file files.File) error {
}

dagnode := dag.NodeWithData(sdata)
dagnode.SetPrefix(adder.Prefix)
dagnode.SetCidOpts(adder.CidOpts)
err = adder.dagService.Add(adder.ctx, dagnode)
if err != nil {
return err
Expand Down Expand Up @@ -511,7 +512,7 @@ func (adder *Adder) addDir(dir files.File) error {
err = mfs.Mkdir(mr, dir.FileName(), mfs.MkdirOpts{
Mkparents: true,
Flush: false,
Prefix: adder.Prefix,
CidOpts: adder.CidOpts,
})
if err != nil {
return err
Expand Down
18 changes: 9 additions & 9 deletions importer/helpers/dagbuilder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@ import (
"os"

dag "github.com/ipfs/go-ipfs/merkledag"
cide "github.com/ipfs/go-ipfs/thirdparty/cidextra"
ft "github.com/ipfs/go-ipfs/unixfs"

chunker "gx/ipfs/QmWo8jYc19ppG7YoTsrr2kEtLRbARTJho5oNXFTR6B7Peq/go-ipfs-chunker"
cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid"
files "gx/ipfs/QmceUdzxkimdYsgtX733uNgzf1DLHyBKN6ehGSp85ayppM/go-ipfs-cmdkit/files"
ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format"
)
Expand All @@ -26,7 +26,7 @@ type DagBuilderHelper struct {
batch *ipld.Batch
fullPath string
stat os.FileInfo
prefix *cid.Prefix
cidOpts *cide.Opts
}

// DagBuilderParams wraps configuration options to create a DagBuilderHelper
Expand All @@ -39,8 +39,8 @@ type DagBuilderParams struct {
// instead of using the unixfs TRaw type
RawLeaves bool

// CID Prefix to use if set
Prefix *cid.Prefix
// CID Options
CidOpts *cide.Opts

// DAGService to write blocks to (required)
Dagserv ipld.DAGService
Expand All @@ -57,7 +57,7 @@ func (dbp *DagBuilderParams) New(spl chunker.Splitter) *DagBuilderHelper {
dserv: dbp.Dagserv,
spl: spl,
rawLeaves: dbp.RawLeaves,
prefix: dbp.Prefix,
cidOpts: dbp.CidOpts,
maxlinks: dbp.Maxlinks,
batch: ipld.NewBatch(context.TODO(), dbp.Dagserv),
}
Expand Down Expand Up @@ -118,7 +118,7 @@ func (db *DagBuilderHelper) NewUnixfsNode() *UnixfsNode {
node: new(dag.ProtoNode),
ufmt: &ft.FSNode{Type: ft.TFile},
}
n.SetPrefix(db.prefix)
n.SetCidOpts(db.cidOpts)
return n
}

Expand All @@ -128,7 +128,7 @@ func (db *DagBuilderHelper) newUnixfsBlock() *UnixfsNode {
node: new(dag.ProtoNode),
ufmt: &ft.FSNode{Type: ft.TRaw},
}
n.SetPrefix(db.prefix)
n.SetCidOpts(db.cidOpts)
return n
}

Expand Down Expand Up @@ -169,13 +169,13 @@ func (db *DagBuilderHelper) GetNextDataNode() (*UnixfsNode, error) {
}

if db.rawLeaves {
if db.prefix == nil {
if db.cidOpts == nil {
return &UnixfsNode{
rawnode: dag.NewRawNode(data),
raw: true,
}, nil
}
rawnode, err := dag.NewRawNodeWPrefix(data, *db.prefix)
rawnode, err := dag.NewRawNodeWOpts(data, *db.cidOpts)
if err != nil {
return nil, err
}
Expand Down
10 changes: 5 additions & 5 deletions importer/helpers/helpers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@ import (
"os"

dag "github.com/ipfs/go-ipfs/merkledag"
cide "github.com/ipfs/go-ipfs/thirdparty/cidextra"
ft "github.com/ipfs/go-ipfs/unixfs"
pi "gx/ipfs/Qmb3jLEFAQrqdVgWUajqEyuuDoavkSq1XQXz6tWdFWF995/go-ipfs-posinfo"

cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid"
pi "gx/ipfs/Qmb3jLEFAQrqdVgWUajqEyuuDoavkSq1XQXz6tWdFWF995/go-ipfs-posinfo"
ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format"
)

Expand Down Expand Up @@ -60,9 +60,9 @@ func NewUnixfsNodeFromDag(nd *dag.ProtoNode) (*UnixfsNode, error) {
}, nil
}

// SetPrefix sets the CID Prefix
func (n *UnixfsNode) SetPrefix(prefix *cid.Prefix) {
n.node.SetPrefix(prefix)
// SetCidOpts sets the CID Prefix
func (n *UnixfsNode) SetCidOpts(opts *cide.Opts) {
n.node.SetCidOpts(opts)
}

// NumChildren returns the number of children referenced by this UnixfsNode.
Expand Down
Loading