Add NodeReader interface

.travis.yml

@@ -5,10 +5,10 @@ os:
 language: go
 
 go:
-  - 1.4
   - 1.5
 
 script:
+  - go get -v ./...
   - go test -race -cpu 5 ./...
 
 sudo: false #docker containers for CI

Makefile

@@ -0,0 +1,4 @@
+.PHONY: test
+test:
+	go test ./...
+

codec.go

@@ -0,0 +1,5 @@
+package ipld
+
+const (
+	CodecKey = "@codec" // used to determine which multicodec to use
+)

coding/Makefile

@@ -1,19 +1,41 @@
 
-pb/bin/multicodec:
-	$(MAKE) -C pb bin/multicodec
+bin/multicodec:
+	mkdir -p bin
+	go get -d github.com/jbenet/go-multicodec/multicodec
+	go build -o "$@" github.com/jbenet/go-multicodec/multicodec
+
+bin/json2cbor:
+	mkdir -p bin
+	go get -d github.com/whyrusleeping/cbor/go/json2cbor
+	go build -o "$@" github.com/whyrusleeping/cbor/go/json2cbor
 
 bin/convert:
+	mkdir -p bin
 	cd bin; go build convert.go
 
-json.testfile: pb/bin/multicodec Makefile
+bin/msgio:
+	mkdir -p bin
+	go get -d github.com/jbenet/go-msgio/msgio
+	go build -o "$@" github.com/jbenet/go-msgio/msgio
+
+json.testfile: bin/multicodec Makefile test1.json
 	: >$@
-	pb/bin/multicodec header /multicodec >>$@
-	pb/bin/multicodec header /json >>$@
-	echo '{"@codec":"/json","abc":{"mlink":"QmXg9Pp2ytZ14xgmQjYEiHjVjMFXzCVVEcRTWJBmLgR39V"}}' >>$@
+	bin/multicodec header /mdagv1 >>$@
+	bin/multicodec header /json >>$@
+	cat test1.json >>$@
 
-cbor.testfile: json.testfile
-	pb/bin/multicodec header /multicodec >$@
+cbor.testfile: bin/multicodec bin/json2cbor Makefile test1.json
 	./convert -i $< -o $@.tmp -c '/cbor'
-	cat $@.tmp >>$@
-	rm -f $@.tmp
+	: >$@
+	bin/multicodec header /mdagv1 >>$@
+	bin/multicodec header /cbor >>$@
+	bin/json2cbor -i test1.json -o - >>$@
 
+protobuf.testfile: bin/multicodec bin/msgio
+	bin/multicodec header /mdagv1 >$@
+	bin/multicodec header /protobuf/msgio >>$@
+	mkdir -p dir
+	echo a >dir/a
+	echo b >dir/b
+	hash=`ipfs add -q -r dir | tail -n1` && \
+		ipfs object get "$$hash" --enc=protobuf | bin/msgio wrap >>$@

coding/base/base_reader.go

@@ -0,0 +1,55 @@
+package base
+
+import (
+	"github.com/ipfs/go-ipld/coding/stream"
+)
+
+type BaseReader struct {
+	Callback  stream.ReadFun
+	cbEnabled []bool
+	path      []interface{}
+	tokens    []stream.ReaderToken
+}
+
+func CreateBaseReader(cb stream.ReadFun) BaseReader {
+	return BaseReader{cb, []bool{}, []interface{}{}, []stream.ReaderToken{}}
+}
+
+// Executes the callback and stay in scope for sub elements
+func (p *BaseReader) ExecCallback(token stream.ReaderToken, value interface{}) error {
+	var err error
+	enabled := !p.Skipping()
+	if enabled {
+		err = p.Callback(p.path, token, value)
+		if err == stream.NodeReadSkip {
+			enabled = false
+			err = nil
+		}
+	}
+	p.cbEnabled = append(p.cbEnabled, enabled)
+	return err
+}
+
+// Return true if a parent callback wants to skip processing of its children
+func (p *BaseReader) Skipping() bool {
+	enabled := true
+	if len(p.cbEnabled) > 0 {
+		enabled = p.cbEnabled[len(p.cbEnabled)-1]
+	}
+	return !enabled
+}
+
+// Must be called after all sub elements below a ExecCallback are processed
+func (p *BaseReader) Descope() {
+	p.cbEnabled = p.cbEnabled[:len(p.cbEnabled)-1]
+}
+
+// Push a path element
+func (p *BaseReader) PushPath(elem interface{}) {
+	p.path = append(p.path, elem)
+}
+
+// Pop a path element
+func (p *BaseReader) PopPath() {
+	p.path = p.path[:len(p.path)-1]
+}

coding/bin/.gitignore

@@ -0,0 +1,5 @@
+/convert
+/json2cbor
+/msgio
+/multicodec
+

coding/cbor.testfile

@@ -1,3 +1,3 @@
-/multicodec
+/mdagv1
 /cbor
-�cabc�emlinkx.QmXg9Pp2ytZ14xgmQjYEiHjVjMFXzCVVEcRTWJBmLgR39V
+�cabc�emlinkx.QmXg9Pp2ytZ14xgmQjYEiHjVjMFXzCVVEcRTWJBmLgR39Vf@codece/json

coding/cbor/encode.go

@@ -0,0 +1,84 @@
+package cbor
+
+import (
+	"io"
+
+	ipld "github.com/ipfs/go-ipld"
+	ma "github.com/jbenet/go-multiaddr"
+	cbor "github.com/whyrusleeping/cbor/go"
+)
+
+// Encode to CBOR, add the multicodec header
+func Encode(w io.Writer, node ipld.NodeIterator, tags bool) error {
+	_, err := w.Write(Header)
+	if err != nil {
+		return err
+	}
+
+	return RawEncode(w, node, tags)
+}
+
+func encodeFilter(val interface{}) interface{} {
+	var ok bool
+	var object map[string]interface{}
+	var link interface{}
+	var linkStr string
+	var newObject map[string]interface{} = map[string]interface{}{}
+	var linkPath interface{}
+	var linkObject interface{}
+
+	object, ok = val.(map[string]interface{})
+	if !ok {
+		return val
+	}
+
+	link, ok = object[ipld.LinkKey]
+	if !ok {
+		return val
+	}
+
+	linkStr, ok = link.(string)
+	if !ok {
+		return val
+	}
+
+	maddr, err := ma.NewMultiaddr(linkStr)
+	for k, v := range object {
+		if k != ipld.LinkKey {
+			newObject[k] = v
+		}
+	}
+	if err != nil || maddr.String() != linkStr {
+		linkPath = linkStr
+	} else {
+		linkPath = maddr.Bytes()
+	}
+
+	if len(newObject) > 0 {
+		linkObject = []interface{}{linkPath, newObject}
+	} else {
+		linkObject = linkPath
+	}
+
+	return &cbor.CBORTag{
+		Tag:           TagIPLDLink,
+		WrappedObject: linkObject,
+	}
+}
+
+// Encode to CBOR, do not add the multicodec header
+func RawEncode(w io.Writer, node ipld.NodeIterator, tags bool) error {
+	enc := cbor.NewEncoder(w)
+	if tags {
+		enc.SetFilter(encodeFilter)
+	}
+
+	// Buffering to memory is absolutely necessary to normalize the CBOR data file
+	// (pairs order in a map)
+	mem, err := ipld.ToMemory(node)
+	if err != nil {
+		return err
+	}
+
+	return enc.Encode(mem)
+}