feat(manifest): generate a manifest for a given reference

actions/manifest.go

@@ -0,0 +1,56 @@
+package actions
+
+import (
+	"context"
+
+	"github.com/qri-io/qri/manifest"
+	"github.com/qri-io/qri/p2p"
+
+	"gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid"
+	ipld "gx/ipfs/QmR7TcHkR9nxkUorfi8XMTAMLUK7GiP64TWWBzY3aacc1o/go-ipld-format"
+	"gx/ipfs/QmUJYo4etAQqFfSS2rarFAE97eNGB8ej64YkRT2SmsYD4r/go-ipfs/core/coreapi"
+	coreiface "gx/ipfs/QmUJYo4etAQqFfSS2rarFAE97eNGB8ej64YkRT2SmsYD4r/go-ipfs/core/coreapi/interface"
+)
+
+// NewManifest generates a manifest for a given node
+func NewManifest(node *p2p.QriNode, path string) (*manifest.Manifest, error) {
+	ipfsn, err := node.IPFSNode()
+	if err != nil {
+		return nil, err
+	}
+
+	id, err := cid.Parse(path)
+	if err != nil {
+		return nil, err
+	}
+
+	ng := &nodeGetter{dag: coreapi.NewCoreAPI(ipfsn).Dag()}
+	return manifest.NewManifest(node.Context(), ng, id)
+}
+
+type nodeGetter struct {
+	dag coreiface.DagAPI
+}
+
+// Get retrieves nodes by CID. Depending on the NodeGetter
+// implementation, this may involve fetching the Node from a remote
+// machine; consider setting a deadline in the context.
+func (ng *nodeGetter) Get(ctx context.Context, id cid.Cid) (ipld.Node, error) {
+	path, err := coreiface.ParsePath(id.String())
+	if err != nil {
+		return nil, err
+	}
+	return ng.dag.Get(ctx, path)
+}
+
+// GetMany returns a channel of NodeOptions given a set of CIDs.
+func (ng *nodeGetter) GetMany(ctx context.Context, cids []cid.Cid) <-chan *ipld.NodeOption {
+	ch := make(chan *ipld.NodeOption)
+	go func() {
+		for _, id := range cids {
+			n, err := ng.Get(ctx, id)
+			ch <- &ipld.NodeOption{Err: err, Node: n}
+		}
+	}()
+	return ch
+}

cmd/manifest.go

@@ -0,0 +1,89 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"encoding/hex"
+	"encoding/json"
+
+	"github.com/ghodss/yaml"
+	"github.com/qri-io/ioes"
+	"github.com/qri-io/qri/lib"
+	"github.com/qri-io/qri/manifest"
+	"github.com/spf13/cobra"
+)
+
+// NewManifestCommand creates a new `qri search` command that searches for datasets
+func NewManifestCommand(f Factory, ioStreams ioes.IOStreams) *cobra.Command {
+	o := &ManifestOptions{IOStreams: ioStreams}
+	cmd := &cobra.Command{
+		Use:    "manifest",
+		Hidden: true,
+		Short:  "generate a qri manifest",
+		// Annotations: map[string]string{
+		// 	"group": "dataset",
+		// },
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if err := o.Complete(f, args); err != nil {
+				return err
+			}
+			return o.Run()
+		},
+	}
+
+	cmd.Flags().StringVarP(&o.Format, "format", "f", "json", "set output format [json, yaml, cbor]")
+	cmd.Flags().BoolVar(&o.Pretty, "pretty", false, "print output without indentation, only applies to json format")
+
+	return cmd
+}
+
+// ManifestOptions encapsulates state for the get command
+type ManifestOptions struct {
+	ioes.IOStreams
+
+	Refs   []string
+	Format string
+	Pretty bool
+
+	DatasetRequests *lib.DatasetRequests
+}
+
+// Complete adds any missing configuration that can only be added just before calling Run
+func (o *ManifestOptions) Complete(f Factory, args []string) (err error) {
+	o.Refs = args
+	o.DatasetRequests, err = f.DatasetRequests()
+	return
+}
+
+// Run executes the get command
+func (o *ManifestOptions) Run() (err error) {
+	mf := &manifest.Manifest{}
+	for _, refstr := range o.Refs {
+		if err = o.DatasetRequests.Manifest(&refstr, mf); err != nil {
+			return err
+		}
+
+		var buffer []byte
+		switch strings.ToLower(o.Format) {
+		case "json":
+			if !o.Pretty {
+				buffer, err = json.Marshal(mf)
+			} else {
+				buffer, err = json.MarshalIndent(mf, "", " ")
+			}
+		case "yaml":
+			buffer, err = yaml.Marshal(mf)
+		case "cbor":
+			var raw []byte
+			raw, err = mf.MarshalCBOR()
+			buffer = []byte(hex.EncodeToString(raw))
+		}
+		if err != nil {
+			return fmt.Errorf("error getting config: %s", err)
+		}
+		_, err = o.Out.Write(buffer)
+	}
+
+	return err
+}

cmd/qri.go

@@ -56,6 +56,7 @@ https://github.com/qri-io/qri/issues`,
 		NewInfoCommand(opt, ioStreams),
 		NewListCommand(opt, ioStreams),
 		NewLogCommand(opt, ioStreams),
+		NewManifestCommand(opt, ioStreams),
 		NewPublishCommand(opt, ioStreams),
 		NewPeersCommand(opt, ioStreams),
 		NewRegistryCommand(opt, ioStreams),

lib/datasets.go

@@ -10,6 +10,7 @@ import (
 	"github.com/qri-io/dsdiff"
 	"github.com/qri-io/jsonschema"
 	"github.com/qri-io/qri/actions"
+	"github.com/qri-io/qri/manifest"
 	"github.com/qri-io/qri/p2p"
 	"github.com/qri-io/qri/repo"
 )
@@ -470,3 +471,26 @@ func (r *DatasetRequests) Diff(p *DiffParams, diffs *map[string]*dsdiff.SubDiff)
 	*diffs, err = actions.DiffDatasets(r.node, p.Left, p.Right, p.DiffAll, p.DiffComponents)
 	return
 }
+
+// Manifest generates a manifest for a dataset path
+func (r *DatasetRequests) Manifest(refstr *string, m *manifest.Manifest) (err error) {
+	if r.cli != nil {
+		return r.cli.Call("DatasetRequests.Manifest", refstr, m)
+	}
+
+	ref, err := repo.ParseDatasetRef(*refstr)
+	if err != nil {
+		return err
+	}
+	if err = repo.CanonicalizeDatasetRef(r.node.Repo, &ref); err != nil {
+		return
+	}
+
+	var mf *manifest.Manifest
+	mf, err = actions.NewManifest(r.node, ref.Path)
+	if err != nil {
+		return
+	}
+	*m = *mf
+	return
+}

manifest/manifest.go

@@ -0,0 +1,135 @@
+package manifest
+
+import (
+	"bytes"
+	"context"
+
+	"github.com/ugorji/go/codec"
+
+	"gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid"
+	format "gx/ipfs/QmR7TcHkR9nxkUorfi8XMTAMLUK7GiP64TWWBzY3aacc1o/go-ipld-format"
+)
+
+// Node is a subset of the ipld format.Node interface
+type Node interface {
+	// pulled from blocks.Block format
+	Cid() cid.Cid
+	// Links is a helper function that returns all links within this object
+	Links() []*format.Link
+	// Size returns the size in bytes of the serialized object
+	Size() (uint64, error)
+}
+
+// Manifest is a DAG of only block names and links (no content)
+// node identifiers are stored in a slice "nodes", all other slices reference
+// cids by index positions
+type Manifest struct {
+	Root  int              `json:"root"`           // index if CID in nodes list this manifest is about. The subject of the manifest
+	Nodes []string         `json:"nodes"`          // list if CIDS contained in the root dag
+	Links [][2]int         `json:"links"`          // links between nodes
+	Sizes []uint64         `json:"sizes"`          // sizes of nodes in bytes
+	Meta  map[string][]int `json:"meta,omitempty"` // meta are lists of logical sub-DAGs by positions in the nodes list
+}
+
+// NewManifest generates a manifest from an ipld node
+func NewManifest(ctx context.Context, ng format.NodeGetter, id cid.Cid) (*Manifest, error) {
+	ms := &mstate{
+		ctx:  ctx,
+		ng:   ng,
+		cids: map[string]int{},
+		// by convention root is zero b/c root is first node to be added
+		m: &Manifest{},
+	}
+
+	node, err := ng.Get(ctx, id)
+	if err != nil {
+		return nil, err
+	}
+
+	if _, err := ms.addNode(node); err != nil {
+		return nil, err
+	}
+	return ms.m, nil
+}
+
+// MarshalCBOR encodes this manifest as CBOR data
+func (m *Manifest) MarshalCBOR() (data []byte, err error) {
+	buf := &bytes.Buffer{}
+	err = codec.NewEncoder(buf, &codec.CborHandle{}).Encode(m)
+	data = buf.Bytes()
+	return
+}
+
+// UnmarshalCBOR decodes a manifest from a byte slice
+func UnmarshalCBOR(data []byte) (m *Manifest, err error) {
+	m = &Manifest{}
+	err = codec.NewDecoder(bytes.NewReader(data), &codec.CborHandle{}).Decode(m)
+	return
+}
+
+// mstate is a state machine for generating a manifest
+type mstate struct {
+	ctx  context.Context
+	ng   format.NodeGetter
+	idx  int
+	cids map[string]int // lookup table of already-added cids
+	m    *Manifest
+}
+
+// addNode places a node in the manifest & state machine, recursively adding linked nodes
+// addNode returns early if this node is already added to the manifest
+func (ms *mstate) addNode(node Node) (int, error) {
+	id := node.Cid().String()
+
+	if idx, ok := ms.cids[id]; ok {
+		return idx, nil
+	}
+
+	// add the node
+	idx := ms.idx
+	ms.idx++
+
+	ms.cids[id] = idx
+	ms.m.Nodes = append(ms.m.Nodes, id)
+
+	// ignore size errors b/c uint64 has no way to represent
+	// errored size state as an int (-1), hopefully implementations default to 0
+	// when erroring :/
+	size, _ := node.Size()
+
+	ms.m.Sizes = append(ms.m.Sizes, size)
+
+	for _, link := range node.Links() {
+		linkNode, err := link.GetNode(ms.ctx, ms.ng)
+		if err != nil {
+			return -1, err
+		}
+
+		nodeIdx, err := ms.addNode(linkNode)
+		if err != nil {
+			return -1, err
+		}
+
+		ms.m.Links = append(ms.m.Links, [2]int{idx, nodeIdx})
+	}
+
+	return idx, nil
+}
+
+// Present produces a list of nodes that are present in a nodeGetter
+// func (m *Manifest) Present(ctx context.Context, ng format.NodeGetter) ([]string, error) {
+// 	present := make([]string, len(m.Nodes))
+
+// 	i := 0
+// 	for _, id := range m.Nodes {
+// 		cid := cid.Par
+// 		node, err := ng.Get(ctx, cid)
+// 		if err != nil {
+// 			return present, err
+// 		} else {
+// 			present[i] = cid
+// 			i++
+// 		}
+// 	}
+// 	return present[:i], nil
+// }