From e4d9e273c21b9e29f2d99b7d9d33859572fe941c Mon Sep 17 00:00:00 2001 From: Kasey Date: Tue, 12 Mar 2019 19:02:55 -0400 Subject: [PATCH] feat(daginfo): add `qri daginfo` command that returns a dag.Info of a dataset 1) adds DAGInfo.go file to cmd 2) adds base NewDAGInfo, that you give a context, store, nodegetter, and id, and it returns a labeled DAGInfo with the Manifest of the dataset, the sizes of each node. The labels correspond to the different dataset components: body, meta, structure, viz, transform, commit. 3) adds action and lib functions to wire the `qri daginfo` command together --- actions/manifest.go | 10 +++ base/manifest.go | 59 +++++++++++++++ cmd/DAGInfo.go | 175 ++++++++++++++++++++++++++++++++++++++++++++ cmd/qri.go | 1 + lib/datasets.go | 23 ++++++ 5 files changed, 268 insertions(+) create mode 100644 cmd/DAGInfo.go diff --git a/actions/manifest.go b/actions/manifest.go index 58502c799..24a4a0d15 100644 --- a/actions/manifest.go +++ b/actions/manifest.go @@ -30,6 +30,16 @@ func Missing(node *p2p.QriNode, m *dag.Manifest) (missing *dag.Manifest, err err return dag.Missing(node.Context(), ng, m) } +// NewDAGInfo generates a DAGInfo for a given node +func NewDAGInfo(node *p2p.QriNode, path string) (*dag.Info, error) { + ng, err := newNodeGetter(node) + if err != nil { + return nil, err + } + + return base.NewDAGInfo(node.Context(), node.Repo.Store(), ng, path) +} + // newNodeGetter generates an ipld.NodeGetter from a QriNode func newNodeGetter(node *p2p.QriNode) (ng ipld.NodeGetter, err error) { ipfsn, err := node.IPFSNode() diff --git a/base/manifest.go b/base/manifest.go index 26275465e..625ab7df2 100644 --- a/base/manifest.go +++ b/base/manifest.go @@ -4,6 +4,8 @@ import ( "context" "github.com/qri-io/dag" + "github.com/qri-io/dataset/dsfs" + "github.com/qri-io/qfs/cafs" "gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid" ipld "gx/ipfs/QmR7TcHkR9nxkUorfi8XMTAMLUK7GiP64TWWBzY3aacc1o/go-ipld-format" @@ -18,3 +20,60 @@ func NewManifest(ctx context.Context, ng ipld.NodeGetter, path string) (*dag.Man return dag.NewManifest(ctx, ng, id) } + +// NewDAGInfo generates a DAGInfo for a given node +func NewDAGInfo(ctx context.Context, store cafs.Filestore, ng ipld.NodeGetter, path string) (*dag.Info, error) { + id, err := cid.Parse(path) + if err != nil { + return nil, err + } + + info, err := dag.NewInfo(ctx, ng, id) + if err != nil { + return nil, err + } + // get referenced version of dataset + ds, err := dsfs.LoadDatasetRefs(store, path) + if err != nil { + return nil, err + } + info.Labels = map[string]int{} + prefix := store.PathPrefix() + if ds.BodyPath != "" { + err := info.AddLabelByID("body", dsfs.GetHashBase(ds.BodyPath, prefix)) + if err != nil { + return nil, err + } + } + if ds.Viz != nil { + err := info.AddLabelByID("viz", dsfs.GetHashBase(ds.Viz.Path, prefix)) + if err != nil { + return nil, err + } + } + if ds.Transform != nil { + err := info.AddLabelByID("transform", dsfs.GetHashBase(ds.Transform.Path, prefix)) + if err != nil { + return nil, err + } + } + if ds.Meta != nil { + err := info.AddLabelByID("meta", dsfs.GetHashBase(ds.Meta.Path, prefix)) + if err != nil { + return nil, err + } + } + if ds.Structure != nil { + err := info.AddLabelByID("structure", dsfs.GetHashBase(ds.Structure.Path, prefix)) + if err != nil { + return nil, err + } + } + if ds.Commit != nil { + err := info.AddLabelByID("commit", dsfs.GetHashBase(ds.Commit.Path, prefix)) + if err != nil { + return nil, err + } + } + return info, nil +} diff --git a/cmd/DAGInfo.go b/cmd/DAGInfo.go new file mode 100644 index 000000000..565d09440 --- /dev/null +++ b/cmd/DAGInfo.go @@ -0,0 +1,175 @@ +package cmd + +import ( + "encoding/hex" + "encoding/json" + "fmt" + // "io/ioutil" + // "path/filepath" + "strings" + + "github.com/ghodss/yaml" + "github.com/qri-io/dag" + "github.com/qri-io/ioes" + "github.com/qri-io/qri/lib" + "github.com/spf13/cobra" +) + +// NewDAGInfoCommand creates a new `qri daginfo` command that generates a daginfo for a given +// dataset reference. Referenced dataset must be stored in local CAFS +func NewDAGInfoCommand(f Factory, ioStreams ioes.IOStreams) *cobra.Command { + o := &DAGInfoOptions{IOStreams: ioStreams} + cmd := &cobra.Command{ + Use: "daginfo", + Hidden: true, + Short: "dataset daginfo interaction", + } + + get := &cobra.Command{ + Use: "get", + Short: "get one or more DAG info for a given reference", + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Complete(f, args); err != nil { + return err + } + return o.Get() + }, + } + + // missing := &cobra.Command{ + // Use: "missing", + // Short: "list blocks not present in this repo for a given daginfo", + // RunE: func(cmd *cobra.Command, args []string) error { + // if err := o.Complete(f, args); err != nil { + // return err + // } + // return o.Missing() + // }, + // } + + get.Flags().StringVar(&o.Format, "format", "json", "set output format [json, yaml, cbor]") + get.Flags().BoolVar(&o.Pretty, "pretty", false, "print output without indentation, only applies to json format") + get.Flags().BoolVar(&o.Hex, "hex", false, "hex-encode output") + + // missing.Flags().StringVar(&o.Format, "format", "json", "set output format [json, yaml, cbor]") + // missing.Flags().BoolVar(&o.Pretty, "pretty", false, "print output without indentation, only applies to json format") + // missing.Flags().BoolVar(&o.Hex, "hex", false, "hex-encode output") + // missing.Flags().StringVar(&o.File, "file", "", "daginfo file") + + cmd.AddCommand(get) + // cmd.AddCommand(get, missing) + + return cmd +} + +// DAGInfoOptions encapsulates state for the daginfo command +type DAGInfoOptions struct { + ioes.IOStreams + + Refs []string + Format string + Pretty bool + Hex bool + File string + + DatasetRequests *lib.DatasetRequests +} + +// Complete adds any missing configuration that can only be added just before calling Run +func (o *DAGInfoOptions) Complete(f Factory, args []string) (err error) { + o.Refs = args + o.DatasetRequests, err = f.DatasetRequests() + return +} + +// Get executes the get command +func (o *DAGInfoOptions) Get() (err error) { + info := &dag.Info{} + for _, refstr := range o.Refs { + if err = o.DatasetRequests.DAGInfo(&refstr, info); err != nil { + return err + } + + var buffer []byte + switch strings.ToLower(o.Format) { + case "json": + if !o.Pretty { + buffer, err = json.Marshal(info) + } else { + buffer, err = json.MarshalIndent(info, "", " ") + } + case "yaml": + buffer, err = yaml.Marshal(info) + // case "cbor": + // buffer, err = info.MarshalCBOR() + } + if err != nil { + return fmt.Errorf("err encoding daginfo: %s", err) + } + if o.Hex { + buffer = []byte(hex.EncodeToString(buffer)) + } + _, err = o.Out.Write(buffer) + } + + return err +} + +// Missing executes the missing command +// func (o *DAGInfoOptions) Missing() error { +// if o.File == "" { +// return fmt.Errorf("daginfo file is required") +// } + +// in := &dag.DAGInfo{} +// data, err := ioutil.ReadFile(o.File) +// if err != nil { +// return err +// } + +// switch strings.ToLower(filepath.Ext(o.File)) { +// case ".yaml": +// err = yaml.Unmarshal(data, in) +// case ".json": +// err = json.Unmarshal(data, in) +// case ".cbor": +// // TODO - detect hex input? +// // data, err = hex.DecodeString(string(data)) +// // if err != nil { +// // return err +// // } +// in, err = dag.UnmarshalCBORDAGInfo(data) +// } + +// if err != nil { +// return err +// } + +// info := &dag.DAGInfo{} +// if err = o.DatasetRequests.DAGInfoMissing(in, info); err != nil { +// return err +// } + +// var buffer []byte +// switch strings.ToLower(o.Format) { +// case "json": +// if !o.Pretty { +// buffer, err = json.Marshal(info) +// } else { +// buffer, err = json.MarshalIndent(info, "", " ") +// } +// case "yaml": +// buffer, err = yaml.Marshal(info) +// case "cbor": +// buffer, err = info.MarshalCBOR() +// } +// if err != nil { +// return fmt.Errorf("error encoding daginfo: %s", err) +// } +// if o.Hex { +// buffer = []byte(hex.EncodeToString(buffer)) +// } +// _, err = o.Out.Write(buffer) + +// return err +// } diff --git a/cmd/qri.go b/cmd/qri.go index 92cb224d1..ebcf74e95 100644 --- a/cmd/qri.go +++ b/cmd/qri.go @@ -72,6 +72,7 @@ https://github.com/qri-io/qri/issues`, NewUpdateCommand(opt, ioStreams), NewValidateCommand(opt, ioStreams), NewVersionCommand(opt, ioStreams), + NewDAGInfoCommand(opt, ioStreams), ) for _, sub := range cmd.Commands() { diff --git a/lib/datasets.go b/lib/datasets.go index 713468032..a561ae2a7 100644 --- a/lib/datasets.go +++ b/lib/datasets.go @@ -625,3 +625,26 @@ func (r *DatasetRequests) ManifestMissing(a, b *dag.Manifest) (err error) { *b = *mf return } + +// DAGInfo generates a manifest for a dataset path +func (r *DatasetRequests) DAGInfo(refstr *string, i *dag.Info) (err error) { + if r.cli != nil { + return r.cli.Call("DatasetRequests.DAGInfo", refstr, i) + } + + ref, err := repo.ParseDatasetRef(*refstr) + if err != nil { + return err + } + if err = repo.CanonicalizeDatasetRef(r.node.Repo, &ref); err != nil { + return + } + + var info *dag.Info + info, err = actions.NewDAGInfo(r.node, ref.Path) + if err != nil { + return + } + *i = *info + return +}