feat(registry cmd): add commands for working with registries

b5 · b5 · commit 85d68928122a · 2018-06-06T12:30:06.000-04:00
I've refactored a bunch of the logic around working with registries
to make the process more explicit &amp; controllable.
diff --git a/cmd/add.go b/cmd/add.go
@@ -24,6 +24,7 @@ var (
 	addDsPassive        bool
 	addDsShowValidation bool
 	addDsPrivate        bool
+	addDsNoRegistry     bool
 	addDsSecrets        []string
 )
 
@@ -149,6 +150,7 @@ continue?`, true) {
 	p := &core.SaveParams{
 		Dataset: dsp,
 		Private: addDsPrivate,
+		Publish: !addDsNoRegistry,
 	}
 
 	req, err := datasetRequests(false)
@@ -187,6 +189,7 @@ func init() {
 	datasetAddCmd.Flags().StringVarP(&addDsMessage, "message", "m", "", "commit message")
 	datasetAddCmd.Flags().BoolVarP(&addDsPrivate, "private", "", false, "make dataset private. WARNING: not yet implimented. Please refer to https://github.com/qri-io/qri/issues/291 for updates")
 	datasetAddCmd.Flags().StringSliceVar(&addDsSecrets, "secrets", nil, "transform secrets as comma separated key,value,key,value,... sequence")
+	datasetAddCmd.Flags().BoolVarP(&addDsNoRegistry, "no-registry", "n", false, "don't publish this dataset to the registry")
 	// datasetAddCmd.Flags().BoolVarP(&addDsShowValidation, "show-validation", "s", false, "display a list of validation errors upon adding")
 	RootCmd.AddCommand(datasetAddCmd)
 }
diff --git a/cmd/cmd_test.go b/cmd/cmd_test.go
@@ -162,6 +162,8 @@ func TestCommandsIntegration(t *testing.T) {
 		{"diff", "me/movies", "me/movies2", "-d", "detail"},
 		{"export", "--dataset", "-o" + path, "me/movies"},
 		{"export", "--all", "-o" + path, "--format=json", "--data-format=json", "me/movies"},
+		{"registry", "unpublish", "me/movies"},
+		{"registry", "publish", "me/movies"},
 		{"rename", "me/movies", "me/movie"},
 		{"data", "--limit=1", "--data-format=cbor", "me/movie"},
 		{"validate", "me/movie"},
diff --git a/cmd/registry.go b/cmd/registry.go
@@ -0,0 +1,97 @@
+package cmd
+
+import (
+	"github.com/qri-io/qri/repo"
+	"github.com/spf13/cobra"
+)
+
+// RegistryCmd is the subcommand for working with configured registries
+var RegistryCmd = &cobra.Command{
+	Use:   "registry",
+	Short: "commands for working with a qri registry",
+	Long: `Registries are federated public records of datasets and peers.
+These records form a public facing central lookup for your datasets, so others
+can find them through search tools and via web links. You can use registry 
+commands to control how your datasets are published to registries, opting out
+on a dataset-by-dataset basis.
+
+By default qri is configured to publish to https://registry.qri.io,
+the main public collection of datasets & peers. "qri add" and "qri update"
+default to publishing to a registry as part of dataset creation unless run 
+with the "no-registry" flag.
+
+Unpublished dataset info will be held locally so you can still interact
+with it. And your datasets will be available to others peers when you run 
+"qri connect", but will not show up in search results, and will not be 
+displayed on lists of registry datasets.
+
+Qri is designed to work without a registry should you want to opt out of
+centralized listing entirely, but know that peers who *do* participate in
+registries may choose to deprioritize connections with you. Opting out of a
+registry entirely is better left to advanced users.
+
+You can opt out of registries entirely by running:
+$ qri config set registry.location ""`,
+
+	Annotations: map[string]string{
+		"group": "network",
+	},
+}
+
+// publishCmd represents the export command
+var publishCmd = &cobra.Command{
+	Use:   "publish",
+	Short: "publish dataset info to the registry",
+	Example: `  Publish a dataset you've created to the registry:
+	$ qri registry publish me/dataset_name`,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		loadConfig()
+	},
+	Args: cobra.MinimumNArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		req, err := registryRequests(false)
+		ExitIfErr(err)
+
+		var res bool
+		for _, arg := range args {
+			ref, err := repo.ParseDatasetRef(arg)
+			ExitIfErr(err)
+
+			err = req.Publish(&ref, &res)
+			ExitIfErr(err)
+			printInfo("published dataset %s", ref)
+		}
+	},
+}
+
+// unpublishCmd represents the export command
+var unpublishCmd = &cobra.Command{
+	Use:   "unpublish",
+	Short: "remove dataset info from the registry",
+	Example: `  Remove a dataset from the registry:
+  $ qri registry unpublish me/dataset_name`,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		loadConfig()
+	},
+	Args: cobra.MinimumNArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		req, err := registryRequests(false)
+		ExitIfErr(err)
+
+		var res bool
+		for _, arg := range args {
+			ref, err := repo.ParseDatasetRef(arg)
+			ExitIfErr(err)
+
+			err = req.Unpublish(&ref, &res)
+			ExitIfErr(err)
+			printInfo("unpublished dataset %s", ref)
+		}
+	},
+}
+
+func init() {
+	RegistryCmd.AddCommand(publishCmd)
+	RegistryCmd.AddCommand(unpublishCmd)
+	RootCmd.AddCommand(RegistryCmd)
+}
diff --git a/cmd/repo.go b/cmd/repo.go
@@ -120,6 +120,28 @@ func datasetRequests(online bool) (*core.DatasetRequests, error) {
 	return req, nil
 }
 
+func registryRequests(online bool) (*core.RegistryRequests, error) {
+	if cli := rpcConn(); cli != nil {
+		return core.NewRegistryRequests(nil, cli), nil
+	}
+
+	if !online {
+		// TODO - make this not terrible
+		r, cli, err := repoOrClient(online)
+		if err != nil {
+			return nil, err
+		}
+		return core.NewRegistryRequests(r, cli), nil
+	}
+
+	n, err := qriNode(online)
+	if err != nil {
+		return nil, err
+	}
+
+	return core.NewRegistryRequests(n.Repo, nil), nil
+}
+
 func renderRequests(online bool) (*core.RenderRequests, error) {
 	if cli := rpcConn(); cli != nil {
 		return core.NewRenderRequests(nil, cli), nil
diff --git a/cmd/save.go b/cmd/save.go
@@ -23,6 +23,7 @@ var (
 	savePassive        bool
 	saveRescursive     bool
 	saveShowValidation bool
+	saveNoRegistry     bool
 	saveSecrets        []string
 )
 
@@ -113,6 +114,7 @@ continue?`, true) {
 		p := &core.SaveParams{
 			Dataset: dsp,
 			Private: false,
+			Publish: !saveNoRegistry,
 		}
 
 		req, err := datasetRequests(false)
@@ -150,5 +152,6 @@ func init() {
 	saveCmd.Flags().StringVarP(&saveDataPath, "data", "", "", "path to file or url to initialize from")
 	saveCmd.Flags().BoolVarP(&saveShowValidation, "show-validation", "s", false, "display a list of validation errors upon adding")
 	saveCmd.Flags().StringSliceVar(&saveSecrets, "secrets", nil, "transform secrets as comma separated key,value,key,value,... sequence")
+	saveCmd.Flags().BoolVarP(&saveNoRegistry, "no-registry", "n", false, "don't publish this dataset to the registry")
 	RootCmd.AddCommand(saveCmd)
 }
diff --git a/core/core.go b/core/core.go
@@ -27,6 +27,7 @@ func Receivers(node *p2p.QriNode) []Requests {
 
 	return []Requests{
 		NewDatasetRequestsWithNode(r, nil, node),
+		NewRegistryRequests(r, nil),
 		NewHistoryRequests(r, nil),
 		NewPeerRequests(node, nil),
 		NewProfileRequests(r, nil),
diff --git a/core/core_test.go b/core/core_test.go
@@ -18,8 +18,8 @@ func TestReceivers(t *testing.T) {
 	}
 
 	reqs := Receivers(node)
-	if len(reqs) != 6 {
-		t.Errorf("unexpected number of receivers returned. expected: %d. got: %d", 6, len(reqs))
+	if len(reqs) != 7 {
+		t.Errorf("unexpected number of receivers returned. expected: %d. got: %d\nhave you added/removed a receiver?", 7, len(reqs))
 		return
 	}
 }
diff --git a/core/datasets.go b/core/datasets.go
@@ -263,6 +263,7 @@ func (r *DatasetRequests) Get(p *repo.DatasetRef, res *repo.DatasetRef) (err err
 type SaveParams struct {
 	Dataset *dataset.DatasetPod // dataset to create
 	Private bool                // option to make dataset private. private data is not currently implimented, see https://github.com/qri-io/qri/issues/291 for updates
+	Publish bool
 }
 
 // Init creates a new qri dataset from a source of data
@@ -363,6 +364,15 @@ func (r *DatasetRequests) Init(p *SaveParams, res *repo.DatasetRef) (err error)
 		return err
 	}
 
+	if p.Publish {
+		fmt.Println("posting dataset to registry ...")
+		var done bool
+		if err = NewRegistryRequests(r.repo, nil).Publish(res, &done); err != nil {
+			return err
+		}
+		fmt.Println("done")
+	}
+
 	return r.repo.ReadDataset(res)
 }
 
@@ -476,6 +486,15 @@ func (r *DatasetRequests) Save(p *SaveParams, res *repo.DatasetRef) (err error)
 	}
 	ref.Dataset = ds.Encode()
 
+	if p.Publish {
+		fmt.Println("posting dataset to registry ...")
+		var done bool
+		if err = NewRegistryRequests(r.repo, nil).Publish(&ref, &done); err != nil {
+			return err
+		}
+		fmt.Println("done")
+	}
+
 	*res = ref
 	return nil
 }
diff --git a/core/registry.go b/core/registry.go
@@ -0,0 +1,47 @@
+package core
+
+import (
+	"fmt"
+	"net/rpc"
+
+	"github.com/qri-io/qri/repo"
+	"github.com/qri-io/qri/repo/actions"
+)
+
+// RegistryRequests defines business logic for working with registries
+type RegistryRequests struct {
+	repo actions.Registry
+	cli  *rpc.Client
+}
+
+// CoreRequestsName implements the Requests interface
+func (RegistryRequests) CoreRequestsName() string { return "registry" }
+
+// NewRegistryRequests creates a DatasetRequests pointer from either a repo
+// or an rpc.Client
+func NewRegistryRequests(r repo.Repo, cli *rpc.Client) *RegistryRequests {
+	if r != nil && cli != nil {
+		panic(fmt.Errorf("both repo and client supplied to NewRegistryRequests"))
+	}
+
+	return &RegistryRequests{
+		repo: actions.Registry{r},
+		cli:  cli,
+	}
+}
+
+// Publish a dataset to a registry
+func (r *RegistryRequests) Publish(ref *repo.DatasetRef, done *bool) error {
+	if r.cli != nil {
+		return r.cli.Call("RegistryRequests.Publish", ref, done)
+	}
+	return r.repo.Publish(*ref)
+}
+
+// Unpublish a dataset from a registry
+func (r *RegistryRequests) Unpublish(ref *repo.DatasetRef, done *bool) error {
+	if r.cli != nil {
+		return r.cli.Call("RegistryRequests.Unpublish", ref, done)
+	}
+	return r.repo.Unpublish(*ref)
+}
diff --git a/repo/actions/dataset.go b/repo/actions/dataset.go
@@ -69,18 +69,6 @@ func (act Dataset) CreateDataset(name string, ds *dataset.Dataset, data cafs.Fil
 		return
 	}
 
-	if rc := act.Registry(); rc != nil {
-		log.Debugf("posting dataset to registry: %s/%s", pro.Peername, name)
-		dse := ds.Encode()
-		// TODO - this should be set be dsfs.CreateDataset:
-		dse.Path = path.String()
-
-		if e := rc.PutDataset(pro.Peername, name, dse, pro.PrivKey.GetPublic()); e != nil {
-			// ignore registry errors
-			log.Errorf("registering dataset: %s", e.Error())
-		}
-	}
-
 	if err = act.LogEvent(repo.ETDsCreated, ref); err != nil {
 		return
 	}
diff --git a/repo/actions/registry.go b/repo/actions/registry.go
@@ -0,0 +1,85 @@
+package actions
+
+import (
+	"fmt"
+
+	"github.com/ipfs/go-datastore"
+	"github.com/libp2p/go-libp2p-crypto"
+	"github.com/qri-io/dataset"
+	"github.com/qri-io/dataset/dsfs"
+	"github.com/qri-io/qri/repo"
+	"github.com/qri-io/qri/repo/profile"
+	"github.com/qri-io/registry/regclient"
+)
+
+// Registry wraps a repo.Repo, adding actions related to working
+// with registries
+type Registry struct {
+	repo.Repo
+}
+
+// Publish a dataset to a repo's specified registry
+func (act Registry) Publish(ref repo.DatasetRef) (err error) {
+	cli, pub, ds, err := act.dsParams(&ref)
+	if err != nil {
+		return err
+	}
+	if err = act.permission(ref); err != nil {
+		return
+	}
+	return cli.PutDataset(ref.Peername, ref.Name, ds.Encode(), pub)
+}
+
+// Unpublish a dataset from a repo's specified registry
+func (act Registry) Unpublish(ref repo.DatasetRef) (err error) {
+	cli, pub, ds, err := act.dsParams(&ref)
+	if err != nil {
+		return err
+	}
+	if err = act.permission(ref); err != nil {
+		return
+	}
+	return cli.DeleteDataset(ref.Peername, ref.Name, ds.Encode(), pub)
+}
+
+// dsParams is a convenience func that collects params for registry dataset interaction
+func (act Registry) dsParams(ref *repo.DatasetRef) (cli *regclient.Client, pub crypto.PubKey, ds *dataset.Dataset, err error) {
+	if cli = act.Registry(); cli == nil {
+		err = fmt.Errorf("no configured registry")
+		return
+	}
+
+	pk := act.PrivateKey()
+	if pk == nil {
+		err = fmt.Errorf("repo has no configured private key")
+		return
+	}
+	pub = pk.GetPublic()
+
+	if err = repo.CanonicalizeDatasetRef(act, ref); err != nil {
+		err = fmt.Errorf("canonicalizing dataset reference: %s", err.Error())
+		return
+	}
+
+	if ref.Path == "" {
+		if *ref, err = act.GetRef(*ref); err != nil {
+			return
+		}
+	}
+
+	ds, err = dsfs.LoadDataset(act.Store(), datastore.NewKey(ref.Path))
+	return
+}
+
+// permission returns an error if a repo's configured user does not have the right
+// to publish ref to a registry
+func (act Registry) permission(ref repo.DatasetRef) (err error) {
+	var pro *profile.Profile
+	if pro, err = act.Profile(); err != nil {
+		return err
+	}
+	if pro.Peername != ref.Peername {
+		return fmt.Errorf("peername mismatch. '%s' doesn't have permission to publish a dataset created by '%s'", pro.Peername, ref.Peername)
+	}
+	return nil
+}

Original file line number	Diff line number	Diff line change
`@@ -23,6 +23,7 @@ var (`
`23`	`23`	`savePassive bool`
`24`	`24`	`saveRescursive bool`
`25`	`25`	`saveShowValidation bool`
	`26`	`+ saveNoRegistry bool`
`26`	`27`	`saveSecrets []string`
`27`	`28`	`)`
`28`	`29`
@@ -113,6 +114,7 @@ continue?`, true) {
`113`	`114`	`p := &core.SaveParams{`
`114`	`115`	`Dataset: dsp,`
`115`	`116`	`Private: false,`
	`117`	`+ Publish: !saveNoRegistry,`
`116`	`118`	`}`
`117`	`119`
`118`	`120`	`req, err := datasetRequests(false)`
`@@ -150,5 +152,6 @@ func init() {`
`150`	`152`	`saveCmd.Flags().StringVarP(&saveDataPath, "data", "", "", "path to file or url to initialize from")`
`151`	`153`	`saveCmd.Flags().BoolVarP(&saveShowValidation, "show-validation", "s", false, "display a list of validation errors upon adding")`
`152`	`154`	`saveCmd.Flags().StringSliceVar(&saveSecrets, "secrets", nil, "transform secrets as comma separated key,value,key,value,... sequence")`
	`155`	`+ saveCmd.Flags().BoolVarP(&saveNoRegistry, "no-registry", "n", false, "don't publish this dataset to the registry")`
`153`	`156`	`RootCmd.AddCommand(saveCmd)`
`154`	`157`	`}`
Original file line number	Diff line number	Diff line change
`@@ -18,8 +18,8 @@ func TestReceivers(t *testing.T) {`
`18`	`18`	`}`
`19`	`19`
`20`	`20`	`reqs := Receivers(node)`
`21`		`- if len(reqs) != 6 {`
`22`		`- t.Errorf("unexpected number of receivers returned. expected: %d. got: %d", 6, len(reqs))`
	`21`	`+ if len(reqs) != 7 {`
	`22`	`+ t.Errorf("unexpected number of receivers returned. expected: %d. got: %d\nhave you added/removed a receiver?", 7, len(reqs))`
`23`	`23`	`return`
`24`	`24`	`}`
`25`	`25`	`}`