feat(remote): Beginning of "remote" mode implementation

On the remote side, the following is being added:
* The flag --remote-mode which enables remote mode
* An api /dataset that can be posted to, only in remote mode
* Receive method that calculates size of dag, and fails if too large

On the client side:
* The `qri publish` command has a "--remote" to control where to publish
* Publishing to a remote sends a dag.Info instead of dataset head
* Remove call to CanonicalizeDatasetRef in base/

Missing:
* Remote mode should act like read-only
* dag.Info is a stub for now
* `qri publish --remote ...` is ignoring name of remote
* dSync when a push to a remote succeeds

api/api.go

@@ -227,6 +227,11 @@ func NewServerRoutes(s *Server) *http.ServeMux {
 
  dsh := NewDatasetHandlers(s.qriNode, s.cfg.API.ReadOnly)
 
+ if s.cfg.API.RemoteMode {
+ remh := NewRemoteHandlers(s.qriNode)
+ m.Handle("/dataset", s.middleware(remh.ReceiveHandler))
+ }
+
  m.Handle("/list", s.middleware(dsh.ListHandler))
  m.Handle("/list/", s.middleware(dsh.PeerListHandler))
  m.Handle("/save", s.middleware(dsh.SaveHandler))

api/datasets.go

@@ -622,18 +622,18 @@ func (h DatasetHandlers) publishHandler(w http.ResponseWriter, r *http.Request,
  return
  }
 
- ref.Published = publish
  p := &lib.SetPublishStatusParams{
- Ref: &ref,
+ Ref: ref.String(),
+ PublishStatus: publish,
  UpdateRegistry: r.FormValue("no_registry") != "true",
  UpdateRegistryPin: r.FormValue("no_pin") != "true",
  }
- var ok bool
- if err := h.DatasetRequests.SetPublishStatus(p, &ok); err != nil {
+ var publishedRef repo.DatasetRef
+ if err := h.DatasetRequests.SetPublishStatus(p, &publishedRef); err != nil {
  util.WriteErrResponse(w, http.StatusInternalServerError, err)
  return
  }
- util.WriteResponse(w, ref)
+ util.WriteResponse(w, publishedRef)
 }
 
 func (h DatasetHandlers) updateHandler(w http.ResponseWriter, r *http.Request) {

api/remote.go

@@ -0,0 +1,52 @@
+package api
+
+import (
+ "io/ioutil"
+ "net/http"
+
+ util "github.com/datatogether/api/apiutil"
+ "github.com/qri-io/qri/lib"
+ "github.com/qri-io/qri/p2p"
+)
+
+// RemoteHandlers wraps a request struct to interface with http.HandlerFunc
+type RemoteHandlers struct {
+ lib.RemoteRequests
+}
+
+// NewRemoteHandlers allocates a RemoteHandlers pointer
+func NewRemoteHandlers(node *p2p.QriNode) *RemoteHandlers {
+ req := lib.NewRemoteRequests(node, nil)
+ return &RemoteHandlers{*req}
+}
+
+// ReceiveHandler is the endpoint for remotes to receive daginfo
+func (h *RemoteHandlers) ReceiveHandler(w http.ResponseWriter, r *http.Request) {
+ switch r.Method {
+ case "POST":
+ h.receiveDataset(w, r)
+ default:
+ util.NotFoundHandler(w, r)
+ }
+}
+
+func (h *RemoteHandlers) receiveDataset(w http.ResponseWriter, r *http.Request) {
+ content, err := ioutil.ReadAll(r.Body)
+ if err != nil {
+ util.WriteErrResponse(w, http.StatusInternalServerError, err)
+ return
+ }
+ var result bool
+ params := lib.ReceiveParams{Body: string(content)}
+ err = h.Receive(&params, &result)
+ if err != nil {
+ util.WriteErrResponse(w, http.StatusInternalServerError, err)
+ return
+ }
+ // TODO(dlong): Perform dsync
+ if result {
+ util.WriteResponse(w, "Accepted")
+ return
+ }
+ util.WriteResponse(w, "Denied")
+}

base/ref.go

@@ -18,10 +18,6 @@ func InLocalNamespace(r repo.Repo, ref *repo.DatasetRef) bool {
 
 // SetPublishStatus updates the Published field of a dataset ref
 func SetPublishStatus(r repo.Repo, ref *repo.DatasetRef, published bool) error {
- if err := repo.CanonicalizeDatasetRef(r, ref); err != nil {
- return err
- }
-
  if !InLocalNamespace(r, ref) {
  return fmt.Errorf("can't publish datasets that are not in your namespace")
  }

cmd/connect.go

@@ -53,6 +53,7 @@ peers & swapping data.`,
 
  cmd.Flags().BoolVarP(&o.Setup, "setup", "", false, "run setup if necessary, reading options from environment variables")
  cmd.Flags().BoolVarP(&o.ReadOnly, "read-only", "", false, "run qri in read-only mode, limits the api endpoints")
+ cmd.Flags().BoolVarP(&o.RemoteMode, "remote-mode", "", false, "run qri in remote mode")
  cmd.Flags().StringVarP(&o.Registry, "registry", "", "", "specify registry to setup with. only works when --setup is true")
 
  return cmd
@@ -72,9 +73,10 @@ type ConnectOptions struct {
  DisableWebapp bool
  DisableP2P bool
 
- Registry string
- Setup bool
- ReadOnly bool
+ Registry string
+ Setup bool
+ ReadOnly bool
+ RemoteMode bool
 
  Node *p2p.QriNode
  Config *config.Config
@@ -133,6 +135,9 @@ func (o *ConnectOptions) Run() (err error) {
  if o.ReadOnly {
  cfg.API.ReadOnly = true
  }
+ if o.RemoteMode {
+ cfg.API.RemoteMode = true
+ }
  if o.DisableP2P {
  cfg.P2P.Enabled = false
  }

cmd/factory.go

@@ -26,6 +26,7 @@ type Factory interface {
  ConnectionNode() (*p2p.QriNode, error)
 
  DatasetRequests() (*lib.DatasetRequests, error)
+ RemoteRequests() (*lib.RemoteRequests, error)
  RegistryRequests() (*lib.RegistryRequests, error)
  LogRequests() (*lib.LogRequests, error)
  ExportRequests() (*lib.ExportRequests, error)

cmd/factory_test.go

@@ -105,6 +105,11 @@ func (t TestFactory) DatasetRequests() (*lib.DatasetRequests, error) {
  return lib.NewDatasetRequests(t.node, t.rpc), nil
 }
 
+// RemoteRequests generates a lib.RemoteRequests from internal state
+func (t TestFactory) RemoteRequests() (*lib.RemoteRequests, error) {
+ return lib.NewRemoteRequests(t.node, t.rpc), nil
+}
+
 // RegistryRequests generates a lib.RegistryRequests from internal state
 func (t TestFactory) RegistryRequests() (*lib.RegistryRequests, error) {
  return lib.NewRegistryRequests(t.node, t.rpc), nil

cmd/publish.go

@@ -43,6 +43,7 @@ to a published dataset will be immediately visible to connected peers.
  cmd.Flags().BoolVarP(&o.Unpublish, "unpublish", "", false, "unpublish a dataset")
  cmd.Flags().BoolVarP(&o.NoRegistry, "no-registry", "", false, "don't publish to registry")
  cmd.Flags().BoolVarP(&o.NoPin, "no-pin", "", false, "don't pin dataset to registry")
+ cmd.Flags().StringVarP(&o.RemoteName, "remote", "", "", "name of remote to publish to")
 
  return cmd
 }
@@ -55,38 +56,50 @@ type PublishOptions struct {
  Unpublish bool
  NoRegistry bool
  NoPin bool
+ RemoteName string
 
  DatasetRequests *lib.DatasetRequests
+ RemoteRequests *lib.RemoteRequests
 }
 
 // Complete adds any missing configuration that can only be added just before calling Run
 func (o *PublishOptions) Complete(f Factory, args []string) (err error) {
  o.Refs = args
  o.DatasetRequests, err = f.DatasetRequests()
+ o.RemoteRequests, err = f.RemoteRequests()
  return
 }
 
 // Run executes the publish command
 func (o *PublishOptions) Run() error {
- var res bool
-
- for _, arg := range o.Refs {
- ref, err := repo.ParseDatasetRef(arg)
- if err != nil {
- return err
+ for _, ref := range o.Refs {
+ if o.RemoteName != "" {
+ // Publish for a "Remote".
+ p := lib.PushParams{
+ Ref: ref,
+ RemoteName: o.RemoteName,
+ }
+ var res bool
+ if err := o.RemoteRequests.PushToRemote(&p, &res); err != nil {
+ return err
+ }
+ // TODO(dlong): Check if the operation succeeded or failed. Perform dsync.
+ return nil
  }
 
- ref.Published = !o.Unpublish
- p := &lib.SetPublishStatusParams{
- Ref: &ref,
+ // Publish for the legacy Registry server.
+ p := lib.SetPublishStatusParams{
+ Ref: ref,
+ PublishStatus: !o.Unpublish,
  UpdateRegistry: !o.NoRegistry,
  UpdateRegistryPin: !o.NoPin,
  }
 
- if err = o.DatasetRequests.SetPublishStatus(p, &res); err != nil {
+ var publishedRef repo.DatasetRef
+ if err := o.DatasetRequests.SetPublishStatus(&p, &publishedRef); err != nil {
  return err
  }
- printInfo(o.Out, "published dataset %s", ref)
+ printInfo(o.Out, "published dataset %s", publishedRef)
  }
  return nil
 }

cmd/qri.go

@@ -239,6 +239,14 @@ func (o *QriOptions) DatasetRequests() (*lib.DatasetRequests, error) {
  return lib.NewDatasetRequests(o.node, o.rpc), nil
 }
 
+// RemoteRequests generates a lib.RemoteRequests from internal state
+func (o *QriOptions) RemoteRequests() (*lib.RemoteRequests, error) {
+ if err := o.Init(); err != nil {
+ return nil, err
+ }
+ return lib.NewRemoteRequests(o.node, o.rpc), nil
+}
+
 // RegistryRequests generates a lib.RegistryRequests from internal state
 func (o *QriOptions) RegistryRequests() (*lib.RegistryRequests, error) {
  if err := o.Init(); err != nil {

config/api.go

@@ -17,6 +17,8 @@ type API struct {
  Port int `json:"port"`
  // read-only mode
  ReadOnly bool `json:"readonly"`
+ // remote mode
+ RemoteMode bool `json:"remotemode"`
  // URLRoot is the base url for this server
  URLRoot string `json:"urlroot"`
  // TLS enables https via letsEyncrypt

lib/datasets.go

@@ -284,12 +284,13 @@ func (r *DatasetRequests) Save(p *SaveParams, res *repo.DatasetRef) (err error)
  }
 
  if p.Publish {
- var done bool
+ var publishedRef repo.DatasetRef
  err = r.SetPublishStatus(&SetPublishStatusParams{
- Ref: &ref,
+ Ref: ref.String(),
+ PublishStatus: true,
  UpdateRegistry: true,
  UpdateRegistryPin: true,
- }, &done)
+ }, &publishedRef)
 
  if err != nil {
  return err
@@ -372,42 +373,52 @@ func (r *DatasetRequests) Update(p *UpdateParams, res *repo.DatasetRef) error {
 
 // SetPublishStatusParams encapsulates parameters for setting the publication status of a dataset
 type SetPublishStatusParams struct {
- Ref *repo.DatasetRef
+ Ref string
+ PublishStatus bool
  UpdateRegistry bool
  UpdateRegistryPin bool
 }
 
 // SetPublishStatus updates the publicity of a reference in the peer's namespace
-func (r *DatasetRequests) SetPublishStatus(p *SetPublishStatusParams, res *bool) (err error) {
+func (r *DatasetRequests) SetPublishStatus(p *SetPublishStatusParams, publishedRef *repo.DatasetRef) (err error) {
  if r.cli != nil {
- return r.cli.Call("DatasetRequests.SetPublishStatus", p, res)
+ return r.cli.Call("DatasetRequests.SetPublishStatus", p, publishedRef)
+ }
+
+ ref, err := repo.ParseDatasetRef(p.Ref)
+ if err != nil {
+ return err
+ }
+ if err = repo.CanonicalizeDatasetRef(r.node.Repo, &ref); err != nil {
+ return err
  }
 
- ref := p.Ref
- res = &ref.Published
- if err = actions.SetPublishStatus(r.node, ref, ref.Published); err != nil {
+ ref.Published = p.PublishStatus
+ if err = actions.SetPublishStatus(r.node, &ref, ref.Published); err != nil {
  return err
  }
 
+ *publishedRef = ref
+
  if p.UpdateRegistry && r.node.Repo.Registry() != nil {
  var done bool
  rr := NewRegistryRequests(r.node, nil)
 
  if ref.Published {
- if err = rr.Publish(ref, &done); err != nil {
+ if err = rr.Publish(&ref, &done); err != nil {
  return
  }
 
  if p.UpdateRegistryPin {
- return rr.Pin(ref, &done)
+ return rr.Pin(&ref, &done)
  }
  } else {
- if err = rr.Unpublish(ref, &done); err != nil {
+ if err = rr.Unpublish(&ref, &done); err != nil {
  return
  }
 
  if p.UpdateRegistryPin {
- return rr.Unpin(ref, &done)
+ return rr.Unpin(&ref, &done)
  }
  }
  }

lib/params.go

@@ -66,3 +66,14 @@ func (lp ListParams) Page() util.Page {
  number = lp.Offset/size + 1
  return util.NewPage(number, size)
 }
+
+// PushParams holds parameters for pushing daginfo to remotes
+type PushParams struct {
+ Ref string
+ RemoteName string
+}
+
+// ReceiveParams hold parameters for receiving daginfo's when running as a remote
+type ReceiveParams struct {
+ Body string
+}