diff --git a/api/api.go b/api/api.go index 2ace6f3bf..11e663c43 100644 --- a/api/api.go +++ b/api/api.go @@ -212,9 +212,12 @@ func NewServerRoutes(s Server) *mux.Router { m.Handle(lib.AEStatus.String(), s.Middleware(fsih.StatusHandler(lib.AEStatus.NoTrailingSlash()))) m.Handle(lib.AEWhatChanged.String(), s.Middleware(fsih.WhatChangedHandler(lib.AEWhatChanged.NoTrailingSlash()))) m.Handle(lib.AEInit.String(), s.Middleware(fsih.InitHandler(lib.AEInit.NoTrailingSlash()))) + m.Handle(lib.AECanInitDatasetWorkDir.String(), s.Middleware(fsih.CanInitDatasetWorkDirHandler(lib.AECanInitDatasetWorkDir.NoTrailingSlash()))) m.Handle(lib.AECheckout.String(), s.Middleware(fsih.CheckoutHandler(lib.AECheckout.NoTrailingSlash()))) m.Handle(lib.AERestore.String(), s.Middleware(fsih.RestoreHandler(lib.AERestore.NoTrailingSlash()))) m.Handle(lib.AEFSIWrite.String(), s.Middleware(fsih.WriteHandler(lib.AEFSIWrite.NoTrailingSlash()))) + m.Handle(lib.AEFSICreateLink.String(), s.Middleware(fsih.CreateLinkHandler(lib.AEFSICreateLink.NoTrailingSlash()))) + m.Handle(lib.AEFSIUnlink.String(), s.Middleware(fsih.UnlinkHandler(lib.AEFSIUnlink.NoTrailingSlash()))) renderh := NewRenderHandlers(s.Instance) m.Handle(lib.AERender.String(), s.Middleware(renderh.RenderHandler)) diff --git a/api/fsi.go b/api/fsi.go index 22ae4ece3..212036997 100644 --- a/api/fsi.go +++ b/api/fsi.go @@ -1,23 +1,17 @@ package api import ( - "encoding/json" - "errors" - "fmt" "net/http" + "strings" - "github.com/qri-io/dataset" "github.com/qri-io/qri/api/util" - "github.com/qri-io/qri/fsi" + "github.com/qri-io/qri/dsref" "github.com/qri-io/qri/lib" - "github.com/qri-io/qri/profile" - "github.com/qri-io/qri/repo" - reporef "github.com/qri-io/qri/repo/ref" ) // FSIHandlers connects HTTP requests to the FSI subsystem type FSIHandlers struct { - lib.FSIMethods + inst *lib.Instance dsm *lib.DatasetMethods ReadOnly bool } @@ -25,9 +19,9 @@ type FSIHandlers struct { // NewFSIHandlers creates handlers that talk to qri's filesystem integration func NewFSIHandlers(inst *lib.Instance, readOnly bool) FSIHandlers { return FSIHandlers{ - FSIMethods: *lib.NewFSIMethods(inst), - dsm: lib.NewDatasetMethods(inst), - ReadOnly: readOnly, + inst: inst, + dsm: lib.NewDatasetMethods(inst), + ReadOnly: readOnly, } } @@ -42,7 +36,7 @@ func (h *FSIHandlers) StatusHandler(routePrefix string) http.HandlerFunc { } switch r.Method { - case http.MethodGet: + case http.MethodGet, http.MethodPost: handleStatus(w, r) default: util.NotFoundHandler(w, r) @@ -52,20 +46,26 @@ func (h *FSIHandlers) StatusHandler(routePrefix string) http.HandlerFunc { func (h *FSIHandlers) statusHandler(routePrefix string) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - ref, err := lib.DsRefFromPath(r.URL.Path[len(routePrefix):]) + method := "fsi.status" + p := h.inst.NewInputParam(method) + + // TODO(dustmop): Add this to UnmarshalParams for methods that can + // receive a refstr in the URL, or annotate the param struct with + // a tag and marshal the url to that field + err := addDsRefFromURL(r, routePrefix) if err != nil { - util.WriteErrResponse(w, http.StatusBadRequest, fmt.Errorf("bad reference: %s", err.Error())) + util.WriteErrResponse(w, http.StatusBadRequest, err) return } - res := []lib.StatusItem{} - alias := ref.Alias() - err = h.StatusForAlias(&alias, &res) - if err == fsi.ErrNoLink { - util.WriteErrResponse(w, http.StatusBadRequest, fmt.Errorf("no working directory: %s", alias)) + if err := UnmarshalParams(r, p); err != nil { + util.WriteErrResponse(w, http.StatusBadRequest, err) return - } else if err != nil { - util.WriteErrResponse(w, http.StatusInternalServerError, fmt.Errorf("error getting status: %s", err.Error())) + } + + res, err := h.inst.Dispatch(r.Context(), method, p) + if err != nil { + util.RespondWithError(w, err) return } util.WriteResponse(w, res) @@ -94,24 +94,21 @@ func (h *FSIHandlers) WhatChangedHandler(routePrefix string) http.HandlerFunc { func (h *FSIHandlers) whatChangedHandler(routePrefix string) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - ref, err := lib.DsRefFromPath(r.URL.Path[len(routePrefix):]) - if err != nil { - util.WriteErrResponse(w, http.StatusBadRequest, fmt.Errorf("bad reference: %s", err.Error())) + method := "fsi.whatchanged" + p := h.inst.NewInputParam(method) + + if err := UnmarshalParams(r, p); err != nil { + util.WriteErrResponse(w, http.StatusBadRequest, err) return } - res := []lib.StatusItem{} - refStr := ref.String() - err = h.WhatChanged(&refStr, &res) + res, err := h.inst.Dispatch(r.Context(), method, p) if err != nil { - if err == repo.ErrNoHistory { - util.WriteErrResponse(w, http.StatusUnprocessableEntity, err) - return - } - util.WriteErrResponse(w, http.StatusInternalServerError, fmt.Errorf("error getting status: %s", err.Error())) + util.RespondWithError(w, err) return } util.WriteResponse(w, res) + return } } @@ -136,51 +133,59 @@ func (h *FSIHandlers) InitHandler(routePrefix string) http.HandlerFunc { func (h *FSIHandlers) initHandler(routePrefix string) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - p := &lib.InitDatasetParams{ - TargetDir: r.FormValue("targetdir"), - Name: r.FormValue("name"), - Format: r.FormValue("format"), - BodyPath: r.FormValue("bodypath"), - } + method := "fsi.init" + p := h.inst.NewInputParam(method) - var name string - if err := h.InitDataset(r.Context(), p, &name); err != nil { + if err := UnmarshalParams(r, p); err != nil { util.WriteErrResponse(w, http.StatusBadRequest, err) return } - // Get code taken - // taken from ./root.go - gp := lib.GetParams{ - Refstr: name, - } - - res, err := h.dsm.Get(r.Context(), &gp) + res, err := h.inst.Dispatch(r.Context(), method, p) if err != nil { - if err == repo.ErrNotFound { - util.NotFoundHandler(w, r) - return - } - util.WriteErrResponse(w, http.StatusInternalServerError, err) + util.RespondWithError(w, err) return } - if res.Dataset == nil || res.Dataset.IsEmpty() { - util.WriteErrResponse(w, http.StatusNotFound, errors.New("cannot find peer dataset")) + util.WriteResponse(w, res) + return + } +} + +// CanInitDatasetWorkDirHandler returns whether a directory can be initialized +func (h *FSIHandlers) CanInitDatasetWorkDirHandler(routePrefix string) http.HandlerFunc { + handleCanInit := h.canInitDatasetWorkDirHandler(routePrefix) + + return func(w http.ResponseWriter, r *http.Request) { + if h.ReadOnly { + readOnlyResponse(w, "/caninitdatasetworkdir") return } - // TODO (b5) - why is this necessary? - ref := reporef.DatasetRef{ - Peername: res.Dataset.Peername, - ProfileID: profile.IDB58DecodeOrEmpty(res.Dataset.ProfileID), - Name: res.Dataset.Name, - Path: res.Dataset.Path, - FSIPath: res.FSIPath, - Published: res.Published, - Dataset: res.Dataset, + switch r.Method { + case http.MethodPost: + handleCanInit(w, r) + default: + util.NotFoundHandler(w, r) + } + } +} + +func (h *FSIHandlers) canInitDatasetWorkDirHandler(routePrefix string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + method := "fsi.caninitdatasetworkdir" + p := h.inst.NewInputParam(method) + + if err := UnmarshalParams(r, p); err != nil { + util.WriteErrResponse(w, http.StatusBadRequest, err) + return } - util.WriteResponse(w, ref) + res, err := h.inst.Dispatch(r.Context(), method, p) + if err != nil { + util.RespondWithError(w, err) + return + } + util.WriteResponse(w, res) return } } @@ -205,30 +210,124 @@ func (h *FSIHandlers) WriteHandler(routePrefix string) http.HandlerFunc { func (h *FSIHandlers) writeHandler(routePrefix string) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - ref, err := lib.DsRefFromPath(r.URL.Path[len(routePrefix):]) + method := "fsi.write" + p := h.inst.NewInputParam(method) + + // TODO(dustmop): Add this to UnmarshalParams for methods that can + // receive a refstr in the URL, or annotate the param struct with + // a tag and marshal the url to that field + err := addDsRefFromURL(r, routePrefix) + if err != nil { + util.WriteErrResponse(w, http.StatusBadRequest, err) + return + } + + if err := UnmarshalParams(r, p); err != nil { + util.WriteErrResponse(w, http.StatusBadRequest, err) + return + } + + res, err := h.inst.Dispatch(r.Context(), method, p) + if err != nil { + util.RespondWithError(w, err) + return + } + util.WriteResponse(w, res) + return + } +} + +// CreateLinkHandler creates an fsi link +func (h *FSIHandlers) CreateLinkHandler(routePrefix string) http.HandlerFunc { + handler := h.createLinkHandler(routePrefix) + return func(w http.ResponseWriter, r *http.Request) { + if h.ReadOnly { + readOnlyResponse(w, routePrefix) + return + } + + switch r.Method { + case http.MethodPost: + handler(w, r) + default: + util.NotFoundHandler(w, r) + } + } +} + +func (h *FSIHandlers) createLinkHandler(routePrefix string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + method := "fsi.createlink" + p := h.inst.NewInputParam(method) + + // TODO(dustmop): Add this to UnmarshalParams for methods that can + // receive a refstr in the URL, or annotate the param struct with + // a tag and marshal the url to that field + err := addDsRefFromURL(r, routePrefix) if err != nil { - util.WriteErrResponse(w, http.StatusBadRequest, fmt.Errorf("bad reference: %s", err.Error())) + util.WriteErrResponse(w, http.StatusBadRequest, err) return } - ds := &dataset.Dataset{} - if err := json.NewDecoder(r.Body).Decode(ds); err != nil { + if err := UnmarshalParams(r, p); err != nil { util.WriteErrResponse(w, http.StatusBadRequest, err) return } - p := &lib.FSIWriteParams{ - Ref: ref.Alias(), - Ds: ds, + res, err := h.inst.Dispatch(r.Context(), method, p) + if err != nil { + util.RespondWithError(w, err) + return + } + util.WriteResponse(w, res) + return + } +} + +// UnlinkHandler unlinks a working directory +func (h *FSIHandlers) UnlinkHandler(routePrefix string) http.HandlerFunc { + handler := h.unlinkHandler(routePrefix) + return func(w http.ResponseWriter, r *http.Request) { + if h.ReadOnly { + readOnlyResponse(w, routePrefix) + return + } + + switch r.Method { + case http.MethodPost: + handler(w, r) + default: + util.NotFoundHandler(w, r) + } + } +} + +func (h *FSIHandlers) unlinkHandler(routePrefix string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + method := "fsi.unlink" + p := h.inst.NewInputParam(method) + + // TODO(dustmop): Add this to UnmarshalParams for methods that can + // receive a refstr in the URL, or annotate the param struct with + // a tag and marshal the url to that field + err := addDsRefFromURL(r, routePrefix) + if err != nil { + util.WriteErrResponse(w, http.StatusBadRequest, err) + return } - out := []lib.StatusItem{} - if err := h.Write(p, &out); err != nil { + if err := UnmarshalParams(r, p); err != nil { util.WriteErrResponse(w, http.StatusBadRequest, err) return } - util.WriteResponse(w, out) + res, err := h.inst.Dispatch(r.Context(), method, p) + if err != nil { + util.RespondWithError(w, err) + return + } + util.WriteResponse(w, res) + return } } @@ -252,24 +351,30 @@ func (h *FSIHandlers) CheckoutHandler(routePrefix string) http.HandlerFunc { func (h *FSIHandlers) checkoutHandler(routePrefix string) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - ref, err := lib.DsRefFromPath(r.URL.Path[len(routePrefix):]) + method := "fsi.checkout" + p := h.inst.NewInputParam(method) + + // TODO(dustmop): Add this to UnmarshalParams for methods that can + // receive a refstr in the URL, or annotate the param struct with + // a tag and marshal the url to that field + err := addDsRefFromURL(r, routePrefix) if err != nil { - util.WriteErrResponse(w, http.StatusBadRequest, fmt.Errorf("bad reference: %s", err.Error())) + util.WriteErrResponse(w, http.StatusBadRequest, err) return } - p := &lib.CheckoutParams{ - Dir: r.FormValue("dir"), - Ref: ref.String(), + if err := UnmarshalParams(r, p); err != nil { + util.WriteErrResponse(w, http.StatusBadRequest, err) + return } - var res string - if err := h.Checkout(p, &res); err != nil { - util.WriteErrResponse(w, http.StatusInternalServerError, err) + res, err := h.inst.Dispatch(r.Context(), method, p) + if err != nil { + util.RespondWithError(w, err) return } - util.WriteResponse(w, res) + return } } @@ -293,27 +398,53 @@ func (h *FSIHandlers) RestoreHandler(routePrefix string) http.HandlerFunc { func (h *FSIHandlers) restoreHandler(routePrefix string) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - ref, err := lib.DsRefFromPath(r.URL.Path[len(routePrefix):]) + method := "fsi.restore" + p := h.inst.NewInputParam(method) + + // TODO(dustmop): Add this to UnmarshalParams for methods that can + // receive a refstr in the URL, or annotate the param struct with + // a tag and marshal the url to that field + err := addDsRefFromURL(r, routePrefix) if err != nil { - util.WriteErrResponse(w, http.StatusBadRequest, fmt.Errorf("bad reference: %s", err.Error())) + util.WriteErrResponse(w, http.StatusBadRequest, err) return } - // Add the path for the version to restore - ref.Path = r.FormValue("path") - - p := &lib.RestoreParams{ - Dir: r.FormValue("dir"), - Ref: ref.String(), - Component: r.FormValue("component"), + if err := UnmarshalParams(r, p); err != nil { + util.WriteErrResponse(w, http.StatusBadRequest, err) + return } - var res string - if err := h.Restore(p, &res); err != nil { - util.WriteErrResponse(w, http.StatusInternalServerError, err) + res, err := h.inst.Dispatch(r.Context(), method, p) + if err != nil { + util.RespondWithError(w, err) return } - util.WriteResponse(w, res) + return + } +} + +// If the route has a dataset reference in the url, parse that ref, and +// add it to the request object using the field "refstr". +func addDsRefFromURL(r *http.Request, routePrefix string) error { + // routePrefix looks like "/route/{path:.*}" and we only want "/route/" + pos := strings.LastIndex(routePrefix, "/") + if pos > 1 { + routePrefix = routePrefix[:pos+1] + } + + // Parse the ref, then reencode it and attach back on the url + url := r.URL.Path[len(routePrefix):] + ref, err := lib.DsRefFromPath(url) + if err != nil { + if err == dsref.ErrEmptyRef { + return nil + } + return err } + q := r.URL.Query() + q.Add("refstr", ref.String()) + r.URL.RawQuery = q.Encode() + return nil } diff --git a/api/fsi_test.go b/api/fsi_test.go index 6fe8f9e5d..0a936fa08 100644 --- a/api/fsi_test.go +++ b/api/fsi_test.go @@ -271,7 +271,11 @@ func TestFSIWrite(t *testing.T) { t.Errorf("expected body %s, got %s", expectBody, actualBody) } - status, strRes := JSONAPICallWithBody("POST", "/me/write_test", &dataset.Dataset{Meta: &dataset.Meta{Title: "oh hai there"}}, fsiHandler.WriteHandler("")) + p := lib.FSIWriteParams{ + Refstr: "peer/write_test", + Ds: &dataset.Dataset{Meta: &dataset.Meta{Title: "oh hai there"}}, + } + status, strRes := JSONAPICallWithBody("POST", "/fsi/write/me/write_test", p, fsiHandler.WriteHandler("/fsi/write/")) if status != http.StatusOK { t.Errorf("status code mismatch. expected: %d, got: %d", http.StatusOK, status) diff --git a/api/testdata/api.snapshot b/api/testdata/api.snapshot index c958db755..3487d95b6 100755 Binary files a/api/testdata/api.snapshot and b/api/testdata/api.snapshot differ diff --git a/api/util/errors.go b/api/util/errors.go index 80d29785c..27d3e3ac9 100644 --- a/api/util/errors.go +++ b/api/util/errors.go @@ -8,6 +8,7 @@ import ( golog "github.com/ipfs/go-log" "github.com/qri-io/qfs" "github.com/qri-io/qri/dsref" + "github.com/qri-io/qri/fsi" "github.com/qri-io/qri/repo" ) @@ -35,6 +36,14 @@ func RespondWithError(w http.ResponseWriter, err error) { WriteErrResponse(w, http.StatusNotFound, err) return } + if errors.Is(err, repo.ErrNotFound) { + WriteErrResponse(w, http.StatusNotFound, err) + return + } + if errors.Is(err, fsi.ErrNoLink) { + WriteErrResponse(w, http.StatusBadRequest, err) + return + } if errors.Is(err, repo.ErrNoHistory) { WriteErrResponse(w, http.StatusUnprocessableEntity, err) return diff --git a/cmd/checkout.go b/cmd/checkout.go index 037fe0941..c752911df 100644 --- a/cmd/checkout.go +++ b/cmd/checkout.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "fmt" "strings" @@ -40,21 +41,21 @@ func NewCheckoutCommand(f Factory, ioStreams ioes.IOStreams) *cobra.Command { type CheckoutOptions struct { ioes.IOStreams - Refs *RefSelect - - FSIMethods *lib.FSIMethods + Instance *lib.Instance - Dir string + Refs *RefSelect + Dir string } // Complete configures the checkout command func (o *CheckoutOptions) Complete(f Factory, args []string) (err error) { - o.FSIMethods, err = f.FSIMethods() + o.Instance = f.Instance() + if err != nil { return err } - o.Refs, err = GetCurrentRefSelect(f, args, 1, EnsureFSIAgrees(o.FSIMethods)) + o.Refs, err = GetCurrentRefSelect(f, args, 1, EnsureFSIAgrees(o.Instance)) if err != nil { return err } @@ -70,6 +71,9 @@ func (o *CheckoutOptions) Complete(f Factory, args []string) (err error) { // Run executes the `checkout` command func (o *CheckoutOptions) Run() (err error) { + ctx := context.TODO() + inst := o.Instance + if !o.Refs.IsExplicit() { return fmt.Errorf("checkout requires an explicitly provided dataset ref") } @@ -91,8 +95,7 @@ func (o *CheckoutOptions) Run() (err error) { return err } - var res string - err = o.FSIMethods.Checkout(&lib.CheckoutParams{Dir: o.Dir, Ref: ref}, &res) + _, err = inst.Filesys().Checkout(ctx, &lib.LinkParams{Dir: o.Dir, Refstr: ref}) if err != nil { return err } diff --git a/cmd/factory.go b/cmd/factory.go index 8d99f0b40..218ee74dc 100644 --- a/cmd/factory.go +++ b/cmd/factory.go @@ -35,7 +35,6 @@ type Factory interface { ProfileMethods() (*lib.ProfileMethods, error) SearchMethods() (*lib.SearchMethods, error) SQLMethods() (*lib.SQLMethods, error) - FSIMethods() (*lib.FSIMethods, error) RenderMethods() (*lib.RenderMethods, error) TransformMethods() (*lib.TransformMethods, error) } diff --git a/cmd/factory_test.go b/cmd/factory_test.go index f45b74ebb..779fa301d 100644 --- a/cmd/factory_test.go +++ b/cmd/factory_test.go @@ -167,11 +167,6 @@ func (t TestFactory) ProfileMethods() (*lib.ProfileMethods, error) { return lib.NewProfileMethods(t.inst), nil } -// FSIMethods generates a lib.FSIMethods from internal state -func (t TestFactory) FSIMethods() (*lib.FSIMethods, error) { - return lib.NewFSIMethods(t.inst), nil -} - // SearchMethods generates a lib.SearchMethods from internal state func (t TestFactory) SearchMethods() (*lib.SearchMethods, error) { return lib.NewSearchMethods(t.inst), nil diff --git a/cmd/fsi.go b/cmd/fsi.go index e657016e9..98bda7708 100644 --- a/cmd/fsi.go +++ b/cmd/fsi.go @@ -1,10 +1,10 @@ package cmd import ( + "context" "path/filepath" "github.com/qri-io/ioes" - "github.com/qri-io/qri/dsref" "github.com/qri-io/qri/lib" "github.com/spf13/cobra" ) @@ -45,7 +45,8 @@ func NewFSICommand(f Factory, ioStreams ioes.IOStreams) *cobra.Command { if err := o.Complete(f, args); err != nil { return err } - return o.Unlink() + ctx := context.TODO() + return o.Unlink(ctx) }, } @@ -57,25 +58,23 @@ func NewFSICommand(f Factory, ioStreams ioes.IOStreams) *cobra.Command { type FSIOptions struct { ioes.IOStreams - Refs *RefSelect - Path string - FSIMethods *lib.FSIMethods + Instance *lib.Instance + + Refs *RefSelect + Path string } // Complete adds any missing configuration that can only be added just before // calling Run func (o *FSIOptions) Complete(f Factory, args []string) (err error) { - o.FSIMethods, err = f.FSIMethods() - if err != nil { - return err - } + o.Instance = f.Instance() if len(args) > 1 { o.Path = args[1] args = args[:1] } - if o.Refs, err = GetCurrentRefSelect(f, args, 1, EnsureFSIAgrees(o.FSIMethods)); err != nil { + if o.Refs, err = GetCurrentRefSelect(f, args, 1, EnsureFSIAgrees(o.Instance)); err != nil { return err } @@ -89,12 +88,15 @@ func (o *FSIOptions) Link() (err error) { return err } + ctx := context.TODO() + inst := o.Instance + p := &lib.LinkParams{ - Dir: o.Path, - Ref: o.Refs.Ref(), + Dir: o.Path, + Refstr: o.Refs.Ref(), } - res := &dsref.VersionInfo{} - if err := o.FSIMethods.CreateLink(p, res); err != nil { + res, err := inst.Filesys().CreateLink(ctx, p) + if err != nil { return err } @@ -103,18 +105,18 @@ func (o *FSIOptions) Link() (err error) { } // Unlink executes the fsi unlink command -func (o *FSIOptions) Unlink() error { - var res string +func (o *FSIOptions) Unlink(ctx context.Context) error { + inst := o.Instance for _, ref := range o.Refs.RefList() { printRefSelect(o.ErrOut, o.Refs) p := &lib.LinkParams{ - Ref: ref, + Refstr: ref, } - if err := o.FSIMethods.Unlink(p, &res); err != nil { - printErr(o.ErrOut, err) - return nil + res, err := inst.Filesys().Unlink(ctx, p) + if err != nil { + return err } printSuccess(o.Out, "unlinked: %s", res) diff --git a/cmd/fsi_integration_test.go b/cmd/fsi_integration_test.go index 9968d1cec..ce366aa9b 100644 --- a/cmd/fsi_integration_test.go +++ b/cmd/fsi_integration_test.go @@ -1718,10 +1718,10 @@ func TestUnlinkLinkFileButNoFSIPath(t *testing.T) { run.ClearFSIPath(t, "test_peer_file_but_no_fsi_path/unlink_me") // Unlink the dataset - output := run.MustExecCombinedOutErr(t, "qri workdir unlink me/unlink_me") - expect := "test_peer_file_but_no_fsi_path/unlink_me is not linked to a working directory\n" - if expect != output { - t.Errorf("output mismatch. expected %q got %q", expect, output) + outErr := run.ExecCommandCombinedOutErr("qri workdir unlink me/unlink_me") + expect := "test_peer_file_but_no_fsi_path/unlink_me is not linked to a working directory" + if expect != outErr.Error() { + t.Errorf("output mismatch. expected %q got %q", expect, outErr) } // Verify that .qri-ref still exists @@ -1729,6 +1729,7 @@ func TestUnlinkLinkFileButNoFSIPath(t *testing.T) { t.Errorf("expected .qri-ref link file to still exist") } + // Figure out why this is failing and restore this check // Verify that reference in refstore does not have FSIPath vinfo := run.LookupVersionInfo(t, "me/unlink_me") if vinfo == nil { @@ -1795,10 +1796,10 @@ func TestUnlinkDirectoryButRefNotFound(t *testing.T) { run.MustExec(t, "qri save") // Unlink the dataset - output := run.MustExecCombinedOutErr(t, "qri workdir unlink me/not_found") - expect := "reference not found\n" - if output != expect { - t.Errorf("output mismatch. expected %q got %q", expect, output) + outErr := run.ExecCommandCombinedOutErr("qri workdir unlink me/not_found") + expect := "reference not found" + if outErr.Error() != expect { + t.Errorf("output mismatch. expected %q got %q", expect, outErr) } // Verify that .qri-ref still exists diff --git a/cmd/init.go b/cmd/init.go index 44f7df9e5..f249fee8f 100644 --- a/cmd/init.go +++ b/cmd/init.go @@ -2,6 +2,7 @@ package cmd import ( "context" + "os" "path/filepath" "strings" @@ -57,6 +58,8 @@ already existing body file using the 'body' flag.`, type InitOptions struct { ioes.IOStreams + Instance *lib.Instance + // Name of the dataset that will be created Name string // Format of the body @@ -69,7 +72,6 @@ type InitOptions struct { UseDscache bool DatasetMethods *lib.DatasetMethods - FSIMethods *lib.FSIMethods } // Complete completes a dataset reference @@ -77,7 +79,7 @@ func (o *InitOptions) Complete(f Factory, args []string) (err error) { if o.DatasetMethods, err = f.DatasetMethods(); err != nil { return err } - o.FSIMethods, err = f.FSIMethods() + o.Instance = f.Instance() if len(args) > 0 { o.TargetDir = args[0] } @@ -86,12 +88,20 @@ func (o *InitOptions) Complete(f Factory, args []string) (err error) { // Run executes the `init` command func (o *InitOptions) Run() (err error) { + ctx := context.TODO() + inst := o.Instance + + // An empty dir means the current dir + if o.TargetDir == "" { + o.TargetDir, _ = os.Getwd() + } + // First, check if the directory can be init'd, before prompting for any input canInitParams := lib.InitDatasetParams{ TargetDir: o.TargetDir, BodyPath: o.BodyPath, } - if err = o.FSIMethods.CanInitDatasetWorkDir(&canInitParams, nil); err != nil { + if err = inst.Filesys().CanInitDatasetWorkDir(ctx, &canInitParams); err != nil { return err } @@ -127,9 +137,8 @@ func (o *InitOptions) Run() (err error) { UseDscache: o.UseDscache, } - ctx := context.TODO() - var refstr string - if err = o.FSIMethods.InitDataset(ctx, p, &refstr); err != nil { + refstr, err := inst.Filesys().Init(ctx, p) + if err != nil { return err } diff --git a/cmd/qri.go b/cmd/qri.go index d31dca73e..f33f51d3f 100644 --- a/cmd/qri.go +++ b/cmd/qri.go @@ -321,15 +321,6 @@ func (o *QriOptions) ConfigMethods() (m *lib.ConfigMethods, err error) { return lib.NewConfigMethods(o.inst), nil } -// FSIMethods generates a lib.FSIMethods from internal state -func (o *QriOptions) FSIMethods() (m *lib.FSIMethods, err error) { - if err = o.Init(); err != nil { - return - } - - return lib.NewFSIMethods(o.inst), nil -} - // Shutdown closes the instance func (o *QriOptions) Shutdown() <-chan error { if o.inst == nil { diff --git a/cmd/ref_select.go b/cmd/ref_select.go index 507640df0..f4fceacce 100644 --- a/cmd/ref_select.go +++ b/cmd/ref_select.go @@ -211,11 +211,11 @@ func DefaultSelectedRefList(f Factory) ([]string, error) { // This is useful if a user has a working directory, and then manually deletes the .qri-ref (which // will unlink the dataset), or renames / moves the directory and then runs a command in that // directory (which will update the repository with the new working directory's path). -func EnsureFSIAgrees(f *lib.FSIMethods) *FSIRefLinkEnsurer { - if f == nil { +func EnsureFSIAgrees(inst *lib.Instance) *FSIRefLinkEnsurer { + if inst == nil { return nil } - return &FSIRefLinkEnsurer{FSIMethods: f} + return &FSIRefLinkEnsurer{FSIMethods: inst.Filesys()} } // FSIRefLinkEnsurer is a simple wrapper for ensuring the linkfile agrees with the repository. We @@ -233,10 +233,8 @@ func (e *FSIRefLinkEnsurer) EnsureRef(refs *RefSelect) error { if e == nil { return nil } - p := lib.EnsureParams{Dir: refs.Dir(), Ref: refs.Ref()} - info := dsref.VersionInfo{} + p := lib.LinkParams{Dir: refs.Dir(), Refstr: refs.Ref()} ctx := context.TODO() - // Lib call matches the gorpc method signature, but `out` is not used - err := e.FSIMethods.EnsureRef(ctx, &p, &info) + _, err := e.FSIMethods.EnsureRef(ctx, &p) return err } diff --git a/cmd/restore.go b/cmd/restore.go index 2f393a463..65f868985 100644 --- a/cmd/restore.go +++ b/cmd/restore.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "fmt" "strings" @@ -58,11 +59,11 @@ not ` + "`structure.json`" + `)`, type RestoreOptions struct { ioes.IOStreams + Instance *lib.Instance + Refs *RefSelect Path string ComponentName string - - FSIMethods *lib.FSIMethods } // Complete configures the restore command @@ -71,6 +72,8 @@ func (o *RestoreOptions) Complete(f Factory, args []string) (err error) { o.Path = "" o.ComponentName = "" + o.Instance = f.Instance() + // TODO(dlong): Add low-level utilities that parse strings like "peername/ds_name", and // "/ipfs/QmFoo", "meta.description", etc and use those everywhere. Use real regexs so // that we properly handle user input everywhere. Too much code is duplicating half working @@ -111,10 +114,7 @@ func (o *RestoreOptions) Complete(f Factory, args []string) (err error) { return fmt.Errorf("unknown argument \"%s\"", arg) } - if o.FSIMethods, err = f.FSIMethods(); err != nil { - return err - } - if o.Refs, err = GetCurrentRefSelect(f, dsRefList, 1, EnsureFSIAgrees(o.FSIMethods)); err != nil { + if o.Refs, err = GetCurrentRefSelect(f, dsRefList, 1, EnsureFSIAgrees(o.Instance)); err != nil { return err } return nil @@ -124,17 +124,21 @@ func (o *RestoreOptions) Complete(f Factory, args []string) (err error) { func (o *RestoreOptions) Run() (err error) { printRefSelect(o.ErrOut, o.Refs) + ctx := context.TODO() + inst := o.Instance + ref := o.Refs.Ref() if o.Path != "" { ref += o.Path } - var res string - err = o.FSIMethods.Restore(&lib.RestoreParams{ - Ref: ref, + params := lib.RestoreParams{ + Refstr: ref, Dir: o.Refs.Dir(), Component: o.ComponentName, - }, &res) + } + + _, err = inst.Filesys().Restore(ctx, ¶ms) if err != nil { return err } diff --git a/cmd/save.go b/cmd/save.go index c07a58dcd..e74ce3424 100644 --- a/cmd/save.go +++ b/cmd/save.go @@ -112,7 +112,6 @@ type SaveOptions struct { UseDscache bool DatasetMethods *lib.DatasetMethods - FSIMethods *lib.FSIMethods } // Complete adds any missing configuration that can only be added just before calling Run diff --git a/cmd/status.go b/cmd/status.go index 2fa286755..171efb19d 100644 --- a/cmd/status.go +++ b/cmd/status.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "fmt" "path/filepath" "strings" @@ -54,18 +55,15 @@ a commit alongside a dataset like: type StatusOptions struct { ioes.IOStreams + Instance *lib.Instance + Refs *RefSelect ShowMtime bool - - FSIMethods *lib.FSIMethods } // Complete adds any missing configuration that can only be added just before calling Run func (o *StatusOptions) Complete(f Factory, args []string) (err error) { - o.FSIMethods, err = f.FSIMethods() - if err != nil { - return err - } + o.Instance = f.Instance() // Cannot pass explicit reference, must be run in a working directory if len(args) > 0 { @@ -73,7 +71,7 @@ func (o *StatusOptions) Complete(f Factory, args []string) (err error) { return fmt.Errorf("can only get status of the current working directory") } - o.Refs, err = GetCurrentRefSelect(f, args, 0, EnsureFSIAgrees(o.FSIMethods)) + o.Refs, err = GetCurrentRefSelect(f, args, 0, EnsureFSIAgrees(o.Instance)) if err != nil { return err } @@ -88,16 +86,18 @@ const ColumnPositionForMtime = 40 func (o *StatusOptions) Run() (err error) { printRefSelect(o.ErrOut, o.Refs) - res := []lib.StatusItem{} - dir := o.Refs.Dir() - if err := o.FSIMethods.Status(&dir, &res); err != nil { - printErr(o.ErrOut, err) - return nil + ctx := context.TODO() + inst := o.Instance + + params := lib.LinkParams{Dir: o.Refs.Dir()} + items, err := inst.Filesys().Status(ctx, ¶ms) + if err != nil { + return err } clean := true valid := true - for _, si := range res { + for _, si := range items { line := "" switch si.Type { case fsi.STRemoved: diff --git a/cmd/test_runner_test.go b/cmd/test_runner_test.go index f99629dd3..22a5d87f0 100644 --- a/cmd/test_runner_test.go +++ b/cmd/test_runner_test.go @@ -238,7 +238,7 @@ func (run *TestRunner) ExecCommandWithStdin(ctx context.Context, cmdText, stdinT return err } - return timedShutdown(fmt.Sprintf("ExecCommandCombinedOutErr: %q\n", cmdText), shutdown) + return timedShutdown(fmt.Sprintf("ExecCommandWithStdin: %q\n", cmdText), shutdown) } // ExecCommandCombinedOutErr executes the command with a combined stdout and stderr stream @@ -247,6 +247,10 @@ func (run *TestRunner) ExecCommandCombinedOutErr(cmdText string) error { var shutdown func() <-chan error run.CmdR, shutdown = run.CreateCommandRunnerCombinedOutErr(ctx) if err := executeCommand(run.CmdR, cmdText); err != nil { + shutDownErr := <-shutdown() + if shutDownErr != nil { + log.Errorf("error shutting down %q: %q", cmdText, shutDownErr) + } cancel() return err } diff --git a/cmd/whatchanged.go b/cmd/whatchanged.go index f3f02e2e5..b91770ff2 100644 --- a/cmd/whatchanged.go +++ b/cmd/whatchanged.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "fmt" "github.com/qri-io/ioes" @@ -35,16 +36,15 @@ except only available for dataset versions in history.`, type WhatChangedOptions struct { ioes.IOStreams - Refs *RefSelect - FSIMethods *lib.FSIMethods + Instance *lib.Instance + + Refs *RefSelect } // Complete adds any missing configuration that can only be added just before calling Run func (o *WhatChangedOptions) Complete(f Factory, args []string) (err error) { - if o.FSIMethods, err = f.FSIMethods(); err != nil { - return err - } - o.Refs, err = GetCurrentRefSelect(f, args, 1, EnsureFSIAgrees(o.FSIMethods)) + o.Instance = f.Instance() + o.Refs, err = GetCurrentRefSelect(f, args, 1, EnsureFSIAgrees(o.Instance)) return nil } @@ -52,9 +52,12 @@ func (o *WhatChangedOptions) Complete(f Factory, args []string) (err error) { func (o *WhatChangedOptions) Run() (err error) { printRefSelect(o.ErrOut, o.Refs) - res := []lib.StatusItem{} - ref := o.Refs.Ref() - if err := o.FSIMethods.WhatChanged(&ref, &res); err != nil { + ctx := context.TODO() + inst := o.Instance + + params := lib.LinkParams{Refstr: o.Refs.Ref()} + res, err := inst.Filesys().WhatChanged(ctx, ¶ms) + if err != nil { printErr(o.ErrOut, err) return nil } diff --git a/fsi/status.go b/fsi/status.go index bfe7bde42..cb390d9e0 100644 --- a/fsi/status.go +++ b/fsi/status.go @@ -65,8 +65,7 @@ func (fsi *FSI) Status(ctx context.Context, dir string) (changes []StatusItem, e fs := fsi.repo.Filesystem() ref, ok := GetLinkedFilesysRef(dir) if !ok { - err = fmt.Errorf("not a linked directory") - return nil, err + return nil, ErrNoLink } var stored *dataset.Dataset diff --git a/lib/api.go b/lib/api.go index fefb36ce0..92bfdca3d 100644 --- a/lib/api.go +++ b/lib/api.go @@ -106,12 +106,18 @@ const ( AEWhatChanged = APIEndpoint("/whatchanged/{path:.*}") // AEInit invokes a dataset initialization on the filesystem AEInit = APIEndpoint("/init/{path:.*}") + // AECanInitDatasetWorkDir returns whether a dataset can be initialized + AECanInitDatasetWorkDir = APIEndpoint("/caninitdatasetworkdir/{path:.*}") // AECheckout invokes a dataset checkout to the filesystem AECheckout = APIEndpoint("/checkout/{path:.*}") // AERestore invokes a restore AERestore = APIEndpoint("/restore/{path:.*}") // AEFSIWrite writes input data to the filesystem AEFSIWrite = APIEndpoint("/fsi/write/{path:.*}") + // AEFSICreateLink creates an fsi link + AEFSICreateLink = APIEndpoint("/fsi/createlink/{path:.*}") + // AEFSIUnlink removes the fsi link + AEFSIUnlink = APIEndpoint("/fsi/unlink/{path:.*}") // other endpoints diff --git a/lib/datasets.go b/lib/datasets.go index 8ea8ebb31..1a60d50a7 100644 --- a/lib/datasets.go +++ b/lib/datasets.go @@ -1133,7 +1133,7 @@ func (m *DatasetMethods) Remove(ctx context.Context, p *RemoveParams) (*RemoveRe log.Debugf("Remove, IsWorkingDirectoryDirty") return nil, ErrCantRemoveDirectoryDirty } - if strings.Contains(wdErr.Error(), "not a linked directory") { + if errors.Is(wdErr, fsi.ErrNoLink) || strings.Contains(wdErr.Error(), "not a linked directory") { // If the working directory has been removed (or renamed), could not get the // status. However, don't let this stop the remove operation, since the files // are already gone, and therefore won't be removed. @@ -1307,13 +1307,12 @@ func (m *DatasetMethods) Pull(ctx context.Context, p *PullParams) (*dataset.Data *res = *ds if p.LinkDir != "" { - checkoutp := &CheckoutParams{ - Ref: ref.Human(), - Dir: p.LinkDir, + checkoutp := &LinkParams{ + Refstr: ref.Human(), + Dir: p.LinkDir, } - m := NewFSIMethods(m.inst) - checkoutRes := "" - if err = m.Checkout(checkoutp, &checkoutRes); err != nil { + fsiMethods := m.inst.Filesys() + if _, err = fsiMethods.Checkout(ctx, checkoutp); err != nil { return nil, err } } diff --git a/lib/datasets_test.go b/lib/datasets_test.go index 619f9849a..ce42854c3 100644 --- a/lib/datasets_test.go +++ b/lib/datasets_test.go @@ -588,13 +588,12 @@ func TestDatasetRequestsGetFSIPath(t *testing.T) { defer os.RemoveAll(tempDir) dsDir := filepath.Join(tempDir, "movies") - fsim := NewFSIMethods(inst) - p := &CheckoutParams{ - Dir: dsDir, - Ref: "peer/movies", + fsim := inst.Filesys() + p := &LinkParams{ + Dir: dsDir, + Refstr: "peer/movies", } - out := "" - if err := fsim.Checkout(p, &out); err != nil { + if _, err := fsim.Checkout(ctx, p); err != nil { t.Fatalf("error checking out dataset: %s", err) } @@ -800,12 +799,16 @@ func TestRenameNoHistory(t *testing.T) { Name: "rename_no_history", Format: "csv", } - var refstr string - if err := NewFSIMethods(tr.Instance).InitDataset(ctx, initP, &refstr); err != nil { + refstr, err := tr.Instance.Filesys().Init(ctx, initP) + if err != nil { t.Fatal(err) } - // // Read .qri-ref file, it contains the reference this directory is linked to + if refstr != "peer/rename_no_history" { + t.Errorf("init returned bad refstring %q", refstr) + } + + // Read .qri-ref file, it contains the reference this directory is linked to actual := tr.MustReadFile(t, filepath.Join(workDir, ".qri-ref")) expect := "peer/rename_no_history" if diff := cmp.Diff(expect, actual); diff != "" { @@ -817,7 +820,7 @@ func TestRenameNoHistory(t *testing.T) { Current: "me/rename_no_history", Next: "me/rename_second_name", } - _, err := NewDatasetMethods(tr.Instance).Rename(ctx, renameP) + _, err = NewDatasetMethods(tr.Instance).Rename(ctx, renameP) if err != nil { t.Fatal(err) } @@ -848,7 +851,7 @@ func TestDatasetRequestsRemove(t *testing.T) { allRevs := &dsref.Rev{Field: "ds", Gen: -1} // we need some fsi stuff to fully test remove - fsim := NewFSIMethods(inst) + fsim := inst.Filesys() // create datasets working directory datasetsDir, err := ioutil.TempDir("", "QriTestDatasetRequestsRemove") if err != nil { @@ -862,18 +865,16 @@ func TestDatasetRequestsRemove(t *testing.T) { TargetDir: filepath.Join(datasetsDir, "no_history"), Format: "csv", } - var noHistoryName string - if err := fsim.InitDataset(ctx, initp, &noHistoryName); err != nil { + if _, err := fsim.Init(ctx, initp); err != nil { t.Fatal(err) } // link cities dataset with a checkout - checkoutp := &CheckoutParams{ - Dir: filepath.Join(datasetsDir, "cities"), - Ref: "me/cities", + checkoutp := &LinkParams{ + Dir: filepath.Join(datasetsDir, "cities"), + Refstr: "me/cities", } - var out string - if err := fsim.Checkout(checkoutp, &out); err != nil { + if _, err := fsim.Checkout(ctx, checkoutp); err != nil { t.Fatal(err) } @@ -884,11 +885,11 @@ func TestDatasetRequestsRemove(t *testing.T) { } // link craigslist with a checkout - checkoutp = &CheckoutParams{ - Dir: filepath.Join(datasetsDir, "craigslist"), - Ref: "me/craigslist", + checkoutp = &LinkParams{ + Dir: filepath.Join(datasetsDir, "craigslist"), + Refstr: "me/craigslist", } - if err := fsim.Checkout(checkoutp, &out); err != nil { + if _, err := fsim.Checkout(ctx, checkoutp); err != nil { t.Fatal(err) } @@ -1146,8 +1147,8 @@ func TestDatasetRequestsValidateFSI(t *testing.T) { TargetDir: filepath.Join(workDir, "validate_test"), Format: "csv", } - var refstr string - if err := NewFSIMethods(tr.Instance).InitDataset(ctx, initP, &refstr); err != nil { + refstr, err := tr.Instance.Filesys().Init(ctx, initP) + if err != nil { t.Fatal(err) } diff --git a/lib/dispatch.go b/lib/dispatch.go new file mode 100644 index 000000000..46c580321 --- /dev/null +++ b/lib/dispatch.go @@ -0,0 +1,230 @@ +package lib + +import ( + "context" + "fmt" + "reflect" + "strings" +) + +// Dispatch is a system for handling calls to lib. Should only be called by top-level lib methods. +// +// When programs are using qri as a library (such as the `cmd` package), calls to `lib` will +// arrive at dispatch, before being routed to the actual implementation routine. This solves +// a few problems: +// 1) Multiple methods can be running on qri at once, dispatch will schedule as needed (TODO) +// 2) Access to core qri data structures (like logbook) can be handled safetly (TODO) +// 3) User identity, permissions, etc is scoped to a single call, not the global process (TODO) +// 4) The qri http api maps directly onto dispatch's behavior, leading to a simpler api +// 5) A `qri connect` process can be transparently forwarded a method call with little work +// +// At construction time, the Instance registers all methods that dispatch can access, as well +// as the input and output parameters for those methods, and associates a string name for each +// method. Dispatch works by looking up that method name, constructing the necessary input, +// then invoking the actual implementation. +func (inst *Instance) Dispatch(ctx context.Context, method string, param interface{}) (res interface{}, err error) { + if inst == nil { + return nil, fmt.Errorf("instance is nil, cannot dispatch") + } + + // If the http rpc layer is engaged, use it to dispatch methods + // This happens when another process is running `qri connect` + if inst.http != nil { + if c, ok := inst.regMethods.lookup(method); ok { + // TODO(dustmop): This is always using the "POST" verb currently. We need some + // mechanism of tagging methods as being read-only and "GET"-able. Once that + // exists, use it here to lookup the verb that should be used to invoke the rpc. + out := reflect.New(c.OutType) + res = out.Interface() + err = inst.http.Call(ctx, methodEndpoint(method), param, res) + if err != nil { + return nil, err + } + out = reflect.ValueOf(res) + out = out.Elem() + return out.Interface(), nil + } + return nil, fmt.Errorf("method %q not found", method) + } + + // Look up the method for the given signifier + if c, ok := inst.regMethods.lookup(method); ok { + // Construct the isolated scope for this call + // TODO(dustmop): Add user authentication, profile, identity, etc + // TODO(dustmop): Also determine if the method is read-only vs read-write, + // and only execute a single read-write method at a time + // Eventually, the data that lives in scope should be immutable for its lifetime, + // or use copy-on-write semantics, so that one method running at the same time as + // another cannot modify the out-of-scope data of the other. This will mostly + // involve making copies of the right things + scope := scope{ + ctx: ctx, + inst: inst, + } + + // Construct the parameter list for the function call, then call it + args := make([]reflect.Value, 3) + args[0] = reflect.ValueOf(c.Impl) + args[1] = reflect.ValueOf(scope) + args[2] = reflect.ValueOf(param) + outVals := c.Func.Call(args) + + // TODO(dustmop): If the method wrote to our internal data structures, like + // refstore, logbook, etc, serialize and commit those changes here + + // Validate the return values. This shouldn't fail as long as all method + // implementations are declared correctly + if len(outVals) != 2 { + return nil, fmt.Errorf("wrong number of return args: %d", len(outVals)) + } + + // Extract the concrete typed values from the method return + var out interface{} + out = outVals[0].Interface() + errVal := outVals[1].Interface() + if errVal == nil { + return out, nil + } + if err, ok := errVal.(error); ok { + return out, err + } + return nil, fmt.Errorf("second return value should be an error, got: %v", errVal) + } + return nil, fmt.Errorf("method %q not found", method) +} + +// NewInputParam takes a method name that has been registered, and constructs +// an instance of that input parameter +func (inst *Instance) NewInputParam(method string) interface{} { + if c, ok := inst.regMethods.lookup(method); ok { + obj := reflect.New(c.InType) + return obj.Interface() + } + return nil +} + +// regMethodSet represents a set of registered methods +type regMethodSet struct { + reg map[string]callable +} + +// lookup finds the callable structure with the given method name +func (r *regMethodSet) lookup(method string) (*callable, bool) { + if c, ok := r.reg[method]; ok { + return &c, true + } + return nil, false +} + +type callable struct { + Impl interface{} + Func reflect.Value + InType reflect.Type + OutType reflect.Type +} + +// RegisterMethods iterates the methods provided by the lib API, and makes them visible to dispatch +func (inst *Instance) RegisterMethods() { + reg := make(map[string]callable) + // TODO(dustmop): Change registerOne to take both the MethodSet and the Impl, validate + // that their signatures agree. + inst.registerOne("fsi", &FSIImpl{}, reg) + inst.regMethods = ®MethodSet{reg: reg} +} + +func (inst *Instance) registerOne(ourName string, impl interface{}, reg map[string]callable) { + implType := reflect.TypeOf(impl) + // Iterate methods on the implementation, register those that have the right signature + num := implType.NumMethod() + for k := 0; k < num; k++ { + m := implType.Method(k) + lowerName := strings.ToLower(m.Name) + funcName := fmt.Sprintf("%s.%s", ourName, lowerName) + + // Validate the parameters to the method + // should have 3 input parameters: (receiver, scope, input struct) + // should have 2 output parametres: (output value, error) + // TODO(dustmop): allow variadic returns: error only, cursor for pagination + f := m.Type + if f.NumIn() != 3 { + log.Fatalf("%s: bad number of inputs: %d", funcName, f.NumIn()) + } + if f.NumOut() != 2 { + log.Fatalf("%s: bad number of outputs: %d", funcName, f.NumOut()) + } + // First input must be the receiver + inType := f.In(0) + if inType != implType { + log.Fatalf("%s: first input param should be impl, got %v", funcName, inType) + } + // Second input must be a scope + inType = f.In(1) + if inType.Name() != "scope" { + log.Fatalf("%s: second input param should be scope, got %v", funcName, inType) + } + // Third input is a pointer to the input struct + inType = f.In(2) + if inType.Kind() != reflect.Ptr { + log.Fatalf("%s: third input param must be a struct pointer, got %v", funcName, inType) + } + inType = inType.Elem() + if inType.Kind() != reflect.Struct { + log.Fatalf("%s: third input param must be a struct pointer, got %v", funcName, inType) + } + // First output is anything + outType := f.Out(0) + // Second output must be an error + outErrType := f.Out(1) + if outErrType.Name() != "error" { + log.Fatalf("%s: second output param should be error, got %v", funcName, outErrType) + } + + // Save the method to the registration table + reg[funcName] = callable{ + Impl: impl, + Func: m.Func, + InType: inType, + OutType: outType, + } + log.Debugf("%d: registered %s(*%s) %v", k, funcName, inType, outType) + } +} + +// MethodSet represents a set of methods to be registered +type MethodSet interface { + Name() string +} + +func dispatchMethodName(m MethodSet, funcName string) string { + lowerName := strings.ToLower(funcName) + return fmt.Sprintf("%s.%s", m.Name(), lowerName) +} + +// methodEndpoint returns a method name and returns the API endpoint for it +func methodEndpoint(method string) APIEndpoint { + // TODO(dustmop): This is here temporarily. /fsi/write/ works differently than + // other methods; their http API endpoints are only their method name, for + // exmaple /status/. This should be replaced with an explicit mapping from + // method names to endpoints. + if method == "fsi.write" { + return "/fsi/write/" + } + if method == "fsi.createlink" { + return "/fsi/createlink/" + } + if method == "fsi.unlink" { + return "/fsi/unlink/" + } + pos := strings.Index(method, ".") + prefix := method[:pos] + _ = prefix + res := "/" + method[pos+1:] + "/" + return APIEndpoint(res) +} + +func dispatchReturnError(got interface{}, err error) error { + if got != nil { + log.Errorf("type mismatch: %v of type %s", got, reflect.TypeOf(got)) + } + return err +} diff --git a/lib/fsi.go b/lib/fsi.go index 88f3af0e5..db7efe617 100644 --- a/lib/fsi.go +++ b/lib/fsi.go @@ -17,314 +17,402 @@ import ( "github.com/qri-io/qri/repo" ) -// FSIMethods encapsulates filesystem integrations methods +// FSIMethods groups together methods for FSI type FSIMethods struct { inst *Instance } -// NewFSIMethods creates a fsi handle from an instance -func NewFSIMethods(inst *Instance) *FSIMethods { - return &FSIMethods{inst: inst} +// Name returns the name of this method group +func (m *FSIMethods) Name() string { + return "fsi" } -// CoreRequestsName specifies this is a fsi handle -func (m FSIMethods) CoreRequestsName() string { return "fsi" } +// Filesys returns the FSIMethods that Instance has registered +func (inst *Instance) Filesys() *FSIMethods { + return &FSIMethods{inst: inst} +} -// LinkParams encapsulate parameters to the link method +// LinkParams encapsulate parameters for linked datasets type LinkParams struct { - Dir string - Ref string + Dir string + Refstr string +} + +// FSIWriteParams encapsultes arguments for writing to an FSI-linked directory +type FSIWriteParams struct { + Refstr string + Ds *dataset.Dataset } +// RestoreParams provides parameters to the restore method. +type RestoreParams struct { + Dir string + Refstr string + Path string + Component string +} + +// InitDatasetParams proxies parameters to initialization +type InitDatasetParams = fsi.InitParams + +// StatusItem is an alias for an fsi.StatusItem +type StatusItem = fsi.StatusItem + // CreateLink creates a connection between a working drirectory and a dataset history -func (m *FSIMethods) CreateLink(p *LinkParams, res *dsref.VersionInfo) (err error) { - // absolutize path name - path, err := filepath.Abs(p.Dir) +func (m *FSIMethods) CreateLink(ctx context.Context, p *LinkParams) (*dsref.VersionInfo, error) { + got, err := m.inst.Dispatch(ctx, dispatchMethodName(m, "createlink"), p) + if res, ok := got.(*dsref.VersionInfo); ok { + return res, err + } + return nil, dispatchReturnError(got, err) +} + +// Unlink removes the connection between a working directory and a dataset. If given only a +// directory, will remove the link file from that directory. If given only a reference, +// will remove the fsi path from that reference, and remove the link file from that fsi path +func (m *FSIMethods) Unlink(ctx context.Context, p *LinkParams) (string, error) { + got, err := m.inst.Dispatch(ctx, dispatchMethodName(m, "unlink"), p) + if res, ok := got.(string); ok { + return res, err + } + return "", dispatchReturnError(got, err) +} + +// Status checks for any modifications or errors in a linked directory against its previous +// version in the repo. Must only be called if FSI is enabled for this dataset. +func (m *FSIMethods) Status(ctx context.Context, p *LinkParams) ([]StatusItem, error) { + // TODO(dustmop): Have Dispatch perform this AbsPath call automatically + err := qfs.AbsPath(&p.Dir) if err != nil { - return err + return nil, err + } + got, err := m.inst.Dispatch(ctx, dispatchMethodName(m, "status"), p) + if res, ok := got.([]StatusItem); ok { + return res, err + } + return nil, dispatchReturnError(got, err) +} + +// WhatChanged gets changes that happened at a particular version in the history of the given +// dataset reference. +func (m *FSIMethods) WhatChanged(ctx context.Context, p *LinkParams) ([]StatusItem, error) { + got, err := m.inst.Dispatch(ctx, dispatchMethodName(m, "whatchanged"), p) + if res, ok := got.([]StatusItem); ok { + return res, err } - p.Dir = path + return nil, dispatchReturnError(got, err) +} - if m.inst.rpc != nil { - return checkRPCError(m.inst.rpc.Call("FSIMethods.CreateLink", p, res)) +// Checkout method writes a dataset to a directory as individual files. +func (m *FSIMethods) Checkout(ctx context.Context, p *LinkParams) (string, error) { + got, err := m.inst.Dispatch(ctx, dispatchMethodName(m, "checkout"), p) + if res, ok := got.(string); ok { + return res, err } + return "", dispatchReturnError(got, err) +} - ctx := context.TODO() - ref, _, err := m.inst.ParseAndResolveRef(ctx, p.Ref, "local") +// Write mutates a linked dataset on the filesystem +func (m *FSIMethods) Write(ctx context.Context, p *FSIWriteParams) ([]StatusItem, error) { + got, err := m.inst.Dispatch(ctx, dispatchMethodName(m, "write"), p) + if res, ok := got.([]StatusItem); ok { + return res, err + } + return nil, dispatchReturnError(got, err) +} + +// Restore method restores a component or all of the component files of a dataset from the repo +func (m *FSIMethods) Restore(ctx context.Context, p *RestoreParams) (string, error) { + got, err := m.inst.Dispatch(ctx, dispatchMethodName(m, "restore"), p) + if res, ok := got.(string); ok { + return res, err + } + return "", dispatchReturnError(got, err) +} + +// Init initializes a new working directory for a linked dataset +func (m *FSIMethods) Init(ctx context.Context, p *InitDatasetParams) (string, error) { + // TODO(dustmop): Have Dispatch perform these AbsPath calls automatically + err := qfs.AbsPath(&p.TargetDir) + if err != nil { + return "", err + } + err = qfs.AbsPath(&p.BodyPath) if err != nil { - return err + return "", err } + got, err := m.inst.Dispatch(ctx, dispatchMethodName(m, "init"), p) + if res, ok := got.(string); ok { + return res, err + } + return "", dispatchReturnError(got, err) +} - res, _, err = m.inst.fsi.CreateLink(ctx, p.Dir, ref) +// CanInitDatasetWorkDir returns nil if the directory can init a dataset, or an error if not +func (m *FSIMethods) CanInitDatasetWorkDir(ctx context.Context, p *InitDatasetParams) error { + // TODO(dustmop): This method is cheating a bit; its type signature does not match the + // implementation. Instead, dispatch should allow methods to only return 1 value, if that + // value is an error + _, err := m.inst.Dispatch(ctx, dispatchMethodName(m, "caninitdatasetworkdir"), p) return err } +// EnsureRef will modify the directory path in the repo for the given reference +func (m *FSIMethods) EnsureRef(ctx context.Context, p *LinkParams) (*dsref.VersionInfo, error) { + got, err := m.inst.Dispatch(ctx, dispatchMethodName(m, "ensureref"), p) + if res, ok := got.(*dsref.VersionInfo); ok { + return res, err + } + return nil, dispatchReturnError(got, err) +} + +// Implementations for FSI methods follow. +// TODO(dustmop): Perhaps consider moving these methods to /lib/impl/*.go +// TODO(dustmop): If it's not too hard, look into writing a custom lint or vet rule +// that validates methods are using a compatible signature with the implementations + +// FSIImpl holds the method implementations for FSI +type FSIImpl struct{} + +// CreateLink creates a connection between a working drirectory and a dataset history +func (*FSIImpl) CreateLink(scope scope, p *LinkParams) (*dsref.VersionInfo, error) { + ctx := scope.Context() + + ref, _, err := scope.ParseAndResolveRef(ctx, p.Refstr, "local") + if err != nil { + return nil, err + } + vinfo, _, err := scope.FSISubsystem().CreateLink(ctx, p.Dir, ref) + return vinfo, err +} + // Unlink removes the connection between a working directory and a dataset. If given only a // directory, will remove the link file from that directory. If given only a reference, // will remove the fsi path from that reference, and remove the link file from that fsi path -func (m *FSIMethods) Unlink(p *LinkParams, res *string) (err error) { - if m.inst.rpc != nil { - return checkRPCError(m.inst.rpc.Call("FSIMethods.Unlink", p, res)) - } - ctx := context.TODO() +func (*FSIImpl) Unlink(scope scope, p *LinkParams) (string, error) { + ctx := scope.Context() - if p.Dir != "" && p.Ref != "" { - return fmt.Errorf("Unlink should be called with either Dir or Ref, not both") + if p.Dir != "" && p.Refstr != "" { + return "", fmt.Errorf("Unlink should be called with either Dir or Ref, not both") } var ref dsref.Ref if p.Dir == "" { // If only ref provided, canonicalize it to get its ref - ref, _, err = m.inst.ParseAndResolveRef(ctx, p.Ref, "local") + var err error + ref, _, err = scope.ParseAndResolveRef(ctx, p.Refstr, "local") if err != nil { - return err + return "", err } - vi, err := repo.GetVersionInfoShim(m.inst.repo, ref) + // NOTE: GetVersionInfoShim is in the process of being removed. Try not to add + // new callers. + vi, err := scope.GetVersionInfoShim(ref) if err != nil { - return err + return "", err } if vi.FSIPath == "" { - return fmt.Errorf("%s is not linked to a working directory", ref.Human()) + return "", fmt.Errorf("%s is not linked to a working directory", ref.Human()) } p.Dir = vi.FSIPath } - if err := m.inst.fsi.Unlink(ctx, p.Dir, ref); err != nil { - return err + if err := scope.FSISubsystem().Unlink(ctx, p.Dir, ref); err != nil { + return "", err } - *res = ref.Alias() - return nil + return ref.Alias(), nil } -// StatusItem is an alias for an fsi.StatusItem -type StatusItem = fsi.StatusItem - // Status checks for any modifications or errors in a linked directory against its previous // version in the repo. Must only be called if FSI is enabled for this dataset. -func (m *FSIMethods) Status(dir *string, res *[]StatusItem) (err error) { - if m.inst.rpc != nil { - return checkRPCError(m.inst.rpc.Call("FSIMethods.Status", dir, res)) - } - ctx := context.TODO() +func (*FSIImpl) Status(scope scope, p *LinkParams) ([]StatusItem, error) { + ctx := scope.Context() - *res, err = m.inst.fsi.Status(ctx, *dir) - return err -} + if p.Dir == "" && p.Refstr == "" { + return nil, fmt.Errorf("either Dir or Refstr required for status") + } -// StatusForAlias receives an alias for a dataset that must be linked to the filesystem, and checks -// the status of its current working directory. It is an error to call this for a reference that -// is not linked. -func (m *FSIMethods) StatusForAlias(alias *string, res *[]StatusItem) (err error) { - if m.inst.rpc != nil { - return checkRPCError(m.inst.rpc.Call("FSIMethods.AliasStatus", alias, res)) + // If the directory is given, get the status of the linked dataset + if p.Dir != "" { + return scope.FSISubsystem().Status(ctx, p.Dir) } - ctx := context.TODO() - // If only ref provided, canonicalize it to get its ref - ref, err := dsref.ParseHumanFriendly(*alias) + // Otherwise, get the file system path by looking up the ref + ref, err := dsref.ParseHumanFriendly(p.Refstr) if err != nil { - return err + return nil, err } - vi, err := repo.GetVersionInfoShim(m.inst.repo, ref) + vi, err := scope.GetVersionInfoShim(ref) if err != nil { - return err + return nil, err } - *res, err = m.inst.fsi.Status(ctx, vi.FSIPath) - return err + return scope.FSISubsystem().Status(ctx, vi.FSIPath) } // WhatChanged gets changes that happened at a particular version in the history of the given -// dataset reference. Not used for FSI. -func (m *FSIMethods) WhatChanged(refstr *string, res *[]StatusItem) (err error) { - if m.inst.rpc != nil { - return checkRPCError(m.inst.rpc.Call("FSIMethods.WhatChanged", refstr, res)) - } - ctx := context.TODO() +// dataset reference. +func (*FSIImpl) WhatChanged(scope scope, p *LinkParams) ([]StatusItem, error) { + ctx := scope.Context() - ref, _, err := m.inst.ParseAndResolveRef(ctx, *refstr, "local") + ref, _, err := scope.ParseAndResolveRef(ctx, p.Refstr, "local") if err != nil { - return err + return nil, err } - *res, err = m.inst.fsi.StatusAtVersion(ctx, ref) - return err -} - -// CheckoutParams provides parameters to the Checkout method. -type CheckoutParams struct { - Dir string - Ref string + return scope.FSISubsystem().StatusAtVersion(ctx, ref) } // Checkout method writes a dataset to a directory as individual files. -func (m *FSIMethods) Checkout(p *CheckoutParams, out *string) (err error) { - if m.inst.rpc != nil { - return checkRPCError(m.inst.rpc.Call("FSIMethods.Checkout", p, out)) - } - ctx := context.TODO() +// TODO(dustmop): Returned string is not used, remove it once dispatch is compatible +func (*FSIImpl) Checkout(scope scope, p *LinkParams) (string, error) { + ctx := scope.Context() // Require a non-empty, absolute path for the checkout if p.Dir == "" || !filepath.IsAbs(p.Dir) { - return fmt.Errorf("need Dir to be a non-empty, absolute path") + return "", fmt.Errorf("need Dir to be a non-empty, absolute path") } log.Debugf("Checkout started, stat'ing %q", p.Dir) // If directory exists, error. - if _, err = os.Stat(p.Dir); !os.IsNotExist(err) { - return fmt.Errorf("directory with name \"%s\" already exists", p.Dir) + if _, err := os.Stat(p.Dir); !os.IsNotExist(err) { + return "", fmt.Errorf("directory with name \"%s\" already exists", p.Dir) } // Handle the ref to checkout - ref, source, err := m.inst.ParseAndResolveRef(ctx, p.Ref, "") + ref, source, err := scope.ParseAndResolveRef(ctx, p.Refstr, "") if err != nil { - return err + return "", err } if source != "" { - return fmt.Errorf("auto-adding on checkout is not yet supported, please run `qri add %q` first", ref.Human()) + return "", fmt.Errorf("auto-adding on checkout is not yet supported, please run `qri add %q` first", ref.Human()) } log.Debugf("Checkout for ref %q", ref) // Fail early if link already exists - if err := m.inst.fsi.EnsureRefNotLinked(ref); err != nil { - return err + if err := scope.FSISubsystem().EnsureRefNotLinked(ref); err != nil { + return "", err } // Load dataset that is being checked out. - ds, err := m.inst.LoadDataset(ctx, ref, "") + ds, err := scope.LoadDataset(ctx, ref, "") if err != nil { log.Debugf("Checkout, dsfs.LoadDataset failed, error: %s", err) - return err + return "", err } // Create a directory. if err := os.Mkdir(p.Dir, os.ModePerm); err != nil { log.Debugf("Checkout, Mkdir failed, error: %s", ref) - return err + return "", err } log.Debugf("Checkout made directory %q", p.Dir) // Create the link file, containing the dataset reference. - if _, _, err = m.inst.fsi.CreateLink(ctx, p.Dir, ref); err != nil { + if _, _, err = scope.FSISubsystem().CreateLink(ctx, p.Dir, ref); err != nil { log.Debugf("Checkout, fsi.CreateLink failed, error: %s", ref) - return err + return "", err } - log.Debugf("Checkout created link for %q <-> %q", p.Dir, p.Ref) + log.Debugf("Checkout created link for %q <-> %q", p.Dir, p.Refstr) // Write components of the dataset to the working directory. - err = fsi.WriteComponents(ds, p.Dir, m.inst.node.Repo.Filesystem()) + err = fsi.WriteComponents(ds, p.Dir, scope.Filesystem()) if err != nil { log.Debugf("Checkout, fsi.WriteComponents failed, error: %s", ref) } log.Debugf("Checkout wrote components, successfully checked out dataset") log.Debugf("Checkout successfully checked out dataset") - return nil -} - -// FSIWriteParams encapsultes arguments for writing to an FSI-linked directory -type FSIWriteParams struct { - Ref string - Ds *dataset.Dataset + return "", nil } // Write mutates a linked dataset on the filesystem -func (m *FSIMethods) Write(p *FSIWriteParams, res *[]StatusItem) (err error) { - if m.inst.rpc != nil { - return checkRPCError(m.inst.rpc.Call("FSIMethods.Write", p, res)) - } - ctx := context.TODO() +func (*FSIImpl) Write(scope scope, p *FSIWriteParams) ([]StatusItem, error) { + ctx := scope.Context() if p.Ds == nil { - return fmt.Errorf("dataset is required") + return nil, fmt.Errorf("dataset is required") } - ref, _, err := m.inst.ParseAndResolveRef(ctx, p.Ref, "local") + ref, _, err := scope.ParseAndResolveRef(ctx, p.Refstr, "local") if err != nil { - return err + return nil, err } - vi, err := repo.GetVersionInfoShim(m.inst.node.Repo, ref) + vi, err := scope.GetVersionInfoShim(ref) if err != nil && err != repo.ErrNoHistory { - return err + return nil, err } // Directory to write components to can be determined from FSIPath of versionInfo if vi.FSIPath == "" { - return fsi.ErrNoLink + return nil, fsi.ErrNoLink } // Write components of the dataset to the working directory - err = fsi.WriteComponents(p.Ds, vi.FSIPath, m.inst.node.Repo.Filesystem()) + err = fsi.WriteComponents(p.Ds, vi.FSIPath, scope.Filesystem()) if err != nil { - return err + return nil, err } - *res, err = m.inst.fsi.Status(ctx, vi.FSIPath) - return err -} - -// RestoreParams provides parameters to the restore method. -type RestoreParams struct { - Dir string - Ref string - Component string + return scope.FSISubsystem().Status(ctx, vi.FSIPath) } // Restore method restores a component or all of the component files of a dataset from the repo -func (m *FSIMethods) Restore(p *RestoreParams, out *string) (err error) { - if m.inst.rpc != nil { - return checkRPCError(m.inst.rpc.Call("FSIMethods.Restore", p, out)) - } - ctx := context.TODO() +// TODO(dustmop): Returned string is not used, remove it once dispatch is compatible +func (*FSIImpl) Restore(scope scope, p *RestoreParams) (string, error) { + ctx := scope.Context() - ref, _, err := m.inst.ParseAndResolveRef(ctx, p.Ref, "local") + ref, _, err := scope.ParseAndResolveRef(ctx, p.Refstr, "local") if err != nil { - return err + return "", err + } + + if p.Path != "" { + ref.Path = p.Path } if p.Dir == "" { fsiRef := ref.Copy() - if err = m.inst.fsi.ResolvedPath(&fsiRef); err != nil { - return err + if err = scope.FSISubsystem().ResolvedPath(&fsiRef); err != nil { + return "", err } p.Dir = fsi.FilesystemPathToLocal(fsiRef.Path) } if p.Dir == "" { - return fmt.Errorf("no FSIPath or Dir given") + return "", fmt.Errorf("no FSIPath or Dir given") } ds := &dataset.Dataset{} if ref.Path != "" { // Read the previous version of the dataset from the repo - ds, err = dsfs.LoadDataset(ctx, m.inst.node.Repo.Filesystem(), ref.Path) + ds, err = dsfs.LoadDataset(ctx, scope.Filesystem(), ref.Path) if err != nil { - return fmt.Errorf("loading dataset: %s", err) + return "", fmt.Errorf("loading dataset: %s", err) } - if err = base.OpenDataset(ctx, m.inst.node.Repo.Filesystem(), ds); err != nil { - return + if err = base.OpenDataset(ctx, scope.Filesystem(), ds); err != nil { + return "", err } } // Build component container from the dataset from the repo. - repoContainer := component.ConvertDatasetToComponents(ds, m.inst.node.Repo.Filesystem()) + repoContainer := component.ConvertDatasetToComponents(ds, scope.Filesystem()) repoContainer.Base().RemoveSubcomponent("commit") repoContainer.DropDerivedValues() // Build component container from FSI directory. diskContainer, err := component.ListDirectoryComponents(p.Dir) if err != nil { - return err + return "", err } - err = component.ExpandListedComponents(diskContainer, m.inst.node.Repo.Filesystem()) + err = component.ExpandListedComponents(diskContainer, scope.Filesystem()) if err != nil { - return err + return "", err } for _, compName := range component.AllSubcomponentNames() { @@ -336,66 +424,38 @@ func (m *FSIMethods) Restore(p *RestoreParams, out *string) (err error) { } } } - return nil + return "", nil } -// InitDatasetParams proxies parameters to initialization -type InitDatasetParams = fsi.InitParams - -// InitDataset creates a new dataset in a working directory -func (m *FSIMethods) InitDataset(ctx context.Context, p *InitDatasetParams, refstr *string) (err error) { - if err = qfs.AbsPath(&p.BodyPath); err != nil { - return err - } - - if p.TargetDir == "" { - p.TargetDir = "." - } - if err = qfs.AbsPath(&p.TargetDir); err != nil { - return err - } - - if m.inst.rpc != nil { - return checkRPCError(m.inst.rpc.Call("FSIMethods.InitDataset", p, refstr)) - } +// Init creates a new dataset in a working directory +func (*FSIImpl) Init(scope scope, p *InitDatasetParams) (string, error) { + ctx := scope.Context() - // If the dscache doesn't exist yet, it will only be created if the appropriate flag enables it. if p.UseDscache { - m.inst.Dscache().CreateNewEnabled = true + scope.Dscache().CreateNewEnabled = true } - - ref, err := m.inst.fsi.InitDataset(ctx, *p) - *refstr = ref.Human() - return err + ref, err := scope.FSISubsystem().InitDataset(ctx, *p) + refstr := ref.Human() + return refstr, err } // CanInitDatasetWorkDir returns nil if the directory can init a dataset, or an error if not -func (m *FSIMethods) CanInitDatasetWorkDir(p *InitDatasetParams, ok *bool) error { - targetPath := p.TargetDir - bodyPath := p.BodyPath - return m.inst.fsi.CanInitDatasetWorkDir(targetPath, bodyPath) -} - -// EnsureParams holds values for EnsureRef call -type EnsureParams struct { - Dir string - Ref string +func (*FSIImpl) CanInitDatasetWorkDir(scope scope, p *InitDatasetParams) (bool, error) { + // TODO(dustmop): Change dispatch so that implementations can only return 1 value + // if that value is an error + return true, scope.FSISubsystem().CanInitDatasetWorkDir(p.TargetDir, p.BodyPath) } // EnsureRef will modify the directory path in the repo for the given reference -func (m *FSIMethods) EnsureRef(ctx context.Context, p *EnsureParams, out *dsref.VersionInfo) error { - if m.inst.rpc != nil { - return checkRPCError(m.inst.rpc.Call("FSIMethods.EnsureRef", p, out)) - } +func (*FSIImpl) EnsureRef(scope scope, p *LinkParams) (*dsref.VersionInfo, error) { + ctx := scope.Context() - ref, err := dsref.Parse(p.Ref) + ref, err := dsref.Parse(p.Refstr) if err != nil { - return err + return nil, err } - vi, err := m.inst.fsi.ModifyLinkDirectory(ctx, p.Dir, ref) - *out = *vi - return err + return scope.FSISubsystem().ModifyLinkDirectory(ctx, p.Dir, ref) } // PathJoinPosix joins two paths, and makes it explicitly clear we want POSIX slashes diff --git a/lib/fsi_test.go b/lib/fsi_test.go index a1c95ec9b..126332453 100644 --- a/lib/fsi_test.go +++ b/lib/fsi_test.go @@ -35,7 +35,7 @@ func TestFSIMethodsWrite(t *testing.T) { inst := NewInstanceFromConfigAndNode(ctx, testcfg.DefaultConfigForTesting(), node) // we need some fsi stuff to fully test remove - methods := NewFSIMethods(inst) + methods := inst.Filesys() // create datasets working directory datasetsDir, err := ioutil.TempDir("", "QriTestDatasetRequestsRemove") if err != nil { @@ -49,27 +49,26 @@ func TestFSIMethodsWrite(t *testing.T) { TargetDir: filepath.Join(datasetsDir, "no_history"), Format: "csv", } - var noHistoryName string - if err := methods.InitDataset(ctx, initp, &noHistoryName); err != nil { + noHistoryName, err := methods.Init(ctx, initp) + if err != nil { t.Fatal(err) } // link cities dataset with a checkout - checkoutp := &CheckoutParams{ - Dir: filepath.Join(datasetsDir, "cities"), - Ref: "me/cities", + checkoutp := &LinkParams{ + Dir: filepath.Join(datasetsDir, "cities"), + Refstr: "me/cities", } - var out string - if err := methods.Checkout(checkoutp, &out); err != nil { + if _, err := methods.Checkout(ctx, checkoutp); err != nil { t.Fatal(err) } // link craigslist with a checkout - checkoutp = &CheckoutParams{ - Dir: filepath.Join(datasetsDir, "craigslist"), - Ref: "me/craigslist", + checkoutp = &LinkParams{ + Dir: filepath.Join(datasetsDir, "craigslist"), + Refstr: "me/craigslist", } - if err := methods.Checkout(checkoutp, &out); err != nil { + if _, err := methods.Checkout(ctx, checkoutp); err != nil { t.Fatal(err) } @@ -77,19 +76,17 @@ func TestFSIMethodsWrite(t *testing.T) { err string params FSIWriteParams }{ - {"dataset is required", FSIWriteParams{Ref: "abc/movies"}}, - {`"" is not a valid dataset reference: empty reference`, FSIWriteParams{Ref: "", Ds: &dataset.Dataset{}}}, - {`"abc/ABC" is not a valid dataset reference: dataset name may not contain any upper-case letters`, FSIWriteParams{Ref: "abc/ABC", Ds: &dataset.Dataset{}}}, - {`"👋" is not a valid dataset reference: unexpected character at position 0: 'ð'`, FSIWriteParams{Ref: "👋", Ds: &dataset.Dataset{}}}, - {"reference not found", FSIWriteParams{Ref: "abc/movies", Ds: &dataset.Dataset{}}}, - {"dataset is not linked to the filesystem", FSIWriteParams{Ref: "peer/movies", Ds: &dataset.Dataset{}}}, + {"dataset is required", FSIWriteParams{Refstr: "abc/movies"}}, + {`"" is not a valid dataset reference: empty reference`, FSIWriteParams{Refstr: "", Ds: &dataset.Dataset{}}}, + {`"abc/ABC" is not a valid dataset reference: dataset name may not contain any upper-case letters`, FSIWriteParams{Refstr: "abc/ABC", Ds: &dataset.Dataset{}}}, + {`"👋" is not a valid dataset reference: unexpected character at position 0: 'ð'`, FSIWriteParams{Refstr: "👋", Ds: &dataset.Dataset{}}}, + {"reference not found", FSIWriteParams{Refstr: "abc/movies", Ds: &dataset.Dataset{}}}, + {"dataset is not linked to the filesystem", FSIWriteParams{Refstr: "peer/movies", Ds: &dataset.Dataset{}}}, } for _, c := range badCases { t.Run(fmt.Sprintf("bad_case_%s", c.err), func(t *testing.T) { - res := []StatusItem{} - err := methods.Write(&c.params, &res) - + _, err := methods.Write(ctx, &c.params) if err == nil { t.Errorf("expected error. got nil") return @@ -105,7 +102,7 @@ func TestFSIMethodsWrite(t *testing.T) { res []StatusItem }{ {"update cities structure", - FSIWriteParams{Ref: "me/cities", Ds: &dataset.Dataset{Structure: &dataset.Structure{Format: "json"}}}, + FSIWriteParams{Refstr: "me/cities", Ds: &dataset.Dataset{Structure: &dataset.Structure{Format: "json"}}}, []StatusItem{ {Component: "meta", Type: "unmodified"}, {Component: "structure", Type: "modified"}, @@ -118,7 +115,7 @@ func TestFSIMethodsWrite(t *testing.T) { // []StatusItem{}, // }, {"set title for no history dataset", - FSIWriteParams{Ref: noHistoryName, Ds: &dataset.Dataset{Meta: &dataset.Meta{Title: "Changed Title"}}}, + FSIWriteParams{Refstr: noHistoryName, Ds: &dataset.Dataset{Meta: &dataset.Meta{Title: "Changed Title"}}}, []StatusItem{ {Component: "meta", Type: "add"}, {Component: "structure", Type: "add"}, @@ -129,8 +126,7 @@ func TestFSIMethodsWrite(t *testing.T) { for _, c := range goodCases { t.Run(fmt.Sprintf("good_case_%s", c.description), func(t *testing.T) { - res := []StatusItem{} - err := methods.Write(&c.params, &res) + res, err := methods.Write(ctx, &c.params) if err != nil { t.Errorf("unexpected error: %s", err) @@ -306,11 +302,12 @@ func TestInitWithCurrentDir(t *testing.T) { t.Fatal(err) } - // Don't pass the working directory path, `init` will use the current directory + // Don't pass the full working directory path, "." will use the current directory err := run.InitWithParams( &InitDatasetParams{ - Name: "new_ds", - Format: "csv", + Name: "new_ds", + Format: "csv", + TargetDir: ".", }, ) if err != nil { diff --git a/lib/http.go b/lib/http.go index a1b1d92e3..0c3cea0eb 100644 --- a/lib/http.go +++ b/lib/http.go @@ -141,7 +141,7 @@ func (c HTTPClient) do(ctx context.Context, addr string, httpMethod string, mime err = json.Unmarshal(body, &resData) if err != nil { log.Debugf("HTTPClient unmarshal err: %s", err.Error()) - return err + return fmt.Errorf("HTTPClient unmarshal err: %s", err) } return c.checkError(resData.Meta) } diff --git a/lib/lib.go b/lib/lib.go index 166541a4a..d81f0d0ab 100644 --- a/lib/lib.go +++ b/lib/lib.go @@ -399,6 +399,8 @@ func NewInstance(ctx context.Context, repoPath string, opts ...Option) (qri *Ins log.Debugf("--log-all set: turning on logging for all activity") } + inst.RegisterMethods() + // check if we're operating over RPC if cfg.RPC.Enabled { addr, err := ma.NewMultiaddr(cfg.RPC.Address) @@ -663,6 +665,7 @@ func NewInstanceFromConfigAndNodeAndBus(ctx context.Context, cfg *config.Config, logbook: r.Logbook(), transform: transform.NewService(ctx), } + inst.RegisterMethods() inst.stats = stats.New(nil) @@ -698,6 +701,8 @@ type Instance struct { repoPath string cfg *config.Config + regMethods *regMethodSet + streams ioes.IOStreams repo repo.Repo node *p2p.QriNode diff --git a/lib/lib_test.go b/lib/lib_test.go index 5d983405d..1262921c0 100644 --- a/lib/lib_test.go +++ b/lib/lib_test.go @@ -187,7 +187,7 @@ func TestReceivers(t *testing.T) { inst := &Instance{node: node, cfg: cfg} reqs := Receivers(inst) - expect := 11 + expect := 10 if len(reqs) != expect { t.Errorf("unexpected number of receivers returned. expected: %d. got: %d\nhave you added/removed a receiver?", expect, len(reqs)) return diff --git a/lib/rpc.go b/lib/rpc.go index 9548bc227..1c48b45a3 100644 --- a/lib/rpc.go +++ b/lib/rpc.go @@ -41,7 +41,6 @@ func Receivers(inst *Instance) []Methods { NewSearchMethods(inst), NewSQLMethods(inst), NewRenderMethods(inst), - NewFSIMethods(inst), } } diff --git a/lib/scope.go b/lib/scope.go new file mode 100644 index 000000000..736ee50a7 --- /dev/null +++ b/lib/scope.go @@ -0,0 +1,80 @@ +package lib + +import ( + "context" + + "github.com/qri-io/dataset" + "github.com/qri-io/qfs/muxfs" + "github.com/qri-io/qri/dscache" + "github.com/qri-io/qri/dsref" + "github.com/qri-io/qri/event" + "github.com/qri-io/qri/fsi" + "github.com/qri-io/qri/repo" +) + +// scope represents the lifetime of a method call, abstractly connected to the caller of +// that method, such that the implementation is unaware of how it has been invoked. +// Using scope instead of the global data on Instance lets us control user identity, +// permissions, and configuration, while also setting us up to properly run multiple +// operations at the same time to support multi-tenancy and multi-processing. +type scope struct { + ctx context.Context + inst *Instance + // TODO(dustmop): Additional information, such as user identity, their profile, keys +} + +// Context returns the context for this scope. Though this pattern is usually discouraged, +// we're following http.Request's lead, as scope plays the same role. The lifetime of a +// single scope matches the lifetime of the Context; this ownership is not long-lived +func (s *scope) Context() context.Context { + return s.ctx +} + +// FSISubsystem returns a reference to the FSI subsystem +// TODO(dustmop): This subsystem contains global data, we should move that data out and +// into scope +func (s *scope) FSISubsystem() *fsi.FSI { + return s.inst.fsi +} + +// Bus returns the event bus +func (s *scope) Bus() event.Bus { + // TODO(dustmop): Filter only events for this scope. + return s.inst.bus +} + +// Filesystem returns a filesystem +func (s *scope) Filesystem() *muxfs.Mux { + return s.inst.qfs +} + +// Dscache returns the dscache +func (s *scope) Dscache() *dscache.Dscache { + return s.inst.Dscache() +} + +// ParseAndResolveRef parses a reference and resolves it +func (s *scope) ParseAndResolveRef(ctx context.Context, refStr, source string) (dsref.Ref, string, error) { + return s.inst.ParseAndResolveRef(ctx, refStr, source) +} + +// ParseAndResolveRefWithWorkingDir parses a reference and resolves it with FSI info attached +func (s *scope) ParseAndResolveRefWithWorkingDir(ctx context.Context, refstr, source string) (dsref.Ref, string, error) { + return s.inst.ParseAndResolveRefWithWorkingDir(ctx, refstr, source) +} + +// LoadDataset loads a dataset +func (s *scope) LoadDataset(ctx context.Context, ref dsref.Ref, source string) (*dataset.Dataset, error) { + return s.inst.LoadDataset(ctx, ref, source) +} + +// Loader returns the default dataset ref loader +func (s *scope) Loader() dsref.ParseResolveLoad { + return NewParseResolveLoadFunc("", s.inst.defaultResolver(), s.inst) +} + +// GetVersionInfoShim is in the process of being removed. Try not to add new callers. +func (s *scope) GetVersionInfoShim(ref dsref.Ref) (*dsref.VersionInfo, error) { + r := s.inst.Repo() + return repo.GetVersionInfoShim(r, ref) +} diff --git a/lib/test_runner_test.go b/lib/test_runner_test.go index 6f1e5e2d4..e224f3a28 100644 --- a/lib/test_runner_test.go +++ b/lib/test_runner_test.go @@ -222,34 +222,35 @@ func (tr *testRunner) DiffWithParams(p *DiffParams) (string, error) { } func (tr *testRunner) Init(refstr, format string) error { - ctx := context.Background() + ctx := tr.Ctx ref, err := dsref.Parse(refstr) if err != nil { return err } - m := NewFSIMethods(tr.Instance) - out := "" + m := tr.Instance.Filesys() p := InitDatasetParams{ Name: ref.Name, TargetDir: tr.WorkDir, Format: format, } - return m.InitDataset(ctx, &p, &out) + _, err = m.Init(ctx, &p) + return err } func (tr *testRunner) InitWithParams(p *InitDatasetParams) error { - ctx := context.Background() - m := NewFSIMethods(tr.Instance) - out := "" - return m.InitDataset(ctx, p, &out) + ctx := tr.Ctx + m := tr.Instance.Filesys() + _, err := m.Init(ctx, p) + return err } func (tr *testRunner) Checkout(refstr, dir string) error { - m := NewFSIMethods(tr.Instance) - out := "" - p := CheckoutParams{ - Ref: refstr, - Dir: dir, + ctx := tr.Ctx + m := tr.Instance.Filesys() + p := LinkParams{ + Refstr: refstr, + Dir: dir, } - return m.Checkout(&p, &out) + _, err := m.Checkout(ctx, &p) + return err }