feat(fsi): add fsimethods.Write to write directly to the linked files…

…ystem

Merge pull request #937 from qri-io/feat_api_write_fsi

api/api.go

@@ -284,6 +284,7 @@ func NewServerRoutes(s Server) *http.ServeMux {
  m.Handle("/init/", s.middleware(fsih.InitHandler("/init")))
  m.Handle("/checkout/", s.middleware(fsih.CheckoutHandler("/checkout")))
  m.Handle("/restore/", s.middleware(fsih.RestoreHandler("/restore")))
+ m.Handle("/fsi/write/", s.middleware(fsih.WriteHandler("/fsi/write")))
 
  renderh := NewRenderHandlers(node.Repo)
  m.Handle("/render/", s.middleware(renderh.RenderHandler))

api/fsi.go

@@ -1,11 +1,13 @@
 package api
 
 import (
+ "encoding/json"
  "errors"
  "fmt"
  "net/http"
 
  util "github.com/qri-io/apiutil"
+ "github.com/qri-io/dataset"
  "github.com/qri-io/qri/lib"
  "github.com/qri-io/qri/repo"
  "github.com/qri-io/qri/repo/profile"
@@ -164,6 +166,55 @@ func (h *FSIHandlers) initHandler(routePrefix string) http.HandlerFunc {
  }
 }
 
+// WriteHandler writes input data to the local filesystem link
+func (h *FSIHandlers) WriteHandler(routePrefix string) http.HandlerFunc {
+ handler := h.writeHandler(routePrefix)
+ return func(w http.ResponseWriter, r *http.Request) {
+ if h.ReadOnly {
+ readOnlyResponse(w, routePrefix)
+ return
+ }
+
+ switch r.Method {
+ case "OPTIONS":
+ util.EmptyOkHandler(w, r)
+ case "POST":
+ handler(w, r)
+ default:
+ util.NotFoundHandler(w, r)
+ }
+ }
+}
+
+func (h *FSIHandlers) writeHandler(routePrefix string) http.HandlerFunc {
+ return func(w http.ResponseWriter, r *http.Request) {
+ ref, err := DatasetRefFromPath(r.URL.Path[len(routePrefix):])
+ if err != nil {
+ util.WriteErrResponse(w, http.StatusBadRequest, fmt.Errorf("bad reference: %s", err.Error()))
+ return
+ }
+
+ ds := &dataset.Dataset{}
+ if err := json.NewDecoder(r.Body).Decode(ds); err != nil {
+ util.WriteErrResponse(w, http.StatusBadRequest, err)
+ return
+ }
+
+ p := &lib.FSIWriteParams{
+ Ref: ref.AliasString(),
+ Ds: ds,
+ }
+
+ out := []lib.StatusItem{}
+ if err := h.Write(p, &out); err != nil {
+ util.WriteErrResponse(w, http.StatusBadRequest, err)
+ return
+ }
+
+ util.WriteResponse(w, out)
+ }
+}
+
 // CheckoutHandler invokes checkout via an API call
 func (h *FSIHandlers) CheckoutHandler(routePrefix string) http.HandlerFunc {
  handleCheckout := h.checkoutHandler(routePrefix)

api/fsi_test.go

@@ -1,6 +1,7 @@
 package api
 
 import (
+ "bytes"
  "encoding/json"
  "fmt"
  "io/ioutil"
@@ -236,6 +237,93 @@ func TestNoHistory(t *testing.T) {
  t.Errorf("expected body %v, got %v\ndiff:%v", expectNoHistoryBody, gotBody, diff)
  }
 }
+
+func TestFSIWrite(t *testing.T) {
+ node, teardown := newTestNode(t)
+ defer teardown()
+
+ inst := newTestInstanceWithProfileFromNode(node)
+
+ tmpDir := os.TempDir()
+ workSubdir := "write_test"
+ workDir := filepath.Join(tmpDir, workSubdir)
+ // Don't create the work directory, it must not exist for checkout to work. Remove if it
+ // already exists.
+ _ = os.RemoveAll(workDir)
+
+ dr := lib.NewDatasetRequests(node, nil)
+ fsiHandler := NewFSIHandlers(inst, false)
+
+ // Save version 1
+ saveParams := lib.SaveParams{
+ Ref: "me/write_test",
+ Dataset: &dataset.Dataset{
+ Meta: &dataset.Meta{
+ Title: "title one",
+ },
+ },
+ BodyPath: "testdata/cities/data.csv",
+ }
+ res := repo.DatasetRef{}
+ if err := dr.Save(&saveParams, &res); err != nil {
+ t.Fatal(err)
+ }
+
+ // Checkout the dataset
+ actualStatusCode, actualBody := APICallWithParams(
+ "POST",
+ "/checkout/peer/write_test",
+ map[string]string{
+ "dir": workDir,
+ },
+ fsiHandler.CheckoutHandler("/checkout"))
+ if actualStatusCode != 200 {
+ t.Errorf("expected status code 200, got %d", actualStatusCode)
+ }
+ expectBody := `{"data":"","meta":{"code":200}}`
+ if expectBody != actualBody {
+ t.Errorf("expected body %s, got %s", expectBody, actualBody)
+ }
+
+ status, strRes := JSONAPICallWithBody("POST", "/me/write_test", &dataset.Dataset{Meta: &dataset.Meta{Title: "oh hai there"}}, fsiHandler.WriteHandler(""))
+
+ if status != http.StatusOK {
+ t.Errorf("status code mismatch. expected: %d, got: %d", http.StatusOK, status)
+ }
+
+ exp := struct {
+ Data []struct {
+ // ignore mtime & path fields by only deserializing component & type from JSON
+ Component string
+ Type string
+ }
+ }{
+ Data: []struct {
+ Component string
+ Type string
+ }{
+ {Component: "meta", Type: "modified"},
+ {Component: "structure", Type: "unmodified"},
+ {Component: "schema", Type: "unmodified"},
+ {Component: "body", Type: "unmodified"},
+ },
+ }
+
+ got := struct {
+ Data []struct {
+ Component string
+ Type string
+ }
+ }{}
+ if err := json.NewDecoder(strings.NewReader(strRes)).Decode(&got); err != nil {
+ t.Fatal(err)
+ }
+
+ if diff := cmp.Diff(exp, got); diff != "" {
+ t.Errorf("response data mistmach (-want +got):\n%s", diff)
+ }
+}
+
 func TestCheckoutAndRestore(t *testing.T) {
  node, teardown := newTestNode(t)
  defer teardown()
@@ -424,3 +512,24 @@ func APICallWithParams(method, reqURL string, params map[string]string, hf http.
  }
  return statusCode, string(bodyBytes)
 }
+
+func JSONAPICallWithBody(method, reqURL string, data interface{}, hf http.HandlerFunc) (int, string) {
+ enc, err := json.Marshal(data)
+ if err != nil {
+ panic(err)
+ }
+
+ req := httptest.NewRequest(method, reqURL, bytes.NewReader(enc))
+ // Set form-encoded header so server will find the parameters
+ req.Header.Add("Content-Type", "application/json")
+ w := httptest.NewRecorder()
+ hf(w, req)
+ res := w.Result()
+ statusCode := res.StatusCode
+
+ bodyBytes, err := ioutil.ReadAll(res.Body)
+ if err != nil {
+ panic(err)
+ }
+ return statusCode, string(bodyBytes)
+}

lib/fsi.go

@@ -185,6 +185,48 @@ func (m *FSIMethods) Checkout(p *CheckoutParams, out *string) (err error) {
  return err
 }
 
+// FSIWriteParams encapsultes arguments for writing to an FSI-linked directory
+type FSIWriteParams struct {
+ Ref string
+ Ds *dataset.Dataset
+}
+
+// Write mutates a linked dataset on the filesystem
+func (m *FSIMethods) Write(p *FSIWriteParams, res *[]StatusItem) (err error) {
+ if m.inst.rpc != nil {
+ return m.inst.rpc.Call("FSIMethods.Write", p, res)
+ }
+ ctx := context.TODO()
+
+ if p.Ref == "" {
+ return repo.ErrEmptyRef
+ }
+ if p.Ds == nil {
+ return fmt.Errorf("dataset is required")
+ }
+ ref, err := repo.ParseDatasetRef(p.Ref)
+ if err != nil {
+ return fmt.Errorf("'%s' is not a valid dataset reference", p.Ref)
+ }
+ err = repo.CanonicalizeDatasetRef(m.inst.node.Repo, &ref)
+ if err != nil && err != repo.ErrNoHistory {
+ return err
+ }
+
+ // Directory to write components to can be determined from FSIPath of ref.
+ if ref.FSIPath == "" {
+ return fsi.ErrNoLink
+ }
+
+ // Write components of the dataset to the working directory
+ if err = fsi.WriteComponents(p.Ds, ref.FSIPath); err != nil {
+ return err
+ }
+
+ *res, err = m.inst.fsi.Status(ctx, ref.FSIPath)
+ return err
+}
+
 // RestoreParams provides parameters to the restore method.
 type RestoreParams struct {
  Dir string

lib/fsi_test.go

@@ -0,0 +1,142 @@
+package lib
+
+import (
+ "fmt"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "testing"
+
+ cmp "github.com/google/go-cmp/cmp"
+ cmpopts "github.com/google/go-cmp/cmp/cmpopts"
+ "github.com/qri-io/dataset"
+ "github.com/qri-io/qri/config"
+ "github.com/qri-io/qri/p2p"
+ testrepo "github.com/qri-io/qri/repo/test"
+)
+
+func TestFSIMethodsWrite(t *testing.T) {
+ mr, err := testrepo.NewTestRepo()
+ if err != nil {
+ t.Fatalf("error allocating test repo: %s", err.Error())
+ }
+ node, err := p2p.NewQriNode(mr, config.DefaultP2PForTesting())
+ if err != nil {
+ t.Fatal(err.Error())
+ }
+
+ inst := NewInstanceFromConfigAndNode(config.DefaultConfigForTesting(), node)
+
+ // we need some fsi stuff to fully test remove
+ methods := NewFSIMethods(inst)
+ // create datasets working directory
+ datasetsDir, err := ioutil.TempDir("", "QriTestDatasetRequestsRemove")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer os.RemoveAll(datasetsDir)
+
+ // initialize an example no-history dataset
+ initp := &InitFSIDatasetParams{
+ Name: "no_history",
+ Dir: datasetsDir,
+ Format: "csv",
+ Mkdir: "no_history",
+ }
+ var noHistoryName string
+ if err := methods.InitDataset(initp, &noHistoryName); err != nil {
+ t.Fatal(err)
+ }
+
+ // link cities dataset with a checkout
+ checkoutp := &CheckoutParams{
+ Dir: filepath.Join(datasetsDir, "cities"),
+ Ref: "me/cities",
+ }
+ var out string
+ if err := methods.Checkout(checkoutp, &out); err != nil {
+ t.Fatal(err)
+ }
+
+ // link craigslist with a checkout
+ checkoutp = &CheckoutParams{
+ Dir: filepath.Join(datasetsDir, "craigslist"),
+ Ref: "me/craigslist",
+ }
+ if err := methods.Checkout(checkoutp, &out); err != nil {
+ t.Fatal(err)
+ }
+
+ badCases := []struct {
+ err string
+ params FSIWriteParams
+ }{
+ {"repo: empty dataset reference", FSIWriteParams{Ref: ""}},
+ {"dataset is required", FSIWriteParams{Ref: "abc/ABC"}},
+ // TODO (b5) - this is a bug. should return: "'👋' is not a valid dataset reference"
+ {"repo: not found", FSIWriteParams{Ref: "👋", Ds: &dataset.Dataset{}}},
+ {"repo: not found", FSIWriteParams{Ref: "abc/ABC", Ds: &dataset.Dataset{}}},
+ {"dataset is not linked to the filesystem", FSIWriteParams{Ref: "peer/movies", Ds: &dataset.Dataset{}}},
+ // TODO (b5) - this is also a bug, and should be allowed. body should map to a file or something
+ {"body is defined in two places: body.json and dataset.json. please remove one",
+ FSIWriteParams{Ref: "me/craigslist", Ds: &dataset.Dataset{Body: []interface{}{[]interface{}{"foo", "bar", "baz"}}}}},
+ }
+
+ for _, c := range badCases {
+ t.Run(fmt.Sprintf("bad_case_%s", c.err), func(t *testing.T) {
+ res := []StatusItem{}
+ err := methods.Write(&c.params, &res)
+
+ if err == nil {
+ t.Errorf("expected error. got nil")
+ return
+ } else if c.err != err.Error() {
+ t.Errorf("error mismatch: expected: %s, got: %s", c.err, err)
+ }
+ })
+ }
+
+ goodCases := []struct {
+ description string
+ params FSIWriteParams
+ res []StatusItem
+ }{
+ {"update cities structure",
+ FSIWriteParams{Ref: "me/cities", Ds: &dataset.Dataset{Structure: &dataset.Structure{Format: "json"}}},
+ []StatusItem{
+ {Component: "meta", Type: "unmodified"},
+ {Component: "structure", Type: "unmodified"},
+ {Component: "schema", Type: "unmodified"},
+ {Component: "body", Type: "unmodified"},
+ },
+ },
+ // TODO (b5) - doesn't work yet
+ // {"overwrite craigslist body",
+ // FSIWriteParams{Ref: "me/craigslist", Ds: &dataset.Dataset{Body: []interface{}{[]interface{}{"foo", "bar", "baz"}}}},
+ // []StatusItem{},
+ // },
+ {"set title for no history dataset",
+ FSIWriteParams{Ref: noHistoryName, Ds: &dataset.Dataset{Meta: &dataset.Meta{Title: "Changed Title"}}},
+ []StatusItem{
+ {Component: "meta", Type: "add"},
+ {Component: "body", Type: "add"},
+ },
+ },
+ }
+
+ for _, c := range goodCases {
+ t.Run(fmt.Sprintf("good_case_%s", c.description), func(t *testing.T) {
+ res := []StatusItem{}
+ err := methods.Write(&c.params, &res)
+
+ if err != nil {
+ t.Errorf("unexpected error: %s", err)
+ return
+ }
+
+ if diff := cmp.Diff(c.res, res, cmpopts.IgnoreFields(StatusItem{}, "Mtime", "SourceFile")); diff != "" {
+ t.Errorf("response mismatch (-want +got):\n%s", diff)
+ }
+ })
+ }
+}