From d1d9670586ffd1754cd09802b2173a3685eb0f59 Mon Sep 17 00:00:00 2001 From: Dustin Long Date: Wed, 28 Aug 2019 16:20:54 -0400 Subject: [PATCH] feat(restore): Add api for restore. Test for checkout and restore apis. Use restore like: ``` curl -X POST -d "component=body" localhost:2503/restore/peername/ds_name ``` --- api/api.go | 1 + api/fsi.go | 47 +++++ api/fsi_test.go | 184 ++++++++++++++++++- cmd/{fsi_test.go => fsi_integration_test.go} | 81 +++++++- cmd/restore.go | 14 +- fsi/mapping.go | 2 +- lib/fsi.go | 18 +- 7 files changed, 330 insertions(+), 17 deletions(-) rename cmd/{fsi_test.go => fsi_integration_test.go} (89%) diff --git a/api/api.go b/api/api.go index 6c42f95e7..85b76104a 100644 --- a/api/api.go +++ b/api/api.go @@ -286,6 +286,7 @@ func NewServerRoutes(s Server) *http.ServeMux { m.Handle("/status/", s.middleware(fsih.StatusHandler("/status"))) m.Handle("/init/", s.middleware(fsih.InitHandler("/init"))) m.Handle("/checkout/", s.middleware(fsih.CheckoutHandler("/checkout"))) + m.Handle("/restore/", s.middleware(fsih.RestoreHandler("/restore"))) renderh := NewRenderHandlers(node.Repo) m.Handle("/render/", s.middleware(renderh.RenderHandler)) diff --git a/api/fsi.go b/api/fsi.go index 63d3ac87d..11e6445f1 100644 --- a/api/fsi.go +++ b/api/fsi.go @@ -163,3 +163,50 @@ func (h *FSIHandlers) checkoutHandler(routePrefix string) http.HandlerFunc { util.WriteResponse(w, res) } } + +// RestoreHandler invokes restore via an API call +func (h *FSIHandlers) RestoreHandler(routePrefix string) http.HandlerFunc { + handleRestore := h.restoreHandler(routePrefix) + return func(w http.ResponseWriter, r *http.Request) { + if h.ReadOnly { + readOnlyResponse(w, routePrefix) + return + } + + switch r.Method { + case "OPTIONS": + util.EmptyOkHandler(w, r) + case "POST": + handleRestore(w, r) + default: + util.NotFoundHandler(w, r) + } + } +} + +func (h *FSIHandlers) restoreHandler(routePrefix string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + ref, err := DatasetRefFromPath(r.URL.Path[len(routePrefix):]) + if err != nil { + util.WriteErrResponse(w, http.StatusBadRequest, fmt.Errorf("bad reference: %s", err.Error())) + return + } + + // Add the path for the version to restore + ref.Path = r.FormValue("path") + + p := &lib.RestoreParams{ + Dir: r.FormValue("dir"), + Ref: ref.String(), + Component: r.FormValue("component"), + } + + var res string + if err := h.Restore(p, &res); err != nil { + util.WriteErrResponse(w, http.StatusInternalServerError, err) + return + } + + util.WriteResponse(w, res) + } +} diff --git a/api/fsi_test.go b/api/fsi_test.go index 068937153..f549e11fd 100644 --- a/api/fsi_test.go +++ b/api/fsi_test.go @@ -1,16 +1,22 @@ package api import ( - "bytes" "fmt" "io/ioutil" "net/http" "net/http/httptest" + "net/url" "os" "path/filepath" + "strconv" + "strings" "testing" + "github.com/google/go-cmp/cmp" + "github.com/qri-io/dataset" "github.com/qri-io/qri/fsi" + "github.com/qri-io/qri/lib" + "github.com/qri-io/qri/repo" ) func TestFSIHandlers(t *testing.T) { @@ -66,11 +72,13 @@ func TestNoHistory(t *testing.T) { inst := newTestInstanceWithProfileFromNode(node) - initDir := "fsi_tests/init_dir" + tmpDir := os.TempDir() + initSubdir := "fsi_init_dir" + initDir := filepath.Join(tmpDir, initSubdir) if err := os.MkdirAll(initDir, os.ModePerm); err != nil { panic(err) } - defer os.RemoveAll(filepath.Join("fsi_tests")) + defer os.RemoveAll(initDir) // Create a linked dataset without saving, it has no versions in the repository f := fsi.NewFSI(node.Repo) @@ -104,9 +112,11 @@ func TestNoHistory(t *testing.T) { if actualStatusCode != 200 { t.Errorf("expected status code 200, got %d", actualStatusCode) } - expectBody = `{"data":{"peername":"peer","name":"test_ds","fsiPath":"fsi_tests/init_dir","dataset":{"bodyPath":"fsi_tests/init_dir/body.csv","meta":{"keywords":[],"qri":"md:0"},"name":"test_ds","peername":"peer","qri":"ds:0","structure":{"format":"csv","qri":"st:0","schema":{"items":{"items":[{"title":"name","type":"string"},{"title":"describe","type":"string"},{"title":"quantity","type":"integer"}],"type":"array"},"type":"array"}}},"published":false},"meta":{"code":200}}` - if expectBody != actualBody { - t.Errorf("expected body %s, got %s", expectBody, actualBody) + // Handle tempoary directory by replacing the temp part with a shorter string. + resultBody := strings.Replace(actualBody, initDir, initSubdir, -1) + expectBody = `{"data":{"peername":"peer","name":"test_ds","fsiPath":"fsi_init_dir","dataset":{"bodyPath":"fsi_init_dir/body.csv","meta":{"keywords":[],"qri":"md:0"},"name":"test_ds","peername":"peer","qri":"ds:0","structure":{"format":"csv","qri":"st:0","schema":{"items":{"items":[{"title":"name","type":"string"},{"title":"describe","type":"string"},{"title":"quantity","type":"integer"}],"type":"array"},"type":"array"}}},"published":false},"meta":{"code":200}}` + if expectBody != resultBody { + t.Errorf("expected body %s, got %s", expectBody, resultBody) } // Body with no history @@ -146,9 +156,11 @@ func TestNoHistory(t *testing.T) { if actualStatusCode != 200 { t.Errorf("expected status code 200, got %d", actualStatusCode) } - expectBody = `{"data":[{"sourceFile":"fsi_tests/init_dir/meta.json","component":"meta","type":"add","message":""},{"sourceFile":"fsi_tests/init_dir/schema.json","component":"schema","type":"add","message":""},{"sourceFile":"body.csv","component":"body","type":"add","message":""}],"meta":{"code":200}}` - if expectBody != actualBody { - t.Errorf("expected body %s, got %s", expectBody, actualBody) + // Handle tempoary directory by replacing the temp part with a shorter string. + resultBody = strings.Replace(actualBody, initDir, initSubdir, -1) + expectBody = `{"data":[{"sourceFile":"fsi_init_dir/meta.json","component":"meta","type":"add","message":""},{"sourceFile":"fsi_init_dir/schema.json","component":"schema","type":"add","message":""},{"sourceFile":"body.csv","component":"body","type":"add","message":""}],"meta":{"code":200}}` + if expectBody != resultBody { + t.Errorf("expected body %s, got %s", expectBody, resultBody) } logHandler := NewLogHandlers(node) @@ -174,9 +186,161 @@ func TestNoHistory(t *testing.T) { } } +func TestCheckoutAndRestore(t *testing.T) { + node, teardown := newTestNode(t) + defer teardown() + + inst := newTestInstanceWithProfileFromNode(node) + + tmpDir := os.TempDir() + workSubdir := "fsi_checkout_restore" + workDir := filepath.Join(tmpDir, workSubdir) + // Don't create the work directory, it must not exist for checkout to work. Remove if it + // already exists. + _ = os.RemoveAll(workDir) + + dr := lib.NewDatasetRequests(node, nil) + + // Save version 1 + saveParams := lib.SaveParams{ + Ref: "me/fsi_checkout_restore", + Dataset: &dataset.Dataset{ + Meta: &dataset.Meta{ + Title: "title one", + }, + }, + BodyPath: "testdata/cities/data.csv", + } + res := repo.DatasetRef{} + if err := dr.Save(&saveParams, &res); err != nil { + t.Fatal(err) + } + + // Save the path from reference for later. + // TODO(dlong): Support full dataset refs, not just the path. + pos := strings.Index(res.String(), "/map/") + ref1 := res.String()[pos:] + + // Save version 2 with a different title + saveParams = lib.SaveParams{ + Ref: "me/fsi_checkout_restore", + Dataset: &dataset.Dataset{ + Meta: &dataset.Meta{ + Title: "title two", + }, + }, + } + if err := dr.Save(&saveParams, &res); err != nil { + t.Fatal(err) + } + + fsiHandler := NewFSIHandlers(inst, false) + + // Checkout the dataset + actualStatusCode, actualBody := APICallWithParams( + "POST", + "/checkout/peer/fsi_checkout_restore", + map[string]string{ + "dir": workDir, + }, + fsiHandler.CheckoutHandler("/checkout")) + if actualStatusCode != 200 { + t.Errorf("expected status code 200, got %d", actualStatusCode) + } + expectBody := `{"data":"","meta":{"code":200}}` + if expectBody != actualBody { + t.Errorf("expected body %s, got %s", expectBody, actualBody) + } + + // Read meta.json, should have "title two" as the meta title + metaContents, err := ioutil.ReadFile(filepath.Join(workDir, "meta.json")) + if err != nil { + t.Fatalf(err.Error()) + } + expectContents := "{\n \"qri\": \"md:0\",\n \"title\": \"title two\"\n}" + if diff := cmp.Diff(expectContents, string(metaContents)); diff != "" { + t.Errorf("meta.json contents (-want +got):\n%s", diff) + } + + // Overwrite meta so it has a different title + if err = ioutil.WriteFile("meta.json", []byte(`{"title": "hello"}`), os.ModePerm); err != nil { + t.Fatalf(err.Error()) + } + + // Restore the meta component + actualStatusCode, actualBody = APICallWithParams( + "POST", + "/restore/peer/fsi_checkout_restore", + map[string]string{ + "component": "meta", + }, + fsiHandler.RestoreHandler("/restore")) + if actualStatusCode != 200 { + t.Errorf("expected status code 200, got %d", actualStatusCode) + } + expectBody = `{"data":"","meta":{"code":200}}` + if expectBody != actualBody { + t.Errorf("expected body %s, got %s", expectBody, actualBody) + } + + // Read meta.json, should once again have "title two" as the meta title + metaContents, err = ioutil.ReadFile(filepath.Join(workDir, "meta.json")) + if err != nil { + t.Fatalf(err.Error()) + } + expectContents = "{\n \"qri\": \"md:0\",\n \"title\": \"title two\"\n}" + if diff := cmp.Diff(expectContents, string(metaContents)); diff != "" { + t.Errorf("meta.json contents (-want +got):\n%s", diff) + } + + // Restore the previous version of the dataset + actualStatusCode, actualBody = APICallWithParams( + "POST", + "/restore/peer/fsi_checkout_restore", + map[string]string{ + // TODO(dlong): Have to pass "dir" to this method. In the test, the ref does + // not have an FSIPath. Might be because we're using /map/, not sure. + "dir": workDir, + "path": ref1, + }, + fsiHandler.RestoreHandler("/restore")) + if actualStatusCode != 200 { + t.Errorf("expected status code 200, got %d", actualStatusCode) + } + expectBody = `{"data":"","meta":{"code":200}}` + if expectBody != actualBody { + t.Errorf("expected body %s, got %s", expectBody, actualBody) + } + + // Read meta.json, should now have "title one" as the meta title + metaContents, err = ioutil.ReadFile(filepath.Join(workDir, "meta.json")) + if err != nil { + t.Fatalf(err.Error()) + } + expectContents = "{\n \"qri\": \"md:0\",\n \"title\": \"title one\"\n}" + if diff := cmp.Diff(expectContents, string(metaContents)); diff != "" { + t.Errorf("meta.json contents (-want +got):\n%s", diff) + } +} + // APICall calls the api and returns the status code and body func APICall(url string, hf http.HandlerFunc) (int, string) { - req := httptest.NewRequest("GET", url, bytes.NewBuffer(nil)) + return APICallWithParams("GET", url, nil, hf) +} + +// APICallWithParams calls the api and returns the status code and body +func APICallWithParams(method, reqURL string, params map[string]string, hf http.HandlerFunc) (int, string) { + // Add parameters from map + reqParams := url.Values{} + if params != nil { + for key := range params { + reqParams.Set(key, params[key]) + } + } + req := httptest.NewRequest(method, reqURL, strings.NewReader(reqParams.Encode())) + // Set form-encoded header so server will find the parameters + req.Header.Add("Content-Type", "application/x-www-form-urlencoded") + req.Header.Add("Content-Length", strconv.Itoa(len(reqParams.Encode()))) w := httptest.NewRecorder() hf(w, req) res := w.Result() diff --git a/cmd/fsi_test.go b/cmd/fsi_integration_test.go similarity index 89% rename from cmd/fsi_test.go rename to cmd/fsi_integration_test.go index a695e7409..fb439db41 100644 --- a/cmd/fsi_test.go +++ b/cmd/fsi_integration_test.go @@ -16,8 +16,6 @@ import ( "github.com/spf13/cobra" ) -// TODO(dlong): In a future commit, rename this file to fsi_integration_test.go - // FSITestRunner holds test info for fsi integration tests, for convenient cleanup. type FSITestRunner struct { RepoRoot *TestRepoRoot @@ -60,7 +58,7 @@ func NewFSITestRunner(t *testing.T, testName string) *FSITestRunner { } // Delete cleans up after a FSITestRunner is done being used. -func (fr* FSITestRunner) Delete() { +func (fr *FSITestRunner) Delete() { os.Chdir(fr.Pwd) if fr.WorkPath != "" { defer os.RemoveAll(fr.WorkPath) @@ -495,6 +493,7 @@ func TestStatusAtVersion(t *testing.T) { fr := NewFSITestRunner(t, "qri_test_status_at_version") defer fr.Delete() + // First version has only a body err := fr.ExecCommand("qri save --body=testdata/movies/body_two.json me/status_ver") if err != nil { t.Fatalf(err.Error()) @@ -673,6 +672,82 @@ run ` + "`qri save`" + ` to commit this dataset } } +// Test restoring previous version +func TestRestorePreviousVersion(t *testing.T) { + fr := NewFSITestRunner(t, "qri_test_restore_prev_version") + defer fr.Delete() + + // First version has only a body + err := fr.ExecCommand("qri save --body=testdata/movies/body_two.json me/prev_ver") + if err != nil { + t.Fatalf(err.Error()) + } + _ = parseRefFromSave(fr.GetCommandOutput()) + + // Add a meta + err = fr.ExecCommand("qri save --file=testdata/movies/meta_override.yaml me/prev_ver") + if err != nil { + t.Fatalf(err.Error()) + } + ref2 := parseRefFromSave(fr.GetCommandOutput()) + + // Change the meta + err = fr.ExecCommand("qri save --file=testdata/movies/meta_another.yaml me/prev_ver") + if err != nil { + t.Fatalf(err.Error()) + } + _ = parseRefFromSave(fr.GetCommandOutput()) + + fr.ChdirToRoot() + + // Checkout the newly created dataset. + if err := fr.ExecCommand("qri checkout me/prev_ver"); err != nil { + t.Fatalf(err.Error()) + } + + _ = fr.ChdirToWorkDir("prev_ver") + + // Verify that the status is clean + if err = fr.ExecCommand(fmt.Sprintf("qri status")); err != nil { + t.Fatalf(err.Error()) + } + + output := fr.GetCommandOutput() + if diff := cmpTextLines(cleanStatusMessage("test_peer/prev_ver"), output); diff != "" { + t.Errorf("qri status (-want +got):\n%s", diff) + } + + // Read meta.json, contains the contents of meta_another.yaml + metaContents, err := ioutil.ReadFile("meta.json") + if err != nil { + t.Fatalf(err.Error()) + } + expectContents := "{\n \"qri\": \"md:0\",\n \"title\": \"yet another title\"\n}" + if diff := cmp.Diff(expectContents, string(metaContents)); diff != "" { + t.Errorf("meta.json contents (-want +got):\n%s", diff) + } + + // TODO(dlong): Handle full dataset paths, including peername and dataset name. + + pos := strings.Index(ref2, "/ipfs/") + path := ref2[pos:] + + // Restore the previous version + if err = fr.ExecCommand(fmt.Sprintf("qri restore %s", path)); err != nil { + t.Fatalf(err.Error()) + } + + // Read meta.json, due to restore, it has the old data from meta_override.yaml + metaContents, err = ioutil.ReadFile("meta.json") + if err != nil { + t.Fatalf(err.Error()) + } + expectContents = "{\n \"qri\": \"md:0\",\n \"title\": \"different title\"\n}" + if diff := cmp.Diff(expectContents, string(metaContents)); diff != "" { + t.Errorf("meta.json contents (-want +got):\n%s", diff) + } +} + func parseRefFromSave(output string) string { pos := strings.Index(output, "saved: ") ref := output[pos+7:] diff --git a/cmd/restore.go b/cmd/restore.go index 18f3d79a1..59b3ede9a 100644 --- a/cmd/restore.go +++ b/cmd/restore.go @@ -48,9 +48,14 @@ func (o *RestoreOptions) Complete(f Factory, args []string) (err error) { o.Path = "" o.ComponentName = "" + // TODO(dlong): Add low-level utilities that parse strings like "peername/ds_name", and + // "/ipfs/QmFoo", "meta.description", etc and use those everywhere. Use real regexs so + // that we properly handle user input everywhere. Too much code is duplicating half working + // input handling for various stringified identifiers. + // Process arguments to get dataset name, component name, and/or ref path. for _, arg := range args { - if strings.HasPrefix(arg, "@/ipfs/") { + if strings.HasPrefix(arg, "/ipfs/") { if o.Path != "" { return fmt.Errorf("cannot provide more than one ref Path") } @@ -93,6 +98,8 @@ func (o *RestoreOptions) Complete(f Factory, args []string) (err error) { // Run executes the `restore` command func (o *RestoreOptions) Run() (err error) { + printRefSelect(o.Out, o.Refs) + ref := o.Refs.Ref() if o.Path != "" { ref += o.Path @@ -103,8 +110,11 @@ func (o *RestoreOptions) Run() (err error) { if err != nil { return err } - if o.ComponentName != "" { + if o.ComponentName != "" && o.Path == "" { printSuccess(o.Out, fmt.Sprintf("Restored %s of dataset %s", o.ComponentName, ref)) + } else if o.Path != "" && o.ComponentName == "" { + printSuccess(o.Out, fmt.Sprintf("Restored dataset version %s", ref)) } + // TODO(dlong): Print message when both component and path are specified. return nil } diff --git a/fsi/mapping.go b/fsi/mapping.go index c9fc5844d..d3952f7f6 100644 --- a/fsi/mapping.go +++ b/fsi/mapping.go @@ -300,7 +300,7 @@ func WriteComponents(ds *dataset.Dataset, dirPath string) error { case "json": bodyFilename = "body.json" default: - return fmt.Errorf("unknown body format: \"%s\"", bodyFormat) + return fmt.Errorf(`unknown body format: "%s"`, bodyFormat) } err = ioutil.WriteFile(filepath.Join(dirPath, bodyFilename), data, os.ModePerm) if err != nil { diff --git a/lib/fsi.go b/lib/fsi.go index 5780c9568..598aa8f77 100644 --- a/lib/fsi.go +++ b/lib/fsi.go @@ -147,6 +147,9 @@ func (m *FSIMethods) Checkout(p *CheckoutParams, out *string) (err error) { return m.inst.rpc.Call("FSIMethods.Checkout", p, out) } + // TODO(dlong): Fail if Dir is "", should be required to specify a location. Should probably + // only allow absolute paths. Add tests. + // If directory exists, error. if _, err = os.Stat(p.Dir); !os.IsNotExist(err) { return fmt.Errorf("directory with name \"%s\" already exists", p.Dir) @@ -217,6 +220,13 @@ func (m *FSIMethods) Restore(p *RestoreParams, out *string) (err error) { return } + // Directory to write components to can be determined from FSIPath of ref. + if p.Dir == "" && ref.FSIPath != "" { + p.Dir = ref.FSIPath + } + // TODO(dlong): Perhaps disallow empty Dir (without FSIPath override), since relative + // paths cause problems. Test using `qri connect`. + ds, err := dsfs.LoadDataset(m.inst.node.Repo.Store(), ref.Path) if err != nil { return fmt.Errorf("loading dataset: %s", err) @@ -229,12 +239,18 @@ func (m *FSIMethods) Restore(p *RestoreParams, out *string) (err error) { var history dataset.Dataset history.Structure = &dataset.Structure{} history.Structure.Format = ds.Structure.Format - if p.Component == "meta" { + if p.Component == "" { + // Entire dataset. + history.Assign(ds) + } else if p.Component == "meta" { + // Meta component. history.Meta = &dataset.Meta{} history.Meta.Assign(ds.Meta) } else if p.Component == "schema" || p.Component == "structure.schema" { + // Schema is not a "real" component, is short for the structure's schema. history.Structure.Schema = ds.Structure.Schema } else if p.Component == "body" { + // Body of the dataset. df, err := dataset.ParseDataFormatString(history.Structure.Format) if err != nil { return err