diff --git a/cmd/checkout.go b/cmd/checkout.go new file mode 100644 index 000000000..3a046b5f1 --- /dev/null +++ b/cmd/checkout.go @@ -0,0 +1,190 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/qri-io/ioes" + "github.com/qri-io/qri/lib" + "github.com/spf13/cobra" +) + +// NewCheckoutCommand creates new `qri checkout` command that connects a working directory in +// the local filesystem to a dataset your repo. +func NewCheckoutCommand(f Factory, ioStreams ioes.IOStreams) *cobra.Command { + o := &CheckoutOptions{IOStreams: ioStreams} + cmd := &cobra.Command{ + Use: "checkout", + Short: "checkout created a linked directory and writes dataset files to that directory", + Long: ``, + Example: ``, + Annotations: map[string]string{ + "group": "dataset", + }, + RunE: func(cmd *cobra.Command, args []string) error { + o.Complete(f, args) + return o.Run() + }, + } + + return cmd +} + +// CheckoutOptions encapsulates state for the `checkout` command +type CheckoutOptions struct { + ioes.IOStreams + + Args []string + + DatasetRequests *lib.DatasetRequests + FSIMethods *lib.FSIMethods +} + +// Complete completes a the command +func (o *CheckoutOptions) Complete(f Factory, args []string) (err error) { + o.Args = args + if o.DatasetRequests, err = f.DatasetRequests(); err != nil { + return err + } + o.FSIMethods, err = f.FSIMethods() + return err +} + +// Run executes the `checkout` command +func (o *CheckoutOptions) Run() (err error) { + // TODO: Finalize UI for command-line checkout command. + ref := o.Args[0] + + // Derive directory name from the dataset name. + pos := strings.Index(ref, "/") + if pos == -1 { + return fmt.Errorf("expect '/' in dataset ref") + } + folderName := ref[pos+1:] + + // If directory exists, error. + if _, err = os.Stat(folderName); !os.IsNotExist(err) { + return fmt.Errorf("directory with name \"%s\" already exists", folderName) + } + + // Get the dataset from your repo. + p := lib.GetParams{ + Path: ref, + Selector: "", + } + res := lib.GetResult{} + if err = o.DatasetRequests.Get(&p, &res); err != nil { + return err + } + + // Create a directory. + if err = os.Mkdir(folderName, os.ModePerm); err != nil { + return err + } + + // Create the link file, containing the dataset reference. + lnkp := &lib.LinkParams{ + Dir: folderName, + Ref: ref, + } + lnkres := "" + if err = o.FSIMethods.CreateLink(lnkp, &lnkres); err != nil { + return err + } + + // Prepare dataset. + // TODO(dlong): Move most of this into FSI? + ds := res.Dataset + + // Get individual components out of the dataset. + meta := ds.Meta + ds.Meta = nil + schema := ds.Structure.Schema + ds.Structure.Schema = nil + + // Structure is kept in the dataset. + bodyFormat := ds.Structure.Format + ds.Structure.Format = "" + ds.Structure.Qri = "" + + // Commit, viz, transform are never checked out. + ds.Commit = nil + ds.Viz = nil + ds.Transform = nil + + // Meta component. + if meta != nil { + meta.DropDerivedValues() + if !meta.IsEmpty() { + data, err := json.MarshalIndent(meta, "", " ") + if err != nil { + return err + } + err = ioutil.WriteFile(filepath.Join(folderName, "meta.json"), data, os.ModePerm) + if err != nil { + return err + } + } + } + + // Schema component. + if len(schema) > 0 { + data, err := json.MarshalIndent(schema, "", " ") + if err != nil { + return err + } + err = ioutil.WriteFile(filepath.Join(folderName, "schema.json"), data, os.ModePerm) + if err != nil { + return err + } + } + + // Body component. + bf := ds.BodyFile() + data, err := ioutil.ReadAll(bf) + if err != nil { + return err + } + ds.BodyPath = "" + var bodyFilename string + switch bodyFormat { + case "csv": + bodyFilename = "body.csv" + case "json": + bodyFilename = "body.json" + default: + return fmt.Errorf("unknown body format: %s", bodyFormat) + } + err = ioutil.WriteFile(filepath.Join(folderName, bodyFilename), data, os.ModePerm) + if err != nil { + return err + } + + // Dataset (everything else). + ds.DropDerivedValues() + // TODO(dlong): Should more of these move to DropDerivedValues? + ds.Qri = "" + ds.Name = "" + ds.Peername = "" + ds.PreviousPath = "" + if ds.Structure.IsEmpty() { + ds.Structure = nil + } + if !ds.IsEmpty() { + data, err := json.MarshalIndent(ds, "", " ") + if err != nil { + return err + } + err = ioutil.WriteFile(filepath.Join(folderName, "dataset.json"), data, os.ModePerm) + if err != nil { + return err + } + } + + printSuccess(o.Out, "created and linked working directory %s for existing dataset", folderName) + return nil +} diff --git a/cmd/fsi_test.go b/cmd/fsi_test.go new file mode 100644 index 000000000..974754173 --- /dev/null +++ b/cmd/fsi_test.go @@ -0,0 +1,482 @@ +package cmd + +import ( + "context" + "io/ioutil" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" +) + +// Test using "init" to create a new linked directory, using status to see the added files, +// then saving to create the dataset, leading to a clean status in the directory. +func TestInitStatusSave(t *testing.T) { + if err := confirmQriNotRunning(); err != nil { + t.Skip(err.Error()) + } + + r := NewTestRepoRoot(t, "qri_test_init_status_save") + defer r.Delete() + + ctx, done := context.WithCancel(context.Background()) + defer done() + + pwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + + rootPath, err := ioutil.TempDir("", "") + if err != nil { + t.Fatal(err) + } + + workPath := filepath.Join(rootPath, "brand_new") + err = os.MkdirAll(workPath, os.ModePerm) + if err != nil { + t.Fatal(err) + } + + // Change to a temporary directory. + os.Chdir(workPath) + defer os.Chdir(pwd) + + // Init as a linked directory. + cmdR := r.CreateCommandRunner(ctx) + err = executeCommand(cmdR, "qri init --name brand_new --format csv") + if err != nil { + t.Fatalf(err.Error()) + } + + // Verify the directory contains the files that we expect. + dirContents := listDirectory(workPath) + expectContents := []string{".qri-ref", "body.csv", "meta.json", "schema.json"} + if diff := cmp.Diff(dirContents, expectContents); diff != "" { + t.Errorf("directory contents (-want +got):\n%s", diff) + } + + // Status, check that the working directory has added files. + cmdR = r.CreateCommandRunner(ctx) + err = executeCommand(cmdR, "qri status") + if err != nil { + t.Fatalf(err.Error()) + } + + output := r.GetOutput() + expect := `for dataset [test_peer/brand_new] + + add: meta (source: meta.json) + add: schema (source: schema.json) + add: body (source: body.csv) + +run ` + "`qri save`" + ` to commit this dataset +` + if diff := cmpTextLines(output, expect); diff != "" { + t.Errorf("qri status (-want +got):\n%s", diff) + } + + // Save the new dataset. + cmdR = r.CreateCommandRunner(ctx) + err = executeCommand(cmdR, "qri save") + if err != nil { + t.Fatalf(err.Error()) + } + + // TODO: Verify that files are in ipfs repo. + + // Status again, check that the working directory is clean. + cmdR = r.CreateCommandRunner(ctx) + err = executeCommand(cmdR, "qri status") + if err != nil { + t.Fatalf(err.Error()) + } + + output = r.GetOutput() + expect = `for dataset [test_peer/brand_new] + +working directory clean +` + if diff := cmpTextLines(output, expect); diff != "" { + t.Errorf("qri status (-want +got):\n%s", diff) + } +} + +// Test that checkout, used on a simple dataset with a body.json and no meta, creates a +// working directory with a clean status. +func TestCheckoutSimpleStatus(t *testing.T) { + if err := confirmQriNotRunning(); err != nil { + t.Skip(err.Error()) + } + + r := NewTestRepoRoot(t, "qri_test_checkout_simple_status") + defer r.Delete() + + ctx, done := context.WithCancel(context.Background()) + defer done() + + pwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + + rootPath, err := ioutil.TempDir("", "") + if err != nil { + t.Fatal(err) + } + + // Save a dataset containing a body.json, no meta, nothing special. + cmdR := r.CreateCommandRunner(ctx) + err = executeCommand(cmdR, "qri save --body=testdata/movies/body_two.json me/two_movies") + if err != nil { + t.Fatalf(err.Error()) + } + + // Change to a temporary directory. + os.Chdir(rootPath) + defer os.Chdir(pwd) + + // Checkout the newly created dataset. + cmdR = r.CreateCommandRunner(ctx) + err = executeCommand(cmdR, "qri checkout me/two_movies") + if err != nil { + t.Fatalf(err.Error()) + } + + workPath := filepath.Join(rootPath, "two_movies") + os.Chdir(workPath) + defer os.Chdir(pwd) + + // Verify the directory contains the files that we expect. + dirContents := listDirectory(workPath) + expectContents := []string{".qri-ref", "body.json", "schema.json"} + if diff := cmp.Diff(dirContents, expectContents); diff != "" { + t.Errorf("directory contents (-want +got):\n%s", diff) + } + + // Status, check that the working directory is clean. + cmdR = r.CreateCommandRunner(ctx) + err = executeCommand(cmdR, "qri status") + if err != nil { + t.Fatalf(err.Error()) + } + + output := r.GetOutput() + expect := `for dataset [test_peer/two_movies] + +working directory clean +` + if diff := cmpTextLines(output, expect); diff != "" { + t.Errorf("qri status (-want +got):\n%s", diff) + } + + // Modify the body.json file. + modifyFileUsingStringReplace("body.json", "Avatar", "The Avengers") + + // Status again, check that the body is changed. + cmdR = r.CreateCommandRunner(ctx) + err = executeCommand(cmdR, "qri status") + if err != nil { + t.Fatalf(err.Error()) + } + + output = r.GetOutput() + expect = `for dataset [test_peer/two_movies] + + modified: body (source: body.json) + +run ` + "`qri save`" + ` to commit this dataset +` + if diff := cmpTextLines(output, expect); diff != "" { + t.Errorf("qri status (-want +got):\n%s", diff) + } + + // Create meta.json with a title. + if err = ioutil.WriteFile("meta.json", []byte(`{"title": "hello"}`), os.ModePerm); err != nil { + t.Fatalf(err.Error()) + } + + // Status yet again, check that the meta is added. + cmdR = r.CreateCommandRunner(ctx) + err = executeCommand(cmdR, "qri status") + if err != nil { + t.Fatalf(err.Error()) + } + + output = r.GetOutput() + expect = `for dataset [test_peer/two_movies] + + add: meta (source: meta.json) + modified: body (source: body.json) + +run ` + "`qri save`" + ` to commit this dataset +` + if diff := cmpTextLines(output, expect); diff != "" { + t.Errorf("qri status (-want +got):\n%s", diff) + } +} + +// Test checking out a dataset with a schema, and body.csv. +func TestCheckoutWithStructure(t *testing.T) { + if err := confirmQriNotRunning(); err != nil { + t.Skip(err.Error()) + } + + r := NewTestRepoRoot(t, "qri_test_checkout_with_structure") + defer r.Delete() + + ctx, done := context.WithCancel(context.Background()) + defer done() + + pwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + + rootPath, err := ioutil.TempDir("", "") + if err != nil { + t.Fatal(err) + } + + // Save a dataset containing a body.csv, no meta, nothing special. + cmdR := r.CreateCommandRunner(ctx) + err = executeCommand(cmdR, "qri save --body=testdata/movies/body_ten.csv --file=testdata/movies/meta_override.yaml me/ten_movies") + if err != nil { + t.Fatalf(err.Error()) + } + + // Change to a temporary directory. + os.Chdir(rootPath) + defer os.Chdir(pwd) + + // Checkout the newly created dataset. + cmdR = r.CreateCommandRunner(ctx) + err = executeCommand(cmdR, "qri checkout me/ten_movies") + if err != nil { + t.Fatalf(err.Error()) + } + + workPath := filepath.Join(rootPath, "ten_movies") + os.Chdir(workPath) + defer os.Chdir(pwd) + + // Verify the directory contains the files that we expect. + dirContents := listDirectory(workPath) + expectContents := []string{".qri-ref", "body.csv", "dataset.json", "meta.json", "schema.json"} + if diff := cmp.Diff(dirContents, expectContents); diff != "" { + t.Errorf("directory contents (-want +got):\n%s", diff) + } + + // Status, check that the working directory is clean. + cmdR = r.CreateCommandRunner(ctx) + err = executeCommand(cmdR, "qri status") + if err != nil { + t.Fatalf(err.Error()) + } + + output := r.GetOutput() + expect := `for dataset [test_peer/ten_movies] + +working directory clean +` + if diff := cmpTextLines(output, expect); diff != "" { + t.Errorf("qri status (-want +got):\n%s", diff) + } + + // Modify the body.json file. + modifyFileUsingStringReplace("body.csv", "Avatar", "The Avengers") + + // Status again, check that the body is changed. + cmdR = r.CreateCommandRunner(ctx) + err = executeCommand(cmdR, "qri status") + if err != nil { + t.Fatalf(err.Error()) + } + + output = r.GetOutput() + expect = `for dataset [test_peer/ten_movies] + + modified: body (source: body.csv) + +run ` + "`qri save`" + ` to commit this dataset +` + if diff := cmpTextLines(output, expect); diff != "" { + t.Errorf("qri status (-want +got):\n%s", diff) + } + + // Modify meta.json by changing the title. + if err = ioutil.WriteFile("meta.json", []byte(`{"title": "hello"}`), os.ModePerm); err != nil { + t.Fatalf(err.Error()) + } + + // Status yet again, check that the meta is changed. + cmdR = r.CreateCommandRunner(ctx) + err = executeCommand(cmdR, "qri status") + if err != nil { + t.Fatalf(err.Error()) + } + + output = r.GetOutput() + expect = `for dataset [test_peer/ten_movies] + + modified: meta (source: meta.json) + modified: body (source: body.csv) + +run ` + "`qri save`" + ` to commit this dataset +` + if diff := cmpTextLines(output, expect); diff != "" { + t.Errorf("qri status (-want +got):\n%s", diff) + } + + // Remove meta.json. + if err = os.Remove("meta.json"); err != nil { + t.Fatalf(err.Error()) + } + + // Status one last time, check that the meta was removed. + cmdR = r.CreateCommandRunner(ctx) + err = executeCommand(cmdR, "qri status") + if err != nil { + t.Fatalf(err.Error()) + } + + output = r.GetOutput() + expect = `for dataset [test_peer/ten_movies] + + removed: meta + modified: body (source: body.csv) + +run ` + "`qri save`" + ` to commit this dataset +` + if diff := cmpTextLines(output, expect); diff != "" { + t.Errorf("qri status (-want +got):\n%s", diff) + } +} + +// Test checkout and modifying schema, then checking status. +func TestCheckoutAndModifySchema(t *testing.T) { + if err := confirmQriNotRunning(); err != nil { + t.Skip(err.Error()) + } + + r := NewTestRepoRoot(t, "qri_test_checkout_and_modify_schema") + defer r.Delete() + + ctx, done := context.WithCancel(context.Background()) + defer done() + + pwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + + rootPath, err := ioutil.TempDir("", "") + if err != nil { + t.Fatal(err) + } + + // Save a dataset containing a body.csv, no meta, nothing special. + cmdR := r.CreateCommandRunner(ctx) + err = executeCommand(cmdR, "qri save --body=testdata/movies/body_ten.csv me/more_movies") + if err != nil { + t.Fatalf(err.Error()) + } + + // Change to a temporary directory. + os.Chdir(rootPath) + defer os.Chdir(pwd) + + // Checkout the newly created dataset. + cmdR = r.CreateCommandRunner(ctx) + err = executeCommand(cmdR, "qri checkout me/more_movies") + if err != nil { + t.Fatalf(err.Error()) + } + + workPath := filepath.Join(rootPath, "more_movies") + os.Chdir(workPath) + defer os.Chdir(pwd) + + // Verify the directory contains the files that we expect. + dirContents := listDirectory(workPath) + expectContents := []string{".qri-ref", "body.csv", "dataset.json", "schema.json"} + if diff := cmp.Diff(dirContents, expectContents); diff != "" { + t.Errorf("directory contents (-want +got):\n%s", diff) + } + + // Status, check that the working directory is clean. + cmdR = r.CreateCommandRunner(ctx) + err = executeCommand(cmdR, "qri status") + if err != nil { + t.Fatalf(err.Error()) + } + + output := r.GetOutput() + expect := `for dataset [test_peer/more_movies] + +working directory clean +` + if diff := cmpTextLines(output, expect); diff != "" { + t.Errorf("qri status (-want +got):\n%s", diff) + } + + // Create schema.json with a minimal schema. + if err = ioutil.WriteFile("schema.json", []byte(`{"type": "array"}`), os.ModePerm); err != nil { + t.Fatalf(err.Error()) + } + + // Status again, check that the body is changed. + cmdR = r.CreateCommandRunner(ctx) + err = executeCommand(cmdR, "qri status") + if err != nil { + t.Fatalf(err.Error()) + } + + output = r.GetOutput() + // TODO(dlong): structure/dataset.json should not be marked as `modified` + expect = `for dataset [test_peer/more_movies] + + modified: structure (source: dataset.json) + modified: schema (source: schema.json) + +run ` + "`qri save`" + ` to commit this dataset +` + if diff := cmpTextLines(output, expect); diff != "" { + t.Errorf("qri status (-want +got):\n%s", diff) + } +} + +func cmpTextLines(left, right string) string { + lside := strings.Split(left, "\n") + rside := strings.Split(right, "\n") + return cmp.Diff(lside, rside) +} + +func listDirectory(path string) []string { + contents := []string{} + finfos, err := ioutil.ReadDir(path) + if err != nil { + return nil + } + for _, fi := range finfos { + contents = append(contents, fi.Name()) + } + sort.Strings(contents) + return contents +} + +func modifyFileUsingStringReplace(filename, find, replace string) { + data, err := ioutil.ReadFile(filename) + if err != nil { + panic(err) + } + text := string(data) + text = strings.Replace(text, find, replace, -1) + err = ioutil.WriteFile(filename, []byte(text), os.ModePerm) + if err != nil { + panic(err) + } +} diff --git a/cmd/init.go b/cmd/init.go index 6b39661f2..d1c0d7d11 100644 --- a/cmd/init.go +++ b/cmd/init.go @@ -34,7 +34,6 @@ func NewInitCommand(f Factory, ioStreams ioes.IOStreams) *cobra.Command { cmd.Flags().StringVar(&o.Name, "name", "", "name of the dataset") cmd.Flags().StringVar(&o.Format, "format", "", "format of dataset") - cmd.Flags().StringVar(&o.Link, "link", "", "link this directory to an existing dataset") return cmd } @@ -45,7 +44,6 @@ type InitOptions struct { Name string Format string - Link string DatasetRequests *lib.DatasetRequests FSIMethods *lib.FSIMethods @@ -67,19 +65,6 @@ func (o *InitOptions) Run() (err error) { return err } - if o.Link != "" { - p := &lib.LinkParams{ - Dir: pwd, - Ref: o.Link, - } - res := "" - err = o.FSIMethods.CreateLink(p, &res) - if err == nil { - printSuccess(o.ErrOut, "created link: %s", res) - } - return err - } - if _, err := os.Stat(fsi.QriRefFilename); !os.IsNotExist(err) { return fmt.Errorf("working directory is already linked, .qri-ref exists") } diff --git a/cmd/qri.go b/cmd/qri.go index 547d5e55f..b8b44d237 100644 --- a/cmd/qri.go +++ b/cmd/qri.go @@ -40,6 +40,7 @@ https://github.com/qri-io/qri/issues`, cmd.AddCommand( NewAddCommand(opt, ioStreams), + NewCheckoutCommand(opt, ioStreams), NewConfigCommand(opt, ioStreams), NewConnectCommand(opt, ioStreams), NewDAGCommand(opt, ioStreams), diff --git a/fsi/mapping.go b/fsi/mapping.go index 130e9aa0f..5f962e243 100644 --- a/fsi/mapping.go +++ b/fsi/mapping.go @@ -70,19 +70,31 @@ func ReadDir(dir string) (ds *dataset.Dataset, mapping map[string]string, err er // HACK: Detect body format. bodyFormat := "" - if _, err = os.Stat("body.csv"); !os.IsNotExist(err) { + if _, err = os.Stat(filepath.Join(dir, "body.csv")); !os.IsNotExist(err) { bodyFormat = "csv" - ds.BodyPath = "body.csv" } - if _, err = os.Stat("body.json"); !os.IsNotExist(err) { + if _, err = os.Stat(filepath.Join(dir, "body.json")); !os.IsNotExist(err) { if bodyFormat == "csv" { - // Conflict: both body.csv and body.json + return ds, mapping, fmt.Errorf("body.csv and body.json both exist") } bodyFormat = "json" - ds.BodyPath = "body.json" } - for cmpName, cmp := range components { + bodyFilename := "" + if bodyFormat != "" { + bodyFilename = fmt.Sprintf("body.%s", bodyFormat) + if err = addMapping(componentNameBody, bodyFilename); err != nil { + return ds, mapping, err + } + if ds.BodyPath == "" { + ds.BodyPath = filepath.Join(dir, bodyFilename) + } + } + + // Iterate components in a deterministic order, from highest priority to lowest. + for i := 0; i < len(componentListOrder); i++ { + cmpName := componentListOrder[i] + cmp := components[cmpName] for ext, mkDec := range extensions { filename := fmt.Sprintf("%s%s", cmpName, ext) path := filepath.Join(dir, filename) @@ -92,10 +104,9 @@ func ReadDir(dir string) (ds *dataset.Dataset, mapping map[string]string, err er err = fmt.Errorf("reading %s: %s", filename, err) return ds, mapping, err } - } - - if err = addMapping(cmpName, path); err != nil { - return ds, mapping, err + if err = addMapping(cmpName, path); err != nil { + return ds, mapping, err + } } switch cmpName { @@ -158,8 +169,6 @@ func ReadDir(dir string) (ds *dataset.Dataset, mapping map[string]string, err er if ds.BodyPath == "" { ds.BodyPath = path } - // ds.Body = cmp.(*dataset.Body) - // // TODO (b5) - } } } diff --git a/fsi/status.go b/fsi/status.go index fd04c58c7..349ad9942 100644 --- a/fsi/status.go +++ b/fsi/status.go @@ -4,6 +4,9 @@ import ( "bytes" "encoding/json" "fmt" + "io/ioutil" + "os" + "path/filepath" "sort" "github.com/qri-io/dataset" @@ -48,6 +51,17 @@ var componentOrder = map[string]int{ "body": -1, } +var componentListOrder = []string{ + "dataset", + "commit", + "meta", + "structure", + "schema", + "viz", + "transform", + "body", +} + func (si statusItems) Len() int { return len(si) } func (si statusItems) Swap(i, j int) { si[i], si[j] = si[j], si[i] } func (si statusItems) Less(i, j int) bool { @@ -98,23 +112,31 @@ func (fsi *FSI) Status(dir string) (changes []StatusItem, err error) { } stored.DropDerivedValues() + stored.Commit = nil + stored.Transform = nil + stored.Peername = "" - ds, mapping, err := ReadDir(dir) + working, mapping, err := ReadDir(dir) if err != nil { return nil, err } - ds.DropDerivedValues() + working.DropDerivedValues() // if err = validate.Dataset(ds); err != nil { // return nil, fmt.Errorf("dataset is invalid: %s" , err) // } - storedComponents := dsComponents(stored) + storedComponents := dsAllComponents(stored) for cmpName := range storedComponents { // when reporting deletes, ignore "bound" components that must/must-not // exist based on external conditions - if cmpName != componentNameDataset && cmpName != componentNameStructure && cmpName != componentNameCommit { + if cmpName != componentNameDataset && cmpName != componentNameStructure && cmpName != componentNameCommit && cmpName != componentNameViz { + cmp := dsComponent(stored, cmpName) + // If the component was not in the previous version, it can't have been removed. + if cmp == nil { + continue + } if _, ok := mapping[cmpName]; !ok { change := StatusItem{ Component: cmpName, @@ -125,33 +147,86 @@ func (fsi *FSI) Status(dir string) (changes []StatusItem, err error) { } } - for path, sourceFilepath := range mapping { + // Iterate components in a deterministic order, going backwards. + for i := len(componentListOrder) - 1; i >= 0; i-- { + path := componentListOrder[i] + if path == componentNameDataset { + continue + } + + localFilepath, ok := mapping[path] + if !ok { + continue + } + if cmp := dsComponent(stored, path); cmp == nil { change := StatusItem{ - SourceFile: sourceFilepath, + SourceFile: localFilepath, Component: path, Type: STAdd, } changes = append(changes, change) } else { - srcData, err := json.Marshal(cmp) - if err != nil { - return nil, err - } - wdData, err := json.Marshal(dsComponent(ds, path)) - if err != nil { - return nil, err + + var storedData []byte + var workData []byte + if path == componentNameBody { + // Getting data for the body works differently. + if err = stored.OpenBodyFile(fsi.repo.Filesystem()); err != nil { + return nil, err + } + storedBody := stored.BodyFile() + if storedBody == nil { + // Handle the case where there's no previous version. Body is "add"ed, do + // not attempt to read the non-existent stored body. + change := StatusItem{ + SourceFile: localFilepath, + Component: path, + Type: STAdd, + } + changes = append(changes, change) + continue + } else { + // Read body of previous version. + defer storedBody.Close() + storedData, err = ioutil.ReadAll(storedBody) + if err != nil { + return nil, err + } + } + + workingBody, err := os.Open(filepath.Join(dir, localFilepath)) + if err != nil { + return nil, err + } + defer workingBody.Close() + + workData, err = ioutil.ReadAll(workingBody) + if err != nil { + return nil, err + } + } else { + storedData, err = json.Marshal(cmp) + if err != nil { + return nil, err + } + + workData, err = json.Marshal(dsComponent(working, path)) + if err != nil { + return nil, err + } } - if !bytes.Equal(srcData, wdData) { + + if !bytes.Equal(storedData, workData) { change := StatusItem{ - SourceFile: sourceFilepath, + SourceFile: localFilepath, Component: path, Type: STChange, } changes = append(changes, change) } else { change := StatusItem{ - SourceFile: sourceFilepath, + SourceFile: localFilepath, Component: path, Type: STUnmodified, } @@ -165,12 +240,19 @@ func (fsi *FSI) Status(dir string) (changes []StatusItem, err error) { } func dsComponent(ds *dataset.Dataset, cmpName string) interface{} { + // This switch avoids returning interfaces with nil values and non-nil type tags. switch cmpName { case componentNameCommit: + if ds.Commit == nil { + return nil + } return ds.Commit case componentNameDataset: return ds case componentNameMeta: + if ds.Meta == nil { + return nil + } return ds.Meta case componentNameSchema: if ds.Structure == nil { @@ -178,21 +260,29 @@ func dsComponent(ds *dataset.Dataset, cmpName string) interface{} { } return ds.Structure.Schema case componentNameBody: - // TODO (b5) - this isn't going to work properly - return ds.Body + return ds.BodyPath != "" case componentNameStructure: + if ds.Structure == nil { + return nil + } return ds.Structure case componentNameTransform: + if ds.Transform == nil { + return nil + } return ds.Transform case componentNameViz: + if ds.Viz == nil { + return nil + } return ds.Viz default: return nil } } -// dsComponents returns the components of a dataset as a map of component_name: value -func dsComponents(ds *dataset.Dataset) map[string]interface{} { +// dsAllComponents returns the components of a dataset as a map of component_name: value +func dsAllComponents(ds *dataset.Dataset) map[string]interface{} { components := map[string]interface{}{} cmpNames := []string{ componentNameCommit, diff --git a/fsi/status_test.go b/fsi/status_test.go index 554ce78b6..d511853eb 100644 --- a/fsi/status_test.go +++ b/fsi/status_test.go @@ -52,7 +52,7 @@ func TestStatusValid(t *testing.T) { for _, ch := range changes { actual += strings.Replace(fmt.Sprintf("%s", ch), paths.firstDir, ".", 1) } - expect := `{./commit.json commit modified }{./meta.json meta modified }{./schema.json schema add }{./structure.json structure modified }{./transform.json transform modified }{./viz.json viz modified }` + expect := `{./commit.json commit add }{./meta.json meta add }{./schema.json schema add }{./structure.json structure add }{./transform.json transform add }{./viz.json viz add }{body.csv body add }` if actual != expect { t.Errorf("status error didn't match, actual: %s, expect: %s", actual, expect) }