Skip to content

Commit

Permalink
fix(init): add basic structure.json on init
Browse files Browse the repository at this point in the history
When we create a new dataset, use the given format to determine what `formatConfig` options we want to expose to the user.

Create the structure.json based on these options

How do we handle `Separator`? It's a rune and marshals to an integer. Should probably be converted to a string?
  • Loading branch information
ramfox committed Oct 11, 2019
1 parent f13a3dc commit a6f423b
Show file tree
Hide file tree
Showing 6 changed files with 102 additions and 9 deletions.
8 changes: 5 additions & 3 deletions api/fsi_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,8 @@ func TestNoHistory(t *testing.T) {
metaMtime := st.ModTime().Format(time.RFC3339)
st, _ = os.Stat(filepath.Join(initDir, "body.csv"))
bodyMtime := st.ModTime().Format(time.RFC3339)
st, _ = os.Stat(filepath.Join(initDir, "structure.json"))
structureMtime := st.ModTime().Format(time.RFC3339)

dsHandler := NewDatasetHandlers(inst, false)

Expand Down Expand Up @@ -148,7 +150,7 @@ func TestNoHistory(t *testing.T) {
}
// Handle temporary directory by replacing the temp part with a shorter string.
resultBody := strings.Replace(gotBodyString, initDir, initSubdir, -1)
expectBody := `{"data":{"peername":"peer","name":"test_ds","fsiPath":"fsi_init_dir","dataset":{"bodyPath":"fsi_init_dir/body.csv","meta":{"keywords":[],"qri":"md:0"},"name":"test_ds","peername":"peer","qri":"ds:0","structure":{"format":"csv","qri":"st:0","schema":{"items":{"items":[{"title":"field_1","type":"boolean"},{"title":"field_2","type":"boolean"},{"title":"field_3","type":"integer"}],"type":"array"},"type":"array"}}},"published":false},"meta":{"code":200}}`
expectBody := `{"data":{"peername":"peer","name":"test_ds","fsiPath":"fsi_init_dir","dataset":{"bodyPath":"fsi_init_dir/body.csv","meta":{"keywords":[],"qri":"md:0"},"name":"test_ds","peername":"peer","qri":"ds:0","structure":{"format":"csv","formatConfig":{"headerRow":false,"lazyQuotes":false,"variadicFields":false},"qri":"st:0","schema":{"items":{"items":[{"title":"field_1","type":"boolean"},{"title":"field_2","type":"boolean"},{"title":"field_3","type":"integer"}],"type":"array"},"type":"array"}}},"published":false},"meta":{"code":200}}`
if diff := cmp.Diff(expectBody, resultBody); diff != "" {
t.Errorf("api response (-want +got):\n%s", diff)
}
Expand Down Expand Up @@ -198,8 +200,8 @@ func TestNoHistory(t *testing.T) {
}
// Handle temporary directory by replacing the temp part with a shorter string.
resultBody = strings.Replace(gotBodyString, initDir, initSubdir, -1)
templateBody := `{"data":[{"sourceFile":"fsi_init_dir/meta.json","component":"meta","type":"add","message":"","mtime":"%s"},{"sourceFile":"fsi_init_dir/body.csv","component":"body","type":"add","message":"","mtime":"%s"}],"meta":{"code":200}}`
expectBody = fmt.Sprintf(templateBody, metaMtime, bodyMtime)
templateBody := `{"data":[{"sourceFile":"fsi_init_dir/meta.json","component":"meta","type":"add","message":"","mtime":"%s"},{"sourceFile":"fsi_init_dir/structure.json","component":"structure","type":"add","message":"","mtime":"%s"},{"sourceFile":"fsi_init_dir/body.csv","component":"body","type":"add","message":"","mtime":"%s"}],"meta":{"code":200}}`
expectBody = fmt.Sprintf(templateBody, metaMtime, structureMtime, bodyMtime)
if diff := cmp.Diff(expectBody, resultBody); diff != "" {
t.Errorf("api response (-want +got):\n%s", diff)
}
Expand Down
Binary file modified api/testdata/api.snapshot
Binary file not shown.
12 changes: 8 additions & 4 deletions cmd/fsi_integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ func TestInitStatusSave(t *testing.T) {

// Verify the directory contains the files that we expect.
dirContents := listDirectory(workDir)
expectContents := []string{".qri-ref", "body.csv", "meta.json"}
expectContents := []string{".qri-ref", "body.csv", "meta.json", "structure.json"}
if diff := cmp.Diff(expectContents, dirContents); diff != "" {
t.Errorf("directory contents (-want +got):\n%s", diff)
}
Expand All @@ -134,6 +134,7 @@ func TestInitStatusSave(t *testing.T) {
expect := `for linked dataset [test_peer/brand_new]
add: meta (source: meta.json)
add: structure (source: structure.json)
add: body (source: body.csv)
run ` + "`qri save`" + ` to commit this dataset
Expand Down Expand Up @@ -863,7 +864,7 @@ func TestRestoreWithNoHistory(t *testing.T) {

// Verify the directory contains the files that we expect.
dirContents := listDirectory(workDir)
expectContents := []string{".qri-ref", "body.csv"}
expectContents := []string{".qri-ref", "body.csv", "structure.json"}
if diff := cmp.Diff(expectContents, dirContents); diff != "" {
t.Errorf("directory contents (-want +got):\n%s", diff)
}
Expand All @@ -876,6 +877,7 @@ func TestRestoreWithNoHistory(t *testing.T) {
output := fr.GetCommandOutput()
expect := `for linked dataset [test_peer/new_folder]
add: structure (source: structure.json)
add: body (source: body.csv)
run ` + "`qri save`" + ` to commit this dataset
Expand Down Expand Up @@ -904,7 +906,7 @@ func TestInitWithSourceBodyPath(t *testing.T) {

// Verify the directory contains the files that we expect.
dirContents := listDirectory(workDir)
expectContents := []string{".qri-ref", "body.csv", "meta.json"}
expectContents := []string{".qri-ref", "body.csv", "meta.json", "structure.json"}
if diff := cmp.Diff(expectContents, dirContents); diff != "" {
t.Errorf("directory contents (-want +got):\n%s", diff)
}
Expand All @@ -918,6 +920,7 @@ func TestInitWithSourceBodyPath(t *testing.T) {
expect := `for linked dataset [test_peer/init_source]
add: meta (source: meta.json)
add: structure (source: structure.json)
add: body (source: body.csv)
run ` + "`qri save`" + ` to commit this dataset
Expand Down Expand Up @@ -957,7 +960,7 @@ func TestInitWithDirectory(t *testing.T) {

// Verify the directory contains the files that we expect.
dirContents := listDirectory(workDir)
expectContents := []string{".qri-ref", "body.csv", "meta.json"}
expectContents := []string{".qri-ref", "body.csv", "meta.json", "structure.json"}
if diff := cmp.Diff(expectContents, dirContents); diff != "" {
t.Errorf("directory contents (-want +got):\n%s", diff)
}
Expand All @@ -971,6 +974,7 @@ func TestInitWithDirectory(t *testing.T) {
expect := `for linked dataset [test_peer/init_dir]
add: meta (source: meta.json)
add: structure (source: structure.json)
add: body (source: body.csv)
run ` + "`qri save`" + ` to commit this dataset
Expand Down
43 changes: 41 additions & 2 deletions fsi/init.go
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
package fsi

import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path/filepath"

"github.com/qri-io/dataset"
"github.com/qri-io/qri/repo"
)

Expand Down Expand Up @@ -111,9 +113,46 @@ func (fsi *FSI) InitDataset(p InitParams) (name string, err error) {
return "", err
}

// Create basic structure (no derived values) based on Format
structureBytes, err := createBasicStructure(p.Format)
if err != nil {
return "", err
}
// use format to determine basic formatConfig
structureFilename := filepath.Join(targetPath, "structure.json")
if err := ioutil.WriteFile(structureFilename, structureBytes, os.ModePerm); err != nil {
return "", err
}

return name, err
}

func createBasicStructure(format string) ([]byte, error) {
var err error
formatConfigBytes := []byte{}
switch format {
case "csv":
formatConfigBytes, err = json.Marshal(dataset.CSVOptions{})
if err != nil {
return nil, err
}
case "json":
formatConfigBytes, err = json.Marshal(map[string]interface{}{"pretty": false})
if err != nil {
return nil, err
}
case "xlsx":
formatConfigBytes, err = json.Marshal(dataset.XLSXOptions{SheetName: "sheet1"})
if err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("unknown body format '%s'", format)
}
structureStr := fmt.Sprintf(`{"format":"%s","formatConfig":%s}`, format, formatConfigBytes)
return []byte(structureStr), nil
}

func canInitDir(dir string) error {
if _, err := os.Stat(filepath.Join(dir, QriRefFilename)); !os.IsNotExist(err) {
return fmt.Errorf("working directory is already linked, .qri-ref exists")
Expand All @@ -122,8 +161,8 @@ func canInitDir(dir string) error {
// TODO(dlong): Instead, import the meta.json file for the new dataset
return fmt.Errorf("cannot initialize new dataset, meta.json exists")
}
if _, err := os.Stat(filepath.Join(dir, "schema.json")); !os.IsNotExist(err) {
// TODO(dlong): Instead, import the schema.json file for the new dataset
if _, err := os.Stat(filepath.Join(dir, "structure.json")); !os.IsNotExist(err) {
// TODO(dlong): Instead, import the structure.json file for the new dataset
return fmt.Errorf("cannot initialize new dataset, schema.json exists")
}
if _, err := os.Stat(filepath.Join(dir, "body.csv")); !os.IsNotExist(err) {
Expand Down
47 changes: 47 additions & 0 deletions fsi/init_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
package fsi

import (
"bytes"
"fmt"
"testing"
)

func TestCreateBasicStructure(t *testing.T) {
good := []struct {
format string
expectBytes []byte
}{
{"csv", []byte(`{"format":"csv","formatConfig":{"headerRow":false,"lazyQuotes":false,"variadicFields":false}}`)},
{"json", []byte(`{"format":"json","formatConfig":{"pretty":false}}`)},
{"xlsx", []byte(`{"format":"xlsx","formatConfig":{"sheetName":"sheet1"}}`)},
}
for _, c := range good {
t.Run(fmt.Sprintf("good: %s", c.format), func(t *testing.T) {
gotBytes, err := createBasicStructure(c.format)
if err != nil {
t.Errorf("expected no error. got: %s", err)
}
if !bytes.Equal(gotBytes, c.expectBytes) {
t.Errorf("expected '%s', got '%s'", c.expectBytes, gotBytes)
}
})
}
bad := []struct {
format string
err string
}{
{"bad_format", "unknown body format 'bad_format'"},
}
for _, c := range bad {
t.Run(fmt.Sprintf("bad: %s", c.format), func(t *testing.T) {
_, err := createBasicStructure(c.format)
t.Log(err)
if err == nil {
t.Errorf("expected error. got: %s", err)
}
if err.Error() != c.err {
t.Errorf("expected error '%s'. got: '%s'", c.err, err)
}
})
}
}
1 change: 1 addition & 0 deletions lib/fsi_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@ func TestFSIMethodsWrite(t *testing.T) {
FSIWriteParams{Ref: noHistoryName, Ds: &dataset.Dataset{Meta: &dataset.Meta{Title: "Changed Title"}}},
[]StatusItem{
{Component: "meta", Type: "add"},
{Component: "structure", Type: "add"},
{Component: "body", Type: "add"},
},
},
Expand Down

0 comments on commit a6f423b

Please sign in to comment.