feat(fsi): Package fsi defines qri file system integration

initial mapping function for reading datasets from a directory

fsi/fsi.go

@@ -0,0 +1,12 @@
+// Package fsi defines qri file system integration: representing a dataset as
+// files in a directory on a user's computer. Using fsi, users can edit files
+// as an interface for working with qri datasets.
+//
+// A dataset is "linked" to a directory through a `.qri_ref` dotfile that
+// connects the folder to a version history stored in the local qri repository.
+//
+// files in a linked directory follow naming conventions that map to components
+// of a dataset. eg: a file named "meta.json" in a linked directory maps to
+// the dataset meta component. This mapping can be used to construct a dataset
+// for read and write actions
+package fsi

fsi/mapping.go

@@ -0,0 +1,197 @@
+package fsi
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/ghodss/yaml"
+	"github.com/qri-io/dataset"
+)
+
+const (
+	componentNameCommit    = "commit"
+	componentNameDataset   = "dataset"
+	componentNameMeta      = "meta"
+	componentNameSchema    = "schema"
+	componentNameBody      = "body"
+	componentNameStructure = "structure"
+	componentNameTransform = "transform"
+	componentNameViz       = "viz"
+)
+
+var (
+	// ErrNoDatasetFiles indicates no data
+	ErrNoDatasetFiles = fmt.Errorf("no dataset files provided")
+)
+
+// ReadDir parses a directory into a dataset, returning both the dataset and
+// a map of component names to the files they came from. Files can be specified
+// in either JSON or YAML format. It is an error to specify any component more
+// than once
+func ReadDir(dir string) (ds *dataset.Dataset, mapping map[string]string, err error) {
+	mapping = map[string]string{}
+	ds = &dataset.Dataset{}
+	schema := map[string]interface{}{}
+
+	components := map[string]interface{}{
+		componentNameDataset: ds,
+
+		componentNameCommit:    &dataset.Commit{},
+		componentNameMeta:      &dataset.Meta{},
+		componentNameStructure: &dataset.Structure{},
+		componentNameSchema:    &schema,
+		componentNameTransform: &dataset.Transform{},
+		componentNameViz:       &dataset.Viz{},
+
+		// TODO (b5) - deal with dataset bodies
+		// componentNameBody: &dataset,
+	}
+
+	extensions := map[string]decoderFactory{
+		".json": newJSONDecoder,
+		".yaml": newYAMLDecoder,
+		".yml":  newYAMLDecoder,
+	}
+
+	addMapping := func(cmpName, path string) error {
+		if cmpPath, exists := mapping[cmpName]; exists {
+			cmpPath = filepath.Base(cmpPath)
+			path = filepath.Base(path)
+			return fmt.Errorf(`%s is defined in two places: %s and %s. please remove one`, cmpName, cmpPath, path)
+		}
+
+		mapping[cmpName] = path
+		return nil
+	}
+
+	for cmpName, cmp := range components {
+		for ext, mkDec := range extensions {
+			filename := fmt.Sprintf("%s%s", cmpName, ext)
+			path := filepath.Join(dir, filename)
+			if f, e := os.Open(path); e == nil {
+				if err = mkDec(f).Decode(cmp); err != nil {
+					err = fmt.Errorf("reading %s: %s", filename, err)
+					return ds, mapping, err
+				}
+
+				if err = addMapping(cmpName, path); err != nil {
+					return ds, mapping, err
+				}
+
+				switch cmpName {
+				case componentNameDataset:
+					if ds.Commit != nil {
+						if err = addMapping(componentNameCommit, path); err != nil {
+							return
+						}
+					}
+					if ds.Meta != nil {
+						if err = addMapping(componentNameMeta, path); err != nil {
+							return
+						}
+					}
+					if ds.Structure != nil {
+						if err = addMapping(componentNameStructure, path); err != nil {
+							return
+						}
+						if ds.Structure.Schema != nil {
+							if err = addMapping(componentNameSchema, path); err != nil {
+								return
+							}
+						}
+					}
+					if ds.Viz != nil {
+						if err = addMapping(componentNameViz, path); err != nil {
+							return
+						}
+					}
+					if ds.Transform != nil {
+						if err = addMapping(componentNameTransform, path); err != nil {
+							return
+						}
+					}
+					if ds.Body != nil {
+						if err = addMapping(componentNameBody, path); err != nil {
+							return
+						}
+					}
+
+				case componentNameCommit:
+					ds.Commit = cmp.(*dataset.Commit)
+				case componentNameMeta:
+					ds.Meta = cmp.(*dataset.Meta)
+				case componentNameStructure:
+					ds.Structure = cmp.(*dataset.Structure)
+				case componentNameSchema:
+					if ds.Structure == nil {
+						ds.Structure = &dataset.Structure{}
+					}
+					ds.Structure.Schema = *cmp.(*map[string]interface{})
+				case componentNameViz:
+					ds.Viz = cmp.(*dataset.Viz)
+				case componentNameTransform:
+					ds.Transform = cmp.(*dataset.Transform)
+
+					// case componentNameBody:
+					// 	ds.Body = cmp.(*dataset.Body)
+					// 	// TODO (b5) -
+				}
+			}
+		}
+	}
+
+	if len(mapping) == 0 {
+		err = ErrNoDatasetFiles
+	}
+
+	return ds, mapping, err
+}
+
+type decoderFactory func(io.Reader) decoder
+
+type decoder interface {
+	Decode(m interface{}) error
+}
+
+type jsonDecoder struct {
+	dec *json.Decoder
+}
+
+func newJSONDecoder(r io.Reader) decoder {
+	return jsonDecoder{
+		dec: json.NewDecoder(r),
+	}
+}
+
+func (jd jsonDecoder) Decode(v interface{}) error {
+	return jd.dec.Decode(v)
+}
+
+type yamlDecoder struct {
+	rdr io.Reader
+}
+
+func newYAMLDecoder(r io.Reader) decoder {
+	return yamlDecoder{
+		rdr: r,
+	}
+}
+
+func (yd yamlDecoder) Decode(v interface{}) error {
+	// convert yaml input to json as a hack to support yaml input for now
+	yamlData, err := ioutil.ReadAll(yd.rdr)
+	if err != nil {
+		return fmt.Errorf("invalid file: %s", err.Error())
+	}
+
+	jsonData, err := yaml.YAMLToJSON(yamlData)
+	if err != nil {
+		return fmt.Errorf("converting yaml body to json: %s", err.Error())
+	}
+
+	return json.Unmarshal(jsonData, v)
+}

fsi/mapping_test.go

@@ -0,0 +1,44 @@
+package fsi
+
+import (
+	"fmt"
+	"path/filepath"
+	"testing"
+)
+
+func TestReadDir(t *testing.T) {
+	good := []struct {
+		path string
+	}{
+		{"testdata/valid_mappings/all_json_components"},
+		{"testdata/valid_mappings/all_in_dataset"},
+	}
+
+	for _, c := range good {
+		t.Run(fmt.Sprintf("good: %s", filepath.Base(c.path)), func(t *testing.T) {
+			_, _, err := ReadDir(c.path)
+			if err != nil {
+				t.Errorf("expected no error. got: %s", err)
+			}
+		})
+	}
+
+	bad := []struct {
+		path string
+	}{
+		{"testdata/invalid_mappings/two_metas"},
+		{"testdata/invalid_mappings/double_format"},
+		{"testdata/invalid_mappings/bad_yaml"},
+		{"testdata/invalid_mappings/empty"},
+	}
+
+	for _, c := range bad {
+		t.Run(fmt.Sprintf("bad: %s", filepath.Base(c.path)), func(t *testing.T) {
+			_, _, err := ReadDir(c.path)
+			t.Log(err)
+			if err == nil {
+				t.Errorf("expected error. got: %s", err)
+			}
+		})
+	}
+}

fsi/testdata/invalid_mappings/bad_yaml/dataset.yaml

@@ -0,0 +1 @@
+:

fsi/testdata/invalid_mappings/double_format/dataset.json

@@ -0,0 +1,5 @@
+{
+  "meta": {
+    "title": "foo"
+  }
+}

fsi/testdata/invalid_mappings/double_format/dataset.yaml

@@ -0,0 +1,3 @@
+
+meta:
+  title: bar

fsi/testdata/invalid_mappings/two_metas/dataset.yaml

@@ -0,0 +1,4 @@
+
+
+meta:
+  title: foo

fsi/testdata/invalid_mappings/two_metas/meta.json

@@ -0,0 +1,3 @@
+{
+  "title" : "bar"
+}

fsi/testdata/valid_mappings/all_in_dataset/dataset.yaml

@@ -0,0 +1,13 @@
+
+commit:
+  title: this a commit
+meta:
+  title: this a title
+structure:
+  format: json
+schema:
+  type: array
+transform:
+  scriptPath: not_exist.star
+viz:
+  scriptPath: not_exist.html

fsi/testdata/valid_mappings/all_json_components/__expect.json

@@ -0,0 +1,8 @@
+{
+  "commit" : {
+    "title" : "foo"
+  },
+  "meta" : {
+    "title" : "title"
+  }
+}

fsi/testdata/valid_mappings/all_json_components/body.csv

@@ -0,0 +1,2 @@
+1,2,3
+4,5,6

fsi/testdata/valid_mappings/all_json_components/commit.json

@@ -0,0 +1,3 @@
+{
+  "title": "foo"
+}

fsi/testdata/valid_mappings/all_json_components/meta.json

@@ -0,0 +1,3 @@
+{
+  "title" : "title"
+}

fsi/testdata/valid_mappings/all_json_components/schema.json

@@ -0,0 +1,11 @@
+{
+  "type": "array",
+  "items": {
+    "type" : "array",
+    "items": [
+      { "type" : "number" },
+      { "type" : "number" },
+      { "type" : "number" }
+    ]
+  }
+}

fsi/testdata/valid_mappings/all_json_components/structure.json

@@ -0,0 +1,7 @@
+{
+  "format" : "csv",
+  "formatConfig" : {
+    "lazyQuotes" : true,
+    "variadicRows" : true
+  }
+}

fsi/testdata/valid_mappings/all_json_components/transform.json

@@ -0,0 +1,3 @@
+{
+  "scriptPath": "transform.star"
+}

fsi/testdata/valid_mappings/all_json_components/viz.json

@@ -0,0 +1,3 @@
+{
+  "scriptPath" : "template.html"
+}