feat(fsi transform): skeletal support for transform scripts via FSI

Merge pull request #1021 from qri-io/feat_fsi_transform

base/component/component.go

@@ -56,6 +56,13 @@ func ConvertDatasetToComponents(ds *dataset.Dataset, qfilesys qfs.Filesystem) Co
 		rc.Format = "md"
 		dc.Subcomponents["readme"] = &rc
 	}
+	if ds.Transform != nil {
+		dc.Subcomponents["transform"] = &TransformComponent{
+			BaseComponent: BaseComponent{Format: "star"},
+			Resolver:      qfilesys,
+			Value:         ds.Transform,
+		}
+	}
 
 	if ds.Body != nil {
 		bc := BodyComponent{Resolver: qfilesys}
@@ -76,8 +83,8 @@ func ConvertDatasetToComponents(ds *dataset.Dataset, qfilesys qfs.Filesystem) Co
 	return &dc
 }
 
-// ToDataset converts a component to a dataset. Should only be used on a component representing an
-// entire dataset.
+// ToDataset converts a component to a dataset. Should only be used on a
+// component representing an entire dataset.
 func ToDataset(comp Component) (*dataset.Dataset, error) {
 	dsComp := comp.Base().GetSubcomponent("dataset")
 	if dsComp == nil {
@@ -110,6 +117,11 @@ func ToDataset(comp Component) (*dataset.Dataset, error) {
 			return nil, err
 		}
 	}
+	if tfComponent := comp.Base().GetSubcomponent("transform"); tfComponent != nil {
+		if err := tfComponent.LoadAndFill(ds); err != nil {
+			return nil, err
+		}
+	}
 	if bdComponent := comp.Base().GetSubcomponent("body"); bdComponent != nil {
 		if !bdComponent.Base().IsLoaded {
 			ds.BodyPath = bdComponent.Base().SourceFile

base/component/kinds.go

@@ -562,7 +562,7 @@ func SerializeBody(source interface{}, st *dataset.Structure) ([]byte, error) {
 	return buff.Bytes(), nil
 }
 
-// ReadmeComponent represents a meta component
+// ReadmeComponent represents a readme component
 type ReadmeComponent struct {
 	BaseComponent
 	Resolver qfs.PathResolver
@@ -650,6 +650,99 @@ func (rc *ReadmeComponent) StructuredData() (interface{}, error) {
 	return structToMap(rc.Value)
 }
 
+// TransformComponent represents a transform component
+type TransformComponent struct {
+	BaseComponent
+	Resolver qfs.PathResolver
+	Value    *dataset.Transform
+}
+
+// Compare compares to another component
+func (tc *TransformComponent) Compare(compare Component) (bool, error) {
+	other, ok := compare.(*TransformComponent)
+	if !ok {
+		return false, nil
+	}
+	if err := tc.LoadAndFill(nil); err != nil {
+		return false, err
+	}
+	if err := compare.LoadAndFill(nil); err != nil {
+		return false, err
+	}
+
+	// TODO (b5) - for now we're only comparing script bytes, which means changes to transform
+	// configuration won't be detected by things like status, What's more, because stored transforms include
+	// a starlark syntax and a "qri" key, comparing FSI to stored JSON won't be equal
+	// Let's clean this up
+	return bytes.Equal(tc.Value.ScriptBytes, other.Value.ScriptBytes), nil
+}
+
+// WriteTo writes the component as a file to the directory
+func (tc *TransformComponent) WriteTo(dirPath string) (targetFile string, err error) {
+	if err = tc.LoadAndFill(nil); err != nil {
+		return
+	}
+	if tc.Value != nil && !tc.Value.IsEmpty() {
+		targetFile = filepath.Join(dirPath, fmt.Sprintf("transform.%s", tc.Format))
+		if err = ioutil.WriteFile(targetFile, tc.Value.ScriptBytes, os.ModePerm); err != nil {
+			return
+		}
+	}
+	return "", nil
+}
+
+// RemoveFrom removes the component file from the directory
+func (tc *TransformComponent) RemoveFrom(dirPath string) error {
+	// TODO(dlong): Does component have SoutceFile set?
+	if err := os.Remove(filepath.Join(dirPath, fmt.Sprintf("transform.%s", tc.Format))); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	return nil
+}
+
+// DropDerivedValues drops derived values from the component
+func (tc *TransformComponent) DropDerivedValues() {
+	tc.Value.DropDerivedValues()
+}
+
+// LoadAndFill loads data from the component soutce file and assigns it
+func (tc *TransformComponent) LoadAndFill(ds *dataset.Dataset) error {
+	if tc.Base().IsLoaded {
+		return nil
+	}
+	if tc.Value == nil {
+		fields, err := tc.Base().LoadFile()
+		if err != nil {
+			return err
+		}
+		tc.Value = &dataset.Transform{}
+		if err := fill.Struct(fields, tc.Value); err != nil {
+			return err
+		}
+	}
+	tc.Base().IsLoaded = true
+
+	if tc.Resolver != nil {
+		err := tc.Value.InlineScriptFile(context.Background(), tc.Resolver)
+		if err != nil {
+			return err
+		}
+	}
+
+	if ds != nil {
+		ds.Transform = tc.Value
+	}
+	return nil
+}
+
+// StructuredData returns the transform as a map[string]
+func (tc *TransformComponent) StructuredData() (interface{}, error) {
+	if err := tc.LoadAndFill(nil); err != nil {
+		return nil, err
+	}
+	return structToMap(tc.Value)
+}
+
 // Base returns the common base data for the component
 func (bc *BaseComponent) Base() *BaseComponent {
 	return bc
@@ -683,7 +776,7 @@ func (bc *BaseComponent) LoadFile() (map[string]interface{}, error) {
 			return nil, err
 		}
 		return fields, nil
-	case "html", "md":
+	case "html", "md", "star":
 		fields["ScriptBytes"] = data
 		return fields, nil
 	}
@@ -712,6 +805,8 @@ func (bc *BaseComponent) SetSubcomponent(name string, base BaseComponent) Compon
 		component = &StructureComponent{BaseComponent: base}
 	} else if name == "readme" {
 		component = &ReadmeComponent{BaseComponent: base}
+	} else if name == "transform" {
+		component = &TransformComponent{BaseComponent: base}
 	} else if name == "body" {
 		component = &BodyComponent{BaseComponent: base}
 	} else if name == "dataset" {

base/component/list_dir.go

@@ -143,7 +143,15 @@ func ExpandListedComponents(container Component, resolver qfs.Filesystem) error
 			readme.IsLoaded = true
 		}
 	}
-	// TODO: transform
+	if ds.Transform != nil {
+		comp := assignField(filesysComponent, "transform", dsComponent)
+		if comp != nil {
+			readme := comp.(*TransformComponent)
+			readme.Resolver = resolver
+			readme.Value = ds.Transform
+			readme.IsLoaded = true
+		}
+	}
 	if ds.Body != nil {
 		comp := assignField(filesysComponent, "body", dsComponent)
 		if comp != nil {

fsi/status.go

@@ -90,7 +90,6 @@ func (fsi *FSI) Status(ctx context.Context, dir string) (changes []StatusItem, e
 
 	stored.DropDerivedValues()
 	stored.Commit = nil
-	stored.Transform = nil
 	stored.Peername = ""
 
 	working, err := component.ListDirectoryComponents(dir)

go.mod

@@ -45,7 +45,7 @@ require (
 	github.com/multiformats/go-multihash v0.0.8
 	github.com/qri-io/apiutil v0.1.0
 	github.com/qri-io/dag v0.2.1-0.20191025201336-254aa177fbd7
-	github.com/qri-io/dataset v0.1.5-0.20191025195651-c58fba11892c
+	github.com/qri-io/dataset v0.1.5-0.20191115160920-11db070a08eb
 	github.com/qri-io/deepdiff v0.1.1-0.20191101211235-d2c221028259
 	github.com/qri-io/doggos v0.1.0
 	github.com/qri-io/ioes v0.1.0

go.sum

@@ -718,6 +718,8 @@ github.com/qri-io/dag v0.2.1-0.20191025201336-254aa177fbd7 h1:hujyzdufAz3Hi6I0I0
 github.com/qri-io/dag v0.2.1-0.20191025201336-254aa177fbd7/go.mod h1:gMpEjalFfB6tGDCiOx1bmqsKhKhqnPHOnEycsqY4lqc=
 github.com/qri-io/dataset v0.1.5-0.20191025195651-c58fba11892c h1:pjA/TpX6ivOhm302jEMEgmelGAwhd3rpV0L20dbt7K4=
 github.com/qri-io/dataset v0.1.5-0.20191025195651-c58fba11892c/go.mod h1:efJPKDyyGItDnI9CzKuQYmKiEP0xp4mIxIut7o/bG2g=
+github.com/qri-io/dataset v0.1.5-0.20191115160920-11db070a08eb h1:Dd1n36V8oK7P6ECA1eo27XDjkEZ0tlS5075axAvPDhk=
+github.com/qri-io/dataset v0.1.5-0.20191115160920-11db070a08eb/go.mod h1:efJPKDyyGItDnI9CzKuQYmKiEP0xp4mIxIut7o/bG2g=
 github.com/qri-io/deepdiff v0.1.0 h1:1VXd/ePPrSrq4GRINC4YxmoPHuuLufg2ujMKvvEWsDc=
 github.com/qri-io/deepdiff v0.1.0/go.mod h1:/O1HQVAlm3yhgQvNwyzsKaECfM2vVSx6KxttgXQQGFw=
 github.com/qri-io/deepdiff v0.1.1-0.20191101211235-d2c221028259 h1:lDsC5DpspS4Qg9GZsy1Yo0iXOoQGXHudvTvaUD7P8ZY=