feat(base.LogDiff): diff logs against a repo's dataset history

Now a repo can diff logs! Woot!

actions/log.go

@@ -17,5 +17,5 @@ func DatasetLog(node *p2p.QriNode, ref repo.DatasetRef, limit, offset int) (rlog
 		return node.RequestDatasetLog(ref, limit, offset)
 	}
 
-	return base.DatasetLog(node.Repo, ref, limit, offset)
+	return base.DatasetLog(node.Repo, ref, limit, offset, true)
 }

base/base_test.go

@@ -68,6 +68,38 @@ func addCitiesDataset(t *testing.T, r repo.Repo) repo.DatasetRef {
 	return ref
 }
 
+func updateCitiesDataset(t *testing.T, r repo.Repo) repo.DatasetRef {
+	tc, err := dstest.NewTestCaseFromDir(testdataPath("cities"))
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+	pro, err := r.Profile()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ref, err := r.GetRef(repo.DatasetRef{Peername: pro.Peername, Name: tc.Name})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	prevTitle := tc.Input.Meta.Title
+	tc.Input.Meta.Title = "this is the new title"
+	tc.Input.PreviousPath = ref.Path
+	defer func() {
+		// because test cases are cached for performance, we need to clean up any mutation to
+		// testcase input
+		tc.Input.Meta.Title = prevTitle
+		tc.Input.PreviousPath = ""
+	}()
+
+	ref, _, err = CreateDataset(r, ioes.NewDiscardIOStreams(), tc.Name, tc.Input, tc.BodyFile(), false, true)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+	return ref
+}
+
 func addFlourinatedCompoundsDataset(t *testing.T, r repo.Repo) repo.DatasetRef {
 	tc, err := dstest.NewTestCaseFromDir(testdataPath("flourinated_compounds_in_fast_food_packaging"))
 	if err != nil {

base/log.go

@@ -1,18 +1,26 @@
 package base
 
 import (
+	"fmt"
+
 	datastore "github.com/ipfs/go-datastore"
+	"github.com/qri-io/dataset"
 	"github.com/qri-io/dataset/dsfs"
 	"github.com/qri-io/qri/repo"
 )
 
-// DatasetLog fetches the history of changes to a dataset
-func DatasetLog(r repo.Repo, ref repo.DatasetRef, limit, offset int) (rlog []repo.DatasetRef, err error) {
-
+// DatasetLog fetches the history of changes to a dataset, if loadDatasets is true, dataset information will be populated
+func DatasetLog(r repo.Repo, ref repo.DatasetRef, limit, offset int, loadDatasets bool) (rlog []repo.DatasetRef, err error) {
 	for {
-		ds, e := dsfs.LoadDataset(r.Store(), datastore.NewKey(ref.Path))
-		if e != nil {
-			return nil, e
+		var ds *dataset.Dataset
+		if loadDatasets {
+			if ds, err = dsfs.LoadDataset(r.Store(), datastore.NewKey(ref.Path)); err != nil {
+				return
+			}
+		} else {
+			if ds, err = dsfs.LoadDatasetRefs(r.Store(), datastore.NewKey(ref.Path)); err != nil {
+				return
+			}
 		}
 		ref.Dataset = ds.Encode()
 
@@ -32,3 +40,63 @@ func DatasetLog(r repo.Repo, ref repo.DatasetRef, limit, offset int) (rlog []rep
 
 	return rlog, nil
 }
+
+// LogDiffResult is the result of comparing a set of references
+type LogDiffResult struct {
+	Head        repo.DatasetRef
+	Add, Remove []repo.DatasetRef
+}
+
+// LogDiff determines the difference between an input slice of references
+func LogDiff(r repo.Repo, a []repo.DatasetRef) (ldr LogDiffResult, err error) {
+	if len(a) < 1 {
+		return ldr, fmt.Errorf("no references provided for diffing")
+	}
+
+	alias := repo.DatasetRef{Peername: a[0].Peername, Name: a[0].Name}
+	ldr.Head, err = r.GetRef(alias)
+	if err != nil {
+		return ldr, err
+	}
+
+	// TODO - deal with max limit / offset / pagination issuez
+	b, err := DatasetLog(r, ldr.Head, 10000, 0, false)
+	if err != nil {
+		return ldr, err
+	}
+
+	ldr.Add, ldr.Remove = refDiff(a, b)
+
+	return ldr, nil
+}
+
+// refDiff returns a set of additions and removals needed to sync slice a to b
+func refDiff(a, b []repo.DatasetRef) (add, remove []repo.DatasetRef) {
+	var present bool
+	for _, aRef := range a {
+		present = false
+		for _, bRef := range b {
+			if aRef.Equal(bRef) {
+				present = true
+				break
+			}
+		}
+		if !present {
+			remove = append(remove, aRef)
+		}
+	}
+
+	for _, bRef := range b {
+		present = false
+		for _, aRef := range a {
+			if bRef.Equal(aRef) {
+				present = true
+				break
+			}
+		}
+		if !present {
+			add = append(add, bRef)
+		}
+	}
+	return
+}

base/log_test.go

@@ -2,16 +2,106 @@ package base
 
 import (
 	"testing"
+
+	"github.com/qri-io/qri/repo"
 )
 
 func TestDatasetLog(t *testing.T) {
+	r := newTestRepo(t)
+	addCitiesDataset(t, r)
+	head := updateCitiesDataset(t, r)
+	expectLen := 2
+
+	dlog, err := DatasetLog(r, head, 100, 0, true)
+	if err != nil {
+		t.Error(err)
+	}
+	if len(dlog) != expectLen {
+		t.Errorf("log length mismatch. expected: %d, got: %d", expectLen, len(dlog))
+	}
+	if dlog[0].Dataset.Meta.Title != head.Dataset.Meta.Title {
+		t.Errorf("expected log with loadDataset == true to populate datasets")
+	}
+
+	dlog, err = DatasetLog(r, head, 100, 0, false)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if len(dlog) != expectLen {
+		t.Errorf("log length mismatch. expected: %d, got: %d", expectLen, len(dlog))
+	}
+	if dlog[0].Dataset.Meta.Title != "" {
+		t.Errorf("expected log with loadDataset == false to not load a dataset. got: %v", dlog[0].Dataset)
+	}
+}
+
+func TestLogDiff(t *testing.T) {
 	r := newTestRepo(t)
 	ref := addCitiesDataset(t, r)
-	log, err := DatasetLog(r, ref, 100, 0)
+	head := updateCitiesDataset(t, r)
+
+	ldr, err := LogDiff(r, []repo.DatasetRef{})
+	if err == nil {
+		t.Error("expected empty diff to error")
+	}
+
+	ldr, err = LogDiff(r, []repo.DatasetRef{
+		repo.DatasetRef{Peername: "missing", Name: "reference"},
+	})
+	if err == nil {
+		t.Error("expected diff of missing reference to error")
+	}
+
+	ldr, err = LogDiff(r, []repo.DatasetRef{ref})
 	if err != nil {
 		t.Error(err)
 	}
-	if len(log) != 1 {
-		t.Errorf("log length mismatch. expected: %d, got: %d", 1, len(log))
+
+	expectAdd := []repo.DatasetRef{head}
+	if !RefSetEqual(ldr.Add, expectAdd) {
+		t.Errorf("add mismatch. expected: %v\ngot: %v", expectAdd, ldr.Add)
+	}
+}
+
+func TestRefDiff(t *testing.T) {
+	a := []repo.DatasetRef{
+		repo.MustParseDatasetRef("a/b@c/ipfs/d"),
+		repo.MustParseDatasetRef("a/b@c/ipfs/e"),
+		repo.MustParseDatasetRef("a/b@c/ipfs/f"),
+	}
+	b := []repo.DatasetRef{
+		repo.MustParseDatasetRef("a/b@c/ipfs/e"),
+		repo.MustParseDatasetRef("a/b@c/ipfs/f"),
+		repo.MustParseDatasetRef("a/b@c/ipfs/g"),
+		repo.MustParseDatasetRef("a/b@c/ipfs/h"),
+	}
+	expectAdd := []repo.DatasetRef{
+		repo.MustParseDatasetRef("a/b@c/ipfs/g"),
+		repo.MustParseDatasetRef("a/b@c/ipfs/h"),
+	}
+	expectRm := []repo.DatasetRef{
+		repo.MustParseDatasetRef("a/b@c/ipfs/d"),
+	}
+
+	gotAdd, gotRm := refDiff(a, b)
+
+	if !RefSetEqual(expectAdd, gotAdd) {
+		t.Errorf("add mismatch. expected: %v\ngot: %v", expectAdd, gotAdd)
+	}
+	if !RefSetEqual(expectRm, gotRm) {
+		t.Errorf("remove mismatch. expected: %v\ngot: %v", expectRm, gotRm)
+	}
+}
+
+func RefSetEqual(a, b []repo.DatasetRef) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i, ar := range a {
+		if !b[i].Equal(ar) {
+			return false
+		}
 	}
+	return true
 }

base/viz.go

@@ -25,21 +25,3 @@ func prepareViz(ds *dataset.Dataset) (err error) {
 	}
 	return nil
 }
-
-func prepareTransform(ds *dataset.Dataset) (err error) {
-	// remove any empty transform
-	if ds.Transform != nil && ds.Transform.IsEmpty() {
-		ds.Transform = nil
-		return nil
-	}
-
-	if ds.Transform != nil && ds.Transform.ScriptPath != "" {
-		// create a reader of script bytes
-		scriptdata, err := ioutil.ReadFile(ds.Transform.ScriptPath)
-		if err != nil {
-			return err
-		}
-		ds.Transform.Script = bytes.NewReader(scriptdata)
-	}
-	return nil
-}