feat(dscache): Dscache used for fsi init and checkout

Merge pull request #1349 from qri-io/dscache-fsi

api/fsi_test.go

@@ -103,7 +103,7 @@ func TestNoHistory(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if ref != "peer/test_ds" {
+	if ref.Human() != "peer/test_ds" {
 		t.Errorf("expected ref to be \"peer/test_ds\", got \"%s\"", ref)
 	}
 

cmd/fsi_integration_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/qri-io/qri/base/component"
 	"github.com/qri-io/qri/dsref"
 	"github.com/qri-io/qri/fsi"
+	"github.com/qri-io/qri/lib"
 )
 
 // FSITestRunner holds test info for fsi integration tests, for convenient cleanup.
@@ -66,15 +67,24 @@ func (run *FSITestRunner) GetCommandOutput() string {
 	if buffer, ok := run.Streams.Out.(*bytes.Buffer); ok {
 		outputText = buffer.String()
 	}
+	return run.niceifyTempDirs(outputText)
+}
+
+// niceifyTempDirs replaces temporary directories with nice replacements
+func (run *FSITestRunner) niceifyTempDirs(text string) string {
 	realRoot, err := filepath.EvalSymlinks(run.RepoRoot.RootPath)
 	if err == nil {
-		outputText = strings.Replace(outputText, realRoot, "/root", -1)
+		text = strings.Replace(text, realRoot, "/root", -1)
 	}
 	realTmp, err := filepath.EvalSymlinks(run.RootPath)
 	if err == nil {
-		outputText = strings.Replace(outputText, realTmp, "/tmp", -1)
+		text = strings.Replace(text, realTmp, "/tmp", -1)
+	}
+	workPath, err := filepath.EvalSymlinks(run.WorkPath)
+	if err == nil {
+		text = strings.Replace(text, workPath, "/work", -1)
 	}
-	return outputText
+	return text
 }
 
 // NewFSITestRunnerWithMockRemoteClient returns a new FSITestRunner.
@@ -272,14 +282,21 @@ func TestInitDscache(t *testing.T) {
 	}
 
 	// Access the dscache
-	repo, err := run.RepoRoot.Repo()
+	// TODO(dustmop): A hack in place for now. The instance does not have an accessor for the
+	// dscache, and the dscache on the repo is not correct to use here.
+	instCopy, err := lib.NewInstance(
+		run.Context,
+		run.RepoRoot.QriPath,
+		lib.OptStdIOStreams(),
+		lib.OptSetIPFSPath(run.RepoRoot.IPFSPath),
+	)
 	if err != nil {
 		t.Fatal(err)
 	}
-	cache := repo.Dscache()
+	cache := instCopy.Dscache()
 
 	// Dscache should have one reference. It has topIndex 0 because there there is only "init".
-	actual := cache.VerboseString(false)
+	actual := run.niceifyTempDirs(cache.VerboseString(false))
 	expect := `Dscache:
  Dscache.Users:
   0) user=test_peer profileID=QmeL2mdVka1eahKENjehK6tBxkkpk5dNQ1qMcgWi7Hrb4B
@@ -290,6 +307,7 @@ func TestInitDscache(t *testing.T) {
      cursorIndex   = 0
      prettyName    = init_dscache
      commitTime    = -62135596800
+     fsiPath       = /tmp/init_dscache
 `
 	if diff := cmp.Diff(expect, actual); diff != "" {
 		t.Errorf("result mismatch (-want +got):%s\n", diff)

cmd/save_test.go

@@ -548,7 +548,7 @@ func TestSaveDscacheFirstCommit(t *testing.T) {
 	cacheFilename := cache.Filename
 	ctx := context.Background()
 	// TODO(dustmop): Do we need to pass a book?
-	cache = dscache.NewDscache(ctx, fs, nil, cacheFilename)
+	cache = dscache.NewDscache(ctx, fs, nil, run.Username(), cacheFilename)
 
 	// Dscache should have two entries now. They are alphabetized by pretty name, and have all
 	// the expected data.
@@ -628,7 +628,7 @@ func TestSaveDscacheExistingDataset(t *testing.T) {
 	fs := localfs.NewFS()
 	cacheFilename := cache.Filename
 	ctx := context.Background()
-	cache = dscache.NewDscache(ctx, fs, nil, cacheFilename)
+	cache = dscache.NewDscache(ctx, fs, nil, run.Username(), cacheFilename)
 
 	// Dscache should now have one reference. Now topIndex is 2 because there is another "commit".
 	actual = cache.VerboseString(false)
@@ -710,7 +710,7 @@ func TestSaveDscacheThenRemoveAll(t *testing.T) {
 	fs := localfs.NewFS()
 	cacheFilename := cache.Filename
 	ctx := context.Background()
-	cache = dscache.NewDscache(ctx, fs, nil, cacheFilename)
+	cache = dscache.NewDscache(ctx, fs, nil, run.Username(), cacheFilename)
 
 	// Dscache should now have one reference.
 	actual = cache.VerboseString(false)
@@ -786,7 +786,7 @@ func TestSaveDscacheThenRemoveVersions(t *testing.T) {
 	fs := localfs.NewFS()
 	cacheFilename := cache.Filename
 	ctx := context.Background()
-	cache = dscache.NewDscache(ctx, fs, nil, cacheFilename)
+	cache = dscache.NewDscache(ctx, fs, nil, run.Username(), cacheFilename)
 
 	// Dscache should now have one reference.
 	actual = cache.VerboseString(false)

cmd/test_runner_test.go

@@ -249,6 +249,11 @@ func (run *TestRunner) newCommandRunner(ctx context.Context, combineOutErr bool)
 	return cmd
 }
 
+// Username returns the test username from the config's profile
+func (run *TestRunner) Username() string {
+	return run.RepoRoot.GetConfig().Profile.Peername
+}
+
 // IOReset resets the io streams
 func (run *TestRunner) IOReset() {
 	run.InStream.Reset()
@@ -409,6 +414,7 @@ func (run *TestRunner) GetCommandErrOutput() string {
 }
 
 func (run *TestRunner) niceifyTempDirs(text string) string {
+	text = strings.Replace(text, run.RepoRoot.RootPath, "/root", -1)
 	realRoot, err := filepath.EvalSymlinks(run.RepoRoot.RootPath)
 	if err == nil {
 		text = strings.Replace(text, realRoot, "/root", -1)

dscache/dscache.go

@@ -13,7 +13,7 @@ import (
 	"github.com/qri-io/qfs"
 	"github.com/qri-io/qri/dscache/dscachefb"
 	"github.com/qri-io/qri/dsref"
-	"github.com/qri-io/qri/logbook"
+	"github.com/qri-io/qri/event/hook"
 	"github.com/qri-io/qri/repo/profile"
 	reporef "github.com/qri-io/qri/repo/ref"
 )
@@ -40,7 +40,7 @@ type Dscache struct {
 
 // NewDscache will construct a dscache from the given filename, or will construct an empty dscache
 // that will save to the given filename. Using an empty filename will disable loading and saving
-func NewDscache(ctx context.Context, fsys qfs.Filesystem, book *logbook.Book, filename string) *Dscache {
+func NewDscache(ctx context.Context, fsys qfs.Filesystem, hooks []hook.ChangeNotifier, username, filename string) *Dscache {
 	cache := Dscache{Filename: filename}
 	f, err := fsys.Get(ctx, filename)
 	if err == nil {
@@ -54,9 +54,9 @@ func NewDscache(ctx context.Context, fsys qfs.Filesystem, book *logbook.Book, fi
 			cache = Dscache{Filename: filename, Root: root, Buffer: buffer}
 		}
 	}
-	if book != nil {
-		book.Observe(cache.update)
-		cache.DefaultUsername = book.AuthorName()
+	cache.DefaultUsername = username
+	for _, h := range hooks {
+		h.SetChangeHook(cache.update)
 	}
 	return &cache
 }
@@ -211,26 +211,30 @@ func (d *Dscache) validateProfileID(profileID string) bool {
 	return len(profileID) == lengthOfProfileID
 }
 
-func (d *Dscache) update(act *logbook.Action) {
+func (d *Dscache) update(act hook.DsChange) {
 	switch act.Type {
-	case logbook.ActionDatasetNameInit:
+	case hook.DatasetNameInit:
 		if err := d.updateInitDataset(act); err != nil && err != ErrNoDscache {
 			log.Error(err)
 		}
-	case logbook.ActionDatasetCommitChange:
+	case hook.DatasetCommitChange:
 		if err := d.updateChangeCursor(act); err != nil && err != ErrNoDscache {
 			log.Error(err)
 		}
-	case logbook.ActionDatasetDeleteAll:
+	case hook.DatasetDeleteAll:
 		if err := d.updateDeleteDataset(act); err != nil && err != ErrNoDscache {
 			log.Error(err)
 		}
-	case logbook.ActionDatasetRename:
+	case hook.DatasetRename:
 		// TODO(dustmop): Handle renames
+	case hook.DatasetCreateLink:
+		if err := d.updateCreateLink(act); err != nil && err != ErrNoDscache {
+			log.Error(err)
+		}
 	}
 }
 
-func (d *Dscache) updateInitDataset(act *logbook.Action) error {
+func (d *Dscache) updateInitDataset(act hook.DsChange) error {
 	if d.IsEmpty() {
 		// Only create a new dscache if that feature is enabled. This way no one is forced to
 		// use dscache without opting in.
@@ -278,7 +282,7 @@ func (d *Dscache) updateInitDataset(act *logbook.Action) error {
 }
 
 // Copy the entire dscache, except for the matching entry, rebuild that one to modify it
-func (d *Dscache) updateChangeCursor(act *logbook.Action) error {
+func (d *Dscache) updateChangeCursor(act hook.DsChange) error {
 	if d.IsEmpty() {
 		return ErrNoDscache
 	}
@@ -322,7 +326,7 @@ func (d *Dscache) updateChangeCursor(act *logbook.Action) error {
 }
 
 // Copy the entire dscache, except leave out the matching entry.
-func (d *Dscache) updateDeleteDataset(act *logbook.Action) error {
+func (d *Dscache) updateDeleteDataset(act hook.DsChange) error {
 	if d.IsEmpty() {
 		return ErrNoDscache
 	}
@@ -344,6 +348,40 @@ func (d *Dscache) updateDeleteDataset(act *logbook.Action) error {
 	return d.save()
 }
 
+// Copy the entire dscache, except for the matching entry, which is copied then assigned an fsiPath
+func (d *Dscache) updateCreateLink(act hook.DsChange) error {
+	if d.IsEmpty() {
+		return ErrNoDscache
+	}
+	// Flatbuffers for go do not allow mutation (for complex types like strings). So we construct
+	// a new flatbuffer entirely, copying the old one while replacing the entry we care to change.
+	builder := flatbuffers.NewBuilder(0)
+	users := d.copyUserAssociationList(builder)
+	refs := d.copyReferenceListWithReplacement(
+		builder,
+		// Function to match the entry we're looking to replace
+		func(r *dscachefb.RefEntryInfo) bool {
+			if act.InitID != "" {
+				return string(r.InitID()) == act.InitID
+			}
+			return d.DefaultUsername == act.Username && string(r.PrettyName()) == act.PrettyName
+		},
+		// Function to replace the matching entry
+		func(refStartMutationFunc func(builder *flatbuffers.Builder)) {
+			fsiDir := builder.CreateString(string(act.Dir))
+			// Start building a ref object, by mutating an existing ref object.
+			refStartMutationFunc(builder)
+			// For this kind of update, only the fsiDir is modified
+			dscachefb.RefEntryInfoAddFsiPath(builder, fsiDir)
+			// Don't call RefEntryInfoEnd, that is handled by copyReferenceListWithReplacement
+		},
+	)
+	root, serialized := d.finishBuilding(builder, users, refs)
+	d.Root = root
+	d.Buffer = serialized
+	return d.save()
+}
+
 func convertEntryToVersionInfo(r *dscachefb.RefEntryInfo) dsref.VersionInfo {
 	return dsref.VersionInfo{
 		InitID:      string(r.InitID()),

dscache/dscache_test.go

@@ -47,21 +47,22 @@ func TestDscacheAssignSaveAndLoad(t *testing.T) {
 	fs := localfs.NewFS()
 
 	peerInfo := testPeers.GetTestPeerInfo(0)
+	peername := "test_user"
 
 	// Construct a dscache, will not save without a filename
 	builder := NewBuilder()
-	builder.AddUser("test_user", profile.IDFromPeerID(peerInfo.PeerID).String())
+	builder.AddUser(peername, profile.IDFromPeerID(peerInfo.PeerID).String())
 	builder.AddDsVersionInfo(dsref.VersionInfo{InitID: "abcd1"})
 	builder.AddDsVersionInfo(dsref.VersionInfo{InitID: "efgh2"})
 	constructed := builder.Build()
 
 	// A dscache that will save when it is assigned
 	dscacheFile := filepath.Join(tmpdir, "dscache.qfb")
-	saveable := NewDscache(ctx, fs, nil, dscacheFile)
+	saveable := NewDscache(ctx, fs, nil, peername, dscacheFile)
 	saveable.Assign(constructed)
 
 	// Load the dscache from its serialized file, verify it has correct data
-	loadable := NewDscache(ctx, fs, nil, dscacheFile)
+	loadable := NewDscache(ctx, fs, nil, peername, dscacheFile)
 	if loadable.Root.UsersLength() != 1 {
 		t.Errorf("expected, 1 user, got %d users", loadable.Root.UsersLength())
 	}

event/hook/change.go

@@ -0,0 +1,39 @@
+package hook
+
+import (
+	"github.com/qri-io/qri/dsref"
+)
+
+// ChangeType is the type of change that has happened to a dataset
+type ChangeType byte
+
+const (
+	// DatasetNameInit is when a dataset is initialized
+	DatasetNameInit ChangeType = iota
+	// DatasetCommitChange is when a dataset changes its newest commit
+	DatasetCommitChange
+	// DatasetDeleteAll is when a dataset is entirely deleted
+	DatasetDeleteAll
+	// DatasetRename is when a dataset is renamed
+	DatasetRename
+	// DatasetCreateLink is when a dataset is linked to a working directory
+	DatasetCreateLink
+)
+
+// DsChange represents the result of a change to a dataset
+type DsChange struct {
+	Type       ChangeType
+	InitID     string
+	TopIndex   int
+	ProfileID  string
+	Username   string
+	PrettyName string
+	HeadRef    string
+	Info       *dsref.VersionInfo
+	Dir        string
+}
+
+// ChangeNotifier is something that provides a hook which will be called when a dataset changes
+type ChangeNotifier interface {
+	SetChangeHook(func(change DsChange))
+}

fsi/fsi.go

@@ -22,6 +22,7 @@ import (
 	"github.com/qri-io/qri/base/component"
 	"github.com/qri-io/qri/dsref"
 	"github.com/qri-io/qri/event"
+	"github.com/qri-io/qri/event/hook"
 	"github.com/qri-io/qri/fsi/linkfile"
 	"github.com/qri-io/qri/repo"
 	reporef "github.com/qri-io/qri/repo/ref"
@@ -52,8 +53,9 @@ func RepoPath(repoPath string) string {
 // FSI is a repo-side struct for coordinating file system integration
 type FSI struct {
 	// repository for resolving dataset names
-	repo repo.Repo
-	pub  event.Publisher
+	repo         repo.Repo
+	pub          event.Publisher
+	onChangeHook func(hook.DsChange)
 }
 
 // NewFSI creates an FSI instance from a path to a links flatbuffer file
@@ -157,6 +159,15 @@ func (fsi *FSI) CreateLink(dirPath, refStr string) (alias string, rollback func(
 		Dsname:   datasetRef.Name,
 	})
 
+	if fsi.onChangeHook != nil {
+		fsi.onChangeHook(hook.DsChange{
+			Type:       hook.DatasetCreateLink,
+			Username:   datasetRef.Peername,
+			PrettyName: datasetRef.Name,
+			Dir:        dirPath,
+		})
+	}
+
 	return datasetRef.AliasString(), removeLinkAndRemoveRefFunc, err
 }
 
@@ -252,6 +263,11 @@ func (fsi *FSI) RemoveAll(dirPath string) error {
 	return nil
 }
 
+// SetChangeHook assigns a hook that will be called when a dataset changes
+func (fsi *FSI) SetChangeHook(changeHook func(hook.DsChange)) {
+	fsi.onChangeHook = changeHook
+}
+
 func (fsi *FSI) getRepoRef(refStr string) (ref reporef.DatasetRef, err error) {
 	ref, err = repo.ParseDatasetRef(refStr)
 	if err != nil {