Skip to content

Commit

Permalink
Refactor virtual file cache to ensure generators only run once per sy…
Browse files Browse the repository at this point in the history
…nc (#268)

* package/budfs: make target generator absolute to the filesystem

* rework cache to only run generators at most once per sync
  • Loading branch information
matthewmueller authored Sep 2, 2022
1 parent 701c178 commit a017868
Show file tree
Hide file tree
Showing 19 changed files with 342 additions and 376 deletions.
3 changes: 2 additions & 1 deletion example/basic/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ require (
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
github.com/otiai10/copy v1.7.0 // indirect
github.com/timewasted/go-accept-headers v0.0.0-20130320203746-c78f304b1b09 // indirect
golang.org/x/mod v0.5.1 // indirect
github.com/xlab/treeprint v1.1.0 // indirect
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
Expand Down
5 changes: 5 additions & 0 deletions example/basic/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -51,21 +51,26 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/timewasted/go-accept-headers v0.0.0-20130320203746-c78f304b1b09 h1:QVxbx5l/0pzciWYOynixQMtUhPYC3YKD6EcUlOsgGqw=
github.com/timewasted/go-accept-headers v0.0.0-20130320203746-c78f304b1b09/go.mod h1:Uy/Rnv5WKuOO+PuDhuYLEpUiiKIZtss3z519uk67aF0=
github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk=
github.com/xlab/treeprint v1.1.0 h1:G/1DjNkPpfZCFt9CSh6b5/nY4VimlbHF3Rh4obvtzDk=
github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=
go.kuoruan.net/v8go-polyfills v0.5.0 h1:wd2WxsFIXWK/FcrpITw6BOo8Rn24xMmd4qoHofgg8hc=
golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38=
golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f h1:8w7RhxzTVgUzw/AH/9mUV5q0vMgy40SQRursCcfmkCw=
golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/tools v0.1.9 h1:j9KsMiaP1c3B0OTQGth0/k+miLGTgLsAFUCrF2vLcF8=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
Expand Down
7 changes: 5 additions & 2 deletions internal/dsync/dsync_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (

"github.com/livebud/bud/internal/dsync"
"github.com/livebud/bud/internal/is"
"github.com/livebud/bud/internal/virtual/vcache"
"github.com/livebud/bud/package/budfs/genfs"
"github.com/livebud/bud/package/vfs"
)
Expand Down Expand Up @@ -184,7 +185,8 @@ func TestSkipNotExist(t *testing.T) {
vfs.Now = func() time.Time { return after }

// starting points
sourceFS := genfs.New()
cache := vcache.New()
sourceFS := genfs.New(cache)
sourceFS.GenerateFile("bud/generate/main.go", func(file *genfs.File) error {
return fs.ErrNotExist
})
Expand All @@ -203,7 +205,8 @@ func TestErrorGenerator(t *testing.T) {
vfs.Now = func() time.Time { return after }

// starting points
sourceFS := genfs.New()
cache := vcache.New()
sourceFS := genfs.New(cache)
sourceFS.GenerateFile("bud/generate/main.go", func(file *genfs.File) error {
return errors.New("uh oh")
})
Expand Down
14 changes: 14 additions & 0 deletions internal/virtual/vcache/discard.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package vcache

import "github.com/livebud/bud/internal/virtual"

var Discard Cache = discard{}

type discard struct{}

func (discard) Has(path string) (ok bool) { return false }
func (discard) Get(path string) (entry virtual.Entry, ok bool) { return nil, false }
func (discard) Set(path string, entry virtual.Entry) {}
func (discard) Delete(path string) {}
func (discard) Range(fn func(path string, entry virtual.Entry) bool) {}
func (discard) Clear() {}
70 changes: 70 additions & 0 deletions internal/virtual/vcache/memory.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
package vcache

import (
"sync"

"github.com/livebud/bud/internal/virtual"
)

type Cache interface {
Has(path string) (ok bool)
Get(path string) (entry virtual.Entry, ok bool)
Set(path string, entry virtual.Entry)
Delete(path string)
Range(fn func(path string, entry virtual.Entry) bool)
Clear()
}

func New() Cache {
return &memory{}
}

type memory struct {
sm sync.Map
}

func (c *memory) Has(path string) (ok bool) {
_, ok = c.sm.Load(path)
return ok
}

func (c *memory) Set(path string, entry virtual.Entry) {
c.sm.Store(path, entry)
}

func (c *memory) Get(path string) (entry virtual.Entry, ok bool) {
value, ok := c.sm.Load(path)
if !ok {
return nil, false
}
entry, ok = value.(virtual.Entry)
if !ok {
return nil, false
}
return entry, ok
}

func (c *memory) Delete(path string) {
c.sm.Delete(path)
}

func (c *memory) Range(fn func(path string, entry virtual.Entry) bool) {
c.sm.Range(func(key, value interface{}) bool {
path, ok := key.(string)
if !ok {
return true
}
entry, ok := value.(virtual.Entry)
if !ok {
return true
}
return fn(path, entry)
})
}

func (c *memory) Clear() {
c.sm.Range(func(key, value interface{}) bool {
c.sm.Delete(key)
return true
})
}
7 changes: 7 additions & 0 deletions internal/virtual/virtual.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,10 @@ import "io/fs"
type Entry interface {
Open() fs.File
}

// Opener is a utility function that implements fs.FS
type Opener func(name string) (fs.File, error)

func (fn Opener) Open(name string) (fs.File, error) {
return fn(name)
}
22 changes: 1 addition & 21 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

34 changes: 28 additions & 6 deletions package/budfs/budfs.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,18 @@ import (
"github.com/livebud/bud/internal/dag"
"github.com/livebud/bud/internal/dsync"
"github.com/livebud/bud/internal/once"
"github.com/livebud/bud/package/budfs/cachefs"
"github.com/livebud/bud/internal/virtual"
"github.com/livebud/bud/internal/virtual/vcache"
"github.com/livebud/bud/package/budfs/genfs"
"github.com/livebud/bud/package/budfs/mergefs"
"github.com/livebud/bud/package/log"
)

func New(fsys vfs.ReadWritable, log log.Interface) *FileSystem {
gen := genfs.New()
cache := cachefs.New(log)
func New(cache vcache.Cache, fsys vfs.ReadWritable, log log.Interface) *FileSystem {
gen := genfs.New(cache)
merged := mergefs.Merge(gen, fsys)
syncfs := cache.Wrap(merged)
server := http.FileServer(http.FS(merged))
syncfs := wrapFS(cache, merged, log)
return &FileSystem{
cache: cache,
closer: new(once.Closer),
Expand All @@ -36,7 +36,7 @@ func New(fsys vfs.ReadWritable, log log.Interface) *FileSystem {
}

type FileSystem struct {
cache *cachefs.Cache
cache vcache.Cache
closer *once.Closer
dag *dag.Graph
gen *genfs.FileSystem
Expand Down Expand Up @@ -237,3 +237,25 @@ func (f *FileSystem) Delete(filepath string) {
f.cache.Delete(ancestor)
}
}

// wrapFS wraps a filesystem with a cache
func wrapFS(c vcache.Cache, fsys fs.FS, log log.Interface) fs.FS {
return virtual.Opener(func(name string) (fs.File, error) {
entry, ok := c.Get(name)
if ok {
log.Debug("cachefs: cache hit", "file", name)
return entry.Open(), nil
}
log.Debug("cachefs: cache miss", "file", name)
file, err := fsys.Open(name)
if err != nil {
return nil, err
}
entry, err = virtual.From(file)
if err != nil {
return nil, err
}
c.Set(name, entry)
return entry.Open(), nil
})
}
Loading

0 comments on commit a017868

Please sign in to comment.