Skip to content

Commit

Permalink
feat: refactor cache usage
Browse files Browse the repository at this point in the history
Signed-off-by: Brian McGee <[email protected]>
  • Loading branch information
brianmcgee committed Jul 23, 2024
1 parent 9253e50 commit 9cfe7db
Show file tree
Hide file tree
Showing 11 changed files with 407 additions and 311 deletions.
361 changes: 130 additions & 231 deletions cache/cache.go

Large diffs are not rendered by default.

4 changes: 4 additions & 0 deletions cli/cli.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package cli

import (
"git.numtide.com/numtide/treefmt/cache"
"os"

"github.com/gobwas/glob"
Expand Down Expand Up @@ -41,6 +42,9 @@ type Format struct {
formatters map[string]*format.Formatter
globalExcludes []glob.Glob

cache *cache.Cache

walker walker.Walker
fileCh chan *walker.File
formattedCh chan *walker.File
processedCh chan *walker.File
Expand Down
90 changes: 50 additions & 40 deletions cli/format.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,11 @@ import (
"syscall"
"time"

"git.numtide.com/numtide/treefmt/cache"

"git.numtide.com/numtide/treefmt/format"
"git.numtide.com/numtide/treefmt/stats"

"git.numtide.com/numtide/treefmt/cache"
"git.numtide.com/numtide/treefmt/config"
"git.numtide.com/numtide/treefmt/walker"

Expand Down Expand Up @@ -53,13 +54,6 @@ func (f *Format) Run() (err error) {
// create a prefixed logger
log.SetPrefix("format")

// ensure cache is closed on return
defer func() {
if err := cache.Close(); err != nil {
log.Errorf("failed to close cache: %v", err)
}
}()

// find the config file unless specified
if f.ConfigFile == "" {
pwd, err := os.Getwd()
Expand Down Expand Up @@ -119,12 +113,37 @@ func (f *Format) Run() (err error) {
f.formatters[name] = formatter
}

// open the cache if configured
if !f.NoCache {
if err = cache.Open(f.TreeRoot, f.ClearCache, f.formatters); err != nil {
// if we can't open the cache, we log a warning and fallback to no cache
log.Warnf("failed to open cache: %v", err)
f.NoCache = true
// initialise the cache
f.cache, err = cache.Open(f.TreeRoot, f.NoCache)
if err != nil {
return fmt.Errorf("failed to open cache: %w", err)
}

// ensure the cache is closed on shutdown
defer func() {
if err := f.cache.Close(); err != nil {
log.Errorf("failed to close cache: %v", err)
}
log.Debug("successfully closed cache")
}()

// check if the formatters set has changed in any meaningful way
if err = format.CheckFormatters(f.cache, f.formatters); err != nil {
return fmt.Errorf("failed to check formatters: %w", err)
}
log.Debugf("formatters check complete")

// clear out the paths bucket if desired before starting
if f.ClearCache {
err = f.cache.Update(func(tx *cache.Tx) error {
paths, err := tx.Paths()
if err != nil {
return err
}
return paths.DeleteAll()
})
if err != nil {
return fmt.Errorf("failed to clear paths from cache: %w", err)
}
}

Expand Down Expand Up @@ -222,35 +241,26 @@ func (f *Format) walkFilesystem(ctx context.Context) func() error {
}

// create a filesystem walker
wk, err := walker.New(walkerType, f.TreeRoot, f.NoCache, pathCh)
var err error
f.walker, err = walker.New(walkerType, f.TreeRoot, f.cache, pathCh)
if err != nil {
return fmt.Errorf("failed to create walker: %w", err)
}

// close the file channel when we're done walking the file system
defer close(f.fileCh)

// if no cache has been configured, or we are processing from stdin, we invoke the walker directly
if f.NoCache || f.Stdin {
return wk.Walk(ctx, func(file *walker.File, err error) error {
select {
case <-ctx.Done():
return ctx.Err()
default:
stats.Add(stats.Traversed, 1)
stats.Add(stats.Emitted, 1)
f.fileCh <- file
return nil
}
})
}

// otherwise we pass the walker to the cache and have it generate files for processing based on whether or not
// they have been added/changed since the last invocation
if err = cache.ChangeSet(ctx, wk, f.fileCh); err != nil {
return fmt.Errorf("failed to generate change set: %w", err)
}
return nil
//
return f.walker.Walk(ctx, func(file *walker.File, err error) error {
select {
case <-ctx.Done():
return ctx.Err()
default:
stats.Add(stats.Emitted, 1)
f.fileCh <- file
return nil
}
})
}
}

Expand Down Expand Up @@ -423,17 +433,17 @@ func (f *Format) updateCache(ctx context.Context) func() error {

// apply a batch
processBatch := func() error {
// pass the batch to the cache for updating
if err := cache.Update(batch); err != nil {
// let the walker record updated path info
if err := f.walker.UpdatePaths(batch); err != nil {
return err
}
// reset the batch
batch = batch[:0]
return nil
}

// if we are processing from stdin that means we are outputting to stdout, no caching involved
// if f.NoCache is set that means either the user explicitly disabled the cache or we failed to open on
if f.Stdin || f.NoCache {
if f.Stdin {
// do nothing
processBatch = func() error { return nil }
}
Expand Down
6 changes: 3 additions & 3 deletions cli/format_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -412,7 +412,7 @@ func TestBustCacheOnFormatterChange(t *testing.T) {
}

test.WriteConfig(t, configPath, cfg)
args := []string{"--config-file", configPath, "--tree-root", tempDir}
args := []string{"--config-file", configPath, "--tree-root", tempDir, "-vv"}
_, err := cmd(t, args...)
as.NoError(err)
assertStats(t, as, 32, 32, 3, 0)
Expand All @@ -425,8 +425,8 @@ func TestBustCacheOnFormatterChange(t *testing.T) {
assertStats(t, as, 32, 32, 3, 0)

// check cache is working
_, err = cmd(t, args...)
as.NoError(err)
out, err := cmd(t, args...)
as.NoError(err, string(out))
assertStats(t, as, 32, 0, 0, 0)

// tweak mod time of python formatter
Expand Down
93 changes: 93 additions & 0 deletions format/cache.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
package format

import (
"fmt"
"git.numtide.com/numtide/treefmt/cache"
"github.com/charmbracelet/log"
"os"
)

func CheckFormatters(c *cache.Cache, formatters map[string]*Formatter) error {
return c.Update(func(tx *cache.Tx) error {

clearPaths := false

pathsBucket, err := tx.Paths()
if err != nil {
return fmt.Errorf("failed to get paths bucket from cache: %w", err)
}

formattersBucket, err := tx.Formatters()
if err != nil {
return fmt.Errorf("failed to get formatters bucket from cache: %w", err)
}

// check for any newly configured or modified formatters
for name, formatter := range formatters {

stat, err := os.Lstat(formatter.Executable())
if err != nil {
return fmt.Errorf("failed to stat formatter executable %v: %w", formatter.Executable(), err)
}

entry, err := formattersBucket.Get(name)
if err != nil {
return fmt.Errorf("failed to retrieve cache entry for formatter %v: %w", name, err)
}

isNew := entry == nil
hasChanged := entry != nil && !(entry.Size == stat.Size() && entry.Modified == stat.ModTime())

if isNew {
log.Debugf("formatter '%s' is new", name)
} else if hasChanged {
log.Debug("formatter '%s' has changed",
name,
"size", stat.Size(),
"modTime", stat.ModTime(),
"cachedSize", entry.Size,
"cachedModTime", entry.Modified,
)
}

// update overall flag
clearPaths = clearPaths || isNew || hasChanged

// record formatters info
entry = &cache.Entry{
Size: stat.Size(),
Modified: stat.ModTime(),
}

if err = formattersBucket.Put(name, entry); err != nil {
return fmt.Errorf("failed to write cache entry for formatter %v: %w", name, err)
}
}

// check for any removed formatters
if err = formattersBucket.ForEach(func(key string, _ *cache.Entry) error {
_, ok := formatters[key]
if !ok {
// remove the formatter entry from the cache
if err = formattersBucket.Delete(key); err != nil {
return fmt.Errorf("failed to remove cache entry for formatter %v: %w", key, err)
}
// indicate a clean is required
clearPaths = true
}
return nil
}); err != nil {
return fmt.Errorf("failed to check cache for removed formatters: %w", err)
}

if clearPaths {
// remove all path entries
if err := pathsBucket.DeleteAll(); err != nil {
return fmt.Errorf("failed to remove all path entries from cache: %w", err)
}
}

return nil
})

}
20 changes: 2 additions & 18 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -46,12 +46,8 @@ github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+
github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow=
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4=
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
github.com/go-git/go-git/v5 v5.12.0 h1:7Md+ndsjrzZxbddRDZjF14qK+NN56sy6wkqaVrjZtys=
github.com/go-git/go-git/v5 v5.12.0/go.mod h1:FTM9VKtnI2m65hNI/TenDDDnUf2Q9FHnXYjuz9i5OEY=
github.com/go-git/go-git/v5 v5.12.1-0.20240409060936-cd6633c3c665 h1:+M6D6MplKwRqmw8b41MArUGFsGTnl24+/S40I0yrhKs=
github.com/go-git/go-git/v5 v5.12.1-0.20240409060936-cd6633c3c665/go.mod h1:QZbSbsaXQD7v0yvddAhVE2UfW5wGmeqHQ0UnOSr6JYQ=
github.com/go-git/go-git/v5 v5.12.1-0.20240516215126-9cc340a7fc5c h1:Pyuh3Y6kb/+k7yl5nLN6pcQ+7isiJ1PnBj3QSUH1hbQ=
github.com/go-git/go-git/v5 v5.12.1-0.20240516215126-9cc340a7fc5c/go.mod h1:Tzg+feu/PVazFrRFtWHxfvErT+p+Zo3Yg0nzEhJdxW4=
github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4=
github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
Expand Down Expand Up @@ -120,21 +116,15 @@ github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV
github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI=
go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE=
go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0=
go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY=
golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI=
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM=
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
Expand All @@ -146,8 +136,6 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
Expand All @@ -169,28 +157,24 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
Expand Down
4 changes: 2 additions & 2 deletions test/examples/ruby/bundler.rb
Original file line number Diff line number Diff line change
Expand Up @@ -155,11 +155,11 @@ def specs_path
end

def cache
bundle_path.join("cache/bundler")
bundle_path.join("caching/bundler")
end

def user_cache
user_bundle_path.join("cache")
user_bundle_path.join("caching")
end

def root
Expand Down
Loading

0 comments on commit 9cfe7db

Please sign in to comment.