From ac13194ded48eab1d3fa5b500259565da81eaa66 Mon Sep 17 00:00:00 2001 From: Brian McGee Date: Thu, 17 Oct 2024 18:18:54 +0100 Subject: [PATCH] fixup! feat: improve formatter/config change detection Signed-off-by: Brian McGee --- cmd/format/format.go | 7 -- cmd/root_test.go | 14 +-- format/composite.go | 166 ++++++++++++++-------------- go.mod | 5 +- go.sum | 4 - nix/packages/treefmt/gomod2nix.toml | 9 +- walk/cache/bucket.go | 96 ---------------- walk/cache/cache.go | 47 ++++---- walk/cached.go | 35 ++---- walk/cached_test.go | 137 ----------------------- walk/walk.go | 5 +- 11 files changed, 134 insertions(+), 391 deletions(-) delete mode 100644 walk/cache/bucket.go delete mode 100644 walk/cached_test.go diff --git a/cmd/format/format.go b/cmd/format/format.go index 40dcc094..9a4dffab 100644 --- a/cmd/format/format.go +++ b/cmd/format/format.go @@ -161,13 +161,6 @@ func Run(v *viper.Viper, statz *stats.Stats, cmd *cobra.Command, paths []string) return fmt.Errorf("failed to create composite formatter: %w", err) } - if db != nil { - // compare formatters with the db, busting the cache if the formatters have changed - if err := formatter.BustCache(db); err != nil { - return fmt.Errorf("failed to compare formatters: %w", err) - } - } - // create a new walker for traversing the paths walker, err := walk.NewCompositeReader(walkType, cfg.TreeRoot, paths, db, statz) if err != nil { diff --git a/cmd/root_test.go b/cmd/root_test.go index 5421ea87..0b24c774 100644 --- a/cmd/root_test.go +++ b/cmd/root_test.go @@ -656,7 +656,7 @@ func TestBustCacheOnFormatterChange(t *testing.T) { assertStats(t, as, statz, map[stats.Type]int{ stats.Traversed: 32, stats.Matched: 3, - stats.Formatted: 3, + stats.Formatted: 1, stats.Changed: 0, }) @@ -680,7 +680,7 @@ func TestBustCacheOnFormatterChange(t *testing.T) { assertStats(t, as, statz, map[stats.Type]int{ stats.Traversed: 32, stats.Matched: 3, - stats.Formatted: 3, + stats.Formatted: 2, stats.Changed: 0, }) @@ -710,7 +710,7 @@ func TestBustCacheOnFormatterChange(t *testing.T) { assertStats(t, as, statz, map[stats.Type]int{ stats.Traversed: 32, stats.Matched: 4, - stats.Formatted: 4, + stats.Formatted: 1, stats.Changed: 0, }) @@ -736,7 +736,7 @@ func TestBustCacheOnFormatterChange(t *testing.T) { assertStats(t, as, statz, map[stats.Type]int{ stats.Traversed: 32, stats.Matched: 4, - stats.Formatted: 4, + stats.Formatted: 1, stats.Changed: 0, }) @@ -750,7 +750,7 @@ func TestBustCacheOnFormatterChange(t *testing.T) { assertStats(t, as, statz, map[stats.Type]int{ stats.Traversed: 32, stats.Matched: 4, - stats.Formatted: 4, + stats.Formatted: 1, stats.Changed: 0, }) @@ -764,7 +764,7 @@ func TestBustCacheOnFormatterChange(t *testing.T) { assertStats(t, as, statz, map[stats.Type]int{ stats.Traversed: 32, stats.Matched: 2, - stats.Formatted: 2, + stats.Formatted: 0, stats.Changed: 0, }) @@ -789,7 +789,7 @@ func TestBustCacheOnFormatterChange(t *testing.T) { assertStats(t, as, statz, map[stats.Type]int{ stats.Traversed: 32, stats.Matched: 1, - stats.Formatted: 1, + stats.Formatted: 0, stats.Changed: 0, }) diff --git a/format/composite.go b/format/composite.go index 80da316b..4de0d97f 100644 --- a/format/composite.go +++ b/format/composite.go @@ -1,6 +1,7 @@ package format import ( + "bytes" "cmp" "context" "crypto/sha256" @@ -19,8 +20,6 @@ import ( "github.com/numtide/treefmt/config" "github.com/numtide/treefmt/stats" "github.com/numtide/treefmt/walk" - "github.com/numtide/treefmt/walk/cache" - bolt "go.etcd.io/bbolt" "golang.org/x/sync/errgroup" "mvdan.cc/sh/v3/expand" ) @@ -52,36 +51,81 @@ func newBatchKey(formatters []*Formatter) batchKey { return batchKey(strings.Join(components, batchKeySeparator)) } -// batchMap maintains a mapping between batchKey and a slice of pointers to walk.File, used to organize files into -// batches based on the sequence of formatters to be applied. -type batchMap map[batchKey][]*walk.File +type batcher struct { + batchSize int + batches map[batchKey][]*walk.File + formatters map[string]*Formatter -func formatterSortFunc(a, b *Formatter) int { - // sort by priority in ascending order - priorityA := a.Priority() - priorityB := b.Priority() + signatures map[batchKey][]byte +} - result := priorityA - priorityB - if result == 0 { - // formatters with the same priority are sorted lexicographically to ensure a deterministic outcome - result = cmp.Compare(a.Name(), b.Name()) +func (b *batcher) formatSignature(key batchKey, matches []*Formatter) ([]byte, error) { + signature, ok := b.signatures[key] + if ok { + return signature, nil } - return result + h := sha256.New() + for _, f := range matches { + if err := f.Hash(h); err != nil { + return nil, fmt.Errorf("failed to hash formatter %s: %w", f.Name(), err) + } + } + + signature = h.Sum(nil) + b.signatures[key] = signature + + return signature, nil } -// Append adds a file to the batch corresponding to the given sequence of formatters and returns the updated batch. -func (b batchMap) Append(file *walk.File, matches []*Formatter) (key batchKey, batch []*walk.File) { +func (b *batcher) append( + file *walk.File, + matches []*Formatter, +) (appended bool, key batchKey, batch []*walk.File, err error) { slices.SortFunc(matches, formatterSortFunc) // construct a batch key based on the sequence of formatters key = newBatchKey(matches) + // get format signature + formatSignature, err := b.formatSignature(key, matches) + if err != nil { + return false, "", nil, err + } + + // + h := sha256.New() + h.Write(formatSignature) + h.Write([]byte(fmt.Sprintf("%v %v", file.Info.ModTime().Unix(), file.Info.Size()))) + + file.FormatSignature = h.Sum(nil) + + if bytes.Equal(file.FormatSignature, file.CachedFormatSignature) { + return false, key, b.batches[key], nil + } + // append to the batch - b[key] = append(b[key], file) + b.batches[key] = append(b.batches[key], file) + + return true, key, b.batches[key], nil +} + +func (b *batcher) reset(key batchKey) { + b.batches[key] = make([]*walk.File, 0, b.batchSize) +} + +func formatterSortFunc(a, b *Formatter) int { + // sort by priority in ascending order + priorityA := a.Priority() + priorityB := b.Priority() - // return the batch - return key, b[key] + result := priorityA - priorityB + if result == 0 { + // formatters with the same priority are sorted lexicographically to ensure a deterministic outcome + result = cmp.Compare(a.Name(), b.Name()) + } + + return result } // CompositeFormatter handles the application of multiple Formatter instances based on global excludes and individual @@ -99,7 +143,7 @@ type CompositeFormatter struct { formatters map[string]*Formatter eg *errgroup.Group - batches batchMap + batcher *batcher // formatError indicates if at least one formatting error occurred formatError *atomic.Bool @@ -216,16 +260,17 @@ func (c *CompositeFormatter) Apply(ctx context.Context, files []*walk.File) erro // record there was a match c.stats.Add(stats.Matched, 1) - // check if the file is new or has changed when compared to the cache entry - if file.Cache == nil || file.Cache.HasChanged(file.Info) { - // add this file to a batch and if it's full, apply formatters to the batch - if key, batch := c.batches.Append(file, matches); len(batch) == c.batchSize { - c.apply(ctx, newBatchKey(matches), batch) - // reset the batch - c.batches[key] = make([]*walk.File, 0, c.batchSize) - } - } else { - // no further processing to be done, append to the release list + appended, key, batch, batchErr := c.batcher.append(file, matches) + if batchErr != nil { + return fmt.Errorf("failed to append file to batch: %w", batchErr) + } + + if len(batch) == c.batchSize { + c.apply(ctx, key, batch) + c.batcher.reset(key) + } + + if !appended { toRelease = append(toRelease, file) } } @@ -247,7 +292,8 @@ func (c *CompositeFormatter) Apply(ctx context.Context, files []*walk.File) erro // all formatters have completed their tasks. It returns an error if any formatting failures were detected. func (c *CompositeFormatter) Close(ctx context.Context) error { // flush any partial batches that remain - for key, batch := range c.batches { + // todo clean up + for key, batch := range c.batcher.batches { if len(batch) > 0 { c.apply(ctx, key, batch) } @@ -289,54 +335,6 @@ func (c *CompositeFormatter) Hash() (string, error) { return hex.EncodeToString(h.Sum(nil)), nil } -// BustCache compares the current Hash() of this formatter with the last entry stored in the db. -// If it has changed, we remove all the path entries, forcing a fresh cache state. -func (c *CompositeFormatter) BustCache(db *bolt.DB) error { - // determine current hash - currentHash, err := c.Hash() - if err != nil { - return fmt.Errorf("failed to hash formatter: %w", err) - } - - hashKey := []byte("sha256") - bucketKey := []byte("formatters") - - return db.Update(func(tx *bolt.Tx) error { - // get or create the formatters bucket - bucket, err := tx.CreateBucketIfNotExists(bucketKey) - if err != nil { - return fmt.Errorf("failed to create formatters bucket: %w", err) - } - - // load the previous hash which might be nil - prevHash := bucket.Get(hashKey) - - c.log.Debug( - "comparing formatter hash with db", - "prev_hash", string(prevHash), - "current_hash", currentHash, - ) - - // compare with the previous hash - // if they are different, delete all the path entries - if string(prevHash) != currentHash { - c.log.Debug("hash has changed, deleting all paths") - - paths, err := cache.BucketPaths(tx) - if err != nil { - return fmt.Errorf("failed to get paths bucket from cache: %w", err) - } - - if err = paths.DeleteAll(); err != nil { - return fmt.Errorf("failed to delete paths bucket: %w", err) - } - } - - // save the latest hash - return bucket.Put(hashKey, []byte(currentHash)) - }) -} - func NewCompositeFormatter( cfg *config.Config, statz *stats.Stats, @@ -380,6 +378,14 @@ func NewCompositeFormatter( formatters[name] = formatter } + // + batcher := batcher{ + batchSize: batchSize, + formatters: formatters, + batches: make(map[batchKey][]*walk.File), + signatures: make(map[batchKey][]byte), + } + // create an errgroup for asynchronously formatting eg := errgroup.Group{} // we use a simple heuristic to avoid too much contention by limiting the concurrency to runtime.NumCPU() @@ -396,8 +402,8 @@ func NewCompositeFormatter( unmatchedLevel: unmatchedLevel, eg: &eg, + batcher: &batcher, formatters: formatters, - batches: make(batchMap), formatError: new(atomic.Bool), }, nil } diff --git a/go.mod b/go.mod index 7612b490..c41d6525 100644 --- a/go.mod +++ b/go.mod @@ -8,13 +8,14 @@ require ( github.com/charmbracelet/log v0.4.0 github.com/gobwas/glob v0.2.3 github.com/otiai10/copy v1.14.0 + github.com/rogpeppe/go-internal v1.12.0 github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 - github.com/vmihailenco/msgpack/v5 v5.4.1 go.etcd.io/bbolt v1.3.11 golang.org/x/sync v0.8.0 + golang.org/x/sys v0.25.0 mvdan.cc/sh/v3 v3.9.0 ) @@ -43,11 +44,9 @@ require ( github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect - github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect - golang.org/x/sys v0.25.0 // indirect golang.org/x/term v0.24.0 // indirect golang.org/x/text v0.18.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect diff --git a/go.sum b/go.sum index 9cff6309..846246d1 100644 --- a/go.sum +++ b/go.sum @@ -99,10 +99,6 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= -github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8= -github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= -github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= -github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= diff --git a/nix/packages/treefmt/gomod2nix.toml b/nix/packages/treefmt/gomod2nix.toml index 9f6a2cf6..71bb3cdc 100644 --- a/nix/packages/treefmt/gomod2nix.toml +++ b/nix/packages/treefmt/gomod2nix.toml @@ -70,6 +70,9 @@ schema = 3 [mod."github.com/rivo/uniseg"] version = "v0.4.7" hash = "sha256-rDcdNYH6ZD8KouyyiZCUEy8JrjOQoAkxHBhugrfHjFo=" + [mod."github.com/rogpeppe/go-internal"] + version = "v1.12.0" + hash = "sha256-qvDNCe3l84/LgrA8X4O15e1FeDcazyX91m9LmXGXX6M=" [mod."github.com/sagikazarmark/locafero"] version = "v0.4.0" hash = "sha256-7I1Oatc7GAaHgAqBFO6Tv4IbzFiYeU9bJAfJhXuWaXk=" @@ -100,12 +103,6 @@ schema = 3 [mod."github.com/subosito/gotenv"] version = "v1.6.0" hash = "sha256-LspbjTniiq2xAICSXmgqP7carwlNaLqnCTQfw2pa80A=" - [mod."github.com/vmihailenco/msgpack/v5"] - version = "v5.4.1" - hash = "sha256-pDplX6xU6UpNLcFbO1pRREW5vCnSPvSU+ojAwFDv3Hk=" - [mod."github.com/vmihailenco/tagparser/v2"] - version = "v2.0.0" - hash = "sha256-M9QyaKhSmmYwsJk7gkjtqu9PuiqZHSmTkous8VWkWY0=" [mod."go.etcd.io/bbolt"] version = "v1.3.11" hash = "sha256-SVWYZtE9TBgAo8xJSmo9DtSwuNa056N3zGvPLDJgiA8=" diff --git a/walk/cache/bucket.go b/walk/cache/bucket.go deleted file mode 100644 index 16221305..00000000 --- a/walk/cache/bucket.go +++ /dev/null @@ -1,96 +0,0 @@ -package cache - -import ( - "fmt" - - "github.com/vmihailenco/msgpack/v5" - bolt "go.etcd.io/bbolt" -) - -const ( - bucketPaths = "paths" - bucketFormatters = "formatters" -) - -var ErrKeyNotFound = fmt.Errorf("key not found") - -type Bucket[V any] struct { - bucket *bolt.Bucket -} - -func (b *Bucket[V]) Size() int { - return b.bucket.Stats().KeyN -} - -func (b *Bucket[V]) Get(key string) (*V, error) { - bytes := b.bucket.Get([]byte(key)) - if bytes == nil { - return nil, ErrKeyNotFound - } - - var value V - if err := msgpack.Unmarshal(bytes, &value); err != nil { - return nil, fmt.Errorf("failed to unmarshal cache entry for key '%v': %w", key, err) - } - - return &value, nil -} - -func (b *Bucket[V]) Put(key string, value *V) error { - if bytes, err := msgpack.Marshal(value); err != nil { - return fmt.Errorf("failed to marshal cache entry for key %v: %w", key, err) - } else if err = b.bucket.Put([]byte(key), bytes); err != nil { - return fmt.Errorf("failed to put cache entry for key %v: %w", key, err) - } - - return nil -} - -func (b *Bucket[V]) Delete(key string) error { - return b.bucket.Delete([]byte(key)) -} - -func (b *Bucket[V]) DeleteAll() error { - c := b.bucket.Cursor() - for k, v := c.First(); !(k == nil && v == nil); k, v = c.Next() { - if err := c.Delete(); err != nil { - return fmt.Errorf("failed to remove cache entry for key %s: %w", string(k), err) - } - } - - return nil -} - -func (b *Bucket[V]) ForEach(f func(string, *V) error) error { - return b.bucket.ForEach(func(key, bytes []byte) error { - var value V - if err := msgpack.Unmarshal(bytes, &value); err != nil { - return fmt.Errorf("failed to unmarshal cache entry for key '%v': %w", key, err) - } - - return f(string(key), &value) - }) -} - -func BucketPaths(tx *bolt.Tx) (*Bucket[Entry], error) { - return cacheBucket(bucketPaths, tx) -} - -func cacheBucket(name string, tx *bolt.Tx) (*Bucket[Entry], error) { - var ( - err error - b *bolt.Bucket - ) - - if tx.Writable() { - b, err = tx.CreateBucketIfNotExists([]byte(name)) - } else { - b = tx.Bucket([]byte(name)) - } - - if err != nil { - return nil, fmt.Errorf("failed to get/create bucket %s: %w", bucketPaths, err) - } - - return &Bucket[Entry]{b}, nil -} diff --git a/walk/cache/cache.go b/walk/cache/cache.go index ba18af5d..b80cbd38 100644 --- a/walk/cache/cache.go +++ b/walk/cache/cache.go @@ -4,21 +4,15 @@ import ( "crypto/sha1" //nolint:gosec "encoding/hex" "fmt" - "io/fs" "time" "github.com/adrg/xdg" bolt "go.etcd.io/bbolt" ) -type Entry struct { - Size int64 - Modified time.Time -} - -func (e *Entry) HasChanged(info fs.FileInfo) bool { - return !(e.Modified == info.ModTime() && e.Size == info.Size()) -} +const ( + bucketPaths = "paths" +) func Open(root string) (*bolt.DB, error) { var ( @@ -42,25 +36,36 @@ func Open(root string) (*bolt.DB, error) { return nil, err } + // ensure bucket exist + err = db.Update(func(tx *bolt.Tx) error { + _, err := tx.CreateBucketIfNotExists([]byte(bucketPaths)) + + return err + }) + if err != nil { + return nil, fmt.Errorf("failed to create bucket: %w", err) + } + return db, nil } -func EnsureBuckets(db *bolt.DB) error { - // force creation of buckets if they don't already exist - return db.Update(func(tx *bolt.Tx) error { - _, err := BucketPaths(tx) +func PathsBucket(tx *bolt.Tx) *bolt.Bucket { + return tx.Bucket([]byte("paths")) +} - return err - }) +func deleteAll(bucket *bolt.Bucket) error { + c := bucket.Cursor() + for k, v := c.First(); !(k == nil && v == nil); k, v = c.Next() { + if err := c.Delete(); err != nil { + return fmt.Errorf("failed to remove cache entry for key %s: %w", string(k), err) + } + } + + return nil } func Clear(db *bolt.DB) error { return db.Update(func(tx *bolt.Tx) error { - bucket, err := BucketPaths(tx) - if err != nil { - return fmt.Errorf("failed to get paths bucket: %w", err) - } - - return bucket.DeleteAll() + return deleteAll(PathsBucket(tx)) }) } diff --git a/walk/cached.go b/walk/cached.go index cfecc902..c757ede2 100644 --- a/walk/cached.go +++ b/walk/cached.go @@ -52,20 +52,12 @@ func (c *CachedReader) process() error { } return c.db.Update(func(tx *bolt.Tx) error { - // get the paths bucket - bucket, err := cache.BucketPaths(tx) - if err != nil { - return fmt.Errorf("failed to get bucket: %w", err) - } + bucket := cache.PathsBucket(tx) - // for each file in the batch, add a new cache entry with update size and mod time. + // for each file in the batch, update the bucket entry for _, file := range batch { - entry := &cache.Entry{ - Size: file.Info.Size(), - Modified: file.Info.ModTime(), - } - if err = bucket.Put(file.RelPath, entry); err != nil { - return fmt.Errorf("failed to put entry for path %s: %w", file.RelPath, err) + if err := bucket.Put([]byte(file.RelPath), file.FormatSignature); err != nil { + return fmt.Errorf("failed to put format signature for path %s: %w", file.RelPath, err) } } @@ -92,7 +84,8 @@ func (c *CachedReader) process() error { func (c *CachedReader) Read(ctx context.Context, files []*File) (n int, err error) { err = c.db.View(func(tx *bolt.Tx) error { // get paths bucket - bucket, err := cache.BucketPaths(tx) + bucket := cache.PathsBucket(tx) + if err != nil { return fmt.Errorf("failed to get bucket: %w", err) } @@ -104,13 +97,7 @@ func (c *CachedReader) Read(ctx context.Context, files []*File) (n int, err erro for i := 0; i < n; i++ { file := files[i] - // lookup cache entry and append to the file - var bucketErr error - - file.Cache, bucketErr = bucket.Get(file.RelPath) - if !(bucketErr == nil || errors.Is(bucketErr, cache.ErrKeyNotFound)) { - return bucketErr - } + file.CachedFormatSignature = bucket.Get([]byte(file.RelPath)) // set a release function which inserts this file into the update channel file.AddReleaseFunc(func(ctx context.Context) error { @@ -145,13 +132,7 @@ func (c *CachedReader) Close() error { // NewCachedReader creates a cache Reader instance, backed by a bolt DB and delegating reads to delegate. func NewCachedReader(db *bolt.DB, batchSize int, delegate Reader) (*CachedReader, error) { - // force the creation of the necessary buckets if we're dealing with an empty db - if err := cache.EnsureBuckets(db); err != nil { - return nil, fmt.Errorf("failed to create cache buckets: %w", err) - } - - // create an error group for managing the processing loop - eg := &errgroup.Group{} + eg := &errgroup.Group{} // create an error group for managing the processing loop r := &CachedReader{ db: db, diff --git a/walk/cached_test.go b/walk/cached_test.go deleted file mode 100644 index d74f981f..00000000 --- a/walk/cached_test.go +++ /dev/null @@ -1,137 +0,0 @@ -package walk_test - -import ( - "context" - "errors" - "io" - "os" - "path/filepath" - "testing" - "time" - - "github.com/numtide/treefmt/stats" - "github.com/numtide/treefmt/test" - "github.com/numtide/treefmt/walk" - "github.com/numtide/treefmt/walk/cache" - "github.com/stretchr/testify/require" -) - -func TestCachedReader(t *testing.T) { - as := require.New(t) - - batchSize := 1024 - tempDir := test.TempExamples(t) - - readAll := func(path string) (totalCount, newCount, changeCount int) { - statz := stats.New() - - db, err := cache.Open(tempDir) - as.NoError(err) - defer db.Close() - - delegate := walk.NewFilesystemReader(tempDir, path, &statz, batchSize) - reader, err := walk.NewCachedReader(db, batchSize, delegate) - as.NoError(err) - - for { - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) - - files := make([]*walk.File, 8) - n, err := reader.Read(ctx, files) - - totalCount += n - - for idx := 0; idx < n; idx++ { - file := files[idx] - - if file.Cache == nil { - newCount++ - } else if file.Cache.HasChanged(file.Info) { - changeCount++ - } - - as.NoError(file.Release(ctx)) - } - - cancel() - - if errors.Is(err, io.EOF) { - break - } - } - - as.NoError(reader.Close()) - - return totalCount, newCount, changeCount - } - - totalCount, newCount, changeCount := readAll("") - as.Equal(32, totalCount) - as.Equal(32, newCount) - as.Equal(0, changeCount) - - // read again, should be no changes - totalCount, newCount, changeCount = readAll("") - as.Equal(32, totalCount) - as.Equal(0, newCount) - as.Equal(0, changeCount) - - // change mod times on some files and try again - // we subtract a second to account for the 1 second granularity of modtime according to POSIX - modTime := time.Now().Add(-1 * time.Second) - - as.NoError(os.Chtimes(filepath.Join(tempDir, "treefmt.toml"), time.Now(), modTime)) - as.NoError(os.Chtimes(filepath.Join(tempDir, "shell/foo.sh"), time.Now(), modTime)) - as.NoError(os.Chtimes(filepath.Join(tempDir, "haskell/Nested/Foo.hs"), time.Now(), modTime)) - - totalCount, newCount, changeCount = readAll("") - as.Equal(32, totalCount) - as.Equal(0, newCount) - as.Equal(3, changeCount) - - // create some files and try again - _, err := os.Create(filepath.Join(tempDir, "new.txt")) - as.NoError(err) - - _, err = os.Create(filepath.Join(tempDir, "fizz.go")) - as.NoError(err) - - totalCount, newCount, changeCount = readAll("") - as.Equal(34, totalCount) - as.Equal(2, newCount) - as.Equal(0, changeCount) - - // modify some files - f, err := os.OpenFile(filepath.Join(tempDir, "new.txt"), os.O_WRONLY, 0o644) - as.NoError(err) - _, err = f.Write([]byte("foo")) - as.NoError(err) - as.NoError(f.Close()) - - f, err = os.OpenFile(filepath.Join(tempDir, "fizz.go"), os.O_WRONLY, 0o644) - as.NoError(err) - _, err = f.Write([]byte("bla")) - as.NoError(err) - as.NoError(f.Close()) - - totalCount, newCount, changeCount = readAll("") - as.Equal(34, totalCount) - as.Equal(0, newCount) - as.Equal(2, changeCount) - - // read some paths within the root - totalCount, newCount, changeCount = readAll("go") - as.Equal(2, totalCount) - as.Equal(0, newCount) - as.Equal(0, changeCount) - - totalCount, newCount, changeCount = readAll("elm/src") - as.Equal(1, totalCount) - as.Equal(0, newCount) - as.Equal(0, changeCount) - - totalCount, newCount, changeCount = readAll("haskell") - as.Equal(7, totalCount) - as.Equal(0, newCount) - as.Equal(0, changeCount) -} diff --git a/walk/walk.go b/walk/walk.go index 9480563f..c405a481 100644 --- a/walk/walk.go +++ b/walk/walk.go @@ -11,7 +11,6 @@ import ( "time" "github.com/numtide/treefmt/stats" - "github.com/numtide/treefmt/walk/cache" bolt "go.etcd.io/bbolt" ) @@ -35,8 +34,8 @@ type File struct { RelPath string Info fs.FileInfo - // Cache is the latest entry found for this file, if one exists. - Cache *cache.Entry + FormatSignature []byte + CachedFormatSignature []byte releaseFuncs []ReleaseFunc }