improve caching and tracking #34
56
cache/cache.go
vendored
56
cache/cache.go
vendored
|
@ -5,9 +5,7 @@ import (
|
|||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"time"
|
||||
|
||||
|
@ -180,7 +178,7 @@ func putEntry(bucket *bolt.Bucket, path string, entry *Entry) error {
|
|||
|
||||
// ChangeSet is used to walk a filesystem, starting at root, and outputting any new or changed paths using pathsCh.
|
||||
// It determines if a path is new or has changed by comparing against cache entries.
|
||||
func ChangeSet(ctx context.Context, walker walk.Walker, pathsCh chan<- string) error {
|
||||
func ChangeSet(ctx context.Context, walker walk.Walker, filesCh chan<- *walk.File) error {
|
||||
start := time.Now()
|
||||
|
||||
defer func() {
|
||||
|
@ -198,24 +196,21 @@ func ChangeSet(ctx context.Context, walker walk.Walker, pathsCh chan<- string) e
|
|||
}
|
||||
}()
|
||||
|
||||
// for quick removal of tree root from paths
|
||||
relPathOffset := len(walker.Root()) + 1
|
||||
|
||||
return walker.Walk(ctx, func(path string, info fs.FileInfo, err error) error {
|
||||
return walker.Walk(ctx, func(file *walk.File, err error) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: failed to walk path", err)
|
||||
} else if info.IsDir() {
|
||||
} else if file.Info.IsDir() {
|
||||
// ignore directories
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// ignore symlinks
|
||||
if info.Mode()&os.ModeSymlink == os.ModeSymlink {
|
||||
if file.Info.Mode()&os.ModeSymlink == os.ModeSymlink {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
@ -229,13 +224,12 @@ func ChangeSet(ctx context.Context, walker walk.Walker, pathsCh chan<- string) e
|
|||
bucket = tx.Bucket([]byte(pathsBucket))
|
||||
}
|
||||
|
||||
relPath := path[relPathOffset:]
|
||||
cached, err := getEntry(bucket, relPath)
|
||||
cached, err := getEntry(bucket, file.RelPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
changedOrNew := cached == nil || !(cached.Modified == info.ModTime() && cached.Size == info.Size())
|
||||
changedOrNew := cached == nil || !(cached.Modified == file.Info.ModTime() && cached.Size == file.Info.Size())
|
||||
|
||||
stats.Add(stats.Traversed, 1)
|
||||
if !changedOrNew {
|
||||
|
@ -250,7 +244,7 @@ func ChangeSet(ctx context.Context, walker walk.Walker, pathsCh chan<- string) e
|
|||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
pathsCh <- relPath
|
||||
filesCh <- file
|
||||
}
|
||||
|
||||
// close the current tx if we have reached the batch size
|
||||
|
@ -266,47 +260,35 @@ func ChangeSet(ctx context.Context, walker walk.Walker, pathsCh chan<- string) e
|
|||
}
|
||||
|
||||
// Update is used to record updated cache information for the specified list of paths.
|
||||
func Update(treeRoot string, paths []string) (int, error) {
|
||||
func Update(files []*walk.File) error {
|
||||
start := time.Now()
|
||||
defer func() {
|
||||
logger.Infof("finished updating %v paths in %v", len(paths), time.Since(start))
|
||||
logger.Infof("finished processing %v paths in %v", len(files), time.Since(start))
|
||||
}()
|
||||
|
||||
if len(paths) == 0 {
|
||||
return 0, nil
|
||||
if len(files) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var changes int
|
||||
|
||||
return changes, db.Update(func(tx *bolt.Tx) error {
|
||||
return db.Update(func(tx *bolt.Tx) error {
|
||||
bucket := tx.Bucket([]byte(pathsBucket))
|
||||
|
||||
for _, path := range paths {
|
||||
cached, err := getEntry(bucket, path)
|
||||
for _, f := range files {
|
||||
currentInfo, err := os.Stat(f.Path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
pathInfo, err := os.Stat(filepath.Join(treeRoot, path))
|
||||
if err != nil {
|
||||
return err
|
||||
if !(f.Info.ModTime() == currentInfo.ModTime() && f.Info.Size() == currentInfo.Size()) {
|
||||
stats.Add(stats.Formatted, 1)
|
||||
}
|
||||
|
||||
if cached == nil || !(cached.Modified == pathInfo.ModTime() && cached.Size == pathInfo.Size()) {
|
||||
changes += 1
|
||||
} else {
|
||||
// no change to write
|
||||
continue
|
||||
}
|
||||
|
||||
stats.Add(stats.Formatted, 1)
|
||||
|
||||
entry := Entry{
|
||||
Size: pathInfo.Size(),
|
||||
Modified: pathInfo.ModTime(),
|
||||
Size: currentInfo.Size(),
|
||||
Modified: currentInfo.ModTime(),
|
||||
}
|
||||
|
||||
if err = putEntry(bucket, path, &entry); err != nil {
|
||||
if err = putEntry(bucket, f.RelPath, &entry); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
|
|
@ -5,7 +5,6 @@ import (
|
|||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
|
@ -35,8 +34,8 @@ var (
|
|||
globalExcludes []glob.Glob
|
||||
formatters map[string]*format.Formatter
|
||||
pipelines map[string]*format.Pipeline
|
||||
pathsCh chan string
|
||||
processedCh chan string
|
||||
filesCh chan *walk.File
|
||||
processedCh chan *walk.File
|
||||
|
||||
ErrFailOnChange = errors.New("unexpected changes detected, --fail-on-change is enabled")
|
||||
)
|
||||
|
@ -142,10 +141,10 @@ func (f *Format) Run() (err error) {
|
|||
|
||||
// create a channel for paths to be processed
|
||||
// we use a multiple of batch size here to allow for greater concurrency
|
||||
pathsCh = make(chan string, BatchSize*runtime.NumCPU())
|
||||
filesCh = make(chan *walk.File, BatchSize*runtime.NumCPU())
|
||||
|
||||
// create a channel for tracking paths that have been processed
|
||||
processedCh = make(chan string, cap(pathsCh))
|
||||
processedCh = make(chan *walk.File, cap(filesCh))
|
||||
|
||||
// start concurrent processing tasks
|
||||
eg.Go(updateCache(ctx))
|
||||
|
@ -185,26 +184,26 @@ func walkFilesystem(ctx context.Context) func() error {
|
|||
return fmt.Errorf("failed to create walker: %w", err)
|
||||
}
|
||||
|
||||
defer close(pathsCh)
|
||||
defer close(filesCh)
|
||||
|
||||
if Cli.NoCache {
|
||||
return walker.Walk(ctx, func(path string, info fs.FileInfo, err error) error {
|
||||
return walker.Walk(ctx, func(file *walk.File, err error) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
// ignore symlinks and directories
|
||||
if !(info.IsDir() || info.Mode()&os.ModeSymlink == os.ModeSymlink) {
|
||||
if !(file.Info.IsDir() || file.Info.Mode()&os.ModeSymlink == os.ModeSymlink) {
|
||||
stats.Add(stats.Traversed, 1)
|
||||
stats.Add(stats.Emitted, 1)
|
||||
pathsCh <- path
|
||||
filesCh <- file
|
||||
}
|
||||
return nil
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
if err = cache.ChangeSet(ctx, walker, pathsCh); err != nil {
|
||||
if err = cache.ChangeSet(ctx, walker, filesCh); err != nil {
|
||||
return fmt.Errorf("failed to generate change set: %w", err)
|
||||
}
|
||||
return nil
|
||||
|
@ -213,19 +212,11 @@ func walkFilesystem(ctx context.Context) func() error {
|
|||
|
||||
func updateCache(ctx context.Context) func() error {
|
||||
return func() error {
|
||||
batch := make([]string, 0, BatchSize)
|
||||
|
||||
var changes int
|
||||
batch := make([]*walk.File, 0, BatchSize)
|
||||
|
||||
processBatch := func() error {
|
||||
if Cli.NoCache {
|
||||
changes += len(batch)
|
||||
} else {
|
||||
count, err := cache.Update(Cli.TreeRoot, batch)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
changes += count
|
||||
if err := cache.Update(batch); err != nil {
|
||||
return err
|
||||
}
|
||||
batch = batch[:0]
|
||||
return nil
|
||||
|
@ -254,7 +245,7 @@ func updateCache(ctx context.Context) func() error {
|
|||
return err
|
||||
}
|
||||
|
||||
if Cli.FailOnChange && changes != 0 {
|
||||
if Cli.FailOnChange && stats.Value(stats.Formatted) != 0 {
|
||||
return ErrFailOnChange
|
||||
}
|
||||
|
||||
|
@ -265,28 +256,28 @@ func updateCache(ctx context.Context) func() error {
|
|||
|
||||
func applyFormatters(ctx context.Context) func() error {
|
||||
fg, ctx := errgroup.WithContext(ctx)
|
||||
batches := make(map[string][]string)
|
||||
batches := make(map[string][]*walk.File)
|
||||
|
||||
tryApply := func(key string, path string) {
|
||||
tryApply := func(key string, file *walk.File) {
|
||||
batch, ok := batches[key]
|
||||
if !ok {
|
||||
batch = make([]string, 0, BatchSize)
|
||||
batch = make([]*walk.File, 0, BatchSize)
|
||||
}
|
||||
batch = append(batch, path)
|
||||
batch = append(batch, file)
|
||||
batches[key] = batch
|
||||
|
||||
if len(batch) == BatchSize {
|
||||
pipeline := pipelines[key]
|
||||
|
||||
// copy the batch
|
||||
paths := make([]string, len(batch))
|
||||
copy(paths, batch)
|
||||
files := make([]*walk.File, len(batch))
|
||||
copy(files, batch)
|
||||
|
||||
fg.Go(func() error {
|
||||
if err := pipeline.Apply(ctx, paths); err != nil {
|
||||
if err := pipeline.Apply(ctx, files); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, path := range paths {
|
||||
for _, path := range files {
|
||||
processedCh <- path
|
||||
}
|
||||
return nil
|
||||
|
@ -322,17 +313,19 @@ func applyFormatters(ctx context.Context) func() error {
|
|||
close(processedCh)
|
||||
}()
|
||||
|
||||
for path := range pathsCh {
|
||||
for file := range filesCh {
|
||||
var matched bool
|
||||
for key, pipeline := range pipelines {
|
||||
if !pipeline.Wants(path) {
|
||||
if !pipeline.Wants(file) {
|
||||
continue
|
||||
}
|
||||
matched = true
|
||||
tryApply(key, path)
|
||||
tryApply(key, file)
|
||||
}
|
||||
if matched {
|
||||
stats.Add(stats.Matched, 1)
|
||||
} else {
|
||||
processedCh <- file
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -51,42 +51,42 @@ func TestSpecifyingFormatters(t *testing.T) {
|
|||
test.WriteConfig(t, configPath, config2.Config{
|
||||
Formatters: map[string]*config2.Formatter{
|
||||
"elm": {
|
||||
Command: "echo",
|
||||
Command: "touch",
|
||||
Includes: []string{"*.elm"},
|
||||
},
|
||||
"nix": {
|
||||
Command: "echo",
|
||||
Command: "touch",
|
||||
Includes: []string{"*.nix"},
|
||||
},
|
||||
"ruby": {
|
||||
Command: "echo",
|
||||
Command: "touch",
|
||||
Includes: []string{"*.rb"},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
|
||||
_, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
|
||||
as.NoError(err)
|
||||
assertFormatted(t, as, out, 3)
|
||||
assertStats(t, as, 31, 31, 3, 3)
|
||||
|
||||
out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "elm,nix")
|
||||
_, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "elm,nix")
|
||||
as.NoError(err)
|
||||
assertFormatted(t, as, out, 2)
|
||||
assertStats(t, as, 31, 31, 2, 2)
|
||||
|
||||
out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "ruby,nix")
|
||||
_, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "ruby,nix")
|
||||
as.NoError(err)
|
||||
assertFormatted(t, as, out, 2)
|
||||
assertStats(t, as, 31, 31, 2, 2)
|
||||
|
||||
out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "nix")
|
||||
_, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "nix")
|
||||
as.NoError(err)
|
||||
assertFormatted(t, as, out, 1)
|
||||
assertStats(t, as, 31, 31, 1, 1)
|
||||
|
||||
// test bad names
|
||||
|
||||
out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "foo")
|
||||
_, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "foo")
|
||||
as.Errorf(err, "formatter not found in config: foo")
|
||||
|
||||
out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "bar,foo")
|
||||
_, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "bar,foo")
|
||||
as.Errorf(err, "formatter not found in config: bar")
|
||||
}
|
||||
|
||||
|
@ -94,7 +94,7 @@ func TestIncludesAndExcludes(t *testing.T) {
|
|||
as := require.New(t)
|
||||
|
||||
tempDir := test.TempExamples(t)
|
||||
configPath := tempDir + "/echo.toml"
|
||||
configPath := tempDir + "/touch.toml"
|
||||
|
||||
// test without any excludes
|
||||
cfg := config2.Config{
|
||||
|
@ -107,25 +107,25 @@ func TestIncludesAndExcludes(t *testing.T) {
|
|||
}
|
||||
|
||||
test.WriteConfig(t, configPath, cfg)
|
||||
out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
|
||||
_, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
|
||||
as.NoError(err)
|
||||
assertFormatted(t, as, out, 31)
|
||||
assertStats(t, as, 31, 31, 31, 0)
|
||||
|
||||
// globally exclude nix files
|
||||
cfg.Global.Excludes = []string{"*.nix"}
|
||||
|
||||
test.WriteConfig(t, configPath, cfg)
|
||||
out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
|
||||
_, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
|
||||
as.NoError(err)
|
||||
assertFormatted(t, as, out, 30)
|
||||
assertStats(t, as, 31, 31, 30, 0)
|
||||
|
||||
// add haskell files to the global exclude
|
||||
cfg.Global.Excludes = []string{"*.nix", "*.hs"}
|
||||
|
||||
test.WriteConfig(t, configPath, cfg)
|
||||
out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
|
||||
_, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
|
||||
as.NoError(err)
|
||||
assertFormatted(t, as, out, 24)
|
||||
assertStats(t, as, 31, 31, 24, 0)
|
||||
|
||||
echo := cfg.Formatters["echo"]
|
||||
|
||||
|
@ -133,40 +133,40 @@ func TestIncludesAndExcludes(t *testing.T) {
|
|||
echo.Excludes = []string{"*.py"}
|
||||
|
||||
test.WriteConfig(t, configPath, cfg)
|
||||
out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
|
||||
_, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
|
||||
as.NoError(err)
|
||||
assertFormatted(t, as, out, 22)
|
||||
assertStats(t, as, 31, 31, 22, 0)
|
||||
|
||||
// remove go files from the echo formatter
|
||||
echo.Excludes = []string{"*.py", "*.go"}
|
||||
|
||||
test.WriteConfig(t, configPath, cfg)
|
||||
out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
|
||||
_, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
|
||||
as.NoError(err)
|
||||
assertFormatted(t, as, out, 21)
|
||||
assertStats(t, as, 31, 31, 21, 0)
|
||||
|
||||
// adjust the includes for echo to only include elm files
|
||||
echo.Includes = []string{"*.elm"}
|
||||
|
||||
test.WriteConfig(t, configPath, cfg)
|
||||
out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
|
||||
_, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
|
||||
as.NoError(err)
|
||||
assertFormatted(t, as, out, 1)
|
||||
assertStats(t, as, 31, 31, 1, 0)
|
||||
|
||||
// add js files to echo formatter
|
||||
echo.Includes = []string{"*.elm", "*.js"}
|
||||
|
||||
test.WriteConfig(t, configPath, cfg)
|
||||
out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
|
||||
_, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
|
||||
as.NoError(err)
|
||||
assertFormatted(t, as, out, 2)
|
||||
assertStats(t, as, 31, 31, 2, 0)
|
||||
}
|
||||
|
||||
func TestCache(t *testing.T) {
|
||||
as := require.New(t)
|
||||
|
||||
tempDir := test.TempExamples(t)
|
||||
configPath := tempDir + "/echo.toml"
|
||||
configPath := tempDir + "/touch.toml"
|
||||
|
||||
// test without any excludes
|
||||
cfg := config2.Config{
|
||||
|
@ -181,7 +181,7 @@ func TestCache(t *testing.T) {
|
|||
test.WriteConfig(t, configPath, cfg)
|
||||
out, err := cmd(t, "--config-file", configPath, "--tree-root", tempDir)
|
||||
as.NoError(err)
|
||||
assertFormatted(t, as, out, 31)
|
||||
assertStats(t, as, 31, 31, 31, 0)
|
||||
|
||||
out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir)
|
||||
as.NoError(err)
|
||||
|
@ -190,7 +190,7 @@ func TestCache(t *testing.T) {
|
|||
// clear cache
|
||||
out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir, "-c")
|
||||
as.NoError(err)
|
||||
assertFormatted(t, as, out, 31)
|
||||
assertStats(t, as, 31, 31, 31, 0)
|
||||
|
||||
out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir)
|
||||
as.NoError(err)
|
||||
|
@ -199,7 +199,7 @@ func TestCache(t *testing.T) {
|
|||
// clear cache
|
||||
out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir, "-c")
|
||||
as.NoError(err)
|
||||
assertFormatted(t, as, out, 31)
|
||||
assertStats(t, as, 31, 31, 31, 0)
|
||||
|
||||
out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir)
|
||||
as.NoError(err)
|
||||
|
@ -208,7 +208,7 @@ func TestCache(t *testing.T) {
|
|||
// no cache
|
||||
out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir, "--no-cache")
|
||||
as.NoError(err)
|
||||
assertStats(t, as, out, 31, 31, 31, 0)
|
||||
assertStats(t, as, 31, 31, 31, 0)
|
||||
}
|
||||
|
||||
func TestChangeWorkingDirectory(t *testing.T) {
|
||||
|
@ -240,22 +240,22 @@ func TestChangeWorkingDirectory(t *testing.T) {
|
|||
|
||||
// by default, we look for ./treefmt.toml and use the cwd for the tree root
|
||||
// this should fail if the working directory hasn't been changed first
|
||||
out, err := cmd(t, "-C", tempDir)
|
||||
_, err = cmd(t, "-C", tempDir)
|
||||
as.NoError(err)
|
||||
assertFormatted(t, as, out, 31)
|
||||
assertStats(t, as, 31, 31, 31, 0)
|
||||
}
|
||||
|
||||
func TestFailOnChange(t *testing.T) {
|
||||
as := require.New(t)
|
||||
|
||||
tempDir := test.TempExamples(t)
|
||||
configPath := tempDir + "/echo.toml"
|
||||
configPath := tempDir + "/touch.toml"
|
||||
|
||||
// test without any excludes
|
||||
cfg := config2.Config{
|
||||
Formatters: map[string]*config2.Formatter{
|
||||
"echo": {
|
||||
Command: "echo",
|
||||
"touch": {
|
||||
Command: "touch",
|
||||
Includes: []string{"*"},
|
||||
},
|
||||
},
|
||||
|
@ -270,7 +270,7 @@ func TestBustCacheOnFormatterChange(t *testing.T) {
|
|||
as := require.New(t)
|
||||
|
||||
tempDir := test.TempExamples(t)
|
||||
configPath := tempDir + "/echo.toml"
|
||||
configPath := tempDir + "/touch.toml"
|
||||
|
||||
// symlink some formatters into temp dir, so we can mess with their mod times
|
||||
binPath := tempDir + "/bin"
|
||||
|
@ -304,33 +304,33 @@ func TestBustCacheOnFormatterChange(t *testing.T) {
|
|||
|
||||
test.WriteConfig(t, configPath, cfg)
|
||||
args := []string{"--config-file", configPath, "--tree-root", tempDir}
|
||||
out, err := cmd(t, args...)
|
||||
_, err := cmd(t, args...)
|
||||
as.NoError(err)
|
||||
assertFormatted(t, as, out, 3)
|
||||
assertStats(t, as, 31, 31, 3, 0)
|
||||
|
||||
// tweak mod time of elm formatter
|
||||
as.NoError(test.RecreateSymlink(t, binPath+"/"+"elm-format"))
|
||||
|
||||
out, err = cmd(t, args...)
|
||||
_, err = cmd(t, args...)
|
||||
as.NoError(err)
|
||||
assertFormatted(t, as, out, 3)
|
||||
assertStats(t, as, 31, 31, 3, 0)
|
||||
|
||||
// check cache is working
|
||||
out, err = cmd(t, args...)
|
||||
_, err = cmd(t, args...)
|
||||
as.NoError(err)
|
||||
assertFormatted(t, as, out, 0)
|
||||
assertStats(t, as, 31, 0, 0, 0)
|
||||
|
||||
// tweak mod time of python formatter
|
||||
as.NoError(test.RecreateSymlink(t, binPath+"/"+"black"))
|
||||
|
||||
out, err = cmd(t, args...)
|
||||
_, err = cmd(t, args...)
|
||||
as.NoError(err)
|
||||
assertFormatted(t, as, out, 3)
|
||||
assertStats(t, as, 31, 31, 3, 0)
|
||||
|
||||
// check cache is working
|
||||
out, err = cmd(t, args...)
|
||||
_, err = cmd(t, args...)
|
||||
as.NoError(err)
|
||||
assertFormatted(t, as, out, 0)
|
||||
assertStats(t, as, 31, 0, 0, 0)
|
||||
|
||||
// add go formatter
|
||||
cfg.Formatters["go"] = &config2.Formatter{
|
||||
|
@ -340,40 +340,40 @@ func TestBustCacheOnFormatterChange(t *testing.T) {
|
|||
}
|
||||
test.WriteConfig(t, configPath, cfg)
|
||||
|
||||
out, err = cmd(t, args...)
|
||||
_, err = cmd(t, args...)
|
||||
as.NoError(err)
|
||||
assertFormatted(t, as, out, 4)
|
||||
assertStats(t, as, 31, 31, 4, 0)
|
||||
|
||||
// check cache is working
|
||||
out, err = cmd(t, args...)
|
||||
_, err = cmd(t, args...)
|
||||
as.NoError(err)
|
||||
assertFormatted(t, as, out, 0)
|
||||
assertStats(t, as, 31, 0, 0, 0)
|
||||
|
||||
// remove python formatter
|
||||
delete(cfg.Formatters, "python")
|
||||
test.WriteConfig(t, configPath, cfg)
|
||||
|
||||
out, err = cmd(t, args...)
|
||||
_, err = cmd(t, args...)
|
||||
as.NoError(err)
|
||||
assertFormatted(t, as, out, 2)
|
||||
assertStats(t, as, 31, 31, 2, 0)
|
||||
|
||||
// check cache is working
|
||||
out, err = cmd(t, args...)
|
||||
_, err = cmd(t, args...)
|
||||
as.NoError(err)
|
||||
assertFormatted(t, as, out, 0)
|
||||
assertStats(t, as, 31, 0, 0, 0)
|
||||
|
||||
// remove elm formatter
|
||||
delete(cfg.Formatters, "elm")
|
||||
test.WriteConfig(t, configPath, cfg)
|
||||
|
||||
out, err = cmd(t, args...)
|
||||
_, err = cmd(t, args...)
|
||||
as.NoError(err)
|
||||
assertFormatted(t, as, out, 1)
|
||||
assertStats(t, as, 31, 31, 1, 0)
|
||||
|
||||
// check cache is working
|
||||
out, err = cmd(t, args...)
|
||||
_, err = cmd(t, args...)
|
||||
as.NoError(err)
|
||||
assertFormatted(t, as, out, 0)
|
||||
assertStats(t, as, 31, 0, 0, 0)
|
||||
}
|
||||
|
||||
func TestGitWorktree(t *testing.T) {
|
||||
|
@ -407,28 +407,28 @@ func TestGitWorktree(t *testing.T) {
|
|||
wt, err := repo.Worktree()
|
||||
as.NoError(err, "failed to get git worktree")
|
||||
|
||||
run := func(formatted int) {
|
||||
run := func(traversed int, emitted int, matched int, formatted int) {
|
||||
out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
|
||||
as.NoError(err)
|
||||
assertFormatted(t, as, out, formatted)
|
||||
}
|
||||
|
||||
// run before adding anything to the worktree
|
||||
run(0)
|
||||
run(0, 0, 0, 0)
|
||||
|
||||
// add everything to the worktree
|
||||
as.NoError(wt.AddGlob("."))
|
||||
as.NoError(err)
|
||||
run(31)
|
||||
run(31, 31, 31, 0)
|
||||
|
||||
// remove python directory
|
||||
as.NoError(wt.RemoveGlob("python/*"))
|
||||
run(28)
|
||||
run(28, 28, 28, 0)
|
||||
|
||||
// walk with filesystem instead of git
|
||||
out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--walk", "filesystem")
|
||||
_, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--walk", "filesystem")
|
||||
as.NoError(err)
|
||||
assertFormatted(t, as, out, 59)
|
||||
assertStats(t, as, 59, 59, 59, 0)
|
||||
}
|
||||
|
||||
func TestPathsArg(t *testing.T) {
|
||||
|
@ -461,17 +461,17 @@ func TestPathsArg(t *testing.T) {
|
|||
test.WriteConfig(t, configPath, cfg)
|
||||
|
||||
// without any path args
|
||||
out, err := cmd(t, "-C", tempDir)
|
||||
_, err = cmd(t, "-C", tempDir)
|
||||
as.NoError(err)
|
||||
assertFormatted(t, as, out, 31)
|
||||
assertStats(t, as, 31, 31, 31, 0)
|
||||
|
||||
// specify some explicit paths
|
||||
out, err = cmd(t, "-C", tempDir, "-c", "elm/elm.json", "haskell/Nested/Foo.hs")
|
||||
_, err = cmd(t, "-C", tempDir, "-c", "elm/elm.json", "haskell/Nested/Foo.hs")
|
||||
as.NoError(err)
|
||||
assertFormatted(t, as, out, 2)
|
||||
assertStats(t, as, 4, 4, 4, 0)
|
||||
|
||||
// specify a bad path
|
||||
out, err = cmd(t, "-C", tempDir, "-c", "elm/elm.json", "haskell/Nested/Bar.hs")
|
||||
_, err = cmd(t, "-C", tempDir, "-c", "elm/elm.json", "haskell/Nested/Bar.hs")
|
||||
as.ErrorContains(err, "no such file or directory")
|
||||
}
|
||||
|
||||
|
@ -526,9 +526,9 @@ go/main.go
|
|||
_, _ = stdin.Seek(0, 0)
|
||||
}()
|
||||
|
||||
out, err := cmd(t, "-C", tempDir, "--stdin")
|
||||
_, err = cmd(t, "-C", tempDir, "--stdin")
|
||||
as.NoError(err)
|
||||
assertFormatted(t, as, out, 3)
|
||||
assertStats(t, as, 6, 6, 6, 0)
|
||||
}
|
||||
|
||||
func TestDeterministicOrderingInPipeline(t *testing.T) {
|
||||
|
|
|
@ -7,6 +7,8 @@ import (
|
|||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.numtide.com/numtide/treefmt/stats"
|
||||
|
||||
"git.numtide.com/numtide/treefmt/test"
|
||||
|
||||
"github.com/alecthomas/kong"
|
||||
|
@ -70,12 +72,12 @@ func cmd(t *testing.T, args ...string) ([]byte, error) {
|
|||
return out, nil
|
||||
}
|
||||
|
||||
func assertStats(t *testing.T, as *require.Assertions, output []byte, traversed int32, emitted int32, matched int32, formatted int32) {
|
||||
func assertStats(t *testing.T, as *require.Assertions, traversed int32, emitted int32, matched int32, formatted int32) {
|
||||
t.Helper()
|
||||
as.Contains(string(output), fmt.Sprintf("traversed %d files", traversed))
|
||||
as.Contains(string(output), fmt.Sprintf("emitted %d files", emitted))
|
||||
as.Contains(string(output), fmt.Sprintf("matched %d files", matched))
|
||||
as.Contains(string(output), fmt.Sprintf("formatted %d files", formatted))
|
||||
as.Equal(traversed, stats.Value(stats.Traversed))
|
||||
as.Equal(emitted, stats.Value(stats.Emitted))
|
||||
as.Equal(matched, stats.Value(stats.Matched))
|
||||
as.Equal(formatted, stats.Value(stats.Formatted))
|
||||
}
|
||||
|
||||
func assertFormatted(t *testing.T, as *require.Assertions, output []byte, count int) {
|
||||
|
|
|
@ -8,6 +8,8 @@ import (
|
|||
"os/exec"
|
||||
"time"
|
||||
|
||||
"git.numtide.com/numtide/treefmt/walk"
|
||||
|
||||
"git.numtide.com/numtide/treefmt/config"
|
||||
|
||||
"github.com/charmbracelet/log"
|
||||
|
@ -38,7 +40,7 @@ func (f *Formatter) Executable() string {
|
|||
return f.executable
|
||||
}
|
||||
|
||||
func (f *Formatter) Apply(ctx context.Context, paths []string, filter bool) error {
|
||||
func (f *Formatter) Apply(ctx context.Context, files []*walk.File, filter bool) error {
|
||||
start := time.Now()
|
||||
|
||||
// construct args, starting with config
|
||||
|
@ -52,9 +54,9 @@ func (f *Formatter) Apply(ctx context.Context, paths []string, filter bool) erro
|
|||
f.batch = f.batch[:0]
|
||||
|
||||
// filter paths
|
||||
for _, path := range paths {
|
||||
if f.Wants(path) {
|
||||
f.batch = append(f.batch, path)
|
||||
for _, file := range files {
|
||||
if f.Wants(file) {
|
||||
f.batch = append(f.batch, file.RelPath)
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -67,12 +69,14 @@ func (f *Formatter) Apply(ctx context.Context, paths []string, filter bool) erro
|
|||
args = append(args, f.batch...)
|
||||
} else {
|
||||
// exit early if nothing to process
|
||||
if len(paths) == 0 {
|
||||
if len(files) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// append paths to the args
|
||||
args = append(args, paths...)
|
||||
for _, file := range files {
|
||||
args = append(args, file.RelPath)
|
||||
}
|
||||
}
|
||||
|
||||
// execute the command
|
||||
|
@ -88,17 +92,17 @@ func (f *Formatter) Apply(ctx context.Context, paths []string, filter bool) erro
|
|||
|
||||
//
|
||||
|
||||
f.log.Infof("%v files processed in %v", len(paths), time.Now().Sub(start))
|
||||
f.log.Infof("%v files processed in %v", len(files), time.Now().Sub(start))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Wants is used to test if a Formatter wants a path based on it's configured Includes and Excludes patterns.
|
||||
// Returns true if the Formatter should be applied to path, false otherwise.
|
||||
func (f *Formatter) Wants(path string) bool {
|
||||
match := !PathMatches(path, f.excludes) && PathMatches(path, f.includes)
|
||||
func (f *Formatter) Wants(file *walk.File) bool {
|
||||
match := !PathMatches(file.RelPath, f.excludes) && PathMatches(file.RelPath, f.includes)
|
||||
if match {
|
||||
f.log.Debugf("match: %v", path)
|
||||
f.log.Debugf("match: %v", file)
|
||||
}
|
||||
return match
|
||||
}
|
||||
|
|
|
@ -3,6 +3,8 @@ package format
|
|||
import (
|
||||
"context"
|
||||
"slices"
|
||||
|
||||
"git.numtide.com/numtide/treefmt/walk"
|
||||
)
|
||||
|
||||
type Pipeline struct {
|
||||
|
@ -17,7 +19,7 @@ func (p *Pipeline) Add(f *Formatter) {
|
|||
})
|
||||
}
|
||||
|
||||
func (p *Pipeline) Wants(path string) bool {
|
||||
func (p *Pipeline) Wants(path *walk.File) bool {
|
||||
var match bool
|
||||
for _, f := range p.sequence {
|
||||
match = f.Wants(path)
|
||||
|
@ -28,7 +30,7 @@ func (p *Pipeline) Wants(path string) bool {
|
|||
return match
|
||||
}
|
||||
|
||||
func (p *Pipeline) Apply(ctx context.Context, paths []string) error {
|
||||
func (p *Pipeline) Apply(ctx context.Context, paths []*walk.File) error {
|
||||
for _, f := range p.sequence {
|
||||
if err := f.Apply(ctx, paths, len(p.sequence) > 1); err != nil {
|
||||
return err
|
||||
|
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
[formatter.deadnix]
|
||||
command = "deadnix"
|
||||
options = ["--edit"]
|
||||
includes = ["*.nix"]
|
||||
pipeline = "nix"
|
||||
priority = 1
|
||||
|
|
|
@ -1,3 +1,3 @@
|
|||
[formatter.echo]
|
||||
command = "echo"
|
||||
command = "touch"
|
||||
includes = [ "*.*" ]
|
|
@ -2,6 +2,7 @@ package walk
|
|||
|
||||
import (
|
||||
"context"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
@ -15,18 +16,42 @@ func (f filesystemWalker) Root() string {
|
|||
return f.root
|
||||
}
|
||||
|
||||
func (f filesystemWalker) Walk(_ context.Context, fn filepath.WalkFunc) error {
|
||||
func (f filesystemWalker) Walk(_ context.Context, fn WalkFunc) error {
|
||||
relPathOffset := len(f.root) + 1
|
||||
|
||||
relPathFn := func(path string) (relPath string) {
|
||||
if len(path) >= relPathOffset {
|
||||
relPath = path[relPathOffset:]
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
walkFn := func(path string, info fs.FileInfo, err error) error {
|
||||
file := File{
|
||||
Path: path,
|
||||
RelPath: relPathFn(path),
|
||||
Info: info,
|
||||
}
|
||||
return fn(&file, err)
|
||||
}
|
||||
|
||||
if len(f.paths) == 0 {
|
||||
return filepath.Walk(f.root, fn)
|
||||
return filepath.Walk(f.root, walkFn)
|
||||
}
|
||||
|
||||
for _, path := range f.paths {
|
||||
info, err := os.Stat(path)
|
||||
if err = filepath.Walk(path, fn); err != nil {
|
||||
if err = filepath.Walk(path, walkFn); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = fn(path, info, err); err != nil {
|
||||
file := File{
|
||||
Path: path,
|
||||
RelPath: relPathFn(path),
|
||||
Info: info,
|
||||
}
|
||||
|
||||
if err = fn(&file, err); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
|
29
walk/git.go
29
walk/git.go
|
@ -24,7 +24,17 @@ func (g *gitWalker) Root() string {
|
|||
return g.root
|
||||
}
|
||||
|
||||
func (g *gitWalker) Walk(ctx context.Context, fn filepath.WalkFunc) error {
|
||||
func (g *gitWalker) Walk(ctx context.Context, fn WalkFunc) error {
|
||||
// for quick relative paths
|
||||
relPathOffset := len(g.root) + 1
|
||||
|
||||
relPathFn := func(path string) (relPath string) {
|
||||
if len(path) >= relPathOffset {
|
||||
relPath = path[relPathOffset:]
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
idx, err := g.repo.Storer.Index()
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: failed to open index", err)
|
||||
|
@ -49,7 +59,13 @@ func (g *gitWalker) Walk(ctx context.Context, fn filepath.WalkFunc) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
return fn(path, info, err)
|
||||
file := File{
|
||||
Path: path,
|
||||
RelPath: relPathFn(path),
|
||||
Info: info,
|
||||
}
|
||||
|
||||
return fn(&file, err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
|
@ -66,7 +82,14 @@ func (g *gitWalker) Walk(ctx context.Context, fn filepath.WalkFunc) error {
|
|||
|
||||
// stat the file
|
||||
info, err := os.Lstat(path)
|
||||
if err = fn(path, info, err); err != nil {
|
||||
|
||||
file := File{
|
||||
Path: path,
|
||||
RelPath: relPathFn(path),
|
||||
Info: info,
|
||||
}
|
||||
|
||||
if err = fn(&file, err); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
|
|
@ -3,7 +3,7 @@ package walk
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"io/fs"
|
||||
)
|
||||
|
||||
type Type string
|
||||
|
@ -14,9 +14,21 @@ const (
|
|||
Filesystem Type = "filesystem"
|
||||
)
|
||||
|
||||
type File struct {
|
||||
Path string
|
||||
RelPath string
|
||||
Info fs.FileInfo
|
||||
}
|
||||
|
||||
func (f File) String() string {
|
||||
return f.Path
|
||||
}
|
||||
|
||||
type WalkFunc func(file *File, err error) error
|
||||
|
||||
type Walker interface {
|
||||
Root() string
|
||||
Walk(ctx context.Context, fn filepath.WalkFunc) error
|
||||
Walk(ctx context.Context, fn WalkFunc) error
|
||||
}
|
||||
|
||||
func New(walkerType Type, root string, paths []string) (Walker, error) {
|
||||
|
|
Reference in New Issue
Block a user