Compare commits

..

22 Commits

Author SHA1 Message Date
65276c05bb
feat: add flake compat
Signed-off-by: Brian McGee <brian@bmcgee.ie>
2024-05-01 14:24:30 +01:00
c2b6fed5e3
doc: refine installation
Signed-off-by: Brian McGee <brian@bmcgee.ie>
2024-05-01 14:20:32 +01:00
2079d9d0c7
doc: refine faq
Signed-off-by: Brian McGee <brian@bmcgee.ie>
2024-05-01 14:11:41 +01:00
8b21b3a280
doc: refine contributing
Signed-off-by: Brian McGee <brian@bmcgee.ie>
2024-05-01 14:06:26 +01:00
dc63ac923c
doc: refine formatter spec
Signed-off-by: Brian McGee <brian@bmcgee.ie>
2024-05-01 14:00:06 +01:00
3fbe2d1b6f
doc: refine usage
Signed-off-by: Brian McGee <brian@bmcgee.ie>
2024-05-01 13:54:06 +01:00
6898fdfdfe
doc: improve hero gif
Signed-off-by: Brian McGee <brian@bmcgee.ie>
2024-05-01 13:50:04 +01:00
cc88b8a61a
doc: refine overview
Signed-off-by: Brian McGee <brian@bmcgee.ie>
2024-05-01 13:50:04 +01:00
d274ed001a
doc: refine quick start content
Signed-off-by: Brian McGee <brian@bmcgee.ie>
2024-05-01 13:50:04 +01:00
c343ebdd4a
fix: devshell commands for docs
Signed-off-by: Brian McGee <brian@bmcgee.ie>
2024-05-01 13:50:04 +01:00
88b20859b8
doc: fix docs package build
Vitepress cli does some funky stuff with the tty.

Signed-off-by: Brian McGee <brian@bmcgee.ie>
2024-05-01 13:50:03 +01:00
ab2d9ef0f3
doc: add some devshell helpers
Signed-off-by: Brian McGee <brian@bmcgee.ie>
2024-05-01 13:50:03 +01:00
3b623c3307
wip: add focs package
Signed-off-by: Brian McGee <brian@bmcgee.ie>
2024-05-01 13:50:03 +01:00
3b7ba6b60a
doc: move assets into public folder
Fixes issues with built version of the site

Signed-off-by: Brian McGee <brian@bmcgee.ie>
2024-05-01 13:50:03 +01:00
51e14546d1
doc: fix bad formatter spec link
Signed-off-by: Brian McGee <brian@bmcgee.ie>
2024-05-01 13:50:03 +01:00
9c0b26daa3
doc: add footer
Signed-off-by: Brian McGee <brian@bmcgee.ie>
2024-05-01 13:50:03 +01:00
6292c83c2d
doc: some initial experiments with colors
Signed-off-by: Brian McGee <brian@bmcgee.ie>
2024-05-01 13:50:03 +01:00
9414f81d88
doc: remove features on home page
Signed-off-by: Brian McGee <brian@bmcgee.ie>
2024-05-01 13:50:02 +01:00
ef9ed9c339
doc: update github link
Signed-off-by: Brian McGee <brian@bmcgee.ie>
2024-05-01 13:50:02 +01:00
c7ab57f317
doc: port existing content
Signed-off-by: Brian McGee <brian@bmcgee.ie>
2024-05-01 13:50:02 +01:00
5a69a76bcb
doc: configure hero and logo
Signed-off-by: Brian McGee <brian@bmcgee.ie>
2024-05-01 13:50:02 +01:00
1b4f56a552
fix: nix filter for package
Signed-off-by: Brian McGee <brian@bmcgee.ie>
2024-05-01 13:50:02 +01:00
16 changed files with 307 additions and 394 deletions

86
cache/cache.go vendored
View File

@ -5,7 +5,9 @@ import (
"crypto/sha1" "crypto/sha1"
"encoding/hex" "encoding/hex"
"fmt" "fmt"
"io/fs"
"os" "os"
"path/filepath"
"runtime" "runtime"
"time" "time"
@ -55,25 +57,25 @@ func Open(treeRoot string, clean bool, formatters map[string]*format.Formatter)
name := hex.EncodeToString(digest) name := hex.EncodeToString(digest)
path, err := xdg.CacheFile(fmt.Sprintf("treefmt/eval-cache/%v.db", name)) path, err := xdg.CacheFile(fmt.Sprintf("treefmt/eval-cache/%v.db", name))
if err != nil { if err != nil {
return fmt.Errorf("could not resolve local path for the cache: %w", err) return fmt.Errorf("%w: could not resolve local path for the cache", err)
} }
db, err = bolt.Open(path, 0o600, nil) db, err = bolt.Open(path, 0o600, nil)
if err != nil { if err != nil {
return fmt.Errorf("failed to open cache at %v: %w", path, err) return fmt.Errorf("%w: failed to open cache", err)
} }
err = db.Update(func(tx *bolt.Tx) error { err = db.Update(func(tx *bolt.Tx) error {
// create bucket for tracking paths // create bucket for tracking paths
pathsBucket, err := tx.CreateBucketIfNotExists([]byte(pathsBucket)) pathsBucket, err := tx.CreateBucketIfNotExists([]byte(pathsBucket))
if err != nil { if err != nil {
return fmt.Errorf("failed to create paths bucket: %w", err) return fmt.Errorf("%w: failed to create paths bucket", err)
} }
// create bucket for tracking formatters // create bucket for tracking formatters
formattersBucket, err := tx.CreateBucketIfNotExists([]byte(formattersBucket)) formattersBucket, err := tx.CreateBucketIfNotExists([]byte(formattersBucket))
if err != nil { if err != nil {
return fmt.Errorf("failed to create formatters bucket: %w", err) return fmt.Errorf("%w: failed to create formatters bucket", err)
} }
// check for any newly configured or modified formatters // check for any newly configured or modified formatters
@ -81,12 +83,12 @@ func Open(treeRoot string, clean bool, formatters map[string]*format.Formatter)
stat, err := os.Lstat(formatter.Executable()) stat, err := os.Lstat(formatter.Executable())
if err != nil { if err != nil {
return fmt.Errorf("failed to stat formatter executable %v: %w", formatter.Executable(), err) return fmt.Errorf("%w: failed to state formatter executable", err)
} }
entry, err := getEntry(formattersBucket, name) entry, err := getEntry(formattersBucket, name)
if err != nil { if err != nil {
return fmt.Errorf("failed to retrieve cache entry for formatter %v: %w", name, err) return fmt.Errorf("%w: failed to retrieve entry for formatter", err)
} }
clean = clean || entry == nil || !(entry.Size == stat.Size() && entry.Modified == stat.ModTime()) clean = clean || entry == nil || !(entry.Size == stat.Size() && entry.Modified == stat.ModTime())
@ -105,7 +107,7 @@ func Open(treeRoot string, clean bool, formatters map[string]*format.Formatter)
} }
if err = putEntry(formattersBucket, name, entry); err != nil { if err = putEntry(formattersBucket, name, entry); err != nil {
return fmt.Errorf("failed to write cache entry for formatter %v: %w", name, err) return fmt.Errorf("%w: failed to write formatter entry", err)
} }
} }
@ -115,14 +117,14 @@ func Open(treeRoot string, clean bool, formatters map[string]*format.Formatter)
if !ok { if !ok {
// remove the formatter entry from the cache // remove the formatter entry from the cache
if err = formattersBucket.Delete(key); err != nil { if err = formattersBucket.Delete(key); err != nil {
return fmt.Errorf("failed to remove cache entry for formatter %v: %w", key, err) return fmt.Errorf("%w: failed to remove formatter entry", err)
} }
// indicate a clean is required // indicate a clean is required
clean = true clean = true
} }
return nil return nil
}); err != nil { }); err != nil {
return fmt.Errorf("failed to check cache for removed formatters: %w", err) return fmt.Errorf("%w: failed to check for removed formatters", err)
} }
if clean { if clean {
@ -130,7 +132,7 @@ func Open(treeRoot string, clean bool, formatters map[string]*format.Formatter)
c := pathsBucket.Cursor() c := pathsBucket.Cursor()
for k, v := c.First(); !(k == nil && v == nil); k, v = c.Next() { for k, v := c.First(); !(k == nil && v == nil); k, v = c.Next() {
if err = c.Delete(); err != nil { if err = c.Delete(); err != nil {
return fmt.Errorf("failed to remove path entry: %w", err) return fmt.Errorf("%w: failed to remove path entry", err)
} }
} }
} }
@ -155,7 +157,7 @@ func getEntry(bucket *bolt.Bucket, path string) (*Entry, error) {
if b != nil { if b != nil {
var cached Entry var cached Entry
if err := msgpack.Unmarshal(b, &cached); err != nil { if err := msgpack.Unmarshal(b, &cached); err != nil {
return nil, fmt.Errorf("failed to unmarshal cache info for path '%v': %w", path, err) return nil, fmt.Errorf("%w: failed to unmarshal cache info for path '%v'", err, path)
} }
return &cached, nil return &cached, nil
} else { } else {
@ -167,18 +169,18 @@ func getEntry(bucket *bolt.Bucket, path string) (*Entry, error) {
func putEntry(bucket *bolt.Bucket, path string, entry *Entry) error { func putEntry(bucket *bolt.Bucket, path string, entry *Entry) error {
bytes, err := msgpack.Marshal(entry) bytes, err := msgpack.Marshal(entry)
if err != nil { if err != nil {
return fmt.Errorf("failed to marshal cache path %v: %w", path, err) return fmt.Errorf("%w: failed to marshal cache entry", err)
} }
if err = bucket.Put([]byte(path), bytes); err != nil { if err = bucket.Put([]byte(path), bytes); err != nil {
return fmt.Errorf("failed to put cache path %v: %w", path, err) return fmt.Errorf("%w: failed to put cache entry", err)
} }
return nil return nil
} }
// ChangeSet is used to walk a filesystem, starting at root, and outputting any new or changed paths using pathsCh. // ChangeSet is used to walk a filesystem, starting at root, and outputting any new or changed paths using pathsCh.
// It determines if a path is new or has changed by comparing against cache entries. // It determines if a path is new or has changed by comparing against cache entries.
func ChangeSet(ctx context.Context, walker walk.Walker, filesCh chan<- *walk.File) error { func ChangeSet(ctx context.Context, walker walk.Walker, pathsCh chan<- string) error {
start := time.Now() start := time.Now()
defer func() { defer func() {
@ -196,21 +198,24 @@ func ChangeSet(ctx context.Context, walker walk.Walker, filesCh chan<- *walk.Fil
} }
}() }()
return walker.Walk(ctx, func(file *walk.File, err error) error { // for quick removal of tree root from paths
relPathOffset := len(walker.Root()) + 1
return walker.Walk(ctx, func(path string, info fs.FileInfo, err error) error {
select { select {
case <-ctx.Done(): case <-ctx.Done():
return ctx.Err() return ctx.Err()
default: default:
if err != nil { if err != nil {
return fmt.Errorf("failed to walk path: %w", err) return fmt.Errorf("%w: failed to walk path", err)
} else if file.Info.IsDir() { } else if info.IsDir() {
// ignore directories // ignore directories
return nil return nil
} }
} }
// ignore symlinks // ignore symlinks
if file.Info.Mode()&os.ModeSymlink == os.ModeSymlink { if info.Mode()&os.ModeSymlink == os.ModeSymlink {
return nil return nil
} }
@ -219,17 +224,18 @@ func ChangeSet(ctx context.Context, walker walk.Walker, filesCh chan<- *walk.Fil
if tx == nil { if tx == nil {
tx, err = db.Begin(false) tx, err = db.Begin(false)
if err != nil { if err != nil {
return fmt.Errorf("failed to open a new cache read tx: %w", err) return fmt.Errorf("%w: failed to open a new read tx", err)
} }
bucket = tx.Bucket([]byte(pathsBucket)) bucket = tx.Bucket([]byte(pathsBucket))
} }
cached, err := getEntry(bucket, file.RelPath) relPath := path[relPathOffset:]
cached, err := getEntry(bucket, relPath)
if err != nil { if err != nil {
return err return err
} }
changedOrNew := cached == nil || !(cached.Modified == file.Info.ModTime() && cached.Size == file.Info.Size()) changedOrNew := cached == nil || !(cached.Modified == info.ModTime() && cached.Size == info.Size())
stats.Add(stats.Traversed, 1) stats.Add(stats.Traversed, 1)
if !changedOrNew { if !changedOrNew {
@ -244,7 +250,7 @@ func ChangeSet(ctx context.Context, walker walk.Walker, filesCh chan<- *walk.Fil
case <-ctx.Done(): case <-ctx.Done():
return ctx.Err() return ctx.Err()
default: default:
filesCh <- file pathsCh <- relPath
} }
// close the current tx if we have reached the batch size // close the current tx if we have reached the batch size
@ -260,35 +266,47 @@ func ChangeSet(ctx context.Context, walker walk.Walker, filesCh chan<- *walk.Fil
} }
// Update is used to record updated cache information for the specified list of paths. // Update is used to record updated cache information for the specified list of paths.
func Update(files []*walk.File) error { func Update(treeRoot string, paths []string) (int, error) {
start := time.Now() start := time.Now()
defer func() { defer func() {
logger.Infof("finished processing %v paths in %v", len(files), time.Since(start)) logger.Infof("finished updating %v paths in %v", len(paths), time.Since(start))
}() }()
if len(files) == 0 { if len(paths) == 0 {
return nil return 0, nil
} }
return db.Update(func(tx *bolt.Tx) error { var changes int
return changes, db.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(pathsBucket)) bucket := tx.Bucket([]byte(pathsBucket))
for _, f := range files { for _, path := range paths {
currentInfo, err := os.Stat(f.Path) cached, err := getEntry(bucket, path)
if err != nil { if err != nil {
return err return err
} }
if !(f.Info.ModTime() == currentInfo.ModTime() && f.Info.Size() == currentInfo.Size()) { pathInfo, err := os.Stat(filepath.Join(treeRoot, path))
stats.Add(stats.Formatted, 1) if err != nil {
return err
} }
if cached == nil || !(cached.Modified == pathInfo.ModTime() && cached.Size == pathInfo.Size()) {
changes += 1
} else {
// no change to write
continue
}
stats.Add(stats.Formatted, 1)
entry := Entry{ entry := Entry{
Size: currentInfo.Size(), Size: pathInfo.Size(),
Modified: currentInfo.ModTime(), Modified: pathInfo.ModTime(),
} }
if err = putEntry(bucket, f.RelPath, &entry); err != nil { if err = putEntry(bucket, path, &entry); err != nil {
return err return err
} }
} }

View File

@ -26,14 +26,14 @@ type Format struct {
Stdin bool `help:"Format the context passed in via stdin"` Stdin bool `help:"Format the context passed in via stdin"`
} }
func ConfigureLogging() { func (f *Format) Configure() {
log.SetReportTimestamp(false) log.SetReportTimestamp(false)
if Cli.Verbosity == 0 { if f.Verbosity == 0 {
log.SetLevel(log.WarnLevel) log.SetLevel(log.WarnLevel)
} else if Cli.Verbosity == 1 { } else if f.Verbosity == 1 {
log.SetLevel(log.InfoLevel) log.SetLevel(log.InfoLevel)
} else if Cli.Verbosity > 1 { } else if f.Verbosity > 1 {
log.SetLevel(log.DebugLevel) log.SetLevel(log.DebugLevel)
} }
} }

View File

@ -5,10 +5,13 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"io/fs"
"os" "os"
"os/signal" "os/signal"
"path/filepath" "path/filepath"
"runtime" "runtime"
"slices"
"sort"
"strings" "strings"
"syscall" "syscall"
@ -32,17 +35,19 @@ var (
globalExcludes []glob.Glob globalExcludes []glob.Glob
formatters map[string]*format.Formatter formatters map[string]*format.Formatter
pipelines map[string]*format.Pipeline pipelines map[string]*format.Pipeline
filesCh chan *walk.File pathsCh chan string
processedCh chan *walk.File processedCh chan string
ErrFailOnChange = errors.New("unexpected changes detected, --fail-on-change is enabled") ErrFailOnChange = errors.New("unexpected changes detected, --fail-on-change is enabled")
) )
func (f *Format) Run() (err error) { func (f *Format) Run() (err error) {
// create a prefixed logger stats.Init()
Cli.Configure()
l := log.WithPrefix("format") l := log.WithPrefix("format")
// ensure cache is closed on return
defer func() { defer func() {
if err := cache.Close(); err != nil { if err := cache.Close(); err != nil {
l.Errorf("failed to close cache: %v", err) l.Errorf("failed to close cache: %v", err)
@ -50,23 +55,46 @@ func (f *Format) Run() (err error) {
}() }()
// read config // read config
cfg, err := config.ReadFile(Cli.ConfigFile, Cli.Formatters) cfg, err := config.ReadFile(Cli.ConfigFile)
if err != nil { if err != nil {
return fmt.Errorf("failed to read config file %v: %w", Cli.ConfigFile, err) return fmt.Errorf("%w: failed to read config file", err)
} }
// compile global exclude globs
if globalExcludes, err = format.CompileGlobs(cfg.Global.Excludes); err != nil { if globalExcludes, err = format.CompileGlobs(cfg.Global.Excludes); err != nil {
return fmt.Errorf("failed to compile global excludes: %w", err) return fmt.Errorf("%w: failed to compile global globs", err)
} }
// initialise pipelines
pipelines = make(map[string]*format.Pipeline) pipelines = make(map[string]*format.Pipeline)
formatters = make(map[string]*format.Formatter) formatters = make(map[string]*format.Formatter)
// iterate the formatters in lexicographical order // filter formatters
for _, name := range cfg.Names { if len(Cli.Formatters) > 0 {
// init formatter // first check the cli formatter list is valid
for _, name := range Cli.Formatters {
_, ok := cfg.Formatters[name]
if !ok {
return fmt.Errorf("formatter not found in config: %v", name)
}
}
// next we remove any formatter configs that were not specified
for name := range cfg.Formatters {
if !slices.Contains(Cli.Formatters, name) {
delete(cfg.Formatters, name)
}
}
}
// sort the formatter names so that, as we construct pipelines, we add formatters in a determinstic fashion. This
// ensures a deterministic order even when all priority values are the same e.g. 0
names := make([]string, 0, len(cfg.Formatters))
for name := range cfg.Formatters {
names = append(names, name)
}
sort.Strings(names)
// init formatters
for _, name := range names {
formatterCfg := cfg.Formatters[name] formatterCfg := cfg.Formatters[name]
formatter, err := format.NewFormatter(name, Cli.TreeRoot, formatterCfg, globalExcludes) formatter, err := format.NewFormatter(name, Cli.TreeRoot, formatterCfg, globalExcludes)
if errors.Is(err, format.ErrCommandNotFound) && Cli.AllowMissingFormatter { if errors.Is(err, format.ErrCommandNotFound) && Cli.AllowMissingFormatter {
@ -76,12 +104,8 @@ func (f *Format) Run() (err error) {
return fmt.Errorf("%w: failed to initialise formatter: %v", err, name) return fmt.Errorf("%w: failed to initialise formatter: %v", err, name)
} }
// store formatter by name
formatters[name] = formatter formatters[name] = formatter
// If no pipeline is configured, we add the formatter to a nominal pipeline of size 1 with the key being the
// formatter's name. If a pipeline is configured, we add the formatter to a pipeline keyed by
// 'p:<pipeline_name>' in which it is sorted by priority.
if formatterCfg.Pipeline == "" { if formatterCfg.Pipeline == "" {
pipeline := format.Pipeline{} pipeline := format.Pipeline{}
pipeline.Add(formatter) pipeline.Add(formatter)
@ -113,20 +137,17 @@ func (f *Format) Run() (err error) {
cancel() cancel()
}() }()
// initialise stats collection // create some groups for concurrent processing and control flow
stats.Init()
// create an overall error group for executing high level tasks concurrently
eg, ctx := errgroup.WithContext(ctx) eg, ctx := errgroup.WithContext(ctx)
// create a channel for files needing to be processed // create a channel for paths to be processed
// we use a multiple of batch size here as a rudimentary concurrency optimization based on the host machine // we use a multiple of batch size here to allow for greater concurrency
filesCh = make(chan *walk.File, BatchSize*runtime.NumCPU()) pathsCh = make(chan string, BatchSize*runtime.NumCPU())
// create a channel for files that have been processed // create a channel for tracking paths that have been processed
processedCh = make(chan *walk.File, cap(filesCh)) processedCh = make(chan string, cap(pathsCh))
// start concurrent processing tasks in reverse order // start concurrent processing tasks
eg.Go(updateCache(ctx)) eg.Go(updateCache(ctx))
eg.Go(applyFormatters(ctx)) eg.Go(applyFormatters(ctx))
eg.Go(walkFilesystem(ctx)) eg.Go(walkFilesystem(ctx))
@ -135,15 +156,76 @@ func (f *Format) Run() (err error) {
return eg.Wait() return eg.Wait()
} }
func walkFilesystem(ctx context.Context) func() error {
return func() error {
paths := Cli.Paths
if len(paths) == 0 && Cli.Stdin {
cwd, err := os.Getwd()
if err != nil {
return fmt.Errorf("%w: failed to determine current working directory", err)
}
// read in all the paths
scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
path := scanner.Text()
if !strings.HasPrefix(path, "/") {
// append the cwd
path = filepath.Join(cwd, path)
}
paths = append(paths, path)
}
}
walker, err := walk.New(Cli.Walk, Cli.TreeRoot, paths)
if err != nil {
return fmt.Errorf("failed to create walker: %w", err)
}
defer close(pathsCh)
if Cli.NoCache {
return walker.Walk(ctx, func(path string, info fs.FileInfo, err error) error {
select {
case <-ctx.Done():
return ctx.Err()
default:
// ignore symlinks and directories
if !(info.IsDir() || info.Mode()&os.ModeSymlink == os.ModeSymlink) {
stats.Add(stats.Traversed, 1)
stats.Add(stats.Emitted, 1)
pathsCh <- path
}
return nil
}
})
}
if err = cache.ChangeSet(ctx, walker, pathsCh); err != nil {
return fmt.Errorf("failed to generate change set: %w", err)
}
return nil
}
}
func updateCache(ctx context.Context) func() error { func updateCache(ctx context.Context) func() error {
return func() error { return func() error {
// used to batch updates for more efficient txs batch := make([]string, 0, BatchSize)
batch := make([]*walk.File, 0, BatchSize)
var changes int
// apply a batch
processBatch := func() error { processBatch := func() error {
if err := cache.Update(batch); err != nil { if Cli.NoCache {
return err changes += len(batch)
} else {
count, err := cache.Update(Cli.TreeRoot, batch)
if err != nil {
return err
}
changes += count
} }
batch = batch[:0] batch = batch[:0]
return nil return nil
@ -152,17 +234,13 @@ func updateCache(ctx context.Context) func() error {
LOOP: LOOP:
for { for {
select { select {
// detect ctx cancellation
case <-ctx.Done(): case <-ctx.Done():
return ctx.Err() return ctx.Err()
// respond to processed files case path, ok := <-processedCh:
case file, ok := <-processedCh:
if !ok { if !ok {
// channel has been closed, no further files to process
break LOOP break LOOP
} }
// append to batch and process if we have enough batch = append(batch, path)
batch = append(batch, file)
if len(batch) == BatchSize { if len(batch) == BatchSize {
if err := processBatch(); err != nil { if err := processBatch(); err != nil {
return err return err
@ -176,124 +254,48 @@ func updateCache(ctx context.Context) func() error {
return err return err
} }
// if fail on change has been enabled, check that no files were actually formatted, throwing an error if so if Cli.FailOnChange && changes != 0 {
if Cli.FailOnChange && stats.Value(stats.Formatted) != 0 {
return ErrFailOnChange return ErrFailOnChange
} }
// print stats to stdout
stats.Print() stats.Print()
return nil
}
}
func walkFilesystem(ctx context.Context) func() error {
return func() error {
paths := Cli.Paths
// we read paths from stdin if the cli flag has been set and no paths were provided as cli args
if len(paths) == 0 && Cli.Stdin {
// determine the current working directory
cwd, err := os.Getwd()
if err != nil {
return fmt.Errorf("failed to determine current working directory: %w", err)
}
// read in all the paths
scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
path := scanner.Text()
if !strings.HasPrefix(path, "/") {
// append the cwd
path = filepath.Join(cwd, path)
}
// append the fully qualified path to our paths list
paths = append(paths, path)
}
}
// create a filesystem walker
walker, err := walk.New(Cli.Walk, Cli.TreeRoot, paths)
if err != nil {
return fmt.Errorf("failed to create walker: %w", err)
}
// close the files channel when we're done walking the file system
defer close(filesCh)
// if no cache has been configured, we invoke the walker directly
if Cli.NoCache {
return walker.Walk(ctx, func(file *walk.File, err error) error {
select {
case <-ctx.Done():
return ctx.Err()
default:
// ignore symlinks and directories
if !(file.Info.IsDir() || file.Info.Mode()&os.ModeSymlink == os.ModeSymlink) {
stats.Add(stats.Traversed, 1)
stats.Add(stats.Emitted, 1)
filesCh <- file
}
return nil
}
})
}
// otherwise we pass the walker to the cache and have it generate files for processing based on whether or not
// they have been added/changed since the last invocation
if err = cache.ChangeSet(ctx, walker, filesCh); err != nil {
return fmt.Errorf("failed to generate change set: %w", err)
}
return nil return nil
} }
} }
func applyFormatters(ctx context.Context) func() error { func applyFormatters(ctx context.Context) func() error {
// create our own errgroup for concurrent formatting tasks
fg, ctx := errgroup.WithContext(ctx) fg, ctx := errgroup.WithContext(ctx)
batches := make(map[string][]string)
// pre-initialise batches keyed by pipeline tryApply := func(key string, path string) {
batches := make(map[string][]*walk.File) batch, ok := batches[key]
for key := range pipelines { if !ok {
batches[key] = make([]*walk.File, 0, BatchSize) batch = make([]string, 0, BatchSize)
} }
batch = append(batch, path)
batches[key] = batch
// for a given pipeline key, add the provided file to the current batch and trigger a format if the batch size has
// been reached
tryApply := func(key string, file *walk.File) {
// append to batch
batches[key] = append(batches[key], file)
// check if the batch is full
batch := batches[key]
if len(batch) == BatchSize { if len(batch) == BatchSize {
// get the pipeline
pipeline := pipelines[key] pipeline := pipelines[key]
// copy the batch // copy the batch
files := make([]*walk.File, len(batch)) paths := make([]string, len(batch))
copy(files, batch) copy(paths, batch)
// apply to the pipeline
fg.Go(func() error { fg.Go(func() error {
if err := pipeline.Apply(ctx, files); err != nil { if err := pipeline.Apply(ctx, paths); err != nil {
return err return err
} }
for _, path := range files { for _, path := range paths {
processedCh <- path processedCh <- path
} }
return nil return nil
}) })
// reset the batch
batches[key] = batch[:0] batches[key] = batch[:0]
} }
} }
// format any partial batches
flushBatches := func() { flushBatches := func() {
for key, pipeline := range pipelines { for key, pipeline := range pipelines {
@ -320,21 +322,17 @@ func applyFormatters(ctx context.Context) func() error {
close(processedCh) close(processedCh)
}() }()
// iterate the files channel, checking if any pipeline wants it, and attempting to apply if so. for path := range pathsCh {
for file := range filesCh {
var matched bool var matched bool
for key, pipeline := range pipelines { for key, pipeline := range pipelines {
if !pipeline.Wants(file) { if !pipeline.Wants(path) {
continue continue
} }
matched = true matched = true
tryApply(key, file) tryApply(key, path)
} }
if matched { if matched {
stats.Add(stats.Matched, 1) stats.Add(stats.Matched, 1)
} else {
// no match, so we send it direct to the processed channel
processedCh <- file
} }
} }

View File

@ -51,42 +51,42 @@ func TestSpecifyingFormatters(t *testing.T) {
test.WriteConfig(t, configPath, config2.Config{ test.WriteConfig(t, configPath, config2.Config{
Formatters: map[string]*config2.Formatter{ Formatters: map[string]*config2.Formatter{
"elm": { "elm": {
Command: "touch", Command: "echo",
Includes: []string{"*.elm"}, Includes: []string{"*.elm"},
}, },
"nix": { "nix": {
Command: "touch", Command: "echo",
Includes: []string{"*.nix"}, Includes: []string{"*.nix"},
}, },
"ruby": { "ruby": {
Command: "touch", Command: "echo",
Includes: []string{"*.rb"}, Includes: []string{"*.rb"},
}, },
}, },
}) })
_, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err) as.NoError(err)
assertStats(t, as, 31, 31, 3, 3) assertFormatted(t, as, out, 3)
_, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "elm,nix") out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "elm,nix")
as.NoError(err) as.NoError(err)
assertStats(t, as, 31, 31, 2, 2) assertFormatted(t, as, out, 2)
_, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "ruby,nix") out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "ruby,nix")
as.NoError(err) as.NoError(err)
assertStats(t, as, 31, 31, 2, 2) assertFormatted(t, as, out, 2)
_, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "nix") out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "nix")
as.NoError(err) as.NoError(err)
assertStats(t, as, 31, 31, 1, 1) assertFormatted(t, as, out, 1)
// test bad names // test bad names
_, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "foo") out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "foo")
as.Errorf(err, "formatter not found in config: foo") as.Errorf(err, "formatter not found in config: foo")
_, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "bar,foo") out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "bar,foo")
as.Errorf(err, "formatter not found in config: bar") as.Errorf(err, "formatter not found in config: bar")
} }
@ -94,7 +94,7 @@ func TestIncludesAndExcludes(t *testing.T) {
as := require.New(t) as := require.New(t)
tempDir := test.TempExamples(t) tempDir := test.TempExamples(t)
configPath := tempDir + "/touch.toml" configPath := tempDir + "/echo.toml"
// test without any excludes // test without any excludes
cfg := config2.Config{ cfg := config2.Config{
@ -107,25 +107,25 @@ func TestIncludesAndExcludes(t *testing.T) {
} }
test.WriteConfig(t, configPath, cfg) test.WriteConfig(t, configPath, cfg)
_, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err) as.NoError(err)
assertStats(t, as, 31, 31, 31, 0) assertFormatted(t, as, out, 31)
// globally exclude nix files // globally exclude nix files
cfg.Global.Excludes = []string{"*.nix"} cfg.Global.Excludes = []string{"*.nix"}
test.WriteConfig(t, configPath, cfg) test.WriteConfig(t, configPath, cfg)
_, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err) as.NoError(err)
assertStats(t, as, 31, 31, 30, 0) assertFormatted(t, as, out, 30)
// add haskell files to the global exclude // add haskell files to the global exclude
cfg.Global.Excludes = []string{"*.nix", "*.hs"} cfg.Global.Excludes = []string{"*.nix", "*.hs"}
test.WriteConfig(t, configPath, cfg) test.WriteConfig(t, configPath, cfg)
_, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err) as.NoError(err)
assertStats(t, as, 31, 31, 24, 0) assertFormatted(t, as, out, 24)
echo := cfg.Formatters["echo"] echo := cfg.Formatters["echo"]
@ -133,40 +133,40 @@ func TestIncludesAndExcludes(t *testing.T) {
echo.Excludes = []string{"*.py"} echo.Excludes = []string{"*.py"}
test.WriteConfig(t, configPath, cfg) test.WriteConfig(t, configPath, cfg)
_, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err) as.NoError(err)
assertStats(t, as, 31, 31, 22, 0) assertFormatted(t, as, out, 22)
// remove go files from the echo formatter // remove go files from the echo formatter
echo.Excludes = []string{"*.py", "*.go"} echo.Excludes = []string{"*.py", "*.go"}
test.WriteConfig(t, configPath, cfg) test.WriteConfig(t, configPath, cfg)
_, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err) as.NoError(err)
assertStats(t, as, 31, 31, 21, 0) assertFormatted(t, as, out, 21)
// adjust the includes for echo to only include elm files // adjust the includes for echo to only include elm files
echo.Includes = []string{"*.elm"} echo.Includes = []string{"*.elm"}
test.WriteConfig(t, configPath, cfg) test.WriteConfig(t, configPath, cfg)
_, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err) as.NoError(err)
assertStats(t, as, 31, 31, 1, 0) assertFormatted(t, as, out, 1)
// add js files to echo formatter // add js files to echo formatter
echo.Includes = []string{"*.elm", "*.js"} echo.Includes = []string{"*.elm", "*.js"}
test.WriteConfig(t, configPath, cfg) test.WriteConfig(t, configPath, cfg)
_, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err) as.NoError(err)
assertStats(t, as, 31, 31, 2, 0) assertFormatted(t, as, out, 2)
} }
func TestCache(t *testing.T) { func TestCache(t *testing.T) {
as := require.New(t) as := require.New(t)
tempDir := test.TempExamples(t) tempDir := test.TempExamples(t)
configPath := tempDir + "/touch.toml" configPath := tempDir + "/echo.toml"
// test without any excludes // test without any excludes
cfg := config2.Config{ cfg := config2.Config{
@ -181,7 +181,7 @@ func TestCache(t *testing.T) {
test.WriteConfig(t, configPath, cfg) test.WriteConfig(t, configPath, cfg)
out, err := cmd(t, "--config-file", configPath, "--tree-root", tempDir) out, err := cmd(t, "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err) as.NoError(err)
assertStats(t, as, 31, 31, 31, 0) assertFormatted(t, as, out, 31)
out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir) out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err) as.NoError(err)
@ -190,7 +190,7 @@ func TestCache(t *testing.T) {
// clear cache // clear cache
out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir, "-c") out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir, "-c")
as.NoError(err) as.NoError(err)
assertStats(t, as, 31, 31, 31, 0) assertFormatted(t, as, out, 31)
out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir) out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err) as.NoError(err)
@ -199,7 +199,7 @@ func TestCache(t *testing.T) {
// clear cache // clear cache
out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir, "-c") out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir, "-c")
as.NoError(err) as.NoError(err)
assertStats(t, as, 31, 31, 31, 0) assertFormatted(t, as, out, 31)
out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir) out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err) as.NoError(err)
@ -208,7 +208,7 @@ func TestCache(t *testing.T) {
// no cache // no cache
out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir, "--no-cache") out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir, "--no-cache")
as.NoError(err) as.NoError(err)
assertStats(t, as, 31, 31, 31, 0) assertStats(t, as, out, 31, 31, 31, 0)
} }
func TestChangeWorkingDirectory(t *testing.T) { func TestChangeWorkingDirectory(t *testing.T) {
@ -240,22 +240,22 @@ func TestChangeWorkingDirectory(t *testing.T) {
// by default, we look for ./treefmt.toml and use the cwd for the tree root // by default, we look for ./treefmt.toml and use the cwd for the tree root
// this should fail if the working directory hasn't been changed first // this should fail if the working directory hasn't been changed first
_, err = cmd(t, "-C", tempDir) out, err := cmd(t, "-C", tempDir)
as.NoError(err) as.NoError(err)
assertStats(t, as, 31, 31, 31, 0) assertFormatted(t, as, out, 31)
} }
func TestFailOnChange(t *testing.T) { func TestFailOnChange(t *testing.T) {
as := require.New(t) as := require.New(t)
tempDir := test.TempExamples(t) tempDir := test.TempExamples(t)
configPath := tempDir + "/touch.toml" configPath := tempDir + "/echo.toml"
// test without any excludes // test without any excludes
cfg := config2.Config{ cfg := config2.Config{
Formatters: map[string]*config2.Formatter{ Formatters: map[string]*config2.Formatter{
"touch": { "echo": {
Command: "touch", Command: "echo",
Includes: []string{"*"}, Includes: []string{"*"},
}, },
}, },
@ -270,7 +270,7 @@ func TestBustCacheOnFormatterChange(t *testing.T) {
as := require.New(t) as := require.New(t)
tempDir := test.TempExamples(t) tempDir := test.TempExamples(t)
configPath := tempDir + "/touch.toml" configPath := tempDir + "/echo.toml"
// symlink some formatters into temp dir, so we can mess with their mod times // symlink some formatters into temp dir, so we can mess with their mod times
binPath := tempDir + "/bin" binPath := tempDir + "/bin"
@ -304,33 +304,33 @@ func TestBustCacheOnFormatterChange(t *testing.T) {
test.WriteConfig(t, configPath, cfg) test.WriteConfig(t, configPath, cfg)
args := []string{"--config-file", configPath, "--tree-root", tempDir} args := []string{"--config-file", configPath, "--tree-root", tempDir}
_, err := cmd(t, args...) out, err := cmd(t, args...)
as.NoError(err) as.NoError(err)
assertStats(t, as, 31, 31, 3, 0) assertFormatted(t, as, out, 3)
// tweak mod time of elm formatter // tweak mod time of elm formatter
as.NoError(test.RecreateSymlink(t, binPath+"/"+"elm-format")) as.NoError(test.RecreateSymlink(t, binPath+"/"+"elm-format"))
_, err = cmd(t, args...) out, err = cmd(t, args...)
as.NoError(err) as.NoError(err)
assertStats(t, as, 31, 31, 3, 0) assertFormatted(t, as, out, 3)
// check cache is working // check cache is working
_, err = cmd(t, args...) out, err = cmd(t, args...)
as.NoError(err) as.NoError(err)
assertStats(t, as, 31, 0, 0, 0) assertFormatted(t, as, out, 0)
// tweak mod time of python formatter // tweak mod time of python formatter
as.NoError(test.RecreateSymlink(t, binPath+"/"+"black")) as.NoError(test.RecreateSymlink(t, binPath+"/"+"black"))
_, err = cmd(t, args...) out, err = cmd(t, args...)
as.NoError(err) as.NoError(err)
assertStats(t, as, 31, 31, 3, 0) assertFormatted(t, as, out, 3)
// check cache is working // check cache is working
_, err = cmd(t, args...) out, err = cmd(t, args...)
as.NoError(err) as.NoError(err)
assertStats(t, as, 31, 0, 0, 0) assertFormatted(t, as, out, 0)
// add go formatter // add go formatter
cfg.Formatters["go"] = &config2.Formatter{ cfg.Formatters["go"] = &config2.Formatter{
@ -340,40 +340,40 @@ func TestBustCacheOnFormatterChange(t *testing.T) {
} }
test.WriteConfig(t, configPath, cfg) test.WriteConfig(t, configPath, cfg)
_, err = cmd(t, args...) out, err = cmd(t, args...)
as.NoError(err) as.NoError(err)
assertStats(t, as, 31, 31, 4, 0) assertFormatted(t, as, out, 4)
// check cache is working // check cache is working
_, err = cmd(t, args...) out, err = cmd(t, args...)
as.NoError(err) as.NoError(err)
assertStats(t, as, 31, 0, 0, 0) assertFormatted(t, as, out, 0)
// remove python formatter // remove python formatter
delete(cfg.Formatters, "python") delete(cfg.Formatters, "python")
test.WriteConfig(t, configPath, cfg) test.WriteConfig(t, configPath, cfg)
_, err = cmd(t, args...) out, err = cmd(t, args...)
as.NoError(err) as.NoError(err)
assertStats(t, as, 31, 31, 2, 0) assertFormatted(t, as, out, 2)
// check cache is working // check cache is working
_, err = cmd(t, args...) out, err = cmd(t, args...)
as.NoError(err) as.NoError(err)
assertStats(t, as, 31, 0, 0, 0) assertFormatted(t, as, out, 0)
// remove elm formatter // remove elm formatter
delete(cfg.Formatters, "elm") delete(cfg.Formatters, "elm")
test.WriteConfig(t, configPath, cfg) test.WriteConfig(t, configPath, cfg)
_, err = cmd(t, args...) out, err = cmd(t, args...)
as.NoError(err) as.NoError(err)
assertStats(t, as, 31, 31, 1, 0) assertFormatted(t, as, out, 1)
// check cache is working // check cache is working
_, err = cmd(t, args...) out, err = cmd(t, args...)
as.NoError(err) as.NoError(err)
assertStats(t, as, 31, 0, 0, 0) assertFormatted(t, as, out, 0)
} }
func TestGitWorktree(t *testing.T) { func TestGitWorktree(t *testing.T) {
@ -407,28 +407,28 @@ func TestGitWorktree(t *testing.T) {
wt, err := repo.Worktree() wt, err := repo.Worktree()
as.NoError(err, "failed to get git worktree") as.NoError(err, "failed to get git worktree")
run := func(traversed int, emitted int, matched int, formatted int) { run := func(formatted int) {
out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err) as.NoError(err)
assertFormatted(t, as, out, formatted) assertFormatted(t, as, out, formatted)
} }
// run before adding anything to the worktree // run before adding anything to the worktree
run(0, 0, 0, 0) run(0)
// add everything to the worktree // add everything to the worktree
as.NoError(wt.AddGlob(".")) as.NoError(wt.AddGlob("."))
as.NoError(err) as.NoError(err)
run(31, 31, 31, 0) run(31)
// remove python directory // remove python directory
as.NoError(wt.RemoveGlob("python/*")) as.NoError(wt.RemoveGlob("python/*"))
run(28, 28, 28, 0) run(28)
// walk with filesystem instead of git // walk with filesystem instead of git
_, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--walk", "filesystem") out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--walk", "filesystem")
as.NoError(err) as.NoError(err)
assertStats(t, as, 59, 59, 59, 0) assertFormatted(t, as, out, 59)
} }
func TestPathsArg(t *testing.T) { func TestPathsArg(t *testing.T) {
@ -461,17 +461,17 @@ func TestPathsArg(t *testing.T) {
test.WriteConfig(t, configPath, cfg) test.WriteConfig(t, configPath, cfg)
// without any path args // without any path args
_, err = cmd(t, "-C", tempDir) out, err := cmd(t, "-C", tempDir)
as.NoError(err) as.NoError(err)
assertStats(t, as, 31, 31, 31, 0) assertFormatted(t, as, out, 31)
// specify some explicit paths // specify some explicit paths
_, err = cmd(t, "-C", tempDir, "-c", "elm/elm.json", "haskell/Nested/Foo.hs") out, err = cmd(t, "-C", tempDir, "-c", "elm/elm.json", "haskell/Nested/Foo.hs")
as.NoError(err) as.NoError(err)
assertStats(t, as, 4, 4, 4, 0) assertFormatted(t, as, out, 2)
// specify a bad path // specify a bad path
_, err = cmd(t, "-C", tempDir, "-c", "elm/elm.json", "haskell/Nested/Bar.hs") out, err = cmd(t, "-C", tempDir, "-c", "elm/elm.json", "haskell/Nested/Bar.hs")
as.ErrorContains(err, "no such file or directory") as.ErrorContains(err, "no such file or directory")
} }
@ -526,9 +526,9 @@ go/main.go
_, _ = stdin.Seek(0, 0) _, _ = stdin.Seek(0, 0)
}() }()
_, err = cmd(t, "-C", tempDir, "--stdin") out, err := cmd(t, "-C", tempDir, "--stdin")
as.NoError(err) as.NoError(err)
assertStats(t, as, 6, 6, 6, 0) assertFormatted(t, as, out, 3)
} }
func TestDeterministicOrderingInPipeline(t *testing.T) { func TestDeterministicOrderingInPipeline(t *testing.T) {

View File

@ -7,8 +7,6 @@ import (
"path/filepath" "path/filepath"
"testing" "testing"
"git.numtide.com/numtide/treefmt/stats"
"git.numtide.com/numtide/treefmt/test" "git.numtide.com/numtide/treefmt/test"
"github.com/alecthomas/kong" "github.com/alecthomas/kong"
@ -57,12 +55,12 @@ func cmd(t *testing.T, args ...string) ([]byte, error) {
// reset and read the temporary output // reset and read the temporary output
if _, err = tempOut.Seek(0, 0); err != nil { if _, err = tempOut.Seek(0, 0); err != nil {
return nil, fmt.Errorf("failed to reset temp output for reading: %w", err) return nil, fmt.Errorf("%w: failed to reset temp output for reading", err)
} }
out, err := io.ReadAll(tempOut) out, err := io.ReadAll(tempOut)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to read temp output: %w", err) return nil, fmt.Errorf("%w: failed to read temp output", err)
} }
// swap outputs back // swap outputs back
@ -72,12 +70,12 @@ func cmd(t *testing.T, args ...string) ([]byte, error) {
return out, nil return out, nil
} }
func assertStats(t *testing.T, as *require.Assertions, traversed int32, emitted int32, matched int32, formatted int32) { func assertStats(t *testing.T, as *require.Assertions, output []byte, traversed int32, emitted int32, matched int32, formatted int32) {
t.Helper() t.Helper()
as.Equal(traversed, stats.Value(stats.Traversed)) as.Contains(string(output), fmt.Sprintf("traversed %d files", traversed))
as.Equal(emitted, stats.Value(stats.Emitted)) as.Contains(string(output), fmt.Sprintf("emitted %d files", emitted))
as.Equal(matched, stats.Value(stats.Matched)) as.Contains(string(output), fmt.Sprintf("matched %d files", matched))
as.Equal(formatted, stats.Value(stats.Formatted)) as.Contains(string(output), fmt.Sprintf("formatted %d files", formatted))
} }
func assertFormatted(t *testing.T, as *require.Assertions, output []byte, count int) { func assertFormatted(t *testing.T, as *require.Assertions, output []byte, count int) {

View File

@ -1,11 +1,6 @@
package config package config
import ( import "github.com/BurntSushi/toml"
"fmt"
"sort"
"github.com/BurntSushi/toml"
)
// Config is used to represent the list of configured Formatters. // Config is used to represent the list of configured Formatters.
type Config struct { type Config struct {
@ -13,39 +8,11 @@ type Config struct {
// Excludes is an optional list of glob patterns used to exclude certain files from all formatters. // Excludes is an optional list of glob patterns used to exclude certain files from all formatters.
Excludes []string Excludes []string
} }
Names []string `toml:"-"`
Formatters map[string]*Formatter `toml:"formatter"` Formatters map[string]*Formatter `toml:"formatter"`
} }
// ReadFile reads from path and unmarshals toml into a Config instance. // ReadFile reads from path and unmarshals toml into a Config instance.
func ReadFile(path string, names []string) (cfg *Config, err error) { func ReadFile(path string) (cfg *Config, err error) {
if _, err = toml.DecodeFile(path, &cfg); err != nil { _, err = toml.DecodeFile(path, &cfg)
return nil, fmt.Errorf("failed to decode config file: %w", err)
}
// filter formatters based on provided names
if len(names) > 0 {
filtered := make(map[string]*Formatter)
// check if the provided names exist in the config
for _, name := range names {
formatterCfg, ok := cfg.Formatters[name]
if !ok {
return nil, fmt.Errorf("formatter %v not found in config", name)
}
filtered[name] = formatterCfg
}
// updated formatters
cfg.Formatters = filtered
}
// sort the formatter names so that, as we construct pipelines, we add formatters in a determinstic fashion. This
// ensures a deterministic order even when all priority values are the same e.g. 0
for name := range cfg.Formatters {
cfg.Names = append(cfg.Names, name)
}
sort.Strings(cfg.Names)
return return
} }

View File

@ -9,7 +9,7 @@ import (
func TestReadConfigFile(t *testing.T) { func TestReadConfigFile(t *testing.T) {
as := require.New(t) as := require.New(t)
cfg, err := ReadFile("../test/examples/treefmt.toml", nil) cfg, err := ReadFile("../test/examples/treefmt.toml")
as.NoError(err, "failed to read config file") as.NoError(err, "failed to read config file")
as.NotNil(cfg) as.NotNil(cfg)

View File

@ -8,8 +8,6 @@ import (
"os/exec" "os/exec"
"time" "time"
"git.numtide.com/numtide/treefmt/walk"
"git.numtide.com/numtide/treefmt/config" "git.numtide.com/numtide/treefmt/config"
"github.com/charmbracelet/log" "github.com/charmbracelet/log"
@ -40,7 +38,7 @@ func (f *Formatter) Executable() string {
return f.executable return f.executable
} }
func (f *Formatter) Apply(ctx context.Context, files []*walk.File, filter bool) error { func (f *Formatter) Apply(ctx context.Context, paths []string, filter bool) error {
start := time.Now() start := time.Now()
// construct args, starting with config // construct args, starting with config
@ -54,9 +52,9 @@ func (f *Formatter) Apply(ctx context.Context, files []*walk.File, filter bool)
f.batch = f.batch[:0] f.batch = f.batch[:0]
// filter paths // filter paths
for _, file := range files { for _, path := range paths {
if f.Wants(file) { if f.Wants(path) {
f.batch = append(f.batch, file.RelPath) f.batch = append(f.batch, path)
} }
} }
@ -69,14 +67,12 @@ func (f *Formatter) Apply(ctx context.Context, files []*walk.File, filter bool)
args = append(args, f.batch...) args = append(args, f.batch...)
} else { } else {
// exit early if nothing to process // exit early if nothing to process
if len(files) == 0 { if len(paths) == 0 {
return nil return nil
} }
// append paths to the args // append paths to the args
for _, file := range files { args = append(args, paths...)
args = append(args, file.RelPath)
}
} }
// execute the command // execute the command
@ -87,22 +83,22 @@ func (f *Formatter) Apply(ctx context.Context, files []*walk.File, filter bool)
if len(out) > 0 { if len(out) > 0 {
_, _ = fmt.Fprintf(os.Stderr, "%s error:\n%s\n", f.name, out) _, _ = fmt.Fprintf(os.Stderr, "%s error:\n%s\n", f.name, out)
} }
return fmt.Errorf("formatter %s failed to apply: %w", f.name, err) return fmt.Errorf("%w: formatter %s failed to apply", err, f.name)
} }
// //
f.log.Infof("%v files processed in %v", len(files), time.Now().Sub(start)) f.log.Infof("%v files processed in %v", len(paths), time.Now().Sub(start))
return nil return nil
} }
// Wants is used to test if a Formatter wants a path based on it's configured Includes and Excludes patterns. // Wants is used to test if a Formatter wants a path based on it's configured Includes and Excludes patterns.
// Returns true if the Formatter should be applied to path, false otherwise. // Returns true if the Formatter should be applied to path, false otherwise.
func (f *Formatter) Wants(file *walk.File) bool { func (f *Formatter) Wants(path string) bool {
match := !PathMatches(file.RelPath, f.excludes) && PathMatches(file.RelPath, f.includes) match := !PathMatches(path, f.excludes) && PathMatches(path, f.includes)
if match { if match {
f.log.Debugf("match: %v", file) f.log.Debugf("match: %v", path)
} }
return match return match
} }
@ -111,7 +107,7 @@ func (f *Formatter) Wants(file *walk.File) bool {
func NewFormatter( func NewFormatter(
name string, name string,
treeRoot string, treeRoot string,
cfg *config.Formatter, config *config.Formatter,
globalExcludes []glob.Glob, globalExcludes []glob.Glob,
) (*Formatter, error) { ) (*Formatter, error) {
var err error var err error
@ -120,11 +116,11 @@ func NewFormatter(
// capture config and the formatter's name // capture config and the formatter's name
f.name = name f.name = name
f.config = cfg f.config = config
f.workingDir = treeRoot f.workingDir = treeRoot
// test if the formatter is available // test if the formatter is available
executable, err := exec.LookPath(cfg.Command) executable, err := exec.LookPath(config.Command)
if errors.Is(err, exec.ErrNotFound) { if errors.Is(err, exec.ErrNotFound) {
return nil, ErrCommandNotFound return nil, ErrCommandNotFound
} else if err != nil { } else if err != nil {
@ -133,20 +129,20 @@ func NewFormatter(
f.executable = executable f.executable = executable
// initialise internal state // initialise internal state
if cfg.Pipeline == "" { if config.Pipeline == "" {
f.log = log.WithPrefix(fmt.Sprintf("format | %s", name)) f.log = log.WithPrefix(fmt.Sprintf("format | %s", name))
} else { } else {
f.log = log.WithPrefix(fmt.Sprintf("format | %s[%s]", cfg.Pipeline, name)) f.log = log.WithPrefix(fmt.Sprintf("format | %s[%s]", config.Pipeline, name))
} }
f.includes, err = CompileGlobs(cfg.Includes) f.includes, err = CompileGlobs(config.Includes)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to compile formatter '%v' includes: %w", f.name, err) return nil, fmt.Errorf("%w: formatter '%v' includes", err, f.name)
} }
f.excludes, err = CompileGlobs(cfg.Excludes) f.excludes, err = CompileGlobs(config.Excludes)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to compile formatter '%v' excludes: %w", f.name, err) return nil, fmt.Errorf("%w: formatter '%v' excludes", err, f.name)
} }
f.excludes = append(f.excludes, globalExcludes...) f.excludes = append(f.excludes, globalExcludes...)

View File

@ -13,7 +13,7 @@ func CompileGlobs(patterns []string) ([]glob.Glob, error) {
for i, pattern := range patterns { for i, pattern := range patterns {
g, err := glob.Compile(pattern) g, err := glob.Compile(pattern)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to compile include pattern '%v': %w", pattern, err) return nil, fmt.Errorf("%w: failed to compile include pattern '%v'", err, pattern)
} }
globs[i] = g globs[i] = g
} }

View File

@ -3,8 +3,6 @@ package format
import ( import (
"context" "context"
"slices" "slices"
"git.numtide.com/numtide/treefmt/walk"
) )
type Pipeline struct { type Pipeline struct {
@ -19,7 +17,7 @@ func (p *Pipeline) Add(f *Formatter) {
}) })
} }
func (p *Pipeline) Wants(path *walk.File) bool { func (p *Pipeline) Wants(path string) bool {
var match bool var match bool
for _, f := range p.sequence { for _, f := range p.sequence {
match = f.Wants(path) match = f.Wants(path)
@ -30,7 +28,7 @@ func (p *Pipeline) Wants(path *walk.File) bool {
return match return match
} }
func (p *Pipeline) Apply(ctx context.Context, paths []*walk.File) error { func (p *Pipeline) Apply(ctx context.Context, paths []string) error {
for _, f := range p.sequence { for _, f := range p.sequence {
if err := f.Apply(ctx, paths, len(p.sequence) > 1); err != nil { if err := f.Apply(ctx, paths, len(p.sequence) > 1); err != nil {
return err return err

View File

@ -36,6 +36,5 @@ func main() {
} }
ctx := kong.Parse(&cli.Cli) ctx := kong.Parse(&cli.Cli)
cli.ConfigureLogging()
ctx.FatalIfErrorf(ctx.Run()) ctx.FatalIfErrorf(ctx.Run())
} }

View File

@ -1,3 +1,3 @@
[formatter.echo] [formatter.echo]
command = "touch" command = "echo"
includes = [ "*.*" ] includes = [ "*.*" ]

View File

@ -2,7 +2,6 @@
[formatter.deadnix] [formatter.deadnix]
command = "deadnix" command = "deadnix"
options = ["--edit"]
includes = ["*.nix"] includes = ["*.nix"]
pipeline = "nix" pipeline = "nix"
priority = 1 priority = 1

View File

@ -2,7 +2,6 @@ package walk
import ( import (
"context" "context"
"io/fs"
"os" "os"
"path/filepath" "path/filepath"
) )
@ -16,42 +15,18 @@ func (f filesystemWalker) Root() string {
return f.root return f.root
} }
func (f filesystemWalker) Walk(_ context.Context, fn WalkFunc) error { func (f filesystemWalker) Walk(_ context.Context, fn filepath.WalkFunc) error {
relPathOffset := len(f.root) + 1
relPathFn := func(path string) (relPath string) {
if len(path) >= relPathOffset {
relPath = path[relPathOffset:]
}
return
}
walkFn := func(path string, info fs.FileInfo, err error) error {
file := File{
Path: path,
RelPath: relPathFn(path),
Info: info,
}
return fn(&file, err)
}
if len(f.paths) == 0 { if len(f.paths) == 0 {
return filepath.Walk(f.root, walkFn) return filepath.Walk(f.root, fn)
} }
for _, path := range f.paths { for _, path := range f.paths {
info, err := os.Stat(path) info, err := os.Stat(path)
if err = filepath.Walk(path, walkFn); err != nil { if err = filepath.Walk(path, fn); err != nil {
return err return err
} }
file := File{ if err = fn(path, info, err); err != nil {
Path: path,
RelPath: relPathFn(path),
Info: info,
}
if err = fn(&file, err); err != nil {
return err return err
} }
} }

View File

@ -24,20 +24,10 @@ func (g *gitWalker) Root() string {
return g.root return g.root
} }
func (g *gitWalker) Walk(ctx context.Context, fn WalkFunc) error { func (g *gitWalker) Walk(ctx context.Context, fn filepath.WalkFunc) error {
// for quick relative paths
relPathOffset := len(g.root) + 1
relPathFn := func(path string) (relPath string) {
if len(path) >= relPathOffset {
relPath = path[relPathOffset:]
}
return
}
idx, err := g.repo.Storer.Index() idx, err := g.repo.Storer.Index()
if err != nil { if err != nil {
return fmt.Errorf("failed to open git index: %w", err) return fmt.Errorf("%w: failed to open index", err)
} }
if len(g.paths) > 0 { if len(g.paths) > 0 {
@ -59,13 +49,7 @@ func (g *gitWalker) Walk(ctx context.Context, fn WalkFunc) error {
return nil return nil
} }
file := File{ return fn(path, info, err)
Path: path,
RelPath: relPathFn(path),
Info: info,
}
return fn(&file, err)
}) })
if err != nil { if err != nil {
return err return err
@ -82,14 +66,7 @@ func (g *gitWalker) Walk(ctx context.Context, fn WalkFunc) error {
// stat the file // stat the file
info, err := os.Lstat(path) info, err := os.Lstat(path)
if err = fn(path, info, err); err != nil {
file := File{
Path: path,
RelPath: relPathFn(path),
Info: info,
}
if err = fn(&file, err); err != nil {
return err return err
} }
} }
@ -102,7 +79,7 @@ func (g *gitWalker) Walk(ctx context.Context, fn WalkFunc) error {
func NewGit(root string, paths []string) (Walker, error) { func NewGit(root string, paths []string) (Walker, error) {
repo, err := git.PlainOpen(root) repo, err := git.PlainOpen(root)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to open git repo: %w", err) return nil, fmt.Errorf("%w: failed to open git repo", err)
} }
return &gitWalker{root, paths, repo}, nil return &gitWalker{root, paths, repo}, nil
} }

View File

@ -3,7 +3,7 @@ package walk
import ( import (
"context" "context"
"fmt" "fmt"
"io/fs" "path/filepath"
) )
type Type string type Type string
@ -14,21 +14,9 @@ const (
Filesystem Type = "filesystem" Filesystem Type = "filesystem"
) )
type File struct {
Path string
RelPath string
Info fs.FileInfo
}
func (f File) String() string {
return f.Path
}
type WalkFunc func(file *File, err error) error
type Walker interface { type Walker interface {
Root() string Root() string
Walk(ctx context.Context, fn WalkFunc) error Walk(ctx context.Context, fn filepath.WalkFunc) error
} }
func New(walkerType Type, root string, paths []string) (Walker, error) { func New(walkerType Type, root string, paths []string) (Walker, error) {