doc: remove authors list

Signed-off-by: Brian McGee <brian@bmcgee.ie>

LICENSE.md

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2024 Treefmt Contributors
+Copyright (c) 2024 NumTide Sarl and contributors
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -0,0 +1,132 @@
+<h1 align="center">
+  <br>
+  <img src="docs/assets/logo.svg" alt="logo" width="200">
+  <br>
+  treefmt — one CLI to format your repo
+  <br>
+  <br>
+</h1>
+
+[![Support room on Matrix](https://img.shields.io/matrix/treefmt:numtide.com.svg?label=%23treefmt%3Anumtide.com&logo=matrix&server_fqdn=matrix.numtide.com)](https://matrix.to/#/#treefmt:numtide.com)
+
+**Status: beta**
+
+`treefmt` applies all the needed formatters to your project with one command line.
+
+## Motivation
+
+Before making contributions to any project, it’s common to get your code formatted according to the project’s standards. This task seems trivial from the first sight — you can simply set up the required language formatter in your IDE. But contributing to multiple projects requires more effort: you need to change the code formatter configs each time you switch between projects, or call formatters manually.
+
+Formatting requires less effort if a universal formatter for multiple languages is in place, which is also project-specific.
+
+## About treefmt
+
+`treefmt` runs all your formatters with one command. It’s easy to configure and fast to execute.
+
+[![asciicast](https://asciinema.org/a/cwtaWUTdBa8qCKJVp40bTwxf0.svg)](https://asciinema.org/a/cwtaWUTdBa8qCKJVp40bTwxf0)
+
+Its main features are:
+
+-   Providing a unified CLI and output: You don’t need to remember which formatters are required for each project. Once you specify the formatters in the config file, you can trigger all of them with one command and get a standardized output.
+-   Running all the formatters in parallel: A standard script loops over your folders and runs each formatter consequentially. In contrast, `treefmt` runs formatters in parallel. This way, the formatting job takes less time.
+-   Caching the changed files: When formatters are run in a script, they process all the files they encounter, no matter whether the code has changed. This unnecessary work can be eliminated if only the changed files are formatted. `treefmt` caches the changed files and marks them for re-formatting.
+
+Just type `treefmt` in any folder to reformat the whole code tree. All in all, you get a fast and simple formatting solution.
+
+## Installation
+
+You can install the tool by downloading the binary. Find the binaries for different architectures [here](https://github.com/numtide/treefmt/releases). Otherwise, you can install the package from the source code — either with [cargo](https://github.com/rust-lang/cargo), or with help of [nix](https://github.com/NixOS/nix). We describe the installation process in detail in the [docs](https://github.com/numtide/treefmt/wiki).
+
+## Usage
+
+In order to use `treefmt` in your project, make sure the config file `treefmt.toml` is present in the root folder and is edited to your needs. You can generate it with:
+
+```
+$ treefmt --init
+```
+
+You can run `treefmt` in your project root folder like this:
+
+```
+$ treefmt
+```
+
+To explore the tool’s flags and options, type:
+
+```console
+$ treefmt --help
+```
+
+Additionally, there's a special tool called [`treefmt-nix`](https://github.com/numtide/treefmt-nix) for using both `treefmt` and [`nix`](https://github.com/NixOS/nix).
+
+## Configuration
+
+Fomatters are specified in the config file `treefmt.toml`, which is usually located in the project root folder. The generic way to specify a formatter is like this:
+
+```
+[formatter.<name>]
+command = "<formatter-command>"
+options = [“<formatter-option-1>”...]
+includes = ["<glob>"]
+```
+
+For example, if you want to use [nixpkgs-fmt](https://github.com/nix-community/nixpkgs-fmt) on your Nix project and rustfmt on your Rust project, then `treefmt.toml` will look as follows:
+
+```
+[formatter.nix]
+command = "nixpkgs-fmt"
+includes = ["*.nix"]
+
+[formatter.rust]
+command = "rustfmt"
+options = ["--edition", "2018"]
+includes = ["*.rs"]
+```
+
+Before specifying the formatter in the config, make sure it’s installed.
+
+To find and share existing formatter recipes, take a look at the [docs](https://github.com/numtide/treefmt/wiki).
+
+If you are a Nix user, you might also be interested in [treefmt-nix](https://github.com/numtide/treefmt-nix) to use Nix to configure and bring in formatters.
+
+## Compatibility
+
+`Treefmt` works with any formatter that adheres to the [following specification](https://github.com/numtide/treefmt/blob/main/docs/formatters-spec.md). For instance, you can go for:
+
+-   [clang-format](https://clang.llvm.org/docs/ClangFormat.html) for Java
+-   gofmt for Golang
+-   Prettier for JavaScript/HTML/CSS
+
+Find the full list of supported formatters [here](https://numtide.github.io/treefmt/formatters).
+
+## Upcoming features
+
+This project is still pretty new. Down the line we also want to add support for:
+
+-   IDE integration
+-   Pre-commit hooks
+-   Effective support of multiple formatters
+
+## Related projects
+
+-   [EditorConfig](https://editorconfig.org/): unifies file indentations
+    configuration on a per-project basis.
+-   [prettier](https://prettier.io/): an opinionated code formatter for a number of languages.
+-   [Super-Linter](https://github.com/github/super-linter): a project by GitHub to lint all of your code.
+-   [pre-commit](https://pre-commit.com/): a framework for managing and
+    maintaining multi-language pre-commit hooks.
+
+## Contributing
+
+All contributions are welcome! We try to keep the project simple and focused. Please refer to [Contributing](./docs/contributing.md) guidelines for more information.
+
+## Commercial support
+
+Looking for help or customization?
+
+Get in touch with Numtide to get a quote. We make it easy for companies to
+work with Open Source projects: <https://numtide.com/contact>
+
+## License
+
+Unless explicitly stated otherwise, any contribution intentionally submitted for inclusion will be licensed under the [MIT license](LICENSE.md) without any additional terms or conditions.

cache/cache.go

@@ -7,6 +7,8 @@ import (
 	"fmt"
 	"io/fs"
 	"os"
+	"path/filepath"
+	"runtime"
 	"time"
 
 	"git.numtide.com/numtide/treefmt/format"
@@ -22,8 +24,6 @@ import (
 const (
 	pathsBucket      = "paths"
 	formattersBucket = "formatters"
-
-	readBatchSize = 1024
 )
 
 // Entry represents a cache entry, indicating the last size and modified time for a file path.
@@ -32,7 +32,11 @@ type Entry struct {
 	Modified time.Time
 }
 
-var db *bolt.DB
+var (
+	db            *bolt.DB
+	ReadBatchSize = 1024 * runtime.NumCPU()
+	logger        *log.Logger
+)
 
 // Open creates an instance of bolt.DB for a given treeRoot path.
 // If clean is true, Open will delete any existing data in the cache.
@@ -40,7 +44,7 @@ var db *bolt.DB
 // The database will be located in `XDG_CACHE_DIR/treefmt/eval-cache/<id>.db`, where <id> is determined by hashing
 // the treeRoot path. This associates a given treeRoot with a given instance of the cache.
 func Open(treeRoot string, clean bool, formatters map[string]*format.Formatter) (err error) {
-	l := log.WithPrefix("cache")
+	logger = log.WithPrefix("cache")
 
 	// determine a unique and consistent db name for the tree root
 	h := sha1.New()
@@ -85,7 +89,7 @@ func Open(treeRoot string, clean bool, formatters map[string]*format.Formatter)
 			}
 
 			clean = clean || entry == nil || !(entry.Size == stat.Size() && entry.Modified == stat.ModTime())
-			l.Debug(
+			logger.Debug(
 				"checking if formatter has changed",
 				"name", name,
 				"clean", clean,
@@ -174,6 +178,12 @@ func putEntry(bucket *bolt.Bucket, path string, entry *Entry) error {
 // ChangeSet is used to walk a filesystem, starting at root, and outputting any new or changed paths using pathsCh.
 // It determines if a path is new or has changed by comparing against cache entries.
 func ChangeSet(ctx context.Context, walker walk.Walker, pathsCh chan<- string) error {
+	start := time.Now()
+
+	defer func() {
+		logger.Infof("finished generating change set in %v", time.Since(start))
+	}()
+
 	var tx *bolt.Tx
 	var bucket *bolt.Bucket
 	var processed int
@@ -185,6 +195,9 @@ func ChangeSet(ctx context.Context, walker walk.Walker, pathsCh chan<- string) e
 		}
 	}()
 
+	// for quick removal of tree root from paths
+	relPathOffset := len(walker.Root()) + 1
+
 	return walker.Walk(ctx, func(path string, info fs.FileInfo, err error) error {
 		select {
 		case <-ctx.Done():
@@ -213,7 +226,8 @@ func ChangeSet(ctx context.Context, walker walk.Walker, pathsCh chan<- string) e
 			bucket = tx.Bucket([]byte(pathsBucket))
 		}
 
-		cached, err := getEntry(bucket, path)
+		relPath := path[relPathOffset:]
+		cached, err := getEntry(bucket, relPath)
 		if err != nil {
 			return err
 		}
@@ -230,13 +244,15 @@ func ChangeSet(ctx context.Context, walker walk.Walker, pathsCh chan<- string) e
 		case <-ctx.Done():
 			return ctx.Err()
 		default:
-			pathsCh <- path
+			pathsCh <- relPath
 		}
 
 		// close the current tx if we have reached the batch size
 		processed += 1
-		if processed == readBatchSize {
-			return tx.Rollback()
+		if processed == ReadBatchSize {
+			err = tx.Rollback()
+			tx = nil
+			return err
 		}
 
 		return nil
@@ -244,7 +260,12 @@ func ChangeSet(ctx context.Context, walker walk.Walker, pathsCh chan<- string) e
 }
 
 // Update is used to record updated cache information for the specified list of paths.
-func Update(paths []string) (int, error) {
+func Update(treeRoot string, paths []string) (int, error) {
+	start := time.Now()
+	defer func() {
+		logger.Infof("finished updating %v paths in %v", len(paths), time.Since(start))
+	}()
+
 	if len(paths) == 0 {
 		return 0, nil
 	}
@@ -260,7 +281,7 @@ func Update(paths []string) (int, error) {
 				return err
 			}
 
-			pathInfo, err := os.Stat(path)
+			pathInfo, err := os.Stat(filepath.Join(treeRoot, path))
 			if err != nil {
 				return err
 			}

cli/cli.go

@@ -20,6 +20,7 @@ type Format struct {
 	Walk                  walk.Type          `enum:"auto,git,filesystem" default:"auto" help:"The method used to traverse the files within --tree-root. Currently supports 'auto', 'git' or 'filesystem'."`
 	Verbosity             int                `name:"verbose" short:"v" type:"counter" default:"0" env:"LOG_LEVEL" help:"Set the verbosity of logs e.g. -vv."`
 	Version               bool               `name:"version" short:"V" help:"Print version"`
+	Init                  bool               `name:"init" short:"i" help:"Create a new treefmt.toml"`
 
 	Paths []string `name:"paths" arg:"" type:"path" optional:"" help:"Paths to format. Defaults to formatting the whole tree."`
 	Stdin bool     `help:"Format the context passed in via stdin"`

cli/format.go

@@ -8,23 +8,42 @@ import (
 	"io/fs"
 	"os"
 	"os/signal"
+	"path/filepath"
+	"runtime"
+	"slices"
+	"sort"
 	"strings"
 	"syscall"
 	"time"
 
+	"git.numtide.com/numtide/treefmt/format"
+	"github.com/gobwas/glob"
+
 	"git.numtide.com/numtide/treefmt/cache"
 	"git.numtide.com/numtide/treefmt/config"
-	format2 "git.numtide.com/numtide/treefmt/format"
 	"git.numtide.com/numtide/treefmt/walk"
 
 	"github.com/charmbracelet/log"
 	"golang.org/x/sync/errgroup"
 )
 
-var ErrFailOnChange = errors.New("unexpected changes detected, --fail-on-change is enabled")
+const (
+	BatchSize = 1024
+)
 
-func (f *Format) Run() error {
-	start := time.Now()
+var (
+	start          time.Time
+	globalExcludes []glob.Glob
+	formatters     map[string]*format.Formatter
+	pipelines      map[string]*format.Pipeline
+	pathsCh        chan string
+	processedCh    chan string
+
+	ErrFailOnChange = errors.New("unexpected changes detected, --fail-on-change is enabled")
+)
+
+func (f *Format) Run() (err error) {
+	start = time.Now()
 
 	Cli.Configure()
 
@@ -36,86 +55,50 @@ func (f *Format) Run() error {
 		}
 	}()
 
-	// create an overall context
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
 	// read config
 	cfg, err := config.ReadFile(Cli.ConfigFile)
 	if err != nil {
 		return fmt.Errorf("%w: failed to read config file", err)
 	}
 
-	globalExcludes, err := format2.CompileGlobs(cfg.Global.Excludes)
-
-	// create optional formatter filter set
-	formatterSet := make(map[string]bool)
-	for _, name := range Cli.Formatters {
-		_, ok := cfg.Formatters[name]
-		if !ok {
-			return fmt.Errorf("%w: formatter not found in config: %v", err, name)
-		}
-		formatterSet[name] = true
+	if globalExcludes, err = format.CompileGlobs(cfg.Global.Excludes); err != nil {
+		return fmt.Errorf("%w: failed to compile global globs", err)
 	}
 
-	includeFormatter := func(name string) bool {
-		if len(formatterSet) == 0 {
-			return true
-		} else {
-			_, include := formatterSet[name]
-			return include
-		}
-	}
+	pipelines = make(map[string]*format.Pipeline)
+	formatters = make(map[string]*format.Formatter)
 
-	formatters := make(map[string]*format2.Formatter)
-
-	// detect broken dependencies
-	for name, formatterCfg := range cfg.Formatters {
-		before := formatterCfg.Before
-		if before != "" {
-			// check child formatter exists
-			_, ok := cfg.Formatters[before]
+	// filter formatters
+	if len(Cli.Formatters) > 0 {
+		// first check the cli formatter list is valid
+		for _, name := range Cli.Formatters {
+			_, ok := cfg.Formatters[name]
 			if !ok {
-				return fmt.Errorf("formatter %v is before %v but config for %v was not found", name, before, before)
+				return fmt.Errorf("formatter not found in config: %v", name)
+			}
+		}
+		// next we remove any formatter configs that were not specified
+		for name := range cfg.Formatters {
+			if !slices.Contains(Cli.Formatters, name) {
+				delete(cfg.Formatters, name)
 			}
 		}
 	}
 
-	// dependency cycle detection
-	for name, formatterCfg := range cfg.Formatters {
-		var ok bool
-		var history []string
-		childName := name
-		for {
-			// add to history
-			history = append(history, childName)
+	// sort the formatter names so that, as we construct pipelines, we add formatters in a determinstic fashion. This
+	// ensures a deterministic order even when all priority values are the same e.g. 0
 
-			if formatterCfg.Before == "" {
-				break
-			} else if formatterCfg.Before == name {
-				return fmt.Errorf("formatter cycle detected %v", strings.Join(history, " -> "))
-			}
-
-			// load child config
-			childName = formatterCfg.Before
-			formatterCfg, ok = cfg.Formatters[formatterCfg.Before]
-			if !ok {
-				return fmt.Errorf("formatter not found: %v", formatterCfg.Before)
-			}
-		}
+	names := make([]string, 0, len(cfg.Formatters))
+	for name := range cfg.Formatters {
+		names = append(names, name)
 	}
+	sort.Strings(names)
 
 	// init formatters
-	for name, formatterCfg := range cfg.Formatters {
-		if !includeFormatter(name) {
-			// remove this formatter
-			delete(cfg.Formatters, name)
-			l.Debugf("formatter %v is not in formatter list %v, skipping", name, Cli.Formatters)
-			continue
-		}
-
-		formatter, err := format2.NewFormatter(name, formatterCfg, globalExcludes)
-		if errors.Is(err, format2.ErrCommandNotFound) && Cli.AllowMissingFormatter {
+	for _, name := range names {
+		formatterCfg := cfg.Formatters[name]
+		formatter, err := format.NewFormatter(name, Cli.TreeRoot, formatterCfg, globalExcludes)
+		if errors.Is(err, format.ErrCommandNotFound) && Cli.AllowMissingFormatter {
 			l.Debugf("formatter not found: %v", name)
 			continue
 		} else if err != nil {
@@ -123,141 +106,84 @@ func (f *Format) Run() error {
 		}
 
 		formatters[name] = formatter
-	}
 
-	// iterate the initialised formatters configuring parent/child relationships
-	for _, formatter := range formatters {
-		if formatter.Before() != "" {
-			child, ok := formatters[formatter.Before()]
+		if formatterCfg.Pipeline == "" {
+			pipeline := format.Pipeline{}
+			pipeline.Add(formatter)
+			pipelines[name] = &pipeline
+		} else {
+			key := fmt.Sprintf("p:%s", formatterCfg.Pipeline)
+			pipeline, ok := pipelines[key]
 			if !ok {
-				// formatter has been filtered out by the user
-				formatter.ResetBefore()
-				continue
+				pipeline = &format.Pipeline{}
+				pipelines[key] = pipeline
 			}
-			formatter.SetChild(child)
-			child.SetParent(formatter)
+			pipeline.Add(formatter)
 		}
 	}
 
+	// open the cache
 	if err = cache.Open(Cli.TreeRoot, Cli.ClearCache, formatters); err != nil {
 		return err
 	}
 
-	//
-	completedCh := make(chan string, 1024)
+	// create an app context and listen for shutdown
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
 
-	ctx = format2.SetCompletedChannel(ctx, completedCh)
+	go func() {
+		exit := make(chan os.Signal, 1)
+		signal.Notify(exit, os.Interrupt, syscall.SIGTERM)
+		<-exit
+		cancel()
+	}()
 
-	//
+	// create some groups for concurrent processing and control flow
 	eg, ctx := errgroup.WithContext(ctx)
 
-	// start the formatters
-	for name := range formatters {
-		formatter := formatters[name]
-		eg.Go(func() error {
-			return formatter.Run(ctx)
-		})
-	}
+	// create a channel for paths to be processed
+	// we use a multiple of batch size here to allow for greater concurrency
+	pathsCh = make(chan string, BatchSize*runtime.NumCPU())
 
-	// determine paths to be formatted
-	pathsCh := make(chan string, 1024)
+	// create a channel for tracking paths that have been processed
+	processedCh = make(chan string, cap(pathsCh))
 
-	// update cache as paths are completed
-	eg.Go(func() error {
-		batchSize := 1024
-		batch := make([]string, 0, batchSize)
+	// start concurrent processing tasks
+	eg.Go(updateCache(ctx))
+	eg.Go(applyFormatters(ctx))
+	eg.Go(walkFilesystem(ctx))
 
-		var changes int
+	// wait for everything to complete
+	return eg.Wait()
+}
 
-		processBatch := func() error {
-			if Cli.NoCache {
-				changes += len(batch)
-			} else {
-				count, err := cache.Update(batch)
-				if err != nil {
-					return err
-				}
-				changes += count
-			}
-			batch = batch[:0]
-			return nil
-		}
-
-	LOOP:
-		for {
-			select {
-			case <-ctx.Done():
-				return ctx.Err()
-			case path, ok := <-completedCh:
-				if !ok {
-					break LOOP
-				}
-				batch = append(batch, path)
-				if len(batch) == batchSize {
-					if err = processBatch(); err != nil {
-						return err
-					}
-				}
-			}
-		}
-
-		// final flush
-		if err = processBatch(); err != nil {
-			return err
-		}
-
-		if Cli.FailOnChange && changes != 0 {
-			return ErrFailOnChange
-		}
-
-		fmt.Printf("%v files changed in %v\n", changes, time.Now().Sub(start))
-		return nil
-	})
-
-	eg.Go(func() error {
-		// pass paths to each formatter
-		for path := range pathsCh {
-			for _, formatter := range formatters {
-				if formatter.Wants(path) {
-					formatter.Put(path)
-				}
-			}
-		}
-
-		// indicate no more paths for each formatter
-		for _, formatter := range formatters {
-			if formatter.Parent() != nil {
-				// this formatter is not a root, it will be closed by a parent
-				continue
-			}
-			formatter.Close()
-		}
-
-		// await completion
-		for _, formatter := range formatters {
-			formatter.AwaitCompletion()
-		}
-
-		// indicate no more completion events
-		close(completedCh)
-
-		return nil
-	})
-
-	eg.Go(func() (err error) {
+func walkFilesystem(ctx context.Context) func() error {
+	return func() error {
 		paths := Cli.Paths
 
 		if len(paths) == 0 && Cli.Stdin {
+
+			cwd, err := os.Getwd()
+			if err != nil {
+				return fmt.Errorf("%w: failed to determine current working directory", err)
+			}
+
 			// read in all the paths
 			scanner := bufio.NewScanner(os.Stdin)
 			for scanner.Scan() {
-				paths = append(paths, scanner.Text())
+				path := scanner.Text()
+				if !strings.HasPrefix(path, "/") {
+					// append the cwd
+					path = filepath.Join(cwd, path)
+				}
+
+				paths = append(paths, path)
 			}
 		}
 
 		walker, err := walk.New(Cli.Walk, Cli.TreeRoot, paths)
 		if err != nil {
-			return fmt.Errorf("%w: failed to create walker", err)
+			return fmt.Errorf("failed to create walker: %w", err)
 		}
 
 		defer close(pathsCh)
@@ -277,16 +203,140 @@ func (f *Format) Run() error {
 			})
 		}
 
-		return cache.ChangeSet(ctx, walker, pathsCh)
-	})
-
-	// listen for shutdown and call cancel if required
-	go func() {
-		exit := make(chan os.Signal, 1)
-		signal.Notify(exit, os.Interrupt, syscall.SIGTERM)
-		<-exit
-		cancel()
-	}()
-
-	return eg.Wait()
+		if err = cache.ChangeSet(ctx, walker, pathsCh); err != nil {
+			return fmt.Errorf("failed to generate change set: %w", err)
+		}
+		return nil
+	}
+}
+
+func updateCache(ctx context.Context) func() error {
+	return func() error {
+		batch := make([]string, 0, BatchSize)
+
+		var changes int
+
+		processBatch := func() error {
+			if Cli.NoCache {
+				changes += len(batch)
+			} else {
+				count, err := cache.Update(Cli.TreeRoot, batch)
+				if err != nil {
+					return err
+				}
+				changes += count
+			}
+			batch = batch[:0]
+			return nil
+		}
+
+	LOOP:
+		for {
+			select {
+			case <-ctx.Done():
+				return ctx.Err()
+			case path, ok := <-processedCh:
+				if !ok {
+					break LOOP
+				}
+				batch = append(batch, path)
+				if len(batch) == BatchSize {
+					if err := processBatch(); err != nil {
+						return err
+					}
+				}
+			}
+		}
+
+		// final flush
+		if err := processBatch(); err != nil {
+			return err
+		}
+
+		if Cli.FailOnChange && changes != 0 {
+			return ErrFailOnChange
+		}
+
+		fmt.Printf("%v files changed in %v\n", changes, time.Now().Sub(start))
+		return nil
+	}
+}
+
+func applyFormatters(ctx context.Context) func() error {
+	fg, ctx := errgroup.WithContext(ctx)
+	batches := make(map[string][]string)
+
+	tryApply := func(key string, path string) {
+		batch, ok := batches[key]
+		if !ok {
+			batch = make([]string, 0, BatchSize)
+		}
+		batch = append(batch, path)
+		batches[key] = batch
+
+		if len(batch) == BatchSize {
+			pipeline := pipelines[key]
+
+			// copy the batch
+			paths := make([]string, len(batch))
+			copy(paths, batch)
+
+			fg.Go(func() error {
+				if err := pipeline.Apply(ctx, paths); err != nil {
+					return err
+				}
+				for _, path := range paths {
+					processedCh <- path
+				}
+				return nil
+			})
+
+			batches[key] = batch[:0]
+		}
+	}
+
+	flushBatches := func() {
+		for key, pipeline := range pipelines {
+
+			batch := batches[key]
+			pipeline := pipeline // capture for closure
+
+			if len(batch) > 0 {
+				fg.Go(func() error {
+					if err := pipeline.Apply(ctx, batch); err != nil {
+						return fmt.Errorf("%s failure: %w", key, err)
+					}
+					for _, path := range batch {
+						processedCh <- path
+					}
+					return nil
+				})
+			}
+		}
+	}
+
+	return func() error {
+		defer func() {
+			// close processed channel
+			close(processedCh)
+		}()
+
+		for path := range pathsCh {
+			for key, pipeline := range pipelines {
+				if !pipeline.Wants(path) {
+					continue
+				}
+				tryApply(key, path)
+			}
+		}
+
+		// flush any partial batches which remain
+		flushBatches()
+
+		// wait for all outstanding formatting tasks to complete
+		if err := fg.Wait(); err != nil {
+			return fmt.Errorf("pipeline processing failure: %w", err)
+		}
+		return nil
+	}
 }

cli/format_test.go

@@ -1,11 +1,13 @@
 package cli
 
 import (
+	"bufio"
 	"fmt"
 	"os"
 	"os/exec"
 	"path"
 	"path/filepath"
+	"regexp"
 	"testing"
 
 	config2 "git.numtide.com/numtide/treefmt/config"
@@ -23,7 +25,7 @@ import (
 func TestAllowMissingFormatter(t *testing.T) {
 	as := require.New(t)
 
-	tempDir := t.TempDir()
+	tempDir := test.TempExamples(t)
 	configPath := tempDir + "/treefmt.toml"
 
 	test.WriteConfig(t, configPath, config2.Config{
@@ -41,27 +43,6 @@ func TestAllowMissingFormatter(t *testing.T) {
 	as.NoError(err)
 }
 
-func TestDependencyCycle(t *testing.T) {
-	as := require.New(t)
-
-	tempDir := t.TempDir()
-	configPath := tempDir + "/treefmt.toml"
-
-	test.WriteConfig(t, configPath, config2.Config{
-		Formatters: map[string]*config2.Formatter{
-			"a": {Command: "echo", Before: "b"},
-			"b": {Command: "echo", Before: "c"},
-			"c": {Command: "echo", Before: "a"},
-			"d": {Command: "echo", Before: "e"},
-			"e": {Command: "echo", Before: "f"},
-			"f": {Command: "echo"},
-		},
-	})
-
-	_, err := cmd(t, "--config-file", configPath, "--tree-root", tempDir)
-	as.ErrorContains(err, "formatter cycle detected")
-}
-
 func TestSpecifyingFormatters(t *testing.T) {
 	as := require.New(t)
 
@@ -129,7 +110,7 @@ func TestIncludesAndExcludes(t *testing.T) {
 	test.WriteConfig(t, configPath, cfg)
 	out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
 	as.NoError(err)
-	as.Contains(string(out), fmt.Sprintf("%d files changed", 30))
+	as.Contains(string(out), fmt.Sprintf("%d files changed", 31))
 
 	// globally exclude nix files
 	cfg.Global.Excludes = []string{"*.nix"}
@@ -137,7 +118,7 @@ func TestIncludesAndExcludes(t *testing.T) {
 	test.WriteConfig(t, configPath, cfg)
 	out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
 	as.NoError(err)
-	as.Contains(string(out), fmt.Sprintf("%d files changed", 29))
+	as.Contains(string(out), fmt.Sprintf("%d files changed", 30))
 
 	// add haskell files to the global exclude
 	cfg.Global.Excludes = []string{"*.nix", "*.hs"}
@@ -145,7 +126,7 @@ func TestIncludesAndExcludes(t *testing.T) {
 	test.WriteConfig(t, configPath, cfg)
 	out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
 	as.NoError(err)
-	as.Contains(string(out), fmt.Sprintf("%d files changed", 23))
+	as.Contains(string(out), fmt.Sprintf("%d files changed", 24))
 
 	echo := cfg.Formatters["echo"]
 
@@ -155,7 +136,7 @@ func TestIncludesAndExcludes(t *testing.T) {
 	test.WriteConfig(t, configPath, cfg)
 	out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
 	as.NoError(err)
-	as.Contains(string(out), fmt.Sprintf("%d files changed", 21))
+	as.Contains(string(out), fmt.Sprintf("%d files changed", 22))
 
 	// remove go files from the echo formatter
 	echo.Excludes = []string{"*.py", "*.go"}
@@ -163,7 +144,7 @@ func TestIncludesAndExcludes(t *testing.T) {
 	test.WriteConfig(t, configPath, cfg)
 	out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
 	as.NoError(err)
-	as.Contains(string(out), fmt.Sprintf("%d files changed", 20))
+	as.Contains(string(out), fmt.Sprintf("%d files changed", 21))
 
 	// adjust the includes for echo to only include elm files
 	echo.Includes = []string{"*.elm"}
@@ -201,7 +182,7 @@ func TestCache(t *testing.T) {
 	test.WriteConfig(t, configPath, cfg)
 	out, err := cmd(t, "--config-file", configPath, "--tree-root", tempDir)
 	as.NoError(err)
-	as.Contains(string(out), fmt.Sprintf("%d files changed", 30))
+	as.Contains(string(out), fmt.Sprintf("%d files changed", 31))
 
 	out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir)
 	as.NoError(err)
@@ -210,7 +191,7 @@ func TestCache(t *testing.T) {
 	// clear cache
 	out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir, "-c")
 	as.NoError(err)
-	as.Contains(string(out), fmt.Sprintf("%d files changed", 30))
+	as.Contains(string(out), fmt.Sprintf("%d files changed", 31))
 
 	out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir)
 	as.NoError(err)
@@ -219,7 +200,7 @@ func TestCache(t *testing.T) {
 	// clear cache
 	out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir, "-c")
 	as.NoError(err)
-	as.Contains(string(out), fmt.Sprintf("%d files changed", 30))
+	as.Contains(string(out), fmt.Sprintf("%d files changed", 31))
 
 	out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir)
 	as.NoError(err)
@@ -228,7 +209,7 @@ func TestCache(t *testing.T) {
 	// no cache
 	out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir, "--no-cache")
 	as.NoError(err)
-	as.Contains(string(out), fmt.Sprintf("%d files changed", 30))
+	as.Contains(string(out), fmt.Sprintf("%d files changed", 31))
 }
 
 func TestChangeWorkingDirectory(t *testing.T) {
@@ -262,7 +243,7 @@ func TestChangeWorkingDirectory(t *testing.T) {
 	// this should fail if the working directory hasn't been changed first
 	out, err := cmd(t, "-C", tempDir)
 	as.NoError(err)
-	as.Contains(string(out), fmt.Sprintf("%d files changed", 30))
+	as.Contains(string(out), fmt.Sprintf("%d files changed", 31))
 }
 
 func TestFailOnChange(t *testing.T) {
@@ -439,70 +420,16 @@ func TestGitWorktree(t *testing.T) {
 	// add everything to the worktree
 	as.NoError(wt.AddGlob("."))
 	as.NoError(err)
-	run(30)
+	run(31)
 
 	// remove python directory
 	as.NoError(wt.RemoveGlob("python/*"))
-	run(27)
+	run(28)
 
 	// walk with filesystem instead of git
 	out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--walk", "filesystem")
 	as.NoError(err)
-	as.Contains(string(out), fmt.Sprintf("%d files changed", 57))
-}
-
-func TestOrderingFormatters(t *testing.T) {
-	as := require.New(t)
-
-	tempDir := test.TempExamples(t)
-	configPath := path.Join(tempDir, "treefmt.toml")
-
-	// missing child
-	test.WriteConfig(t, configPath, config2.Config{
-		Formatters: map[string]*config2.Formatter{
-			"hs-a": {
-				Command:  "echo",
-				Includes: []string{"*.hs"},
-				Before:   "hs-b",
-			},
-		},
-	})
-
-	out, err := cmd(t, "--config-file", configPath, "--tree-root", tempDir)
-	as.ErrorContains(err, "formatter hs-a is before hs-b but config for hs-b was not found")
-
-	// multiple roots
-	test.WriteConfig(t, configPath, config2.Config{
-		Formatters: map[string]*config2.Formatter{
-			"hs-a": {
-				Command:  "echo",
-				Includes: []string{"*.hs"},
-				Before:   "hs-b",
-			},
-			"hs-b": {
-				Command:  "echo",
-				Includes: []string{"*.hs"},
-				Before:   "hs-c",
-			},
-			"hs-c": {
-				Command:  "echo",
-				Includes: []string{"*.hs"},
-			},
-			"py-a": {
-				Command:  "echo",
-				Includes: []string{"*.py"},
-				Before:   "py-b",
-			},
-			"py-b": {
-				Command:  "echo",
-				Includes: []string{"*.py"},
-			},
-		},
-	})
-
-	out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir)
-	as.NoError(err)
-	as.Contains(string(out), "8 files changed")
+	as.Contains(string(out), fmt.Sprintf("%d files changed", 59))
 }
 
 func TestPathsArg(t *testing.T) {
@@ -537,7 +464,7 @@ func TestPathsArg(t *testing.T) {
 	// without any path args
 	out, err := cmd(t, "-C", tempDir)
 	as.NoError(err)
-	as.Contains(string(out), fmt.Sprintf("%d files changed", 30))
+	as.Contains(string(out), fmt.Sprintf("%d files changed", 31))
 
 	// specify some explicit paths
 	out, err = cmd(t, "-C", tempDir, "-c", "elm/elm.json", "haskell/Nested/Foo.hs")
@@ -604,3 +531,63 @@ go/main.go
 	as.NoError(err)
 	as.Contains(string(out), fmt.Sprintf("%d files changed", 3))
 }
+
+func TestDeterministicOrderingInPipeline(t *testing.T) {
+	as := require.New(t)
+
+	tempDir := test.TempExamples(t)
+	configPath := tempDir + "/treefmt.toml"
+
+	test.WriteConfig(t, configPath, config2.Config{
+		Formatters: map[string]*config2.Formatter{
+			// a and b should execute in lexicographical order as they have default priority 0, with c last since it has
+			// priority 1
+			"fmt-a": {
+				Command:  "test-fmt",
+				Options:  []string{"fmt-a"},
+				Includes: []string{"*.py"},
+				Pipeline: "foo",
+			},
+			"fmt-b": {
+				Command:  "test-fmt",
+				Options:  []string{"fmt-b"},
+				Includes: []string{"*.py"},
+				Pipeline: "foo",
+			},
+			"fmt-c": {
+				Command:  "test-fmt",
+				Options:  []string{"fmt-c"},
+				Includes: []string{"*.py"},
+				Pipeline: "foo",
+				Priority: 1,
+			},
+		},
+	})
+
+	_, err := cmd(t, "-C", tempDir)
+	as.NoError(err)
+
+	matcher := regexp.MustCompile("^fmt-(.*)")
+
+	// check each affected file for the sequence of test statements which should be prepended to the end
+	sequence := []string{"fmt-a", "fmt-b", "fmt-c"}
+	paths := []string{"python/main.py", "python/virtualenv_proxy.py"}
+
+	for _, p := range paths {
+		file, err := os.Open(filepath.Join(tempDir, p))
+		as.NoError(err)
+		scanner := bufio.NewScanner(file)
+
+		idx := 0
+
+		for scanner.Scan() {
+			line := scanner.Text()
+			matches := matcher.FindAllString(line, -1)
+			if len(matches) != 1 {
+				continue
+			}
+			as.Equal(sequence[idx], matches[0])
+			idx += 1
+		}
+	}
+}

config/config_test.go

@@ -59,6 +59,18 @@ func TestReadConfigFile(t *testing.T) {
 	as.Nil(alejandra.Options)
 	as.Equal([]string{"*.nix"}, alejandra.Includes)
 	as.Equal([]string{"examples/nix/sources.nix"}, alejandra.Excludes)
+	as.Equal("nix", alejandra.Pipeline)
+	as.Equal(1, alejandra.Priority)
+
+	// deadnix
+	deadnix, ok := cfg.Formatters["deadnix"]
+	as.True(ok, "deadnix formatter not found")
+	as.Equal("deadnix", deadnix.Command)
+	as.Nil(deadnix.Options)
+	as.Equal([]string{"*.nix"}, deadnix.Includes)
+	as.Nil(deadnix.Excludes)
+	as.Equal("nix", deadnix.Pipeline)
+	as.Equal(2, deadnix.Priority)
 
 	// ruby
 	ruby, ok := cfg.Formatters["ruby"]

config/formatter.go

@@ -9,6 +9,8 @@ type Formatter struct {
 	Includes []string
 	// Excludes is an optional list of glob patterns used to exclude certain files from this Formatter.
 	Excludes []string
-	// Before is the name of another formatter which must process a path after this one
-	Before string
+	// Indicates this formatter should be executed as part of a group of formatters all sharing the same pipeline key.
+	Pipeline string
+	// Indicates the order of precedence when executing as part of a pipeline.
+	Priority int
 }

default.nix

@@ -0,0 +1,15 @@
+# This file provides backward compatibility to nix < 2.4 clients
+{ system ? builtins.currentSystem }:
+let
+  lock = builtins.fromJSON (builtins.readFile ./flake.lock);
+
+  inherit (lock.nodes.flake-compat.locked) owner repo rev narHash;
+
+  flake-compat = fetchTarball {
+    url = "https://github.com/${owner}/${repo}/archive/${rev}.tar.gz";
+    sha256 = narHash;
+  };
+
+  flake = import flake-compat { inherit system; src = ./.; };
+in
+flake.defaultNix

docs/assets/logo.svg

@@ -0,0 +1,194 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   inkscape:export-ydpi="512"
+   inkscape:export-xdpi="512"
+   inkscape:export-filename="/home/basile/dev/treefmt.png"
+   sodipodi:docname="treefmt.svg"
+   inkscape:version="1.0 (4035a4fb49, 2020-05-01)"
+   id="svg16"
+   version="1.1"
+   viewBox="0 0 12.7 12.7"
+   height="48"
+   width="48">
+  <defs
+     id="defs10">
+    <inkscape:path-effect
+       hide_knots="false"
+       only_selected="false"
+       apply_with_radius="true"
+       apply_no_radius="true"
+       use_knot_distance="true"
+       flexible="false"
+       chamfer_steps="1"
+       radius="0.5"
+       mode="F"
+       method="auto"
+       unit="px"
+       satellites_param="F,0,0,1,0,0.13229167,0,1 @ F,0,0,1,0,0.13229167,0,1 @ F,0,0,1,0,0.13229167,0,1 @ F,0,0,1,0,0.13229167,0,1"
+       lpeversion="1"
+       is_visible="true"
+       id="path-effect1012"
+       effect="fillet_chamfer" />
+    <inkscape:path-effect
+       hide_knots="false"
+       only_selected="false"
+       apply_with_radius="true"
+       apply_no_radius="true"
+       use_knot_distance="true"
+       flexible="false"
+       chamfer_steps="1"
+       radius="0.5"
+       mode="F"
+       method="auto"
+       unit="px"
+       satellites_param="F,0,0,1,0,0.13229167,0,1 @ F,0,0,1,0,0.13229167,0,1 @ F,0,0,1,0,0.13229167,0,1 @ F,0,0,1,0,0.13229167,0,1"
+       lpeversion="1"
+       is_visible="true"
+       id="path-effect1010"
+       effect="fillet_chamfer" />
+    <inkscape:path-effect
+       hide_knots="false"
+       only_selected="false"
+       apply_with_radius="true"
+       apply_no_radius="true"
+       use_knot_distance="true"
+       flexible="false"
+       chamfer_steps="1"
+       radius="0.5"
+       mode="F"
+       method="auto"
+       unit="px"
+       satellites_param="F,0,0,1,0,0.13229167,0,1 @ F,0,0,1,0,0.13229167,0,1 @ F,0,0,1,0,0.13229167,0,1 @ F,0,0,1,0,0.13229167,0,1"
+       lpeversion="1"
+       is_visible="true"
+       id="path-effect989"
+       effect="fillet_chamfer" />
+  </defs>
+  <sodipodi:namedview
+     inkscape:window-maximized="1"
+     inkscape:window-y="340"
+     inkscape:window-x="1440"
+     inkscape:window-height="1373"
+     inkscape:window-width="2560"
+     units="px"
+     showgrid="false"
+     inkscape:document-rotation="0"
+     inkscape:current-layer="layer1"
+     inkscape:document-units="mm"
+     inkscape:cy="29.374509"
+     inkscape:cx="-10.431476"
+     inkscape:zoom="11.2"
+     inkscape:pageshadow="2"
+     inkscape:pageopacity="0.0"
+     borderopacity="1.0"
+     bordercolor="#666666"
+     pagecolor="#ffffff"
+     id="base">
+    <inkscape:grid
+       dotted="false"
+       id="grid902"
+       type="xygrid" />
+  </sodipodi:namedview>
+  <metadata
+     id="metadata13">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     id="layer1"
+     inkscape:groupmode="layer"
+     inkscape:label="Layer 1">
+    <path
+       d="m 6.3370673,12.637056 c -0.5007028,0 -5.1699157,-2.6957712 -5.42026711,-3.1293926 C 0.66644877,9.074042 0.66644862,3.6824995 0.91680002,3.2488781 1.1671514,2.8152568 5.8363641,0.11948539 6.337067,0.11948538 c 0.5007028,-2e-8 5.169916,2.69577112 5.420267,3.12939252 0.250351,0.4336213 0.250352,5.8251638 0,6.2587852 -0.250351,0.4336214 -4.9195639,3.1293929 -5.4202667,3.1293929 z"
+       inkscape:randomized="0"
+       inkscape:rounded="0.08"
+       inkscape:flatsided="true"
+       sodipodi:arg2="2.0943951"
+       sodipodi:arg1="1.5707963"
+       sodipodi:r2="5.4202676"
+       sodipodi:r1="6.2587852"
+       sodipodi:cy="6.3782705"
+       sodipodi:cx="6.3370669"
+       sodipodi:sides="6"
+       id="path20"
+       style="fill:#729fcf;stroke-width:0.0142755"
+       sodipodi:type="star" />
+    <path
+       sodipodi:type="star"
+       style="fill:#888a85;stroke-width:0.0128537"
+       id="path20-3"
+       sodipodi:sides="6"
+       sodipodi:cx="6.3652418"
+       sodipodi:cy="6.3777179"
+       sodipodi:r1="5.6354737"
+       sodipodi:r2="4.8804641"
+       sodipodi:arg1="1.5707963"
+       sodipodi:arg2="2.0943951"
+       inkscape:flatsided="true"
+       inkscape:rounded="0.08"
+       inkscape:randomized="0"
+       d="m 6.3652422,12.013192 c -0.4508379,0 -4.6550446,-2.4273 -4.8804635,-2.817737 -0.225419,-0.3904371 -0.2254191,-5.2450367 -2e-7,-5.6354738 0.225419,-0.390437 4.4296255,-2.81773694 4.8804634,-2.81773696 0.4508379,-1e-8 4.6550441,2.42729966 4.8804631,2.81773676 0.225419,0.390437 0.225419,5.2450366 0,5.6354737 -0.225418,0.3904371 -4.4296249,2.8177373 -4.8804628,2.8177373 z" />
+    <path
+       sodipodi:type="rect"
+       d="m 4.6302084,3.8364582 5.0270831,0 A 0.13229167,0.13229167 45 0 1 9.7895832,3.9687499 V 4.4979165 A 0.13229167,0.13229167 135 0 1 9.6572915,4.6302082 l -5.0270831,0 A 0.13229167,0.13229167 45 0 1 4.4979167,4.4979165 V 3.9687499 A 0.13229167,0.13229167 135 0 1 4.6302084,3.8364582 Z"
+       inkscape:path-effect="#path-effect989"
+       y="3.8364582"
+       x="4.4979167"
+       height="0.79374999"
+       width="5.2916665"
+       id="rect898"
+       style="fill:#f57900;stroke-width:0.0121433" />
+    <circle
+       r="0.66145831"
+       style="fill:#73d216;stroke-width:0.0221209"
+       id="path860-3"
+       cx="2.778125"
+       cy="4.2333331" />
+    <circle
+       cy="6.4822917"
+       cx="4.6302085"
+       id="path860-3-5"
+       style="fill:#73d216;stroke-width:0.0221209"
+       r="0.66145831" />
+    <circle
+       r="0.66145831"
+       style="fill:#73d216;stroke-width:0.0221209"
+       id="path860-3-5-6"
+       cx="4.6302085"
+       cy="8.9958334" />
+    <path
+       sodipodi:type="rect"
+       d="M 6.4822916,6.0854168 H 9.6572915 A 0.13229167,0.13229167 45 0 1 9.7895832,6.2177085 V 6.7468751 A 0.13229167,0.13229167 135 0 1 9.6572915,6.8791668 H 6.4822916 A 0.13229167,0.13229167 45 0 1 6.3499999,6.7468751 V 6.2177085 A 0.13229167,0.13229167 135 0 1 6.4822916,6.0854168 Z"
+       inkscape:path-effect="#path-effect1010"
+       style="fill:#f57900;stroke-width:0.00979024"
+       id="rect898-2"
+       width="3.4395833"
+       height="0.79374999"
+       x="6.3499999"
+       y="6.0854168" />
+    <path
+       sodipodi:type="rect"
+       d="m 6.4822916,8.598959 3.1749999,0 A 0.13229167,0.13229167 45 0 1 9.7895832,8.7312506 V 9.2604173 A 0.13229167,0.13229167 135 0 1 9.6572915,9.392709 l -3.1749999,0 A 0.13229167,0.13229167 45 0 1 6.3499999,9.2604173 V 8.7312506 A 0.13229167,0.13229167 135 0 1 6.4822916,8.598959 Z"
+       inkscape:path-effect="#path-effect1012"
+       y="8.598959"
+       x="6.3499999"
+       height="0.79374999"
+       width="3.4395833"
+       id="rect898-2-9"
+       style="fill:#f57900;stroke-width:0.00979024" />
+  </g>
+</svg>

docs/book.toml

@@ -0,0 +1,6 @@
+[book]
+authors = []
+language = "en"
+multilingual = false
+src = "."
+title = "treefmt — one CLI to format your repo"

docs/contributing.md

@@ -0,0 +1,141 @@
+# Contribution guidelines
+
+This file contains instructions that will help you make a contribution.
+
+## Licensing
+
+The treefmt binaries and this user guide are licensed under the [MIT license](https://numtide.github.io/treefmt/LICENSE.html).
+
+## Before you contribute
+
+Here you can take a look at the [existing issues](https://github.com/numtide/treefmt/issues). Feel free to contribute, but make sure you have a [GitHub account](https://github.com/join) first :) .
+
+If you're new to open source, please read GitHub's guide on [How to Contribute to Open Source](https://opensource.guide/how-to-contribute/). It's a quick read, and it's a great way to introduce yourself to how things work behind the scenes in open-source projects.
+
+Before sending a pull request, make sure that you've read all the guidelines. If you don't understand something, please [state your question clearly in an issue](https://github.com/numtide/treefmt/issues/new) or ask the community on the [treefmt matrix server](https://matrix.to/#/#treefmt:numtide.com).
+
+## Creating an issue
+
+If you need to create an issue, make sure to clearly describe it, including:
+
+- The steps to reproduce it if it's a bug
+- The version of treefmt used
+- The database driver and version
+- The database version
+
+The cache database is stored in a toml file the ~/.cache/treefmt directory.
+
+## Making changes
+
+If you want to introduce changes to the project, please follow these steps:
+
+- Fork the repository on GitHub
+- Create a branch on your fork. Don't commit directly to main
+- Add the necessary tests for your changes
+- Run treefmt in the source directory before you commit your changes
+- Push your changes to the branch in your repository fork
+- Submit a pull request to the original repository
+
+Make sure you based your commits on logical and atomic units!
+
+## Examples of git history
+
+Git history that we want to have:
+
+Git history that we are trying to avoid:
+
+<details>
+
+<summary>Git history that we want to have</summary>
+
+```
+
+*   e3ed88b (HEAD -> contribution-guide, upstream/main, origin/main, origin/HEAD, main) Merge pull request #470 from zimbatm/fix_lru_cache
+
+|\
+
+| * 1ab7d9f Use rayon for multithreading command
+
+|/
+
+*   e9c5bb4 Merge pull request #468 from zimbatm/multithread
+
+|\
+
+| * de2d6cf Add lint property for Formatter struct
+
+| * cd2ed17 Fix impl on Formatter get_command() function
+
+|/
+
+*   028c344 Merge pull request #465 from rayon/0.15.0-release
+
+|\
+
+| * 7b619d6 0.15.0 release
+
+|/
+
+*   acdf7df Merge pull request #463 from zimbatm/support-multi-part-namespaces
+
+```
+
+</details>
+
+<details>
+
+<summary>Git history that we are <b>trying</b> to avoid:</summary>
+
+```
+
+*   4c8aca8 Merge pull request #120 from zimbatm/add-rayon
+
+|\
+
+| * fc2b449 use rayon for engine now
+
+| * 2304683 add rayon config
+
+| * 5285bd3 bump base image to F30
+
+* |   4d0fbe2 Merge pull request #114 from rizary/create_method_create_release
+
+|\ \
+
+| * | 36a9396 test changed
+
+| * | 22f681d method create release for github created
+
+* | |   2ef4ea1 Merge pull request #119 from rizary/config.rs
+
+|\ \ \
+
+| |/ /
+
+|/| |
+
+| * | 5f1b8f0 unused functions removed
+
+* | |   a93c361 Merge pull request #117 from zimbatm/add-getreleases-to-abstract
+
+|\ \ \
+
+| |/ /
+
+|/| |
+
+| * | 0a97236 add get_releses for Cargo
+
+| * | 55e4c57 add get_releases/get_release into engine.rs
+
+|/ /
+
+* |   badeddd Merge pull request #101 from zimbatm/extreme-cachin
+
+```
+
+</details>
+
+Additionally, it's always good to work on improving documentation and adding examples.
+
+Thank you for considering contributing to `treefmt`.

docs/faq.md

@@ -0,0 +1,19 @@
+# FAQ
+
+## How does treefmt function?
+
+`Treefmt` traverses all your project's folders, maps files to specific code formatters, and formats the code accordingly. Other tools also traverse the filesystem, but not necessarily starting from the root of the project.
+
+Contrary to other formatters, `treefmt` doesn't preview the changes before writing them to a file. If you want to view the changes, you can always check the diff in your version control (we assume that your project is checked into a version control system). You can also rely on version control if errors were introduced into your code as a result of disruptions in the formatter's work.
+
+## How is the cache organized?
+
+At this moment, the cache is represented by a flat TOML file where file paths are mapped to `mtimes`. The file is located in:
+
+```
+~/.cache/treefmt/<hash-of-the-treefmt.toml-path>.toml
+```
+
+However, we are planning to move the hash file to the destination project's root directory.
+
+At the end of each tool run, the cache file gets overwritten with the last formatting time entries. In this way, we can can compare the last change time of the file to the last formatting time, and figure out which files need re-formatting.

docs/formatters-spec.md

@@ -0,0 +1,51 @@
+# Formatter specification
+
+In order to keep the design of `treefmt` simple, we support only formatters which adhere to a certain standard. This document outlines this standard. If the formatter you would like to use doesn't comply with the rules, it's often possible to create a wrapper script that transforms the usage to match the specification.
+
+In this design, we rely on `treefmt` to do the tree traversal, and only invoke
+the code formatter on the selected files.
+
+## Rules
+
+In order for the formatter to comply to this spec, it MUST follow the
+following rules:
+
+### 1. Files passed as arguments
+
+In order to be integrated to `treefmt`'s workflow, the formatter's CLI must adhere to the following specification:
+
+```
+<command> [options] [...<files>]
+```
+
+Where:
+
+- `<command>` is the name of the formatting tool.
+- `[options]` is any number of flags and options that the formatter accepts.
+- `[...<files>]` is one or more files given to the formatter for processing.
+
+Example:
+
+```
+$ rustfmt --edition 2018 src/main.rs src/lib.rs
+```
+
+It SHOULD processes only the specified files. Files that are not passed SHOULD never be formatted.
+
+### 2. Write to changed files
+
+Whenever there is a change to the code formatting, the code formatter MUST
+write to the changes back to the original location.
+
+If there is no changes to the original file, the formatter MUST NOT write to
+the original location.
+
+### 3. Idempotent
+
+The code formatter SHOULD be indempotent. Meaning that it produces stable
+outputs.
+
+### 4. Reliable
+
+We expect the formatter to be reliable and not break the semantic of the
+formatted files.

docs/formatters.md

@@ -0,0 +1,273 @@
+# Known formatters
+
+Here is a list of all the formatters we tested. Feel free to send a PR to add other ones!
+
+## Contents
+
+Single-language formatters:
+
+- [Cabal](#cabal)
+  - [cabal-fmt](#cabal-fmt)
+- [Elm](#elm)
+  - [elm-format](#elm-format)
+- [Golang](#golang)
+  - [gofmt](#gofmt)
+  - [gofumpt](#gofumpt)
+- [Haskell](#haskell)
+  - [hlint](#hlint)
+  - [ormolu](#ormolu)
+  - [stylish-haskell](#stylish-haskell)
+- [Lua](#lua)
+  - [StyLua](#stylua)
+- [Nix](#nix)
+  - [alejandra](#alejandra)
+  - [nixpkgs-fmt](#nixpkgs-fmt)
+- [OCaml](#ocaml)
+  - [ocamlformat](#ocamlformat)
+- [PureScript](#purescript)
+  - [purs-tidy](#purs-tidy)
+- [Python](#python)
+  - [black](#black)
+- [Ruby](#ruby)
+  - [rufo](#rufo)
+- [Rust](#rust)
+  - [rustfmt](#rustfmt)
+- [Scala](#scala)
+  - [scalafmt](#scalafmt)
+- [Shell](#shell)
+  - [shellcheck](#shellcheck)
+  - [shfmt](#shfmt)
+- [Terraform](#terraform)
+  - [terraform fmt](#terraform-fmt)
+
+Multilanguage formatters:
+
+- [clang-format](#clang-format)
+- [Prettier](#prettier)
+
+## Cabal
+
+### [cabal-fmt](https://github.com/phadej/cabal-fmt)
+
+```
+command = "cabal-fmt"
+options = ["--inplace"]
+includes = ["*.cabal"]
+```
+
+## Elm
+
+### [elm-format](https://numtide.github.io/treefmt/formatters.html#elm)
+
+```
+command = "elm-format"
+options = ["--yes"]
+includes = ["*.elm"]
+```
+
+## Golang
+
+### [gofmt](https://pkg.go.dev/cmd/gofmt)
+
+```
+command = "gofmt"
+options = ["-w"]
+includes = ["*.go"]
+
+```
+
+### [gofumpt](https://github.com/mvdan/gofumpt)
+
+```
+command = "gofumpt"
+includes = ["*.go"]
+
+```
+
+## Haskell
+
+### [hlint](https://github.com/ndmitchell/hlint)
+
+```
+command = "hlint"
+includes = [ "*.hs" ]
+```
+
+### [Ormolu](https://github.com/tweag/ormolu)
+
+Make sure to use ormolu 0.1.4.0+ as older versions don't adhere to the spec.
+
+```
+command = "ormolu"
+options = [
+ "--ghc-opt", "-XBangPatterns",
+ "--ghc-opt", "-XPatternSynonyms",
+ "--ghc-opt", "-XTypeApplications",
+ "--mode", "inplace",
+ "--check-idempotence",
+]
+includes = ["*.hs"]
+```
+
+### [stylish-haskell](https://github.com/jaspervdj/stylish-haskell)
+
+```
+command = "stylish-haskell"
+options = [ "--inplace" ]
+includes = [ "*.hs" ]
+```
+
+## Lua
+
+### [StyLua](https://github.com/JohnnyMorganz/StyLua)
+
+```
+command = "stylua"
+includes = ["*.lua"]
+```
+
+## Nix
+
+### [Alejandra](https://github.com/kamadorueda/alejandra)
+
+```
+command = "alejandra"
+includes = ["*.nix"]
+```
+
+### [nixpkgs-fmt](https://github.com/nix-community/nixpkgs-fmt)
+
+```
+command = "nixpkgs-fmt"
+includes = ["*.nix"]
+```
+
+## OCaml
+
+### [ocamlformat](https://github.com/ocaml-ppx/ocamlformat)
+
+```
+command = "ocamlformat"
+options = ["-i"]
+includes = ["*.ml", "*.mli"]
+```
+
+## PureScript
+
+### [purs-tidy](https://www.npmjs.com/package/purs-tidy)
+
+```
+command = "purs-tidy"
+includes = ["*.purs"]
+```
+
+## Python
+
+### [black](https://github.com/psf/black)
+
+```
+command = "black"
+includes = ["*.py"]
+```
+
+## Ruby
+
+### [rufo](https://github.com/ruby-formatter/rufo)
+
+Rufo is an opinionated ruby formatter. By default it exits with status 3 on file change so you have to pass the `-x` option.
+
+```
+command = "rufo"
+options = ["-x"]
+includes = ["*.rb"]
+```
+
+## Rust
+
+cargo fmt is not supported as it doesn't follow the spec. It doesn't allow to pass arbitrary files to be formatter, an ability which `treefmt` relies on. Use rustfmt instead (which is what cargo fmt uses under the hood).
+
+### [rustfmt](https://github.com/rust-lang/rustfmt)
+
+```
+command = "rustfmt"
+options = ["--edition", "2018"]
+includes = ["*.rs"]
+```
+
+## Scala
+
+### [scalafmt](https://github.com/scalameta/scalafmt)
+
+```
+command = "scalafmt"
+includes = ["*.scala"]
+```
+
+## Shell
+
+### [shellcheck](https://github.com/koalaman/shellcheck)
+
+```
+command = "shellcheck"
+includes = ["*.sh"]
+```
+
+### [shfmt](https://github.com/mvdan/sh)
+
+```
+command = "shfmt"
+options = [
+ "-i",
+ "2",  # indent 2
+ "-s",  # simplify the code
+ "-w",  # write back to the file
+]
+includes = ["*.sh"]
+```
+
+## Terraform
+
+### [terraform](https://numtide.github.io/treefmt/formatters.html#terraform)
+
+Make sure to use terraform 1.3.0 or later versions, as earlier versions format only one file at a time. See the details [here](https://github.com/hashicorp/terraform/pull/28191).
+
+```
+command = "terraform"
+options = ["fmt"]
+includes = ["*.tf"]
+```
+
+## Multi-language formatters
+
+### [clang-format](https://clang.llvm.org/docs/ClangFormat.html)
+
+A tool to format C/C++/Java/JavaScript/Objective-C/Protobuf/C# code.
+
+```
+command = "clang-format"
+options = [ "-i" ]
+includes = [ "*.c", "*.cpp", "*.cc", "*.h", "*.hpp" ]
+```
+
+**Note:** This example focuses on C/C++ but can be modified to be used with other languages.
+
+### [Prettier](https://prettier.io/)
+
+An opinionated code formatter that supports many languages.
+
+```
+command = "prettier"
+options = ["--write"]
+includes = [
+ "*.css",
+ "*.html",
+ "*.js",
+ "*.json",
+ "*.jsx",
+ "*.md",
+ "*.mdx",
+ "*.scss",
+ "*.ts",
+ "*.yaml",
+]
+```

docs/index-formatters.md

@@ -0,0 +1,6 @@
+# Formatters
+
+In order to catch up with all the formatters available for different programming languages, we created two files as our guideline:
+
+1. [Formatter Specifications](./formatters-spec.md) — Guidelines for formatter creators to have a smooth integration with `treefmt`
+1. [Formatter Example](./formatters.md) — List of the available formatters with the corresponding configs that can be inserted into `treefmt.toml`

docs/index.md

@@ -0,0 +1,60 @@
+# About the project
+
+`treefmt` is a formatting tool that saves you time: it provides developers with a universal way to trigger all formatters needed for the project in one place.
+
+## Background
+
+Typically, each project has its own code standards enforced by the project's owner. Any code contributions must match that given standard, i.e. be formatted in a specific manner.
+
+At first glance, the task of code formatting may seem trivial: the formatter can be automatically triggered when you save a file in your IDE. Indeed, formatting doesn't take much effort if you're working on a single project long term: setting up the formatters in your IDE won't take much of your time, and then you're ready to go.
+
+Contrary to that, if you're working on multiple projects at the same time, you may have to update your formatter configs in the IDE each time you switch between the projects. This is because formatter settings aren't project-specific --- they are set up globally for all projects.
+
+Alternatively, you can trigger formatters manually, one-by-one or in a script. Actually, for bigger projects, it's common to have a script that runs over your project's directories and calls formatters consequently. But it takes time to iterate through all the files.
+
+All the solutions take up a significant amount of time which a developer could spend doing the actual work. They also require you to remember which formatters and options are used by each project you are working on.
+
+`treefmt` solves these issues.
+
+## Why treefmt?
+
+`treefmt`'s configuration is project-specific, so you don't need to re-configure formatters each time you switch between projects, like you have to when working with formatters in the IDE.
+
+Contrary to calling formatters from the command line, there's no need to remember all the specific formatters required for each project. Once you set up the config, you can run the tool in any of your project's folders without any additional flags or options.
+
+Typically, formatters have different ways to say there was a specific error. With `treefmt`, you get a standardized output which is easier to understand than the variegated outputs of different formatters, so it takes less time to grasp what's wrong.
+
+In addition, `treefmt` works faster than the custom script solution because the changed files are cached and the formatters run only against them. Moreover, formatters are run in parallel, which makes the tool even faster.
+
+The difference may not be significant for smaller projects, but it gets quite visible as the project grows. For instance, take the caching optimization. It takes 9 seconds to traverse a project of 1507 files and no changes without caching:
+
+```
+traversed 1507 files
+matched 828 files to formatters
+left with 828 files after cache
+of whom 0 files were re-formatted
+all of this in 9s
+```
+
+...while it takes 124 milliseconds to traverse the same project with caching:
+
+```
+traversed 1507 files
+matched 828 files to formatters
+left with 0 files after cache
+of whom 0 files were re-formatted
+all of this in 124ms
+```
+
+The tool can be invoked manually or integrated into your CI. There's currently no integration with IDEs, but the feature is coming soon.
+
+## What we still need help with
+
+- **IDE integration:** Most of developers are used to formatting a file upon save in the IDE. So far, you can't use `treefmt` for this purpose, but we're working on it 😀
+- **Pre-commit hook:** It's good to have your code checked for adherence to the project's standards before commit. `treefmt` pre-commit hook won't let you commit if you have formatting issues.
+- **Support of multiple formatters for one language:** In the current version, we advise you to avoid using multiple formatters for one and the same file type. This is because formatters are run in parallel and therefore may encounter issues while processing files. We are going to fix this issue soon, since there are cases when you may need more than one formatter per language.
+
+As a next step, learn how to [install] and [use] `treefmt`.
+
+[install]: installation.md
+[use]: usage.md

docs/installation.md

@@ -0,0 +1,61 @@
+# Installation
+
+There are two options to install `treefmt`: by downloading the latest binary, or by compiling and building the tool from source.
+
+## Installing with a binary file
+
+You can find the list of the latest binaries [here](https://github.com/numtide/treefmt/releases).
+
+## Building from source
+
+There are several ways to build `treefmt` from source. Your choice will depend on whether you're a [nix](https://github.com/NixOS/nix) user.
+
+### Non-Nix User
+
+To try the project without building it, run:
+
+```
+$ cargo run -- --help
+```
+
+The command will output the manual. You can run the tool in this manner with any other flag or option to format your project.
+
+To build a binary, you need to have rust installed. You can install it with [rustup](https://rustup.rs/). Now, if you want to build the project, switch to the project root folder and run:
+
+```
+$ cargo build
+```
+
+After the successful execution of the cargo build command, you will find the `treefmt` binary in the target folder.
+
+### Nix User
+
+[Nix](https://github.com/NixOS/nix) is a package manager foundational for NixOS. You can use it in NixOS and in any other OS equally.
+
+If you're using both `treefmt` and `nix`, you can go for [`treefmt-nix`](https://github.com/numtide/treefmt-nix), a special tool that makes installation and configuration of `treefmt` with `nix` easier.
+
+**Non-flake user**
+
+Here you also have two options: you can install `treefmt` with plain nix-build , or with nix-shell.
+
+To build the package with nix-build, just run:
+
+```
+$ nix-build -A treefmt
+```
+
+**Nix-flake user**
+
+If you want to use this repository with flakes, please enable the flakes feature first. To run the project with flakes without building it, you can execute the following command in the root folder:
+
+```
+$ nix run . -- --help
+```
+
+To build the project, run the following command in the root folder:
+
+```
+$ nix build
+```
+
+The `treefmt` binary will be available in the result folder.

docs/integrations.md

@@ -0,0 +1,9 @@
+# Integrations
+
+> add your project here.
+
+Here is a list of projects that integrate with treefmt.
+
+## the vim null-ls plugin
+
+See <https://github.com/jose-elias-alvarez/null-ls.nvim/pull/1512> for usage.

docs/quickstart.md

@@ -0,0 +1,14 @@
+# Quickstart
+
+To run `treefmt` in your project, please follow these steps:
+
+1. [Install] the tool.
+2. Install the needed formatters.
+3. Run `treefmt --init`. This will generate the basic configuration file `treefmt.toml` containing the formatting rules.
+4. Edit the configuration (see [here] how).
+5. Run `treefmt` with the needed flags and options. You can check the supported options by executing `treefmt --help`.
+
+In the following sections we will guide you through installing and configuring `treefmt` in detail.
+
+[install]: installation.md
+[here]: formatters-spec.md

docs/treefmt-configuration.md

@@ -0,0 +1,43 @@
+# Configuration
+
+`treefmt` can only be run in the presence of `treefmt.toml` where files are mapped to specific code formatters.
+
+Usually the config file sits in the project root folder. If you're running `treefmt` in one of the project's folders, then `treefmt` will look for the config in the parent folders up until the project's root. However, you can place the config anywhere in your project's file tree and specify the path in the ---config-file flag.
+
+The typical section of `treefmt.toml` looks like this:
+
+```
+[formatter.<name>]
+command = "<formatter-command>"
+options = ["<formatter-option-1>"...]
+includes = ["<glob-pattern>"...]
+```
+
+...where name is just an identifier.
+
+```
+[formatter.elm]
+command = "elm-format"
+options = ["--yes"]
+includes = ["*.elm"]
+```
+
+Make sure you installed all the formatters specified in the config before running `treefmt`. If you don't want to install all formatters, you can still run `treefmt` by specifying the flag `--allow-missing-formatter`. This will make the program not error out if the needed formatter is missing.
+
+## Configuration format
+
+### `[formatter.<name>]`
+
+This section describes the integration between a single formatter and treefmt. "Name" here is a unique ID of your formatter in the config file. It doesn't have to match the formatter name.
+
+- `command`: A list of arguments to be executed. This will be concatenated with the options attribute during invocation. The first argument is the name of the executable to run.
+- `options`: A list of extra arguments to add to the command. These are typically project-specific arguments.
+- `includes`: A list of glob patterns to match file names, including extensions and paths, used to select specific files for formatting. Typically, only file extensions are specified to pick all files written in a specific language. For instance,[`"*.sh"`] selects shell script files. But sometimes, you may need to specify a full file name, like [`"Makefile"`], or a pattern picking files in a specific folder, like [`"/home/user/project/*"`].
+
+- `excludes`: A list of glob patterns to exclude from formatting. If any of these patterns match, the file will be excluded from formatting by a particular formatter.
+
+### `[global]`
+
+This section describes the configuration properties that apply to every formatter.
+
+- `excludes`: A list of glob patterns to deny. If any of these patterns match, the file won't be formatted. This list is appended to the individual formatter's excludes lists.

docs/usage.md

@@ -0,0 +1,115 @@
+# Usage
+
+You can run treefmt by executing:
+
+`$ treefmt`
+
+or, if it's not in your `$PATH`:
+
+`$ ./treefmt`
+
+Treefmt has the following specification:
+
+```
+treefmt [FLAGS] [OPTIONS] [--] [paths]...
+```
+
+## Flags
+
+`--allow-missing-formatter`
+
+> Do not exit with an error if some of the configured formatters are missing.
+
+`--clear-cache`
+
+> Reset the evaluation cache. Invalidation should happen automatically if the formatting tool has been updated, or if the files to format have changed. If cache wasn't properly invalidated, you can use this flag to clear the cache.
+
+`--fail-on-change`
+
+> Exit with error if some files require re-formatting. This is useful for your CI if you want to detect if the contributed code was forgotten to be formatted.
+
+`-h, --help`
+
+> Prints available flags and options
+
+`--init`
+
+> Creates a new config file `treefmt.toml`.
+
+`--no-cache`
+
+> Tells `treefmt` to ignore the evaluation cache entirely. With this flag, you can avoid cache invalidation issues, if any. Typically, the machine that is running treefmt in the CI is starting with a fresh environment each time, so any calculated cache is lost. The `--no-cache` flag eliminates unnecessary work in the CI.
+
+`-q, --quiet`
+
+> Don't print output to stderr.
+
+`--stdin`
+
+> Format the content passed in stdin.
+
+`-V, --version`
+
+> Print version information.
+
+`-v, --verbose`
+
+> Change the log verbosity. Log verbosity is based off the number of 'v' used. With one `-v`, your logs will display `[INFO]` and `[ERROR]` messages, while `-vv` will also show `[DEBUG]` messages.
+
+`--config-file <config-file>`
+
+> Run with the specified config file which is not in the project tree.
+
+`-f, --formatters <formatters>...`
+
+> Only apply selected formatters. Defaults to all formatters.
+
+`-H, --hidden`
+
+> Also traverse hidden files (files that start with a .). This behaviour can be overridden with the `--no-hidden` flag.
+
+`--no-hidden`
+
+> Override the `--hidden` flag. Don't traverse hidden files.
+
+`--tree-root <tree-root>`
+
+> Set the path to the tree root directory where treefmt will look for the files to format. Defaults to the folder holding the `treefmt.toml` file. It’s mostly useful in combination with `--config-file` to specify the project root which won’t coincide with the directory holding `treefmt.toml`.
+
+`-C <work-dir>`
+
+> Run as if `treefmt` was started in `<work-dir>` instead of the current working directory (default: `.`). Equivalent to `cd <work dir>; treefmt`.
+
+## Arguments
+
+`<paths>...`
+
+> Paths to format. Defaults to formatting the whole tree
+
+## CI integration
+
+Typically, you would use treefmt in the CI with the `--fail-on-change` and `--no-cache flags`. Find the explanations above.
+
+You can you set a `treefmt` job in the GitHub pipeline for Ubuntu with nix-shell like this:
+
+```
+name: treefmt
+on:
+  pull_request:
+  push:
+    branches: main
+jobs:
+  formatter:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+    - uses: cachix/install-nix-action@v12
+      with:
+        nix_path: nixpkgs=channel:nixos-unstable
+    - uses: cachix/cachix-action@v10
+      with:
+        name: nix-community
+        authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+    - name: treefmt
+      run: nix-shell --run "treefmt --fail-on-change --no-cache"
+```

flake.lock

@@ -21,6 +21,21 @@
         "type": "github"
       }
     },
+    "flake-compat": {
+      "locked": {
+        "lastModified": 1688025799,
+        "narHash": "sha256-ktpB4dRtnksm9F5WawoIkEneh1nrEvuxb5lJFt1iOyw=",
+        "owner": "nix-community",
+        "repo": "flake-compat",
+        "rev": "8bf105319d44f6b9f0d764efa4fdef9f1cc9ba1c",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
     "flake-parts": {
       "inputs": {
         "nixpkgs-lib": "nixpkgs-lib"
@@ -111,6 +126,26 @@
         "type": "github"
       }
     },
+    "mkdocs-numtide": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1687786869,
+        "narHash": "sha256-KhaNnOTjj9FgPLtRHTFGa1RFXvSc+nF1UPcBiYf/CCY=",
+        "owner": "numtide",
+        "repo": "mkdocs-numtide",
+        "rev": "b3008171c75083f2bf2f1dc4e6781d4737dfaa49",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "mkdocs-numtide",
+        "type": "github"
+      }
+    },
     "nix-filter": {
       "locked": {
         "lastModified": 1705332318,
@@ -163,9 +198,11 @@
     "root": {
       "inputs": {
         "devshell": "devshell",
+        "flake-compat": "flake-compat",
         "flake-parts": "flake-parts",
         "flake-root": "flake-root",
         "gomod2nix": "gomod2nix",
+        "mkdocs-numtide": "mkdocs-numtide",
         "nix-filter": "nix-filter",
         "nixpkgs": "nixpkgs",
         "treefmt-nix": "treefmt-nix"

flake.nix

@@ -18,8 +18,12 @@
       url = "github:nix-community/gomod2nix";
       inputs.nixpkgs.follows = "nixpkgs";
     };
-
     nix-filter.url = "github:numtide/nix-filter";
+    mkdocs-numtide = {
+      url = "github:numtide/mkdocs-numtide";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+    flake-compat.url = "github:nix-community/flake-compat";
   };
 
   outputs = inputs @ {flake-parts, ...}:

format/context.go

@@ -1,21 +0,0 @@
-package format
-
-import (
-	"context"
-)
-
-const (
-	completedChKey = "completedCh"
-)
-
-// SetCompletedChannel is used to set a channel for indication processing completion in the provided context.
-func SetCompletedChannel(ctx context.Context, completedCh chan string) context.Context {
-	return context.WithValue(ctx, completedChKey, completedCh)
-}
-
-// MarkPathComplete is used to indicate that all processing has finished for the provided path.
-// This is done by adding the path to the completion channel which should have already been set using
-// SetCompletedChannel.
-func MarkPathComplete(ctx context.Context, path string) {
-	ctx.Value(completedChKey).(chan string) <- path
-}

format/formatter.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os"
 	"os/exec"
 	"time"
 
@@ -23,33 +24,13 @@ type Formatter struct {
 
 	log        *log.Logger
 	executable string // path to the executable described by Command
-
-	before string
-
-	child  *Formatter
-	parent *Formatter
+	workingDir string
 
 	// internal compiled versions of Includes and Excludes.
 	includes []glob.Glob
 	excludes []glob.Glob
 
-	// inboxCh is used to accept new paths for formatting.
-	inboxCh chan string
-	// completedCh is used to wait for this formatter to finish all processing.
-	completedCh chan interface{}
-
-	// Entries from inboxCh are batched according to batchSize and stored in batch for processing when the batchSize has
-	// been reached or Close is invoked.
-	batch     []string
-	batchSize int
-}
-
-func (f *Formatter) Before() string {
-	return f.before
-}
-
-func (f *Formatter) ResetBefore() {
-	f.before = ""
+	batch []string
 }
 
 // Executable returns the path to the executable defined by Command
@@ -57,15 +38,86 @@ func (f *Formatter) Executable() string {
 	return f.executable
 }
 
+func (f *Formatter) Apply(ctx context.Context, paths []string, filter bool) error {
+	start := time.Now()
+
+	// construct args, starting with config
+	args := f.config.Options
+
+	// If filter is true it indicates we are executing as part of a pipeline.
+	// In such a scenario each formatter must sub filter the paths provided as different formatters might want different
+	// files in a pipeline.
+	if filter {
+		// reset the batch
+		f.batch = f.batch[:0]
+
+		// filter paths
+		for _, path := range paths {
+			if f.Wants(path) {
+				f.batch = append(f.batch, path)
+			}
+		}
+
+		// exit early if nothing to process
+		if len(f.batch) == 0 {
+			return nil
+		}
+
+		// append paths to the args
+		args = append(args, f.batch...)
+	} else {
+		// exit early if nothing to process
+		if len(paths) == 0 {
+			return nil
+		}
+
+		// append paths to the args
+		args = append(args, paths...)
+	}
+
+	// execute the command
+	cmd := exec.CommandContext(ctx, f.config.Command, args...)
+	cmd.Dir = f.workingDir
+
+	if out, err := cmd.CombinedOutput(); err != nil {
+		if len(out) > 0 {
+			_, _ = fmt.Fprintf(os.Stderr, "%s error:\n%s\n", f.name, out)
+		}
+		return fmt.Errorf("%w: formatter %s failed to apply", err, f.name)
+	}
+
+	//
+
+	f.log.Infof("%v files processed in %v", len(paths), time.Now().Sub(start))
+
+	return nil
+}
+
+// Wants is used to test if a Formatter wants a path based on it's configured Includes and Excludes patterns.
+// Returns true if the Formatter should be applied to path, false otherwise.
+func (f *Formatter) Wants(path string) bool {
+	match := !PathMatches(path, f.excludes) && PathMatches(path, f.includes)
+	if match {
+		f.log.Debugf("match: %v", path)
+	}
+	return match
+}
+
 // NewFormatter is used to create a new Formatter.
-func NewFormatter(name string, config *config.Formatter, globalExcludes []glob.Glob) (*Formatter, error) {
+func NewFormatter(
+	name string,
+	treeRoot string,
+	config *config.Formatter,
+	globalExcludes []glob.Glob,
+) (*Formatter, error) {
 	var err error
 
 	f := Formatter{}
-	// capture the name from the config file
+
+	// capture config and the formatter's name
 	f.name = name
 	f.config = config
-	f.before = config.Before
+	f.workingDir = treeRoot
 
 	// test if the formatter is available
 	executable, err := exec.LookPath(config.Command)
@@ -77,11 +129,11 @@ func NewFormatter(name string, config *config.Formatter, globalExcludes []glob.G
 	f.executable = executable
 
 	// initialise internal state
-	f.log = log.WithPrefix("format | " + name)
-	f.batchSize = 1024
-	f.batch = make([]string, 0, f.batchSize)
-	f.inboxCh = make(chan string, f.batchSize)
-	f.completedCh = make(chan interface{}, 1)
+	if config.Pipeline == "" {
+		f.log = log.WithPrefix(fmt.Sprintf("format | %s", name))
+	} else {
+		f.log = log.WithPrefix(fmt.Sprintf("format | %s[%s]", config.Pipeline, name))
+	}
 
 	f.includes, err = CompileGlobs(config.Includes)
 	if err != nil {
@@ -96,140 +148,3 @@ func NewFormatter(name string, config *config.Formatter, globalExcludes []glob.G
 
 	return &f, nil
 }
-
-func (f *Formatter) SetParent(formatter *Formatter) {
-	f.parent = formatter
-}
-
-func (f *Formatter) Parent() *Formatter {
-	return f.parent
-}
-
-func (f *Formatter) SetChild(formatter *Formatter) {
-	f.child = formatter
-}
-
-// Wants is used to test if a Formatter wants path based on it's configured Includes and Excludes patterns.
-// Returns true if the Formatter should be applied to path, false otherwise.
-func (f *Formatter) Wants(path string) bool {
-	if f.parent != nil {
-		// we don't accept this path directly, our parent will forward it
-		return false
-	}
-	match := !PathMatches(path, f.excludes) && PathMatches(path, f.includes)
-	if match {
-		f.log.Debugf("match: %v", path)
-	}
-	return match
-}
-
-// Put add path into this Formatter's inboxCh for processing.
-func (f *Formatter) Put(path string) {
-	f.inboxCh <- path
-}
-
-// Run is the main processing loop for this Formatter.
-// It accepts a context which is used to lookup certain dependencies and for cancellation.
-func (f *Formatter) Run(ctx context.Context) (err error) {
-	defer func() {
-		if f.child != nil {
-			// indicate no further processing for the child formatter
-			f.child.Close()
-		}
-
-		// indicate this formatter has finished processing
-		f.completedCh <- nil
-	}()
-
-LOOP:
-	// keep processing until ctx has been cancelled or inboxCh has been closed
-	for {
-		select {
-
-		case <-ctx.Done():
-			// ctx has been cancelled
-			err = ctx.Err()
-			break LOOP
-
-		case path, ok := <-f.inboxCh:
-			// check if the inboxCh has been closed
-			if !ok {
-				break LOOP
-			}
-
-			// add path to the current batch
-			f.batch = append(f.batch, path)
-
-			if len(f.batch) == f.batchSize {
-				// drain immediately
-				if err := f.apply(ctx); err != nil {
-					break LOOP
-				}
-			}
-		}
-	}
-
-	// check if LOOP was exited due to an error
-	if err != nil {
-		return
-	}
-
-	// processing any lingering batch
-	return f.apply(ctx)
-}
-
-// apply executes Command against the latest batch of paths.
-// It accepts a context which is used to lookup certain dependencies and for cancellation.
-func (f *Formatter) apply(ctx context.Context) error {
-	// empty check
-	if len(f.batch) == 0 {
-		return nil
-	}
-
-	// construct args, starting with config
-	args := f.config.Options
-
-	// append each file path
-	for _, path := range f.batch {
-		args = append(args, path)
-	}
-
-	// execute
-	start := time.Now()
-	cmd := exec.CommandContext(ctx, f.config.Command, args...)
-
-	if out, err := cmd.CombinedOutput(); err != nil {
-		f.log.Debugf("\n%v", string(out))
-		// todo log output
-		return err
-	}
-
-	f.log.Infof("%v files processed in %v", len(f.batch), time.Now().Sub(start))
-
-	if f.child == nil {
-		// mark each path in this batch as completed
-		for _, path := range f.batch {
-			MarkPathComplete(ctx, path)
-		}
-	} else {
-		// otherwise forward each path onto the next formatter for processing
-		for _, path := range f.batch {
-			f.child.Put(path)
-		}
-	}
-
-	// reset batch
-	f.batch = f.batch[:0]
-
-	return nil
-}
-
-// Close is used to indicate that a Formatter should process any remaining paths and then stop it's processing loop.
-func (f *Formatter) Close() {
-	close(f.inboxCh)
-}
-
-func (f *Formatter) AwaitCompletion() {
-	// todo support a timeout
-	<-f.completedCh
-}

format/pipeline.go

@@ -0,0 +1,38 @@
+package format
+
+import (
+	"context"
+	"slices"
+)
+
+type Pipeline struct {
+	sequence []*Formatter
+}
+
+func (p *Pipeline) Add(f *Formatter) {
+	p.sequence = append(p.sequence, f)
+	// sort by priority in ascending order
+	slices.SortFunc(p.sequence, func(a, b *Formatter) int {
+		return a.config.Priority - b.config.Priority
+	})
+}
+
+func (p *Pipeline) Wants(path string) bool {
+	var match bool
+	for _, f := range p.sequence {
+		match = f.Wants(path)
+		if match {
+			break
+		}
+	}
+	return match
+}
+
+func (p *Pipeline) Apply(ctx context.Context, paths []string) error {
+	for _, f := range p.sequence {
+		if err := f.Apply(ctx, paths, len(p.sequence) > 1); err != nil {
+			return err
+		}
+	}
+	return nil
+}

init.toml

@@ -0,0 +1,11 @@
+# One CLI to format the code tree - https://git.numtide.com/numtide/treefmt
+
+[formatter.mylanguage]
+# Formatter to run
+command = "command-to-run"
+# Command-line arguments for the command
+options = []
+# Glob pattern of files to include
+includes = [ "*.<language-extension>" ]
+# Glob patterns of files to exclude
+excludes = []

main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	_ "embed"
 	"fmt"
 	"os"
 
@@ -9,6 +10,11 @@ import (
 	"github.com/alecthomas/kong"
 )
 
+// We embed the sample toml file for use with the init flag.
+//
+//go:embed init.toml
+var initBytes []byte
+
 func main() {
 	// This is to maintain compatibility with 1.0.0 which allows specifying the version with a `treefmt --version` flag
 	// on the 'default' command. With Kong it would be better to have `treefmt version` so it would be treated as a
@@ -18,6 +24,14 @@ func main() {
 		if arg == "--version" || arg == "-V" {
 			fmt.Printf("%s %s\n", build.Name, build.Version)
 			return
+		} else if arg == "--init" || arg == "-i" {
+			if err := os.WriteFile("treefmt.toml", initBytes, 0o644); err != nil {
+				fmt.Printf("Failed to write treefmt.toml: %v\n", err)
+				os.Exit(1)
+			}
+
+			fmt.Printf("Generated treefmt.toml. Now it's your turn to edit it.\n")
+			return
 		}
 	}
 

mkdocs.yml

@@ -0,0 +1,27 @@
+INHERIT: !ENV MKDOCS_NUMTIDE_THEME
+
+### Site metadata ###
+
+site_name: treefmt
+site_description: one CLI to format your repo
+site_url: https://numtide.github.io/treefmt/
+repo_name: "numtide/treefmt"
+repo_url: https://git.numtide.com/numtide/treefmt
+edit_uri: edit/main/docs
+
+### Navigation ###
+
+nav:
+    - Home: index.md
+    - QuickStart:
+          - About: quickstart.md
+          - Installation: installation.md
+          - Configuration: treefmt-configuration.md
+    - Usage: usage.md
+    - Integrations: integrations.md
+    - Formatters:
+          - About: index-formatters.md
+          - Formatter Specification: formatters-spec.md
+          - Known Formatters: formatters.md
+    - FAQ: faq.md
+    - Contributing: contributing.md

nix/devshell.nix

@@ -6,6 +6,7 @@
   config.perSystem = {
     pkgs,
     config,
+    inputs',
     ...
   }: {
     config.devshells.default = {
@@ -25,6 +26,9 @@
           # golang
           go
           delve
+
+          # docs
+          inputs'.mkdocs-numtide.packages.default
         ]
         ++
         # include formatters for development and testing

nix/formatters.nix

@@ -6,6 +6,7 @@ with pkgs; [
   haskellPackages.cabal-fmt
   haskellPackages.ormolu
   mdsh
+  nixpkgs-fmt
   nodePackages.prettier
   python3.pkgs.black
   rufo
@@ -15,4 +16,17 @@ with pkgs; [
   statix
   deadnix
   terraform
+  # util for unit testing
+  (pkgs.writeShellApplication {
+    name = "test-fmt";
+    text = ''
+      VALUE="$1"
+      shift
+
+      # append value to each file
+      for FILE in "$@"; do
+          echo "$VALUE" >> "$FILE"
+      done
+    '';
+  })
 ]

nix/packages.nix

@@ -8,9 +8,15 @@
     inputs',
     lib,
     pkgs,
+    system,
     ...
   }: {
     packages = rec {
+      docs = inputs.mkdocs-numtide.lib.${system}.mkDocs {
+        name = "treefmt-docs";
+        src = ../.;
+      };
+
       treefmt = inputs'.gomod2nix.legacyPackages.buildGoApplication rec {
         pname = "treefmt";
         version = "2.0.0+dev";
@@ -31,8 +37,8 @@
         ldflags = [
           "-s"
           "-w"
-          "-X 'build.Name=${pname}'"
-          "-X 'build.Version=${version}'"
+          "-X git.numtide.com/numtide/treefmt/build.Name=${pname}"
+          "-X git.numtide.com/numtide/treefmt/build.Version=v${version}"
         ];
 
         nativeBuildInputs =

shell.nix

@@ -0,0 +1,15 @@
+# This file provides backward compatibility to nix < 2.4 clients
+{ system ? builtins.currentSystem }:
+let
+  lock = builtins.fromJSON (builtins.readFile ./flake.lock);
+
+  inherit (lock.nodes.flake-compat.locked) owner repo rev narHash;
+
+  flake-compat = fetchTarball {
+    url = "https://github.com/${owner}/${repo}/archive/${rev}.tar.gz";
+    sha256 = narHash;
+  };
+
+  flake = import flake-compat { inherit system; src = ./.; };
+in
+flake.shellNix

test/examples/nixpkgs.toml

@@ -0,0 +1,13 @@
+# One CLI to format the code tree - https://git.numtide.com/numtide/treefmt
+
+[formatter.deadnix]
+command = "deadnix"
+includes = ["*.nix"]
+pipeline = "nix"
+priority = 1
+
+[formatter.nixpkgs-fmt]
+command = "nixpkgs-fmt"
+includes = ["*.nix"]
+pipeline = "nix"
+priority = 2

test/examples/treefmt.toml

@@ -31,12 +31,14 @@ command = "alejandra"
 includes = ["*.nix"]
 # Act as an example on how to exclude specific files
 excludes = ["examples/nix/sources.nix"]
-# Make this run before deadnix
-# Note this formatter determines the file set for any 'downstream' formatters
-before = "deadnix"
+pipeline = "nix"
+priority = 1
 
 [formatter.deadnix]
 command = "deadnix"
+includes = ["*.nix"]
+pipeline = "nix"
+priority = 2
 
 [formatter.ruby]
 command = "rufo"