numtide/treefmt
Archived
8
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: numtide:618f6f7e777a3746671b94ead3e8d7b2f0af0f99
Branches Tags
numtide:main
numtide:feat/improve-caching
numtide:feat/stats
numtide:feat/configure-treefmt-pipeline
numtide:feat/go-walk
numtide:why-riig
numtide:feat/config-package
numtide:feat/composite-formatter
numtide:feat/formatter-ordering
..
compare: numtide:d3cdb698e4ae7448ed54a4ed729be32f6a580c22
Branches Tags
numtide:main
numtide:feat/improve-caching
numtide:feat/stats
numtide:feat/configure-treefmt-pipeline
numtide:feat/go-walk
numtide:why-riig
numtide:feat/config-package
numtide:feat/composite-formatter
numtide:feat/formatter-ordering

No commits in common. "618f6f7e777a3746671b94ead3e8d7b2f0af0f99" and "d3cdb698e4ae7448ed54a4ed729be32f6a580c22" have entirely different histories.
618f6f7e77 ... d3cdb698e4

6 changed files with 64 additions and 155 deletions
Show Stats Expand all files Collapse all files

12
cache/cache.go vendored
View File

@ -11,8 +11,6 @@ import (
"runtime"
"time"
"git.numtide.com/numtide/treefmt/stats"
"git.numtide.com/numtide/treefmt/format"
"git.numtide.com/numtide/treefmt/walk"
@ -35,10 +33,9 @@ type Entry struct {
}
var (
db *bolt.DB
logger *log.Logger
db *bolt.DB
ReadBatchSize = 1024 * runtime.NumCPU()
logger *log.Logger
)
// Open creates an instance of bolt.DB for a given treeRoot path.
@ -237,14 +234,11 @@ func ChangeSet(ctx context.Context, walker walk.Walker, pathsCh chan<- string) e
changedOrNew := cached == nil || !(cached.Modified == info.ModTime() && cached.Size == info.Size())
stats.Add(stats.Traversed, 1)
if !changedOrNew {
// no change
return nil
}
stats.Add(stats.Emitted, 1)
// pass on the path
select {
case <-ctx.Done():
@ -299,8 +293,6 @@ func Update(treeRoot string, paths []string) (int, error) {
continue
}
stats.Add(stats.Formatted, 1)
entry := Entry{
Size: pathInfo.Size(),
Modified: pathInfo.ModTime(),

14
cli/format.go
View File

@ -14,9 +14,9 @@ import (
"sort"
"strings"
"syscall"
"time"
"git.numtide.com/numtide/treefmt/format"
"git.numtide.com/numtide/treefmt/stats"
"github.com/gobwas/glob"
"git.numtide.com/numtide/treefmt/cache"
@ -32,6 +32,7 @@ const (
)
var (
start time.Time
globalExcludes []glob.Glob
formatters map[string]*format.Formatter
pipelines map[string]*format.Pipeline
@ -42,7 +43,7 @@ var (
)
func (f *Format) Run() (err error) {
stats.Init()
start = time.Now()
Cli.Configure()
@ -195,8 +196,6 @@ func walkFilesystem(ctx context.Context) func() error {
default:
// ignore symlinks and directories
if !(info.IsDir() || info.Mode()&os.ModeSymlink == os.ModeSymlink) {
stats.Add(stats.Traversed, 1)
stats.Add(stats.Emitted, 1)
pathsCh <- path
}
return nil
@ -258,7 +257,7 @@ func updateCache(ctx context.Context) func() error {
return ErrFailOnChange
}
stats.Print()
fmt.Printf("%v files changed in %v\n", changes, time.Now().Sub(start))
return nil
}
}
@ -323,17 +322,12 @@ func applyFormatters(ctx context.Context) func() error {
}()
for path := range pathsCh {
var matched bool
for key, pipeline := range pipelines {
if !pipeline.Wants(path) {
continue
}
matched = true
tryApply(key, path)
}
if matched {
stats.Add(stats.Matched, 1)
}
}
// flush any partial batches which remain

73
cli/format_test.go
View File

@ -2,6 +2,7 @@ package cli
import (
"bufio"
"fmt"
"os"
"os/exec"
"path"
@ -67,19 +68,19 @@ func TestSpecifyingFormatters(t *testing.T) {
out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err)
assertFormatted(t, as, out, 3)
as.Contains(string(out), "3 files changed")
out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "elm,nix")
as.NoError(err)
assertFormatted(t, as, out, 2)
as.Contains(string(out), "2 files changed")
out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "ruby,nix")
as.NoError(err)
assertFormatted(t, as, out, 2)
as.Contains(string(out), "2 files changed")
out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "nix")
as.NoError(err)
assertFormatted(t, as, out, 1)
as.Contains(string(out), "1 files changed")
// test bad names
@ -109,7 +110,7 @@ func TestIncludesAndExcludes(t *testing.T) {
test.WriteConfig(t, configPath, cfg)
out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err)
assertFormatted(t, as, out, 31)
as.Contains(string(out), fmt.Sprintf("%d files changed", 31))
// globally exclude nix files
cfg.Global.Excludes = []string{"*.nix"}
@ -117,7 +118,7 @@ func TestIncludesAndExcludes(t *testing.T) {
test.WriteConfig(t, configPath, cfg)
out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err)
assertFormatted(t, as, out, 30)
as.Contains(string(out), fmt.Sprintf("%d files changed", 30))
// add haskell files to the global exclude
cfg.Global.Excludes = []string{"*.nix", "*.hs"}
@ -125,7 +126,7 @@ func TestIncludesAndExcludes(t *testing.T) {
test.WriteConfig(t, configPath, cfg)
out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err)
assertFormatted(t, as, out, 24)
as.Contains(string(out), fmt.Sprintf("%d files changed", 24))
echo := cfg.Formatters["echo"]
@ -135,7 +136,7 @@ func TestIncludesAndExcludes(t *testing.T) {
test.WriteConfig(t, configPath, cfg)
out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err)
assertFormatted(t, as, out, 22)
as.Contains(string(out), fmt.Sprintf("%d files changed", 22))
// remove go files from the echo formatter
echo.Excludes = []string{"*.py", "*.go"}
@ -143,7 +144,7 @@ func TestIncludesAndExcludes(t *testing.T) {
test.WriteConfig(t, configPath, cfg)
out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err)
assertFormatted(t, as, out, 21)
as.Contains(string(out), fmt.Sprintf("%d files changed", 21))
// adjust the includes for echo to only include elm files
echo.Includes = []string{"*.elm"}
@ -151,7 +152,7 @@ func TestIncludesAndExcludes(t *testing.T) {
test.WriteConfig(t, configPath, cfg)
out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err)
assertFormatted(t, as, out, 1)
as.Contains(string(out), fmt.Sprintf("%d files changed", 1))
// add js files to echo formatter
echo.Includes = []string{"*.elm", "*.js"}
@ -159,7 +160,7 @@ func TestIncludesAndExcludes(t *testing.T) {
test.WriteConfig(t, configPath, cfg)
out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err)
assertFormatted(t, as, out, 2)
as.Contains(string(out), fmt.Sprintf("%d files changed", 2))
}
func TestCache(t *testing.T) {
@ -181,34 +182,34 @@ func TestCache(t *testing.T) {
test.WriteConfig(t, configPath, cfg)
out, err := cmd(t, "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err)
assertFormatted(t, as, out, 31)
as.Contains(string(out), fmt.Sprintf("%d files changed", 31))
out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err)
assertFormatted(t, as, out, 0)
as.Contains(string(out), "0 files changed")
// clear cache
out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir, "-c")
as.NoError(err)
assertFormatted(t, as, out, 31)
as.Contains(string(out), fmt.Sprintf("%d files changed", 31))
out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err)
assertFormatted(t, as, out, 0)
as.Contains(string(out), "0 files changed")
// clear cache
out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir, "-c")
as.NoError(err)
assertFormatted(t, as, out, 31)
as.Contains(string(out), fmt.Sprintf("%d files changed", 31))
out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err)
assertFormatted(t, as, out, 0)
as.Contains(string(out), "0 files changed")
// no cache
out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir, "--no-cache")
as.NoError(err)
assertStats(t, as, out, 31, 31, 31, 0)
as.Contains(string(out), fmt.Sprintf("%d files changed", 31))
}
func TestChangeWorkingDirectory(t *testing.T) {
@ -242,7 +243,7 @@ func TestChangeWorkingDirectory(t *testing.T) {
// this should fail if the working directory hasn't been changed first
out, err := cmd(t, "-C", tempDir)
as.NoError(err)
assertFormatted(t, as, out, 31)
as.Contains(string(out), fmt.Sprintf("%d files changed", 31))
}
func TestFailOnChange(t *testing.T) {
@ -306,31 +307,31 @@ func TestBustCacheOnFormatterChange(t *testing.T) {
args := []string{"--config-file", configPath, "--tree-root", tempDir}
out, err := cmd(t, args...)
as.NoError(err)
assertFormatted(t, as, out, 3)
as.Contains(string(out), fmt.Sprintf("%d files changed", 3))
// tweak mod time of elm formatter
as.NoError(test.RecreateSymlink(t, binPath+"/"+"elm-format"))
out, err = cmd(t, args...)
as.NoError(err)
assertFormatted(t, as, out, 3)
as.Contains(string(out), fmt.Sprintf("%d files changed", 3))
// check cache is working
out, err = cmd(t, args...)
as.NoError(err)
assertFormatted(t, as, out, 0)
as.Contains(string(out), "0 files changed")
// tweak mod time of python formatter
as.NoError(test.RecreateSymlink(t, binPath+"/"+"black"))
out, err = cmd(t, args...)
as.NoError(err)
assertFormatted(t, as, out, 3)
as.Contains(string(out), fmt.Sprintf("%d files changed", 3))
// check cache is working
out, err = cmd(t, args...)
as.NoError(err)
assertFormatted(t, as, out, 0)
as.Contains(string(out), "0 files changed")
// add go formatter
cfg.Formatters["go"] = &config2.Formatter{
@ -342,12 +343,12 @@ func TestBustCacheOnFormatterChange(t *testing.T) {
out, err = cmd(t, args...)
as.NoError(err)
assertFormatted(t, as, out, 4)
as.Contains(string(out), fmt.Sprintf("%d files changed", 4))
// check cache is working
out, err = cmd(t, args...)
as.NoError(err)
assertFormatted(t, as, out, 0)
as.Contains(string(out), "0 files changed")
// remove python formatter
delete(cfg.Formatters, "python")
@ -355,12 +356,12 @@ func TestBustCacheOnFormatterChange(t *testing.T) {
out, err = cmd(t, args...)
as.NoError(err)
assertFormatted(t, as, out, 2)
as.Contains(string(out), fmt.Sprintf("%d files changed", 2))
// check cache is working
out, err = cmd(t, args...)
as.NoError(err)
assertFormatted(t, as, out, 0)
as.Contains(string(out), "0 files changed")
// remove elm formatter
delete(cfg.Formatters, "elm")
@ -368,12 +369,12 @@ func TestBustCacheOnFormatterChange(t *testing.T) {
out, err = cmd(t, args...)
as.NoError(err)
assertFormatted(t, as, out, 1)
as.Contains(string(out), fmt.Sprintf("%d files changed", 1))
// check cache is working
out, err = cmd(t, args...)
as.NoError(err)
assertFormatted(t, as, out, 0)
as.Contains(string(out), "0 files changed")
}
func TestGitWorktree(t *testing.T) {
@ -407,10 +408,10 @@ func TestGitWorktree(t *testing.T) {
wt, err := repo.Worktree()
as.NoError(err, "failed to get git worktree")
run := func(formatted int) {
run := func(changed int) {
out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err)
assertFormatted(t, as, out, formatted)
as.Contains(string(out), fmt.Sprintf("%d files changed", changed))
}
// run before adding anything to the worktree
@ -428,7 +429,7 @@ func TestGitWorktree(t *testing.T) {
// walk with filesystem instead of git
out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--walk", "filesystem")
as.NoError(err)
assertFormatted(t, as, out, 59)
as.Contains(string(out), fmt.Sprintf("%d files changed", 59))
}
func TestPathsArg(t *testing.T) {
@ -463,12 +464,12 @@ func TestPathsArg(t *testing.T) {
// without any path args
out, err := cmd(t, "-C", tempDir)
as.NoError(err)
assertFormatted(t, as, out, 31)
as.Contains(string(out), fmt.Sprintf("%d files changed", 31))
// specify some explicit paths
out, err = cmd(t, "-C", tempDir, "-c", "elm/elm.json", "haskell/Nested/Foo.hs")
as.NoError(err)
assertFormatted(t, as, out, 2)
as.Contains(string(out), fmt.Sprintf("%d files changed", 2))
// specify a bad path
out, err = cmd(t, "-C", tempDir, "-c", "elm/elm.json", "haskell/Nested/Bar.hs")
@ -528,7 +529,7 @@ go/main.go
out, err := cmd(t, "-C", tempDir, "--stdin")
as.NoError(err)
assertFormatted(t, as, out, 3)
as.Contains(string(out), fmt.Sprintf("%d files changed", 3))
}
func TestDeterministicOrderingInPipeline(t *testing.T) {

13
cli/helpers_test.go
View File

@ -69,16 +69,3 @@ func cmd(t *testing.T, args ...string) ([]byte, error) {
return out, nil
}
func assertStats(t *testing.T, as *require.Assertions, output []byte, traversed int32, emitted int32, matched int32, formatted int32) {
t.Helper()
as.Contains(string(output), fmt.Sprintf("traversed %d files", traversed))
as.Contains(string(output), fmt.Sprintf("emitted %d files", emitted))
as.Contains(string(output), fmt.Sprintf("matched %d files", matched))
as.Contains(string(output), fmt.Sprintf("formatted %d files", formatted))
}
func assertFormatted(t *testing.T, as *require.Assertions, output []byte, count int) {
t.Helper()
as.Contains(string(output), fmt.Sprintf("formatted %d files", count))
}

42
flake.lock
View File

@ -8,11 +8,11 @@
]
},
"locked": {
"lastModified": 1713532798,
"narHash": "sha256-wtBhsdMJA3Wa32Wtm1eeo84GejtI43pMrFrmwLXrsEc=",
"lastModified": 1705332421,
"narHash": "sha256-USpGLPme1IuqG78JNqSaRabilwkCyHmVWY0M9vYyqEA=",
"owner": "numtide",
"repo": "devshell",
"rev": "12e914740a25ea1891ec619bb53cf5e6ca922e40",
"rev": "83cb93d6d063ad290beee669f4badf9914cc16ec",
"type": "github"
},
"original": {
@ -26,11 +26,11 @@
"nixpkgs-lib": "nixpkgs-lib"
},
"locked": {
"lastModified": 1712014858,
"narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=",
"lastModified": 1706830856,
"narHash": "sha256-a0NYyp+h9hlb7ddVz4LUn1vT/PLwqfrWYcHMvFB1xYg=",
"owner": "hercules-ci",
"repo": "flake-parts",
"rev": "9126214d0a59633752a136528f5f3b9aa8565b7d",
"rev": "b253292d9c0a5ead9bc98c4e9a26c6312e27d69f",
"type": "github"
},
"original": {
@ -41,11 +41,11 @@
},
"flake-root": {
"locked": {
"lastModified": 1713493429,
"narHash": "sha256-ztz8JQkI08tjKnsTpfLqzWoKFQF4JGu2LRz8bkdnYUk=",
"lastModified": 1692742795,
"narHash": "sha256-f+Y0YhVCIJ06LemO+3Xx00lIcqQxSKJHXT/yk1RTKxw=",
"owner": "srid",
"repo": "flake-root",
"rev": "bc748b93b86ee76e2032eecda33440ceb2532fcd",
"rev": "d9a70d9c7a5fd7f3258ccf48da9335e9b47c3937",
"type": "github"
},
"original": {
@ -98,11 +98,11 @@
]
},
"locked": {
"lastModified": 1710154385,
"narHash": "sha256-4c3zQ2YY4BZOufaBJB4v9VBBeN2dH7iVdoJw8SDNCfI=",
"lastModified": 1705314449,
"narHash": "sha256-yfQQ67dLejP0FLK76LKHbkzcQqNIrux6MFe32MMFGNQ=",
"owner": "nix-community",
"repo": "gomod2nix",
"rev": "872b63ddd28f318489c929d25f1f0a3c6039c971",
"rev": "30e3c3a9ec4ac8453282ca7f67fca9e1da12c3e6",
"type": "github"
},
"original": {
@ -113,11 +113,11 @@
},
"nix-filter": {
"locked": {
"lastModified": 1710156097,
"narHash": "sha256-1Wvk8UP7PXdf8bCCaEoMnOT1qe5/Duqgj+rL8sRQsSM=",
"lastModified": 1705332318,
"narHash": "sha256-kcw1yFeJe9N4PjQji9ZeX47jg0p9A0DuU4djKvg1a7I=",
"owner": "numtide",
"repo": "nix-filter",
"rev": "3342559a24e85fc164b295c3444e8a139924675b",
"rev": "3449dc925982ad46246cfc36469baf66e1b64f17",
"type": "github"
},
"original": {
@ -128,11 +128,11 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1714253743,
"narHash": "sha256-mdTQw2XlariysyScCv2tTE45QSU9v/ezLcHJ22f0Nxc=",
"lastModified": 1707689078,
"narHash": "sha256-UUGmRa84ZJHpGZ1WZEBEUOzaPOWG8LZ0yPg1pdDF/yM=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "58a1abdbae3217ca6b702f03d3b35125d88a2994",
"rev": "f9d39fb9aff0efee4a3d5f4a6d7c17701d38a1d8",
"type": "github"
},
"original": {
@ -145,11 +145,11 @@
"nixpkgs-lib": {
"locked": {
"dir": "lib",
"lastModified": 1711703276,
"narHash": "sha256-iMUFArF0WCatKK6RzfUJknjem0H9m4KgorO/p3Dopkk=",
"lastModified": 1706550542,
"narHash": "sha256-UcsnCG6wx++23yeER4Hg18CXWbgNpqNXcHIo5/1Y+hc=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "d8fe5e6c92d0d190646fb9f1056741a229980089",
"rev": "97b17f32362e475016f942bbdfda4a4a72a8a652",
"type": "github"
},
"original": {

65
stats/stats.go
View File

@ -1,65 +0,0 @@
package stats
import (
"fmt"
"strings"
"sync/atomic"
"time"
)
type Type int
const (
Traversed Type = iota
Emitted
Matched
Formatted
)
var (
counters map[Type]*atomic.Int32
start time.Time
)
func Init() {
// record start time
start = time.Now()
// init counters
counters = make(map[Type]*atomic.Int32)
counters[Traversed] = &atomic.Int32{}
counters[Emitted] = &atomic.Int32{}
counters[Matched] = &atomic.Int32{}
counters[Formatted] = &atomic.Int32{}
}
func Add(t Type, delta int32) int32 {
return counters[t].Add(delta)
}
func Value(t Type) int32 {
return counters[t].Load()
}
func Elapsed() time.Duration {
return time.Now().Sub(start)
}
func Print() {
components := []string{
"traversed %d files",
"emitted %d files for processing",
"matched %d files to formatters",
"formatted %d files in %v",
"",
}
fmt.Printf(
strings.Join(components, "\n"),
Value(Traversed),
Value(Emitted),
Value(Matched),
Value(Formatted),
Elapsed(),
)
}