diff --git a/cache/cache.go b/cache/cache.go index 529085d..bdc76bc 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -11,6 +11,8 @@ import ( "runtime" "time" + "git.numtide.com/numtide/treefmt/stats" + "git.numtide.com/numtide/treefmt/format" "git.numtide.com/numtide/treefmt/walk" @@ -33,9 +35,10 @@ type Entry struct { } var ( - db *bolt.DB + db *bolt.DB + logger *log.Logger + ReadBatchSize = 1024 * runtime.NumCPU() - logger *log.Logger ) // Open creates an instance of bolt.DB for a given treeRoot path. @@ -234,11 +237,14 @@ func ChangeSet(ctx context.Context, walker walk.Walker, pathsCh chan<- string) e changedOrNew := cached == nil || !(cached.Modified == info.ModTime() && cached.Size == info.Size()) + stats.Add(stats.Traversed, 1) if !changedOrNew { // no change return nil } + stats.Add(stats.Emitted, 1) + // pass on the path select { case <-ctx.Done(): @@ -293,6 +299,8 @@ func Update(treeRoot string, paths []string) (int, error) { continue } + stats.Add(stats.Formatted, 1) + entry := Entry{ Size: pathInfo.Size(), Modified: pathInfo.ModTime(), diff --git a/cli/format.go b/cli/format.go index 6f0b356..7e2e771 100644 --- a/cli/format.go +++ b/cli/format.go @@ -14,9 +14,9 @@ import ( "sort" "strings" "syscall" - "time" "git.numtide.com/numtide/treefmt/format" + "git.numtide.com/numtide/treefmt/stats" "github.com/gobwas/glob" "git.numtide.com/numtide/treefmt/cache" @@ -32,7 +32,6 @@ const ( ) var ( - start time.Time globalExcludes []glob.Glob formatters map[string]*format.Formatter pipelines map[string]*format.Pipeline @@ -43,7 +42,7 @@ var ( ) func (f *Format) Run() (err error) { - start = time.Now() + stats.Init() Cli.Configure() @@ -196,6 +195,8 @@ func walkFilesystem(ctx context.Context) func() error { default: // ignore symlinks and directories if !(info.IsDir() || info.Mode()&os.ModeSymlink == os.ModeSymlink) { + stats.Add(stats.Traversed, 1) + stats.Add(stats.Emitted, 1) pathsCh <- path } return nil @@ -257,7 +258,7 @@ func updateCache(ctx context.Context) func() error { return ErrFailOnChange } - fmt.Printf("%v files changed in %v\n", changes, time.Now().Sub(start)) + stats.Print() return nil } } @@ -322,12 +323,17 @@ func applyFormatters(ctx context.Context) func() error { }() for path := range pathsCh { + var matched bool for key, pipeline := range pipelines { if !pipeline.Wants(path) { continue } + matched = true tryApply(key, path) } + if matched { + stats.Add(stats.Matched, 1) + } } // flush any partial batches which remain diff --git a/cli/format_test.go b/cli/format_test.go index df02288..1918145 100644 --- a/cli/format_test.go +++ b/cli/format_test.go @@ -2,7 +2,6 @@ package cli import ( "bufio" - "fmt" "os" "os/exec" "path" @@ -68,19 +67,19 @@ func TestSpecifyingFormatters(t *testing.T) { out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - as.Contains(string(out), "3 files changed") + assertFormatted(t, as, out, 3) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "elm,nix") as.NoError(err) - as.Contains(string(out), "2 files changed") + assertFormatted(t, as, out, 2) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "ruby,nix") as.NoError(err) - as.Contains(string(out), "2 files changed") + assertFormatted(t, as, out, 2) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "nix") as.NoError(err) - as.Contains(string(out), "1 files changed") + assertFormatted(t, as, out, 1) // test bad names @@ -110,7 +109,7 @@ func TestIncludesAndExcludes(t *testing.T) { test.WriteConfig(t, configPath, cfg) out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 31)) + assertFormatted(t, as, out, 31) // globally exclude nix files cfg.Global.Excludes = []string{"*.nix"} @@ -118,7 +117,7 @@ func TestIncludesAndExcludes(t *testing.T) { test.WriteConfig(t, configPath, cfg) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 30)) + assertFormatted(t, as, out, 30) // add haskell files to the global exclude cfg.Global.Excludes = []string{"*.nix", "*.hs"} @@ -126,7 +125,7 @@ func TestIncludesAndExcludes(t *testing.T) { test.WriteConfig(t, configPath, cfg) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 24)) + assertFormatted(t, as, out, 24) echo := cfg.Formatters["echo"] @@ -136,7 +135,7 @@ func TestIncludesAndExcludes(t *testing.T) { test.WriteConfig(t, configPath, cfg) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 22)) + assertFormatted(t, as, out, 22) // remove go files from the echo formatter echo.Excludes = []string{"*.py", "*.go"} @@ -144,7 +143,7 @@ func TestIncludesAndExcludes(t *testing.T) { test.WriteConfig(t, configPath, cfg) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 21)) + assertFormatted(t, as, out, 21) // adjust the includes for echo to only include elm files echo.Includes = []string{"*.elm"} @@ -152,7 +151,7 @@ func TestIncludesAndExcludes(t *testing.T) { test.WriteConfig(t, configPath, cfg) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 1)) + assertFormatted(t, as, out, 1) // add js files to echo formatter echo.Includes = []string{"*.elm", "*.js"} @@ -160,7 +159,7 @@ func TestIncludesAndExcludes(t *testing.T) { test.WriteConfig(t, configPath, cfg) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 2)) + assertFormatted(t, as, out, 2) } func TestCache(t *testing.T) { @@ -182,34 +181,34 @@ func TestCache(t *testing.T) { test.WriteConfig(t, configPath, cfg) out, err := cmd(t, "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 31)) + assertFormatted(t, as, out, 31) out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - as.Contains(string(out), "0 files changed") + assertFormatted(t, as, out, 0) // clear cache out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir, "-c") as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 31)) + assertFormatted(t, as, out, 31) out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - as.Contains(string(out), "0 files changed") + assertFormatted(t, as, out, 0) // clear cache out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir, "-c") as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 31)) + assertFormatted(t, as, out, 31) out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - as.Contains(string(out), "0 files changed") + assertFormatted(t, as, out, 0) // no cache out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir, "--no-cache") as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 31)) + assertStats(t, as, out, 31, 31, 31, 0) } func TestChangeWorkingDirectory(t *testing.T) { @@ -243,7 +242,7 @@ func TestChangeWorkingDirectory(t *testing.T) { // this should fail if the working directory hasn't been changed first out, err := cmd(t, "-C", tempDir) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 31)) + assertFormatted(t, as, out, 31) } func TestFailOnChange(t *testing.T) { @@ -307,31 +306,31 @@ func TestBustCacheOnFormatterChange(t *testing.T) { args := []string{"--config-file", configPath, "--tree-root", tempDir} out, err := cmd(t, args...) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 3)) + assertFormatted(t, as, out, 3) // tweak mod time of elm formatter as.NoError(test.RecreateSymlink(t, binPath+"/"+"elm-format")) out, err = cmd(t, args...) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 3)) + assertFormatted(t, as, out, 3) // check cache is working out, err = cmd(t, args...) as.NoError(err) - as.Contains(string(out), "0 files changed") + assertFormatted(t, as, out, 0) // tweak mod time of python formatter as.NoError(test.RecreateSymlink(t, binPath+"/"+"black")) out, err = cmd(t, args...) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 3)) + assertFormatted(t, as, out, 3) // check cache is working out, err = cmd(t, args...) as.NoError(err) - as.Contains(string(out), "0 files changed") + assertFormatted(t, as, out, 0) // add go formatter cfg.Formatters["go"] = &config2.Formatter{ @@ -343,12 +342,12 @@ func TestBustCacheOnFormatterChange(t *testing.T) { out, err = cmd(t, args...) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 4)) + assertFormatted(t, as, out, 4) // check cache is working out, err = cmd(t, args...) as.NoError(err) - as.Contains(string(out), "0 files changed") + assertFormatted(t, as, out, 0) // remove python formatter delete(cfg.Formatters, "python") @@ -356,12 +355,12 @@ func TestBustCacheOnFormatterChange(t *testing.T) { out, err = cmd(t, args...) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 2)) + assertFormatted(t, as, out, 2) // check cache is working out, err = cmd(t, args...) as.NoError(err) - as.Contains(string(out), "0 files changed") + assertFormatted(t, as, out, 0) // remove elm formatter delete(cfg.Formatters, "elm") @@ -369,12 +368,12 @@ func TestBustCacheOnFormatterChange(t *testing.T) { out, err = cmd(t, args...) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 1)) + assertFormatted(t, as, out, 1) // check cache is working out, err = cmd(t, args...) as.NoError(err) - as.Contains(string(out), "0 files changed") + assertFormatted(t, as, out, 0) } func TestGitWorktree(t *testing.T) { @@ -408,10 +407,10 @@ func TestGitWorktree(t *testing.T) { wt, err := repo.Worktree() as.NoError(err, "failed to get git worktree") - run := func(changed int) { + run := func(formatted int) { out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", changed)) + assertFormatted(t, as, out, formatted) } // run before adding anything to the worktree @@ -429,7 +428,7 @@ func TestGitWorktree(t *testing.T) { // walk with filesystem instead of git out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--walk", "filesystem") as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 59)) + assertFormatted(t, as, out, 59) } func TestPathsArg(t *testing.T) { @@ -464,12 +463,12 @@ func TestPathsArg(t *testing.T) { // without any path args out, err := cmd(t, "-C", tempDir) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 31)) + assertFormatted(t, as, out, 31) // specify some explicit paths out, err = cmd(t, "-C", tempDir, "-c", "elm/elm.json", "haskell/Nested/Foo.hs") as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 2)) + assertFormatted(t, as, out, 2) // specify a bad path out, err = cmd(t, "-C", tempDir, "-c", "elm/elm.json", "haskell/Nested/Bar.hs") @@ -529,7 +528,7 @@ go/main.go out, err := cmd(t, "-C", tempDir, "--stdin") as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 3)) + assertFormatted(t, as, out, 3) } func TestDeterministicOrderingInPipeline(t *testing.T) { diff --git a/cli/helpers_test.go b/cli/helpers_test.go index 920c757..5abeefc 100644 --- a/cli/helpers_test.go +++ b/cli/helpers_test.go @@ -69,3 +69,16 @@ func cmd(t *testing.T, args ...string) ([]byte, error) { return out, nil } + +func assertStats(t *testing.T, as *require.Assertions, output []byte, traversed int32, emitted int32, matched int32, formatted int32) { + t.Helper() + as.Contains(string(output), fmt.Sprintf("traversed %d files", traversed)) + as.Contains(string(output), fmt.Sprintf("emitted %d files", emitted)) + as.Contains(string(output), fmt.Sprintf("matched %d files", matched)) + as.Contains(string(output), fmt.Sprintf("formatted %d files", formatted)) +} + +func assertFormatted(t *testing.T, as *require.Assertions, output []byte, count int) { + t.Helper() + as.Contains(string(output), fmt.Sprintf("formatted %d files", count)) +} diff --git a/stats/stats.go b/stats/stats.go new file mode 100644 index 0000000..63fdf64 --- /dev/null +++ b/stats/stats.go @@ -0,0 +1,65 @@ +package stats + +import ( + "fmt" + "strings" + "sync/atomic" + "time" +) + +type Type int + +const ( + Traversed Type = iota + Emitted + Matched + Formatted +) + +var ( + counters map[Type]*atomic.Int32 + start time.Time +) + +func Init() { + // record start time + start = time.Now() + + // init counters + counters = make(map[Type]*atomic.Int32) + counters[Traversed] = &atomic.Int32{} + counters[Emitted] = &atomic.Int32{} + counters[Matched] = &atomic.Int32{} + counters[Formatted] = &atomic.Int32{} +} + +func Add(t Type, delta int32) int32 { + return counters[t].Add(delta) +} + +func Value(t Type) int32 { + return counters[t].Load() +} + +func Elapsed() time.Duration { + return time.Now().Sub(start) +} + +func Print() { + components := []string{ + "traversed %d files", + "emitted %d files for processing", + "matched %d files to formatters", + "formatted %d files in %v", + "", + } + + fmt.Printf( + strings.Join(components, "\n"), + Value(Traversed), + Value(Emitted), + Value(Matched), + Value(Formatted), + Elapsed(), + ) +}