diff --git a/cache/cache.go b/cache/cache.go index 529085d..bdc76bc 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -11,6 +11,8 @@ import ( "runtime" "time" + "git.numtide.com/numtide/treefmt/stats" + "git.numtide.com/numtide/treefmt/format" "git.numtide.com/numtide/treefmt/walk" @@ -33,9 +35,10 @@ type Entry struct { } var ( - db *bolt.DB + db *bolt.DB + logger *log.Logger + ReadBatchSize = 1024 * runtime.NumCPU() - logger *log.Logger ) // Open creates an instance of bolt.DB for a given treeRoot path. @@ -234,11 +237,14 @@ func ChangeSet(ctx context.Context, walker walk.Walker, pathsCh chan<- string) e changedOrNew := cached == nil || !(cached.Modified == info.ModTime() && cached.Size == info.Size()) + stats.Add(stats.Traversed, 1) if !changedOrNew { // no change return nil } + stats.Add(stats.Emitted, 1) + // pass on the path select { case <-ctx.Done(): @@ -293,6 +299,8 @@ func Update(treeRoot string, paths []string) (int, error) { continue } + stats.Add(stats.Formatted, 1) + entry := Entry{ Size: pathInfo.Size(), Modified: pathInfo.ModTime(), diff --git a/cli/format.go b/cli/format.go index 6f0b356..7e2e771 100644 --- a/cli/format.go +++ b/cli/format.go @@ -14,9 +14,9 @@ import ( "sort" "strings" "syscall" - "time" "git.numtide.com/numtide/treefmt/format" + "git.numtide.com/numtide/treefmt/stats" "github.com/gobwas/glob" "git.numtide.com/numtide/treefmt/cache" @@ -32,7 +32,6 @@ const ( ) var ( - start time.Time globalExcludes []glob.Glob formatters map[string]*format.Formatter pipelines map[string]*format.Pipeline @@ -43,7 +42,7 @@ var ( ) func (f *Format) Run() (err error) { - start = time.Now() + stats.Init() Cli.Configure() @@ -196,6 +195,8 @@ func walkFilesystem(ctx context.Context) func() error { default: // ignore symlinks and directories if !(info.IsDir() || info.Mode()&os.ModeSymlink == os.ModeSymlink) { + stats.Add(stats.Traversed, 1) + stats.Add(stats.Emitted, 1) pathsCh <- path } return nil @@ -257,7 +258,7 @@ func updateCache(ctx context.Context) func() error { return ErrFailOnChange } - fmt.Printf("%v files changed in %v\n", changes, time.Now().Sub(start)) + stats.Print() return nil } } @@ -322,12 +323,17 @@ func applyFormatters(ctx context.Context) func() error { }() for path := range pathsCh { + var matched bool for key, pipeline := range pipelines { if !pipeline.Wants(path) { continue } + matched = true tryApply(key, path) } + if matched { + stats.Add(stats.Matched, 1) + } } // flush any partial batches which remain diff --git a/cli/format_test.go b/cli/format_test.go index df02288..1918145 100644 --- a/cli/format_test.go +++ b/cli/format_test.go @@ -2,7 +2,6 @@ package cli import ( "bufio" - "fmt" "os" "os/exec" "path" @@ -68,19 +67,19 @@ func TestSpecifyingFormatters(t *testing.T) { out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - as.Contains(string(out), "3 files changed") + assertFormatted(t, as, out, 3) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "elm,nix") as.NoError(err) - as.Contains(string(out), "2 files changed") + assertFormatted(t, as, out, 2) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "ruby,nix") as.NoError(err) - as.Contains(string(out), "2 files changed") + assertFormatted(t, as, out, 2) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "nix") as.NoError(err) - as.Contains(string(out), "1 files changed") + assertFormatted(t, as, out, 1) // test bad names @@ -110,7 +109,7 @@ func TestIncludesAndExcludes(t *testing.T) { test.WriteConfig(t, configPath, cfg) out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 31)) + assertFormatted(t, as, out, 31) // globally exclude nix files cfg.Global.Excludes = []string{"*.nix"} @@ -118,7 +117,7 @@ func TestIncludesAndExcludes(t *testing.T) { test.WriteConfig(t, configPath, cfg) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 30)) + assertFormatted(t, as, out, 30) // add haskell files to the global exclude cfg.Global.Excludes = []string{"*.nix", "*.hs"} @@ -126,7 +125,7 @@ func TestIncludesAndExcludes(t *testing.T) { test.WriteConfig(t, configPath, cfg) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 24)) + assertFormatted(t, as, out, 24) echo := cfg.Formatters["echo"] @@ -136,7 +135,7 @@ func TestIncludesAndExcludes(t *testing.T) { test.WriteConfig(t, configPath, cfg) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 22)) + assertFormatted(t, as, out, 22) // remove go files from the echo formatter echo.Excludes = []string{"*.py", "*.go"} @@ -144,7 +143,7 @@ func TestIncludesAndExcludes(t *testing.T) { test.WriteConfig(t, configPath, cfg) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 21)) + assertFormatted(t, as, out, 21) // adjust the includes for echo to only include elm files echo.Includes = []string{"*.elm"} @@ -152,7 +151,7 @@ func TestIncludesAndExcludes(t *testing.T) { test.WriteConfig(t, configPath, cfg) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 1)) + assertFormatted(t, as, out, 1) // add js files to echo formatter echo.Includes = []string{"*.elm", "*.js"} @@ -160,7 +159,7 @@ func TestIncludesAndExcludes(t *testing.T) { test.WriteConfig(t, configPath, cfg) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 2)) + assertFormatted(t, as, out, 2) } func TestCache(t *testing.T) { @@ -182,34 +181,34 @@ func TestCache(t *testing.T) { test.WriteConfig(t, configPath, cfg) out, err := cmd(t, "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 31)) + assertFormatted(t, as, out, 31) out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - as.Contains(string(out), "0 files changed") + assertFormatted(t, as, out, 0) // clear cache out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir, "-c") as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 31)) + assertFormatted(t, as, out, 31) out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - as.Contains(string(out), "0 files changed") + assertFormatted(t, as, out, 0) // clear cache out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir, "-c") as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 31)) + assertFormatted(t, as, out, 31) out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - as.Contains(string(out), "0 files changed") + assertFormatted(t, as, out, 0) // no cache out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir, "--no-cache") as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 31)) + assertStats(t, as, out, 31, 31, 31, 0) } func TestChangeWorkingDirectory(t *testing.T) { @@ -243,7 +242,7 @@ func TestChangeWorkingDirectory(t *testing.T) { // this should fail if the working directory hasn't been changed first out, err := cmd(t, "-C", tempDir) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 31)) + assertFormatted(t, as, out, 31) } func TestFailOnChange(t *testing.T) { @@ -307,31 +306,31 @@ func TestBustCacheOnFormatterChange(t *testing.T) { args := []string{"--config-file", configPath, "--tree-root", tempDir} out, err := cmd(t, args...) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 3)) + assertFormatted(t, as, out, 3) // tweak mod time of elm formatter as.NoError(test.RecreateSymlink(t, binPath+"/"+"elm-format")) out, err = cmd(t, args...) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 3)) + assertFormatted(t, as, out, 3) // check cache is working out, err = cmd(t, args...) as.NoError(err) - as.Contains(string(out), "0 files changed") + assertFormatted(t, as, out, 0) // tweak mod time of python formatter as.NoError(test.RecreateSymlink(t, binPath+"/"+"black")) out, err = cmd(t, args...) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 3)) + assertFormatted(t, as, out, 3) // check cache is working out, err = cmd(t, args...) as.NoError(err) - as.Contains(string(out), "0 files changed") + assertFormatted(t, as, out, 0) // add go formatter cfg.Formatters["go"] = &config2.Formatter{ @@ -343,12 +342,12 @@ func TestBustCacheOnFormatterChange(t *testing.T) { out, err = cmd(t, args...) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 4)) + assertFormatted(t, as, out, 4) // check cache is working out, err = cmd(t, args...) as.NoError(err) - as.Contains(string(out), "0 files changed") + assertFormatted(t, as, out, 0) // remove python formatter delete(cfg.Formatters, "python") @@ -356,12 +355,12 @@ func TestBustCacheOnFormatterChange(t *testing.T) { out, err = cmd(t, args...) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 2)) + assertFormatted(t, as, out, 2) // check cache is working out, err = cmd(t, args...) as.NoError(err) - as.Contains(string(out), "0 files changed") + assertFormatted(t, as, out, 0) // remove elm formatter delete(cfg.Formatters, "elm") @@ -369,12 +368,12 @@ func TestBustCacheOnFormatterChange(t *testing.T) { out, err = cmd(t, args...) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 1)) + assertFormatted(t, as, out, 1) // check cache is working out, err = cmd(t, args...) as.NoError(err) - as.Contains(string(out), "0 files changed") + assertFormatted(t, as, out, 0) } func TestGitWorktree(t *testing.T) { @@ -408,10 +407,10 @@ func TestGitWorktree(t *testing.T) { wt, err := repo.Worktree() as.NoError(err, "failed to get git worktree") - run := func(changed int) { + run := func(formatted int) { out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", changed)) + assertFormatted(t, as, out, formatted) } // run before adding anything to the worktree @@ -429,7 +428,7 @@ func TestGitWorktree(t *testing.T) { // walk with filesystem instead of git out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--walk", "filesystem") as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 59)) + assertFormatted(t, as, out, 59) } func TestPathsArg(t *testing.T) { @@ -464,12 +463,12 @@ func TestPathsArg(t *testing.T) { // without any path args out, err := cmd(t, "-C", tempDir) as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 31)) + assertFormatted(t, as, out, 31) // specify some explicit paths out, err = cmd(t, "-C", tempDir, "-c", "elm/elm.json", "haskell/Nested/Foo.hs") as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 2)) + assertFormatted(t, as, out, 2) // specify a bad path out, err = cmd(t, "-C", tempDir, "-c", "elm/elm.json", "haskell/Nested/Bar.hs") @@ -529,7 +528,7 @@ go/main.go out, err := cmd(t, "-C", tempDir, "--stdin") as.NoError(err) - as.Contains(string(out), fmt.Sprintf("%d files changed", 3)) + assertFormatted(t, as, out, 3) } func TestDeterministicOrderingInPipeline(t *testing.T) { diff --git a/cli/helpers_test.go b/cli/helpers_test.go index 920c757..5abeefc 100644 --- a/cli/helpers_test.go +++ b/cli/helpers_test.go @@ -69,3 +69,16 @@ func cmd(t *testing.T, args ...string) ([]byte, error) { return out, nil } + +func assertStats(t *testing.T, as *require.Assertions, output []byte, traversed int32, emitted int32, matched int32, formatted int32) { + t.Helper() + as.Contains(string(output), fmt.Sprintf("traversed %d files", traversed)) + as.Contains(string(output), fmt.Sprintf("emitted %d files", emitted)) + as.Contains(string(output), fmt.Sprintf("matched %d files", matched)) + as.Contains(string(output), fmt.Sprintf("formatted %d files", formatted)) +} + +func assertFormatted(t *testing.T, as *require.Assertions, output []byte, count int) { + t.Helper() + as.Contains(string(output), fmt.Sprintf("formatted %d files", count)) +} diff --git a/flake.lock b/flake.lock index 8451813..19e8ea0 100644 --- a/flake.lock +++ b/flake.lock @@ -8,11 +8,11 @@ ] }, "locked": { - "lastModified": 1705332421, - "narHash": "sha256-USpGLPme1IuqG78JNqSaRabilwkCyHmVWY0M9vYyqEA=", + "lastModified": 1713532798, + "narHash": "sha256-wtBhsdMJA3Wa32Wtm1eeo84GejtI43pMrFrmwLXrsEc=", "owner": "numtide", "repo": "devshell", - "rev": "83cb93d6d063ad290beee669f4badf9914cc16ec", + "rev": "12e914740a25ea1891ec619bb53cf5e6ca922e40", "type": "github" }, "original": { @@ -26,11 +26,11 @@ "nixpkgs-lib": "nixpkgs-lib" }, "locked": { - "lastModified": 1706830856, - "narHash": "sha256-a0NYyp+h9hlb7ddVz4LUn1vT/PLwqfrWYcHMvFB1xYg=", + "lastModified": 1712014858, + "narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "b253292d9c0a5ead9bc98c4e9a26c6312e27d69f", + "rev": "9126214d0a59633752a136528f5f3b9aa8565b7d", "type": "github" }, "original": { @@ -41,11 +41,11 @@ }, "flake-root": { "locked": { - "lastModified": 1692742795, - "narHash": "sha256-f+Y0YhVCIJ06LemO+3Xx00lIcqQxSKJHXT/yk1RTKxw=", + "lastModified": 1713493429, + "narHash": "sha256-ztz8JQkI08tjKnsTpfLqzWoKFQF4JGu2LRz8bkdnYUk=", "owner": "srid", "repo": "flake-root", - "rev": "d9a70d9c7a5fd7f3258ccf48da9335e9b47c3937", + "rev": "bc748b93b86ee76e2032eecda33440ceb2532fcd", "type": "github" }, "original": { @@ -98,11 +98,11 @@ ] }, "locked": { - "lastModified": 1705314449, - "narHash": "sha256-yfQQ67dLejP0FLK76LKHbkzcQqNIrux6MFe32MMFGNQ=", + "lastModified": 1710154385, + "narHash": "sha256-4c3zQ2YY4BZOufaBJB4v9VBBeN2dH7iVdoJw8SDNCfI=", "owner": "nix-community", "repo": "gomod2nix", - "rev": "30e3c3a9ec4ac8453282ca7f67fca9e1da12c3e6", + "rev": "872b63ddd28f318489c929d25f1f0a3c6039c971", "type": "github" }, "original": { @@ -113,11 +113,11 @@ }, "nix-filter": { "locked": { - "lastModified": 1705332318, - "narHash": "sha256-kcw1yFeJe9N4PjQji9ZeX47jg0p9A0DuU4djKvg1a7I=", + "lastModified": 1710156097, + "narHash": "sha256-1Wvk8UP7PXdf8bCCaEoMnOT1qe5/Duqgj+rL8sRQsSM=", "owner": "numtide", "repo": "nix-filter", - "rev": "3449dc925982ad46246cfc36469baf66e1b64f17", + "rev": "3342559a24e85fc164b295c3444e8a139924675b", "type": "github" }, "original": { @@ -128,11 +128,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1707689078, - "narHash": "sha256-UUGmRa84ZJHpGZ1WZEBEUOzaPOWG8LZ0yPg1pdDF/yM=", + "lastModified": 1714253743, + "narHash": "sha256-mdTQw2XlariysyScCv2tTE45QSU9v/ezLcHJ22f0Nxc=", "owner": "nixos", "repo": "nixpkgs", - "rev": "f9d39fb9aff0efee4a3d5f4a6d7c17701d38a1d8", + "rev": "58a1abdbae3217ca6b702f03d3b35125d88a2994", "type": "github" }, "original": { @@ -145,11 +145,11 @@ "nixpkgs-lib": { "locked": { "dir": "lib", - "lastModified": 1706550542, - "narHash": "sha256-UcsnCG6wx++23yeER4Hg18CXWbgNpqNXcHIo5/1Y+hc=", + "lastModified": 1711703276, + "narHash": "sha256-iMUFArF0WCatKK6RzfUJknjem0H9m4KgorO/p3Dopkk=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "97b17f32362e475016f942bbdfda4a4a72a8a652", + "rev": "d8fe5e6c92d0d190646fb9f1056741a229980089", "type": "github" }, "original": { diff --git a/stats/stats.go b/stats/stats.go new file mode 100644 index 0000000..63fdf64 --- /dev/null +++ b/stats/stats.go @@ -0,0 +1,65 @@ +package stats + +import ( + "fmt" + "strings" + "sync/atomic" + "time" +) + +type Type int + +const ( + Traversed Type = iota + Emitted + Matched + Formatted +) + +var ( + counters map[Type]*atomic.Int32 + start time.Time +) + +func Init() { + // record start time + start = time.Now() + + // init counters + counters = make(map[Type]*atomic.Int32) + counters[Traversed] = &atomic.Int32{} + counters[Emitted] = &atomic.Int32{} + counters[Matched] = &atomic.Int32{} + counters[Formatted] = &atomic.Int32{} +} + +func Add(t Type, delta int32) int32 { + return counters[t].Add(delta) +} + +func Value(t Type) int32 { + return counters[t].Load() +} + +func Elapsed() time.Duration { + return time.Now().Sub(start) +} + +func Print() { + components := []string{ + "traversed %d files", + "emitted %d files for processing", + "matched %d files to formatters", + "formatted %d files in %v", + "", + } + + fmt.Printf( + strings.Join(components, "\n"), + Value(Traversed), + Value(Emitted), + Value(Matched), + Value(Formatted), + Elapsed(), + ) +}