numtide/treefmt
Archived
8
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Stats output similar to treefmt.rs #33

Merged
brianmcgee merged 2 commits from feat/stats into main 2024-05-01 12:49:32 +00:00
Conversation 0 Commits 2 Files Changed 6 +155 -64
6 changed files with 155 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
cache/cache.go vendored
View File

@ -11,6 +11,8 @@ import (
"runtime"
"time"
"git.numtide.com/numtide/treefmt/stats"
"git.numtide.com/numtide/treefmt/format"
"git.numtide.com/numtide/treefmt/walk"
@ -33,9 +35,10 @@ type Entry struct {
}
var (
db *bolt.DB
db *bolt.DB
logger *log.Logger
ReadBatchSize = 1024 * runtime.NumCPU()
logger *log.Logger
)
// Open creates an instance of bolt.DB for a given treeRoot path.
@ -234,11 +237,14 @@ func ChangeSet(ctx context.Context, walker walk.Walker, pathsCh chan<- string) e
changedOrNew := cached == nil || !(cached.Modified == info.ModTime() && cached.Size == info.Size())
stats.Add(stats.Traversed, 1)
if !changedOrNew {
// no change
return nil
}
stats.Add(stats.Emitted, 1)
// pass on the path
select {
case <-ctx.Done():
@ -293,6 +299,8 @@ func Update(treeRoot string, paths []string) (int, error) {
continue
}
stats.Add(stats.Formatted, 1)
entry := Entry{
Size: pathInfo.Size(),
Modified: pathInfo.ModTime(),

14
cli/format.go
View File

@ -14,9 +14,9 @@ import (
"sort"
"strings"
"syscall"
"time"
"git.numtide.com/numtide/treefmt/format"
"git.numtide.com/numtide/treefmt/stats"
"github.com/gobwas/glob"
"git.numtide.com/numtide/treefmt/cache"
@ -32,7 +32,6 @@ const (
)
var (
start time.Time
globalExcludes []glob.Glob
formatters map[string]*format.Formatter
pipelines map[string]*format.Pipeline
@ -43,7 +42,7 @@ var (
)
func (f *Format) Run() (err error) {
start = time.Now()
stats.Init()
Cli.Configure()
@ -196,6 +195,8 @@ func walkFilesystem(ctx context.Context) func() error {
default:
// ignore symlinks and directories
if !(info.IsDir() || info.Mode()&os.ModeSymlink == os.ModeSymlink) {
stats.Add(stats.Traversed, 1)
stats.Add(stats.Emitted, 1)
pathsCh <- path
}
return nil
@ -257,7 +258,7 @@ func updateCache(ctx context.Context) func() error {
return ErrFailOnChange
}
fmt.Printf("%v files changed in %v\n", changes, time.Now().Sub(start))
stats.Print()
return nil
}
}
@ -322,12 +323,17 @@ func applyFormatters(ctx context.Context) func() error {
}()
for path := range pathsCh {
var matched bool
for key, pipeline := range pipelines {
if !pipeline.Wants(path) {
continue
}
matched = true
tryApply(key, path)
}
if matched {
stats.Add(stats.Matched, 1)
}
}
// flush any partial batches which remain

73
cli/format_test.go
View File

@ -2,7 +2,6 @@ package cli
import (
"bufio"
"fmt"
"os"
"os/exec"
"path"
@ -68,19 +67,19 @@ func TestSpecifyingFormatters(t *testing.T) {
out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err)
as.Contains(string(out), "3 files changed")
assertFormatted(t, as, out, 3)
out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "elm,nix")
as.NoError(err)
as.Contains(string(out), "2 files changed")
assertFormatted(t, as, out, 2)
out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "ruby,nix")
as.NoError(err)
as.Contains(string(out), "2 files changed")
assertFormatted(t, as, out, 2)
out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "nix")
as.NoError(err)
as.Contains(string(out), "1 files changed")
assertFormatted(t, as, out, 1)
// test bad names
@ -110,7 +109,7 @@ func TestIncludesAndExcludes(t *testing.T) {
test.WriteConfig(t, configPath, cfg)
out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err)
as.Contains(string(out), fmt.Sprintf("%d files changed", 31))
assertFormatted(t, as, out, 31)
// globally exclude nix files
cfg.Global.Excludes = []string{"*.nix"}
@ -118,7 +117,7 @@ func TestIncludesAndExcludes(t *testing.T) {
test.WriteConfig(t, configPath, cfg)
out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err)
as.Contains(string(out), fmt.Sprintf("%d files changed", 30))
assertFormatted(t, as, out, 30)
// add haskell files to the global exclude
cfg.Global.Excludes = []string{"*.nix", "*.hs"}
@ -126,7 +125,7 @@ func TestIncludesAndExcludes(t *testing.T) {
test.WriteConfig(t, configPath, cfg)
out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err)
as.Contains(string(out), fmt.Sprintf("%d files changed", 24))
assertFormatted(t, as, out, 24)
echo := cfg.Formatters["echo"]
@ -136,7 +135,7 @@ func TestIncludesAndExcludes(t *testing.T) {
test.WriteConfig(t, configPath, cfg)
out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err)
as.Contains(string(out), fmt.Sprintf("%d files changed", 22))
assertFormatted(t, as, out, 22)
// remove go files from the echo formatter
echo.Excludes = []string{"*.py", "*.go"}
@ -144,7 +143,7 @@ func TestIncludesAndExcludes(t *testing.T) {
test.WriteConfig(t, configPath, cfg)
out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err)
as.Contains(string(out), fmt.Sprintf("%d files changed", 21))
assertFormatted(t, as, out, 21)
// adjust the includes for echo to only include elm files
echo.Includes = []string{"*.elm"}
@ -152,7 +151,7 @@ func TestIncludesAndExcludes(t *testing.T) {
test.WriteConfig(t, configPath, cfg)
out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err)
as.Contains(string(out), fmt.Sprintf("%d files changed", 1))
assertFormatted(t, as, out, 1)
// add js files to echo formatter
echo.Includes = []string{"*.elm", "*.js"}
@ -160,7 +159,7 @@ func TestIncludesAndExcludes(t *testing.T) {
test.WriteConfig(t, configPath, cfg)
out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err)
as.Contains(string(out), fmt.Sprintf("%d files changed", 2))
assertFormatted(t, as, out, 2)
}
func TestCache(t *testing.T) {
@ -182,34 +181,34 @@ func TestCache(t *testing.T) {
test.WriteConfig(t, configPath, cfg)
out, err := cmd(t, "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err)
as.Contains(string(out), fmt.Sprintf("%d files changed", 31))
assertFormatted(t, as, out, 31)
out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err)
as.Contains(string(out), "0 files changed")
assertFormatted(t, as, out, 0)
// clear cache
out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir, "-c")
as.NoError(err)
as.Contains(string(out), fmt.Sprintf("%d files changed", 31))
assertFormatted(t, as, out, 31)
out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err)
as.Contains(string(out), "0 files changed")
assertFormatted(t, as, out, 0)
// clear cache
out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir, "-c")
as.NoError(err)
as.Contains(string(out), fmt.Sprintf("%d files changed", 31))
assertFormatted(t, as, out, 31)
out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err)
as.Contains(string(out), "0 files changed")
assertFormatted(t, as, out, 0)
// no cache
out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir, "--no-cache")
as.NoError(err)
as.Contains(string(out), fmt.Sprintf("%d files changed", 31))
assertStats(t, as, out, 31, 31, 31, 0)
}
func TestChangeWorkingDirectory(t *testing.T) {
@ -243,7 +242,7 @@ func TestChangeWorkingDirectory(t *testing.T) {
// this should fail if the working directory hasn't been changed first
out, err := cmd(t, "-C", tempDir)
as.NoError(err)
as.Contains(string(out), fmt.Sprintf("%d files changed", 31))
assertFormatted(t, as, out, 31)
}
func TestFailOnChange(t *testing.T) {
@ -307,31 +306,31 @@ func TestBustCacheOnFormatterChange(t *testing.T) {
args := []string{"--config-file", configPath, "--tree-root", tempDir}
out, err := cmd(t, args...)
as.NoError(err)
as.Contains(string(out), fmt.Sprintf("%d files changed", 3))
assertFormatted(t, as, out, 3)
// tweak mod time of elm formatter
as.NoError(test.RecreateSymlink(t, binPath+"/"+"elm-format"))
out, err = cmd(t, args...)
as.NoError(err)
as.Contains(string(out), fmt.Sprintf("%d files changed", 3))
assertFormatted(t, as, out, 3)
// check cache is working
out, err = cmd(t, args...)
as.NoError(err)
as.Contains(string(out), "0 files changed")
assertFormatted(t, as, out, 0)
// tweak mod time of python formatter
as.NoError(test.RecreateSymlink(t, binPath+"/"+"black"))
out, err = cmd(t, args...)
as.NoError(err)
as.Contains(string(out), fmt.Sprintf("%d files changed", 3))
assertFormatted(t, as, out, 3)
// check cache is working
out, err = cmd(t, args...)
as.NoError(err)
as.Contains(string(out), "0 files changed")
assertFormatted(t, as, out, 0)
// add go formatter
cfg.Formatters["go"] = &config2.Formatter{
@ -343,12 +342,12 @@ func TestBustCacheOnFormatterChange(t *testing.T) {
out, err = cmd(t, args...)
as.NoError(err)
as.Contains(string(out), fmt.Sprintf("%d files changed", 4))
assertFormatted(t, as, out, 4)
// check cache is working
out, err = cmd(t, args...)
as.NoError(err)
as.Contains(string(out), "0 files changed")
assertFormatted(t, as, out, 0)
// remove python formatter
delete(cfg.Formatters, "python")
@ -356,12 +355,12 @@ func TestBustCacheOnFormatterChange(t *testing.T) {
out, err = cmd(t, args...)
as.NoError(err)
as.Contains(string(out), fmt.Sprintf("%d files changed", 2))
assertFormatted(t, as, out, 2)
// check cache is working
out, err = cmd(t, args...)
as.NoError(err)
as.Contains(string(out), "0 files changed")
assertFormatted(t, as, out, 0)
// remove elm formatter
delete(cfg.Formatters, "elm")
@ -369,12 +368,12 @@ func TestBustCacheOnFormatterChange(t *testing.T) {
out, err = cmd(t, args...)
as.NoError(err)
as.Contains(string(out), fmt.Sprintf("%d files changed", 1))
assertFormatted(t, as, out, 1)
// check cache is working
out, err = cmd(t, args...)
as.NoError(err)
as.Contains(string(out), "0 files changed")
assertFormatted(t, as, out, 0)
}
func TestGitWorktree(t *testing.T) {
@ -408,10 +407,10 @@ func TestGitWorktree(t *testing.T) {
wt, err := repo.Worktree()
as.NoError(err, "failed to get git worktree")
run := func(changed int) {
run := func(formatted int) {
out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir)
as.NoError(err)
as.Contains(string(out), fmt.Sprintf("%d files changed", changed))
assertFormatted(t, as, out, formatted)
}
// run before adding anything to the worktree
@ -429,7 +428,7 @@ func TestGitWorktree(t *testing.T) {
// walk with filesystem instead of git
out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--walk", "filesystem")
as.NoError(err)
as.Contains(string(out), fmt.Sprintf("%d files changed", 59))
assertFormatted(t, as, out, 59)
}
func TestPathsArg(t *testing.T) {
@ -464,12 +463,12 @@ func TestPathsArg(t *testing.T) {
// without any path args
out, err := cmd(t, "-C", tempDir)
as.NoError(err)
as.Contains(string(out), fmt.Sprintf("%d files changed", 31))
assertFormatted(t, as, out, 31)
// specify some explicit paths
out, err = cmd(t, "-C", tempDir, "-c", "elm/elm.json", "haskell/Nested/Foo.hs")
as.NoError(err)
as.Contains(string(out), fmt.Sprintf("%d files changed", 2))
assertFormatted(t, as, out, 2)
// specify a bad path
out, err = cmd(t, "-C", tempDir, "-c", "elm/elm.json", "haskell/Nested/Bar.hs")
@ -529,7 +528,7 @@ go/main.go
out, err := cmd(t, "-C", tempDir, "--stdin")
as.NoError(err)
as.Contains(string(out), fmt.Sprintf("%d files changed", 3))
assertFormatted(t, as, out, 3)
}
func TestDeterministicOrderingInPipeline(t *testing.T) {

13
cli/helpers_test.go
View File

@ -69,3 +69,16 @@ func cmd(t *testing.T, args ...string) ([]byte, error) {
return out, nil
}
func assertStats(t *testing.T, as *require.Assertions, output []byte, traversed int32, emitted int32, matched int32, formatted int32) {
t.Helper()
as.Contains(string(output), fmt.Sprintf("traversed %d files", traversed))
as.Contains(string(output), fmt.Sprintf("emitted %d files", emitted))
as.Contains(string(output), fmt.Sprintf("matched %d files", matched))
as.Contains(string(output), fmt.Sprintf("formatted %d files", formatted))
}
func assertFormatted(t *testing.T, as *require.Assertions, output []byte, count int) {
t.Helper()
as.Contains(string(output), fmt.Sprintf("formatted %d files", count))
}

42
flake.lock
View File

@ -8,11 +8,11 @@
]
},
"locked": {
"lastModified": 1705332421,
"narHash": "sha256-USpGLPme1IuqG78JNqSaRabilwkCyHmVWY0M9vYyqEA=",
"lastModified": 1713532798,
"narHash": "sha256-wtBhsdMJA3Wa32Wtm1eeo84GejtI43pMrFrmwLXrsEc=",
"owner": "numtide",
"repo": "devshell",
"rev": "83cb93d6d063ad290beee669f4badf9914cc16ec",
"rev": "12e914740a25ea1891ec619bb53cf5e6ca922e40",
"type": "github"
},
"original": {
@ -26,11 +26,11 @@
"nixpkgs-lib": "nixpkgs-lib"
},
"locked": {
"lastModified": 1706830856,
"narHash": "sha256-a0NYyp+h9hlb7ddVz4LUn1vT/PLwqfrWYcHMvFB1xYg=",
"lastModified": 1712014858,
"narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=",
"owner": "hercules-ci",
"repo": "flake-parts",
"rev": "b253292d9c0a5ead9bc98c4e9a26c6312e27d69f",
"rev": "9126214d0a59633752a136528f5f3b9aa8565b7d",
"type": "github"
},
"original": {
@ -41,11 +41,11 @@
},
"flake-root": {
"locked": {
"lastModified": 1692742795,
"narHash": "sha256-f+Y0YhVCIJ06LemO+3Xx00lIcqQxSKJHXT/yk1RTKxw=",
"lastModified": 1713493429,
"narHash": "sha256-ztz8JQkI08tjKnsTpfLqzWoKFQF4JGu2LRz8bkdnYUk=",
"owner": "srid",
"repo": "flake-root",
"rev": "d9a70d9c7a5fd7f3258ccf48da9335e9b47c3937",
"rev": "bc748b93b86ee76e2032eecda33440ceb2532fcd",
"type": "github"
},
"original": {
@ -98,11 +98,11 @@
]
},
"locked": {
"lastModified": 1705314449,
"narHash": "sha256-yfQQ67dLejP0FLK76LKHbkzcQqNIrux6MFe32MMFGNQ=",
"lastModified": 1710154385,
"narHash": "sha256-4c3zQ2YY4BZOufaBJB4v9VBBeN2dH7iVdoJw8SDNCfI=",
"owner": "nix-community",
"repo": "gomod2nix",
"rev": "30e3c3a9ec4ac8453282ca7f67fca9e1da12c3e6",
"rev": "872b63ddd28f318489c929d25f1f0a3c6039c971",
"type": "github"
},
"original": {
@ -113,11 +113,11 @@
},
"nix-filter": {
"locked": {
"lastModified": 1705332318,
"narHash": "sha256-kcw1yFeJe9N4PjQji9ZeX47jg0p9A0DuU4djKvg1a7I=",
"lastModified": 1710156097,
"narHash": "sha256-1Wvk8UP7PXdf8bCCaEoMnOT1qe5/Duqgj+rL8sRQsSM=",
"owner": "numtide",
"repo": "nix-filter",
"rev": "3449dc925982ad46246cfc36469baf66e1b64f17",
"rev": "3342559a24e85fc164b295c3444e8a139924675b",
"type": "github"
},
"original": {
@ -128,11 +128,11 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1707689078,
"narHash": "sha256-UUGmRa84ZJHpGZ1WZEBEUOzaPOWG8LZ0yPg1pdDF/yM=",
"lastModified": 1714253743,
"narHash": "sha256-mdTQw2XlariysyScCv2tTE45QSU9v/ezLcHJ22f0Nxc=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "f9d39fb9aff0efee4a3d5f4a6d7c17701d38a1d8",
"rev": "58a1abdbae3217ca6b702f03d3b35125d88a2994",
"type": "github"
},
"original": {
@ -145,11 +145,11 @@
"nixpkgs-lib": {
"locked": {
"dir": "lib",
"lastModified": 1706550542,
"narHash": "sha256-UcsnCG6wx++23yeER4Hg18CXWbgNpqNXcHIo5/1Y+hc=",
"lastModified": 1711703276,
"narHash": "sha256-iMUFArF0WCatKK6RzfUJknjem0H9m4KgorO/p3Dopkk=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "97b17f32362e475016f942bbdfda4a4a72a8a652",
"rev": "d8fe5e6c92d0d190646fb9f1056741a229980089",
"type": "github"
},
"original": {

65
stats/stats.go Normal file
View File

@ -0,0 +1,65 @@
package stats
import (
"fmt"
"strings"
"sync/atomic"
"time"
)
type Type int
const (
Traversed Type = iota
Emitted
Matched
Formatted
)
var (
counters map[Type]*atomic.Int32
start time.Time
)
func Init() {
// record start time
start = time.Now()
// init counters
counters = make(map[Type]*atomic.Int32)
counters[Traversed] = &atomic.Int32{}
counters[Emitted] = &atomic.Int32{}
counters[Matched] = &atomic.Int32{}
counters[Formatted] = &atomic.Int32{}
}
func Add(t Type, delta int32) int32 {
return counters[t].Add(delta)
}
func Value(t Type) int32 {
return counters[t].Load()
}
func Elapsed() time.Duration {
return time.Now().Sub(start)
}
func Print() {
components := []string{
"traversed %d files",
"emitted %d files for processing",
"matched %d files to formatters",
"formatted %d files in %v",
"",
}
fmt.Printf(
strings.Join(components, "\n"),
Value(Traversed),
Value(Emitted),
Value(Matched),
Value(Formatted),
Elapsed(),
)
}