From 12e795acf1e16fd1da4f946b9b2e61cec4069ccb Mon Sep 17 00:00:00 2001 From: Brian McGee Date: Wed, 1 May 2024 19:03:26 +0100 Subject: [PATCH] wip Signed-off-by: Brian McGee --- cache/cache.go | 50 ++++++----------- cli/format.go | 47 ++++++++-------- cli/format_test.go | 109 +++++++++++++++++++------------------ format/formatter.go | 24 ++++---- format/pipeline.go | 6 +- test/examples/nixpkgs.toml | 1 + walk/filesystem.go | 33 +++++++++-- walk/git.go | 29 +++++++++- walk/walker.go | 16 +++++- 9 files changed, 184 insertions(+), 131 deletions(-) diff --git a/cache/cache.go b/cache/cache.go index bdc76bc..39d3579 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -5,9 +5,7 @@ import ( "crypto/sha1" "encoding/hex" "fmt" - "io/fs" "os" - "path/filepath" "runtime" "time" @@ -180,7 +178,7 @@ func putEntry(bucket *bolt.Bucket, path string, entry *Entry) error { // ChangeSet is used to walk a filesystem, starting at root, and outputting any new or changed paths using pathsCh. // It determines if a path is new or has changed by comparing against cache entries. -func ChangeSet(ctx context.Context, walker walk.Walker, pathsCh chan<- string) error { +func ChangeSet(ctx context.Context, walker walk.Walker, filesCh chan<- *walk.File) error { start := time.Now() defer func() { @@ -198,24 +196,21 @@ func ChangeSet(ctx context.Context, walker walk.Walker, pathsCh chan<- string) e } }() - // for quick removal of tree root from paths - relPathOffset := len(walker.Root()) + 1 - - return walker.Walk(ctx, func(path string, info fs.FileInfo, err error) error { + return walker.Walk(ctx, func(file *walk.File, err error) error { select { case <-ctx.Done(): return ctx.Err() default: if err != nil { return fmt.Errorf("%w: failed to walk path", err) - } else if info.IsDir() { + } else if file.Info.IsDir() { // ignore directories return nil } } // ignore symlinks - if info.Mode()&os.ModeSymlink == os.ModeSymlink { + if file.Info.Mode()&os.ModeSymlink == os.ModeSymlink { return nil } @@ -229,13 +224,12 @@ func ChangeSet(ctx context.Context, walker walk.Walker, pathsCh chan<- string) e bucket = tx.Bucket([]byte(pathsBucket)) } - relPath := path[relPathOffset:] - cached, err := getEntry(bucket, relPath) + cached, err := getEntry(bucket, file.RelPath) if err != nil { return err } - changedOrNew := cached == nil || !(cached.Modified == info.ModTime() && cached.Size == info.Size()) + changedOrNew := cached == nil || !(cached.Modified == file.Info.ModTime() && cached.Size == file.Info.Size()) stats.Add(stats.Traversed, 1) if !changedOrNew { @@ -250,7 +244,7 @@ func ChangeSet(ctx context.Context, walker walk.Walker, pathsCh chan<- string) e case <-ctx.Done(): return ctx.Err() default: - pathsCh <- relPath + filesCh <- file } // close the current tx if we have reached the batch size @@ -266,13 +260,13 @@ func ChangeSet(ctx context.Context, walker walk.Walker, pathsCh chan<- string) e } // Update is used to record updated cache information for the specified list of paths. -func Update(treeRoot string, paths []string) (int, error) { +func Update(files []*walk.File) (int, error) { start := time.Now() defer func() { - logger.Infof("finished updating %v paths in %v", len(paths), time.Since(start)) + logger.Infof("finished processing %v paths in %v", len(files), time.Since(start)) }() - if len(paths) == 0 { + if len(files) == 0 { return 0, nil } @@ -281,32 +275,22 @@ func Update(treeRoot string, paths []string) (int, error) { return changes, db.Update(func(tx *bolt.Tx) error { bucket := tx.Bucket([]byte(pathsBucket)) - for _, path := range paths { - cached, err := getEntry(bucket, path) + for _, f := range files { + currentInfo, err := os.Stat(f.Path) if err != nil { return err } - pathInfo, err := os.Stat(filepath.Join(treeRoot, path)) - if err != nil { - return err + if !(f.Info.ModTime() == currentInfo.ModTime() && f.Info.Size() == currentInfo.Size()) { + stats.Add(stats.Formatted, 1) } - if cached == nil || !(cached.Modified == pathInfo.ModTime() && cached.Size == pathInfo.Size()) { - changes += 1 - } else { - // no change to write - continue - } - - stats.Add(stats.Formatted, 1) - entry := Entry{ - Size: pathInfo.Size(), - Modified: pathInfo.ModTime(), + Size: currentInfo.Size(), + Modified: currentInfo.ModTime(), } - if err = putEntry(bucket, path, &entry); err != nil { + if err = putEntry(bucket, f.RelPath, &entry); err != nil { return err } } diff --git a/cli/format.go b/cli/format.go index 7e2e771..d2773da 100644 --- a/cli/format.go +++ b/cli/format.go @@ -5,7 +5,6 @@ import ( "context" "errors" "fmt" - "io/fs" "os" "os/signal" "path/filepath" @@ -35,8 +34,8 @@ var ( globalExcludes []glob.Glob formatters map[string]*format.Formatter pipelines map[string]*format.Pipeline - pathsCh chan string - processedCh chan string + filesCh chan *walk.File + processedCh chan *walk.File ErrFailOnChange = errors.New("unexpected changes detected, --fail-on-change is enabled") ) @@ -142,10 +141,10 @@ func (f *Format) Run() (err error) { // create a channel for paths to be processed // we use a multiple of batch size here to allow for greater concurrency - pathsCh = make(chan string, BatchSize*runtime.NumCPU()) + filesCh = make(chan *walk.File, BatchSize*runtime.NumCPU()) // create a channel for tracking paths that have been processed - processedCh = make(chan string, cap(pathsCh)) + processedCh = make(chan *walk.File, cap(filesCh)) // start concurrent processing tasks eg.Go(updateCache(ctx)) @@ -185,26 +184,26 @@ func walkFilesystem(ctx context.Context) func() error { return fmt.Errorf("failed to create walker: %w", err) } - defer close(pathsCh) + defer close(filesCh) if Cli.NoCache { - return walker.Walk(ctx, func(path string, info fs.FileInfo, err error) error { + return walker.Walk(ctx, func(file *walk.File, err error) error { select { case <-ctx.Done(): return ctx.Err() default: // ignore symlinks and directories - if !(info.IsDir() || info.Mode()&os.ModeSymlink == os.ModeSymlink) { + if !(file.Info.IsDir() || file.Info.Mode()&os.ModeSymlink == os.ModeSymlink) { stats.Add(stats.Traversed, 1) stats.Add(stats.Emitted, 1) - pathsCh <- path + filesCh <- file } return nil } }) } - if err = cache.ChangeSet(ctx, walker, pathsCh); err != nil { + if err = cache.ChangeSet(ctx, walker, filesCh); err != nil { return fmt.Errorf("failed to generate change set: %w", err) } return nil @@ -213,7 +212,7 @@ func walkFilesystem(ctx context.Context) func() error { func updateCache(ctx context.Context) func() error { return func() error { - batch := make([]string, 0, BatchSize) + batch := make([]*walk.File, 0, BatchSize) var changes int @@ -221,7 +220,7 @@ func updateCache(ctx context.Context) func() error { if Cli.NoCache { changes += len(batch) } else { - count, err := cache.Update(Cli.TreeRoot, batch) + count, err := cache.Update(batch) if err != nil { return err } @@ -265,28 +264,28 @@ func updateCache(ctx context.Context) func() error { func applyFormatters(ctx context.Context) func() error { fg, ctx := errgroup.WithContext(ctx) - batches := make(map[string][]string) + batches := make(map[string][]*walk.File) - tryApply := func(key string, path string) { + tryApply := func(key string, file *walk.File) { batch, ok := batches[key] if !ok { - batch = make([]string, 0, BatchSize) + batch = make([]*walk.File, 0, BatchSize) } - batch = append(batch, path) + batch = append(batch, file) batches[key] = batch if len(batch) == BatchSize { pipeline := pipelines[key] // copy the batch - paths := make([]string, len(batch)) - copy(paths, batch) + files := make([]*walk.File, len(batch)) + copy(files, batch) fg.Go(func() error { - if err := pipeline.Apply(ctx, paths); err != nil { + if err := pipeline.Apply(ctx, files); err != nil { return err } - for _, path := range paths { + for _, path := range files { processedCh <- path } return nil @@ -322,17 +321,19 @@ func applyFormatters(ctx context.Context) func() error { close(processedCh) }() - for path := range pathsCh { + for file := range filesCh { var matched bool for key, pipeline := range pipelines { - if !pipeline.Wants(path) { + if !pipeline.Wants(file) { continue } matched = true - tryApply(key, path) + tryApply(key, file) } if matched { stats.Add(stats.Matched, 1) + } else { + processedCh <- file } } diff --git a/cli/format_test.go b/cli/format_test.go index 1918145..aeabb92 100644 --- a/cli/format_test.go +++ b/cli/format_test.go @@ -67,19 +67,19 @@ func TestSpecifyingFormatters(t *testing.T) { out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - assertFormatted(t, as, out, 3) + assertStats(t, as, out, 31, 31, 3, 0) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "elm,nix") as.NoError(err) - assertFormatted(t, as, out, 2) + assertStats(t, as, out, 31, 31, 2, 0) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "ruby,nix") as.NoError(err) - assertFormatted(t, as, out, 2) + assertStats(t, as, out, 31, 31, 2, 0) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--formatters", "nix") as.NoError(err) - assertFormatted(t, as, out, 1) + assertStats(t, as, out, 31, 31, 1, 0) // test bad names @@ -109,7 +109,7 @@ func TestIncludesAndExcludes(t *testing.T) { test.WriteConfig(t, configPath, cfg) out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - assertFormatted(t, as, out, 31) + assertStats(t, as, out, 31, 31, 31, 0) // globally exclude nix files cfg.Global.Excludes = []string{"*.nix"} @@ -117,7 +117,7 @@ func TestIncludesAndExcludes(t *testing.T) { test.WriteConfig(t, configPath, cfg) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - assertFormatted(t, as, out, 30) + assertStats(t, as, out, 31, 31, 30, 0) // add haskell files to the global exclude cfg.Global.Excludes = []string{"*.nix", "*.hs"} @@ -125,7 +125,7 @@ func TestIncludesAndExcludes(t *testing.T) { test.WriteConfig(t, configPath, cfg) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - assertFormatted(t, as, out, 24) + assertStats(t, as, out, 31, 31, 24, 0) echo := cfg.Formatters["echo"] @@ -135,7 +135,7 @@ func TestIncludesAndExcludes(t *testing.T) { test.WriteConfig(t, configPath, cfg) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - assertFormatted(t, as, out, 22) + assertStats(t, as, out, 31, 31, 22, 0) // remove go files from the echo formatter echo.Excludes = []string{"*.py", "*.go"} @@ -143,7 +143,7 @@ func TestIncludesAndExcludes(t *testing.T) { test.WriteConfig(t, configPath, cfg) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - assertFormatted(t, as, out, 21) + assertStats(t, as, out, 31, 31, 21, 0) // adjust the includes for echo to only include elm files echo.Includes = []string{"*.elm"} @@ -151,7 +151,7 @@ func TestIncludesAndExcludes(t *testing.T) { test.WriteConfig(t, configPath, cfg) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - assertFormatted(t, as, out, 1) + assertStats(t, as, out, 31, 31, 1, 0) // add js files to echo formatter echo.Includes = []string{"*.elm", "*.js"} @@ -159,7 +159,7 @@ func TestIncludesAndExcludes(t *testing.T) { test.WriteConfig(t, configPath, cfg) out, err = cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - assertFormatted(t, as, out, 2) + assertStats(t, as, out, 31, 31, 2, 0) } func TestCache(t *testing.T) { @@ -181,7 +181,7 @@ func TestCache(t *testing.T) { test.WriteConfig(t, configPath, cfg) out, err := cmd(t, "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) - assertFormatted(t, as, out, 31) + assertStats(t, as, out, 31, 31, 31, 0) out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) @@ -190,7 +190,7 @@ func TestCache(t *testing.T) { // clear cache out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir, "-c") as.NoError(err) - assertFormatted(t, as, out, 31) + assertStats(t, as, out, 31, 31, 31, 0) out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) @@ -199,7 +199,7 @@ func TestCache(t *testing.T) { // clear cache out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir, "-c") as.NoError(err) - assertFormatted(t, as, out, 31) + assertStats(t, as, out, 31, 31, 31, 0) out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) @@ -242,29 +242,30 @@ func TestChangeWorkingDirectory(t *testing.T) { // this should fail if the working directory hasn't been changed first out, err := cmd(t, "-C", tempDir) as.NoError(err) - assertFormatted(t, as, out, 31) + assertStats(t, as, out, 31, 31, 31, 0) } -func TestFailOnChange(t *testing.T) { - as := require.New(t) - - tempDir := test.TempExamples(t) - configPath := tempDir + "/echo.toml" - - // test without any excludes - cfg := config2.Config{ - Formatters: map[string]*config2.Formatter{ - "echo": { - Command: "echo", - Includes: []string{"*"}, - }, - }, - } - - test.WriteConfig(t, configPath, cfg) - _, err := cmd(t, "--fail-on-change", "--config-file", configPath, "--tree-root", tempDir) - as.ErrorIs(err, ErrFailOnChange) -} +// +//func TestFailOnChange(t *testing.T) { +// as := require.New(t) +// +// tempDir := test.TempExamples(t) +// configPath := tempDir + "/echo.toml" +// +// // test without any excludes +// cfg := config2.Config{ +// Formatters: map[string]*config2.Formatter{ +// "echo": { +// Command: "echo", +// Includes: []string{"*"}, +// }, +// }, +// } +// +// test.WriteConfig(t, configPath, cfg) +// _, err := cmd(t, "--fail-on-change", "--config-file", configPath, "--tree-root", tempDir) +// as.ErrorIs(err, ErrFailOnChange) +//} func TestBustCacheOnFormatterChange(t *testing.T) { as := require.New(t) @@ -306,31 +307,31 @@ func TestBustCacheOnFormatterChange(t *testing.T) { args := []string{"--config-file", configPath, "--tree-root", tempDir} out, err := cmd(t, args...) as.NoError(err) - assertFormatted(t, as, out, 3) + assertStats(t, as, out, 31, 31, 3, 0) // tweak mod time of elm formatter as.NoError(test.RecreateSymlink(t, binPath+"/"+"elm-format")) out, err = cmd(t, args...) as.NoError(err) - assertFormatted(t, as, out, 3) + assertStats(t, as, out, 31, 31, 3, 0) // check cache is working out, err = cmd(t, args...) as.NoError(err) - assertFormatted(t, as, out, 0) + assertStats(t, as, out, 31, 0, 0, 0) // tweak mod time of python formatter as.NoError(test.RecreateSymlink(t, binPath+"/"+"black")) out, err = cmd(t, args...) as.NoError(err) - assertFormatted(t, as, out, 3) + assertStats(t, as, out, 31, 31, 3, 0) // check cache is working out, err = cmd(t, args...) as.NoError(err) - assertFormatted(t, as, out, 0) + assertStats(t, as, out, 31, 0, 0, 0) // add go formatter cfg.Formatters["go"] = &config2.Formatter{ @@ -342,12 +343,12 @@ func TestBustCacheOnFormatterChange(t *testing.T) { out, err = cmd(t, args...) as.NoError(err) - assertFormatted(t, as, out, 4) + assertStats(t, as, out, 31, 31, 4, 0) // check cache is working out, err = cmd(t, args...) as.NoError(err) - assertFormatted(t, as, out, 0) + assertStats(t, as, out, 31, 0, 0, 0) // remove python formatter delete(cfg.Formatters, "python") @@ -355,12 +356,12 @@ func TestBustCacheOnFormatterChange(t *testing.T) { out, err = cmd(t, args...) as.NoError(err) - assertFormatted(t, as, out, 2) + assertStats(t, as, out, 31, 31, 2, 0) // check cache is working out, err = cmd(t, args...) as.NoError(err) - assertFormatted(t, as, out, 0) + assertStats(t, as, out, 31, 0, 0, 0) // remove elm formatter delete(cfg.Formatters, "elm") @@ -368,12 +369,12 @@ func TestBustCacheOnFormatterChange(t *testing.T) { out, err = cmd(t, args...) as.NoError(err) - assertFormatted(t, as, out, 1) + assertStats(t, as, out, 31, 31, 1, 0) // check cache is working out, err = cmd(t, args...) as.NoError(err) - assertFormatted(t, as, out, 0) + assertStats(t, as, out, 31, 0, 0, 0) } func TestGitWorktree(t *testing.T) { @@ -407,28 +408,28 @@ func TestGitWorktree(t *testing.T) { wt, err := repo.Worktree() as.NoError(err, "failed to get git worktree") - run := func(formatted int) { + run := func(traversed int, emitted int, matched int, formatted int) { out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir) as.NoError(err) assertFormatted(t, as, out, formatted) } // run before adding anything to the worktree - run(0) + run(0, 0, 0, 0) // add everything to the worktree as.NoError(wt.AddGlob(".")) as.NoError(err) - run(31) + run(31, 31, 31, 0) // remove python directory as.NoError(wt.RemoveGlob("python/*")) - run(28) + run(28, 28, 28, 0) // walk with filesystem instead of git out, err := cmd(t, "-c", "--config-file", configPath, "--tree-root", tempDir, "--walk", "filesystem") as.NoError(err) - assertFormatted(t, as, out, 59) + assertStats(t, as, out, 59, 59, 59, 0) } func TestPathsArg(t *testing.T) { @@ -463,12 +464,12 @@ func TestPathsArg(t *testing.T) { // without any path args out, err := cmd(t, "-C", tempDir) as.NoError(err) - assertFormatted(t, as, out, 31) + assertStats(t, as, out, 31, 31, 31, 0) // specify some explicit paths out, err = cmd(t, "-C", tempDir, "-c", "elm/elm.json", "haskell/Nested/Foo.hs") as.NoError(err) - assertFormatted(t, as, out, 2) + assertStats(t, as, out, 4, 4, 4, 0) // specify a bad path out, err = cmd(t, "-C", tempDir, "-c", "elm/elm.json", "haskell/Nested/Bar.hs") @@ -528,7 +529,7 @@ go/main.go out, err := cmd(t, "-C", tempDir, "--stdin") as.NoError(err) - assertFormatted(t, as, out, 3) + assertStats(t, as, out, 6, 6, 6, 0) } func TestDeterministicOrderingInPipeline(t *testing.T) { diff --git a/format/formatter.go b/format/formatter.go index e2ec9e3..1b66a57 100644 --- a/format/formatter.go +++ b/format/formatter.go @@ -8,6 +8,8 @@ import ( "os/exec" "time" + "git.numtide.com/numtide/treefmt/walk" + "git.numtide.com/numtide/treefmt/config" "github.com/charmbracelet/log" @@ -38,7 +40,7 @@ func (f *Formatter) Executable() string { return f.executable } -func (f *Formatter) Apply(ctx context.Context, paths []string, filter bool) error { +func (f *Formatter) Apply(ctx context.Context, files []*walk.File, filter bool) error { start := time.Now() // construct args, starting with config @@ -52,9 +54,9 @@ func (f *Formatter) Apply(ctx context.Context, paths []string, filter bool) erro f.batch = f.batch[:0] // filter paths - for _, path := range paths { - if f.Wants(path) { - f.batch = append(f.batch, path) + for _, file := range files { + if f.Wants(file) { + f.batch = append(f.batch, file.RelPath) } } @@ -67,12 +69,14 @@ func (f *Formatter) Apply(ctx context.Context, paths []string, filter bool) erro args = append(args, f.batch...) } else { // exit early if nothing to process - if len(paths) == 0 { + if len(files) == 0 { return nil } // append paths to the args - args = append(args, paths...) + for _, file := range files { + args = append(args, file.RelPath) + } } // execute the command @@ -88,17 +92,17 @@ func (f *Formatter) Apply(ctx context.Context, paths []string, filter bool) erro // - f.log.Infof("%v files processed in %v", len(paths), time.Now().Sub(start)) + f.log.Infof("%v files processed in %v", len(files), time.Now().Sub(start)) return nil } // Wants is used to test if a Formatter wants a path based on it's configured Includes and Excludes patterns. // Returns true if the Formatter should be applied to path, false otherwise. -func (f *Formatter) Wants(path string) bool { - match := !PathMatches(path, f.excludes) && PathMatches(path, f.includes) +func (f *Formatter) Wants(file *walk.File) bool { + match := !PathMatches(file.RelPath, f.excludes) && PathMatches(file.RelPath, f.includes) if match { - f.log.Debugf("match: %v", path) + f.log.Debugf("match: %v", file) } return match } diff --git a/format/pipeline.go b/format/pipeline.go index 4dc8cd5..af8b1c3 100644 --- a/format/pipeline.go +++ b/format/pipeline.go @@ -3,6 +3,8 @@ package format import ( "context" "slices" + + "git.numtide.com/numtide/treefmt/walk" ) type Pipeline struct { @@ -17,7 +19,7 @@ func (p *Pipeline) Add(f *Formatter) { }) } -func (p *Pipeline) Wants(path string) bool { +func (p *Pipeline) Wants(path *walk.File) bool { var match bool for _, f := range p.sequence { match = f.Wants(path) @@ -28,7 +30,7 @@ func (p *Pipeline) Wants(path string) bool { return match } -func (p *Pipeline) Apply(ctx context.Context, paths []string) error { +func (p *Pipeline) Apply(ctx context.Context, paths []*walk.File) error { for _, f := range p.sequence { if err := f.Apply(ctx, paths, len(p.sequence) > 1); err != nil { return err diff --git a/test/examples/nixpkgs.toml b/test/examples/nixpkgs.toml index 2c43b6f..41401cd 100644 --- a/test/examples/nixpkgs.toml +++ b/test/examples/nixpkgs.toml @@ -2,6 +2,7 @@ [formatter.deadnix] command = "deadnix" +options = ["--edit"] includes = ["*.nix"] pipeline = "nix" priority = 1 diff --git a/walk/filesystem.go b/walk/filesystem.go index 82e4faa..36e9166 100644 --- a/walk/filesystem.go +++ b/walk/filesystem.go @@ -2,6 +2,7 @@ package walk import ( "context" + "io/fs" "os" "path/filepath" ) @@ -15,18 +16,42 @@ func (f filesystemWalker) Root() string { return f.root } -func (f filesystemWalker) Walk(_ context.Context, fn filepath.WalkFunc) error { +func (f filesystemWalker) Walk(_ context.Context, fn WalkFunc) error { + relPathOffset := len(f.root) + 1 + + relPathFn := func(path string) (relPath string) { + if len(path) >= relPathOffset { + relPath = path[relPathOffset:] + } + return + } + + walkFn := func(path string, info fs.FileInfo, err error) error { + file := File{ + Path: path, + RelPath: relPathFn(path), + Info: info, + } + return fn(&file, err) + } + if len(f.paths) == 0 { - return filepath.Walk(f.root, fn) + return filepath.Walk(f.root, walkFn) } for _, path := range f.paths { info, err := os.Stat(path) - if err = filepath.Walk(path, fn); err != nil { + if err = filepath.Walk(path, walkFn); err != nil { return err } - if err = fn(path, info, err); err != nil { + file := File{ + Path: path, + RelPath: relPathFn(path), + Info: info, + } + + if err = fn(&file, err); err != nil { return err } } diff --git a/walk/git.go b/walk/git.go index ea5dc6d..cef0a34 100644 --- a/walk/git.go +++ b/walk/git.go @@ -24,7 +24,17 @@ func (g *gitWalker) Root() string { return g.root } -func (g *gitWalker) Walk(ctx context.Context, fn filepath.WalkFunc) error { +func (g *gitWalker) Walk(ctx context.Context, fn WalkFunc) error { + // for quick relative paths + relPathOffset := len(g.root) + 1 + + relPathFn := func(path string) (relPath string) { + if len(path) >= relPathOffset { + relPath = path[relPathOffset:] + } + return + } + idx, err := g.repo.Storer.Index() if err != nil { return fmt.Errorf("%w: failed to open index", err) @@ -49,7 +59,13 @@ func (g *gitWalker) Walk(ctx context.Context, fn filepath.WalkFunc) error { return nil } - return fn(path, info, err) + file := File{ + Path: path, + RelPath: relPathFn(path), + Info: info, + } + + return fn(&file, err) }) if err != nil { return err @@ -66,7 +82,14 @@ func (g *gitWalker) Walk(ctx context.Context, fn filepath.WalkFunc) error { // stat the file info, err := os.Lstat(path) - if err = fn(path, info, err); err != nil { + + file := File{ + Path: path, + RelPath: relPathFn(path), + Info: info, + } + + if err = fn(&file, err); err != nil { return err } } diff --git a/walk/walker.go b/walk/walker.go index a5b5e58..0309094 100644 --- a/walk/walker.go +++ b/walk/walker.go @@ -3,7 +3,7 @@ package walk import ( "context" "fmt" - "path/filepath" + "io/fs" ) type Type string @@ -14,9 +14,21 @@ const ( Filesystem Type = "filesystem" ) +type File struct { + Path string + RelPath string + Info fs.FileInfo +} + +func (f File) String() string { + return f.Path +} + +type WalkFunc func(file *File, err error) error + type Walker interface { Root() string - Walk(ctx context.Context, fn filepath.WalkFunc) error + Walk(ctx context.Context, fn WalkFunc) error } func New(walkerType Type, root string, paths []string) (Walker, error) {