support formatter ordering #20
@ -64,6 +64,20 @@ func (f *Format) Run() error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// detect dependency cycles and broken dependencies
|
||||||
|
for name, formatter := range cfg.Formatters {
|
||||||
|
if formatter.Before != "" {
|
||||||
|
// check child formatter exists
|
||||||
|
_, ok := cfg.Formatters[formatter.Before]
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"formatter %v is before %v but config for %v was not found",
|
||||||
|
name, formatter.Before, formatter.Before,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// init formatters
|
// init formatters
|
||||||
for name, formatter := range cfg.Formatters {
|
for name, formatter := range cfg.Formatters {
|
||||||
if !includeFormatter(name) {
|
if !includeFormatter(name) {
|
||||||
@ -83,14 +97,25 @@ func (f *Format) Run() error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx = format.RegisterFormatters(ctx, cfg.Formatters)
|
// iterate the initialised formatters configuring parent/child relationships
|
||||||
|
for _, formatter := range cfg.Formatters {
|
||||||
|
if formatter.Before != "" {
|
||||||
|
child, ok := cfg.Formatters[formatter.Before]
|
||||||
|
if !ok {
|
||||||
|
// formatter has been filtered out by the user
|
||||||
|
formatter.Before = ""
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
formatter.SetChild(child)
|
||||||
|
child.SetParent(formatter)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if err = cache.Open(Cli.TreeRoot, Cli.ClearCache, cfg.Formatters); err != nil {
|
if err = cache.Open(Cli.TreeRoot, Cli.ClearCache, cfg.Formatters); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
//
|
//
|
||||||
pendingCh := make(chan string, 1024)
|
|
||||||
completedCh := make(chan string, 1024)
|
completedCh := make(chan string, 1024)
|
||||||
|
|
||||||
ctx = format.SetCompletedChannel(ctx, completedCh)
|
ctx = format.SetCompletedChannel(ctx, completedCh)
|
||||||
@ -114,20 +139,13 @@ func (f *Format) Run() error {
|
|||||||
batchSize := 1024
|
batchSize := 1024
|
||||||
batch := make([]string, 0, batchSize)
|
batch := make([]string, 0, batchSize)
|
||||||
|
|
||||||
var pending, completed, changes int
|
var changes int
|
||||||
|
|
||||||
LOOP:
|
LOOP:
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return ctx.Err()
|
return ctx.Err()
|
||||||
case _, ok := <-pendingCh:
|
|
||||||
if ok {
|
|
||||||
pending += 1
|
|
||||||
} else if pending == completed {
|
|
||||||
break LOOP
|
|
||||||
}
|
|
||||||
|
|
||||||
case path, ok := <-completedCh:
|
case path, ok := <-completedCh:
|
||||||
if !ok {
|
if !ok {
|
||||||
break LOOP
|
break LOOP
|
||||||
@ -141,12 +159,6 @@ func (f *Format) Run() error {
|
|||||||
changes += count
|
changes += count
|
||||||
batch = batch[:0]
|
batch = batch[:0]
|
||||||
}
|
}
|
||||||
|
|
||||||
completed += 1
|
|
||||||
|
|
||||||
if completed == pending {
|
|
||||||
close(completedCh)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -166,26 +178,32 @@ func (f *Format) Run() error {
|
|||||||
})
|
})
|
||||||
|
|
||||||
eg.Go(func() error {
|
eg.Go(func() error {
|
||||||
count := 0
|
// pass paths to each formatter
|
||||||
|
|
||||||
for path := range pathsCh {
|
for path := range pathsCh {
|
||||||
for _, formatter := range cfg.Formatters {
|
for _, formatter := range cfg.Formatters {
|
||||||
if formatter.Wants(path) {
|
if formatter.Wants(path) {
|
||||||
pendingCh <- path
|
|
||||||
count += 1
|
|
||||||
formatter.Put(path)
|
formatter.Put(path)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// indicate no more paths for each formatter
|
||||||
for _, formatter := range cfg.Formatters {
|
for _, formatter := range cfg.Formatters {
|
||||||
|
if formatter.Parent() != nil {
|
||||||
|
// this formatter is not a root, it will be closed by a parent
|
||||||
|
continue
|
||||||
|
}
|
||||||
formatter.Close()
|
formatter.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
if count == 0 {
|
// await completion
|
||||||
close(completedCh)
|
for _, formatter := range cfg.Formatters {
|
||||||
|
formatter.AwaitCompletion()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// indicate no more completion events
|
||||||
|
close(completedCh)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
|
@ -404,3 +404,57 @@ func TestGitWorktree(t *testing.T) {
|
|||||||
as.NoError(err)
|
as.NoError(err)
|
||||||
as.Contains(string(out), fmt.Sprintf("%d files changed", 55))
|
as.Contains(string(out), fmt.Sprintf("%d files changed", 55))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestOrderingFormatters(t *testing.T) {
|
||||||
|
as := require.New(t)
|
||||||
|
|
||||||
|
tempDir := test.TempExamples(t)
|
||||||
|
configPath := path.Join(tempDir, "treefmt.toml")
|
||||||
|
|
||||||
|
// missing child
|
||||||
|
test.WriteConfig(t, configPath, format.Config{
|
||||||
|
Formatters: map[string]*format.Formatter{
|
||||||
|
"hs-a": {
|
||||||
|
Command: "echo",
|
||||||
|
Includes: []string{"*.hs"},
|
||||||
|
Before: "hs-b",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
out, err := cmd(t, "--config-file", configPath, "--tree-root", tempDir)
|
||||||
|
as.ErrorContains(err, "formatter hs-a is before hs-b but config for hs-b was not found")
|
||||||
|
|
||||||
|
// multiple roots
|
||||||
|
test.WriteConfig(t, configPath, format.Config{
|
||||||
|
Formatters: map[string]*format.Formatter{
|
||||||
|
"hs-a": {
|
||||||
|
Command: "echo",
|
||||||
|
Includes: []string{"*.hs"},
|
||||||
|
Before: "hs-b",
|
||||||
|
},
|
||||||
|
"hs-b": {
|
||||||
|
Command: "echo",
|
||||||
|
Includes: []string{"*.hs"},
|
||||||
|
Before: "hs-c",
|
||||||
|
},
|
||||||
|
"hs-c": {
|
||||||
|
Command: "echo",
|
||||||
|
Includes: []string{"*.hs"},
|
||||||
|
},
|
||||||
|
"py-a": {
|
||||||
|
Command: "echo",
|
||||||
|
Includes: []string{"*.py"},
|
||||||
|
Before: "py-b",
|
||||||
|
},
|
||||||
|
"py-b": {
|
||||||
|
Command: "echo",
|
||||||
|
Includes: []string{"*.py"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
out, err = cmd(t, "--config-file", configPath, "--tree-root", tempDir)
|
||||||
|
as.NoError(err)
|
||||||
|
as.Contains(string(out), "8 files changed")
|
||||||
|
}
|
||||||
|
@ -5,28 +5,17 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
formattersKey = "formatters"
|
|
||||||
completedChKey = "completedCh"
|
completedChKey = "completedCh"
|
||||||
)
|
)
|
||||||
|
|
||||||
// RegisterFormatters is used to set a map of formatters in the provided context.
|
|
||||||
func RegisterFormatters(ctx context.Context, formatters map[string]*Formatter) context.Context {
|
|
||||||
return context.WithValue(ctx, formattersKey, formatters)
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetFormatters is used to retrieve a formatters map from the provided context.
|
|
||||||
func GetFormatters(ctx context.Context) map[string]*Formatter {
|
|
||||||
return ctx.Value(formattersKey).(map[string]*Formatter)
|
|
||||||
}
|
|
||||||
|
|
||||||
// SetCompletedChannel is used to set a channel for indication processing completion in the provided context.
|
// SetCompletedChannel is used to set a channel for indication processing completion in the provided context.
|
||||||
func SetCompletedChannel(ctx context.Context, completedCh chan string) context.Context {
|
func SetCompletedChannel(ctx context.Context, completedCh chan string) context.Context {
|
||||||
return context.WithValue(ctx, completedChKey, completedCh)
|
return context.WithValue(ctx, completedChKey, completedCh)
|
||||||
}
|
}
|
||||||
|
|
||||||
// MarkFormatComplete is used to indicate that all processing has finished for the provided path.
|
// MarkPathComplete is used to indicate that all processing has finished for the provided path.
|
||||||
// This is done by adding the path to the completion channel which should have already been set using
|
// This is done by adding the path to the completion channel which should have already been set using
|
||||||
// SetCompletedChannel.
|
// SetCompletedChannel.
|
||||||
func MarkFormatComplete(ctx context.Context, path string) {
|
func MarkPathComplete(ctx context.Context, path string) {
|
||||||
ctx.Value(completedChKey).(chan string) <- path
|
ctx.Value(completedChKey).(chan string) <- path
|
||||||
}
|
}
|
||||||
|
@ -24,19 +24,26 @@ type Formatter struct {
|
|||||||
Includes []string
|
Includes []string
|
||||||
// Excludes is an optional list of glob patterns used to exclude certain files from this Formatter.
|
// Excludes is an optional list of glob patterns used to exclude certain files from this Formatter.
|
||||||
Excludes []string
|
Excludes []string
|
||||||
|
// Before is the name of another formatter which must process a path after this one
|
||||||
|
Before string
|
||||||
|
|
||||||
name string
|
name string
|
||||||
log *log.Logger
|
log *log.Logger
|
||||||
executable string // path to the executable described by Command
|
executable string // path to the executable described by Command
|
||||||
|
|
||||||
|
parent *Formatter
|
||||||
|
child *Formatter
|
||||||
|
|
||||||
// internal compiled versions of Includes and Excludes.
|
// internal compiled versions of Includes and Excludes.
|
||||||
includes []glob.Glob
|
includes []glob.Glob
|
||||||
excludes []glob.Glob
|
excludes []glob.Glob
|
||||||
|
|
||||||
// inbox is used to accept new paths for formatting.
|
// inboxCh is used to accept new paths for formatting.
|
||||||
inbox chan string
|
inboxCh chan string
|
||||||
|
// completedCh is used to wait for this formatter to finish all processing.
|
||||||
|
completedCh chan interface{}
|
||||||
|
|
||||||
// Entries from inbox are batched according to batchSize and stored in batch for processing when the batchSize has
|
// Entries from inboxCh are batched according to batchSize and stored in batch for processing when the batchSize has
|
||||||
// been reached or Close is invoked.
|
// been reached or Close is invoked.
|
||||||
batch []string
|
batch []string
|
||||||
batchSize int
|
batchSize int
|
||||||
@ -66,9 +73,9 @@ func (f *Formatter) Init(name string, globalExcludes []glob.Glob) error {
|
|||||||
// initialise internal state
|
// initialise internal state
|
||||||
f.log = log.WithPrefix("format | " + name)
|
f.log = log.WithPrefix("format | " + name)
|
||||||
f.batchSize = 1024
|
f.batchSize = 1024
|
||||||
f.inbox = make(chan string, f.batchSize)
|
f.batch = make([]string, 0, f.batchSize)
|
||||||
f.batch = make([]string, f.batchSize)
|
f.inboxCh = make(chan string, f.batchSize)
|
||||||
f.batch = f.batch[:0]
|
f.completedCh = make(chan interface{}, 1)
|
||||||
|
|
||||||
f.includes, err = CompileGlobs(f.Includes)
|
f.includes, err = CompileGlobs(f.Includes)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -84,6 +91,18 @@ func (f *Formatter) Init(name string, globalExcludes []glob.Glob) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (f *Formatter) SetParent(formatter *Formatter) {
|
||||||
|
f.parent = formatter
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *Formatter) Parent() *Formatter {
|
||||||
|
return f.parent
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *Formatter) SetChild(formatter *Formatter) {
|
||||||
|
f.child = formatter
|
||||||
|
}
|
||||||
|
|
||||||
// Wants is used to test if a Formatter wants path based on it's configured Includes and Excludes patterns.
|
// Wants is used to test if a Formatter wants path based on it's configured Includes and Excludes patterns.
|
||||||
// Returns true if the Formatter should be applied to path, false otherwise.
|
// Returns true if the Formatter should be applied to path, false otherwise.
|
||||||
func (f *Formatter) Wants(path string) bool {
|
func (f *Formatter) Wants(path string) bool {
|
||||||
@ -94,16 +113,26 @@ func (f *Formatter) Wants(path string) bool {
|
|||||||
return match
|
return match
|
||||||
}
|
}
|
||||||
|
|
||||||
// Put add path into this Formatter's inbox for processing.
|
// Put add path into this Formatter's inboxCh for processing.
|
||||||
func (f *Formatter) Put(path string) {
|
func (f *Formatter) Put(path string) {
|
||||||
f.inbox <- path
|
f.inboxCh <- path
|
||||||
}
|
}
|
||||||
|
|
||||||
// Run is the main processing loop for this Formatter.
|
// Run is the main processing loop for this Formatter.
|
||||||
// It accepts a context which is used to lookup certain dependencies and for cancellation.
|
// It accepts a context which is used to lookup certain dependencies and for cancellation.
|
||||||
func (f *Formatter) Run(ctx context.Context) (err error) {
|
func (f *Formatter) Run(ctx context.Context) (err error) {
|
||||||
|
defer func() {
|
||||||
|
if f.child != nil {
|
||||||
|
// indicate no further processing for the child formatter
|
||||||
|
f.child.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
// indicate this formatter has finished processing
|
||||||
|
f.completedCh <- nil
|
||||||
|
}()
|
||||||
|
|
||||||
LOOP:
|
LOOP:
|
||||||
// keep processing until ctx has been cancelled or inbox has been closed
|
// keep processing until ctx has been cancelled or inboxCh has been closed
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
|
|
||||||
@ -112,8 +141,8 @@ LOOP:
|
|||||||
err = ctx.Err()
|
err = ctx.Err()
|
||||||
break LOOP
|
break LOOP
|
||||||
|
|
||||||
case path, ok := <-f.inbox:
|
case path, ok := <-f.inboxCh:
|
||||||
// check if the inbox has been closed
|
// check if the inboxCh has been closed
|
||||||
if !ok {
|
if !ok {
|
||||||
break LOOP
|
break LOOP
|
||||||
}
|
}
|
||||||
@ -167,9 +196,16 @@ func (f *Formatter) apply(ctx context.Context) error {
|
|||||||
|
|
||||||
f.log.Infof("%v files processed in %v", len(f.batch), time.Now().Sub(start))
|
f.log.Infof("%v files processed in %v", len(f.batch), time.Now().Sub(start))
|
||||||
|
|
||||||
|
if f.child == nil {
|
||||||
// mark each path in this batch as completed
|
// mark each path in this batch as completed
|
||||||
for _, path := range f.batch {
|
for _, path := range f.batch {
|
||||||
MarkFormatComplete(ctx, path)
|
MarkPathComplete(ctx, path)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// otherwise forward each path onto the next formatter for processing
|
||||||
|
for _, path := range f.batch {
|
||||||
|
f.child.Put(path)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// reset batch
|
// reset batch
|
||||||
@ -180,5 +216,10 @@ func (f *Formatter) apply(ctx context.Context) error {
|
|||||||
|
|
||||||
// Close is used to indicate that a Formatter should process any remaining paths and then stop it's processing loop.
|
// Close is used to indicate that a Formatter should process any remaining paths and then stop it's processing loop.
|
||||||
func (f *Formatter) Close() {
|
func (f *Formatter) Close() {
|
||||||
close(f.inbox)
|
close(f.inboxCh)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *Formatter) AwaitCompletion() {
|
||||||
|
// todo support a timeout
|
||||||
|
<-f.completedCh
|
||||||
}
|
}
|
||||||
|
Reference in New Issue
Block a user