From b6405d07140eb168e179f1af54767c9b8088a822 Mon Sep 17 00:00:00 2001 From: Brian McGee Date: Sun, 24 Dec 2023 11:59:05 +0000 Subject: [PATCH] feat: clean up and documentation --- internal/cache/cache.go | 29 ++++++++++++++---- internal/cache/types.go | 8 ----- internal/format/config.go | 2 ++ internal/format/context.go | 16 ++++------ internal/format/format.go | 61 ++++++++++++++++++++++++-------------- internal/log/writer.go | 21 ------------- 6 files changed, 70 insertions(+), 67 deletions(-) delete mode 100644 internal/cache/types.go delete mode 100644 internal/log/writer.go diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 30ba389..d024120 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -8,6 +8,7 @@ import ( "io/fs" "os" "path/filepath" + "time" "github.com/adrg/xdg" "github.com/juju/errors" @@ -19,8 +20,19 @@ const ( modifiedBucket = "modified" ) +// Entry represents a cache entry, indicating the last size and modified time for a file path. +type Entry struct { + Size int64 + Modified time.Time +} + var db *bolt.DB +// Open creates an instance of bolt.DB for a given treeRoot path. +// If clean is true, Open will delete any existing data in the cache. +// +// The database will be located in `XDG_CACHE_DIR/treefmt/eval-cache/.db`, where is determined by hashing +// the treeRoot path. This associates a given treeRoot with a given instance of the cache. func Open(treeRoot string, clean bool) (err error) { // determine a unique and consistent db name for the tree root h := sha1.New() @@ -30,7 +42,7 @@ func Open(treeRoot string, clean bool) (err error) { name := base32.StdEncoding.EncodeToString(digest) path, err := xdg.CacheFile(fmt.Sprintf("treefmt/eval-cache/%v.db", name)) - // bust the cache if specified + // force a clean of the cache if specified if clean { err := os.Remove(path) if errors.Is(err, os.ErrNotExist) { @@ -60,6 +72,7 @@ func Open(treeRoot string, clean bool) (err error) { return } +// Close closes any open instance of the cache. func Close() error { if db == nil { return nil @@ -67,10 +80,11 @@ func Close() error { return db.Close() } -func getFileInfo(bucket *bolt.Bucket, path string) (*FileInfo, error) { +// getEntry is a helper for reading cache entries from bolt. +func getEntry(bucket *bolt.Bucket, path string) (*Entry, error) { b := bucket.Get([]byte(path)) if b != nil { - var cached FileInfo + var cached Entry if err := msgpack.Unmarshal(b, &cached); err != nil { return nil, errors.Annotatef(err, "failed to unmarshal cache info for path '%v'", path) } @@ -80,6 +94,8 @@ func getFileInfo(bucket *bolt.Bucket, path string) (*FileInfo, error) { } } +// ChangeSet is used to walk a filesystem, starting at root, and outputting any new or changed paths using pathsCh. +// It determines if a path is new or has changed by comparing against cache entries. func ChangeSet(ctx context.Context, root string, pathsCh chan<- string) error { return db.Update(func(tx *bolt.Tx) error { bucket := tx.Bucket([]byte(modifiedBucket)) @@ -99,7 +115,7 @@ func ChangeSet(ctx context.Context, root string, pathsCh chan<- string) error { return nil } - cached, err := getFileInfo(bucket, path) + cached, err := getEntry(bucket, path) if err != nil { return err } @@ -118,6 +134,7 @@ func ChangeSet(ctx context.Context, root string, pathsCh chan<- string) error { }) } +// Update is used to record updated cache information for the specified list of paths. func Update(paths []string) (int, error) { if len(paths) == 0 { return 0, nil @@ -133,7 +150,7 @@ func Update(paths []string) (int, error) { continue } - cached, err := getFileInfo(bucket, path) + cached, err := getEntry(bucket, path) if err != nil { return err } @@ -150,7 +167,7 @@ func Update(paths []string) (int, error) { continue } - cacheInfo := FileInfo{ + cacheInfo := Entry{ Size: pathInfo.Size(), Modified: pathInfo.ModTime(), } diff --git a/internal/cache/types.go b/internal/cache/types.go deleted file mode 100644 index d78e953..0000000 --- a/internal/cache/types.go +++ /dev/null @@ -1,8 +0,0 @@ -package cache - -import "time" - -type FileInfo struct { - Size int64 - Modified time.Time -} diff --git a/internal/format/config.go b/internal/format/config.go index 6b786c3..6287da0 100644 --- a/internal/format/config.go +++ b/internal/format/config.go @@ -2,10 +2,12 @@ package format import "github.com/BurntSushi/toml" +// Config is used to represent the list of configured Formatters. type Config struct { Formatters map[string]*Formatter `toml:"formatter"` } +// ReadConfigFile reads from path and unmarshals toml into a Config instance. func ReadConfigFile(path string) (cfg *Config, err error) { _, err = toml.DecodeFile(path, &cfg) return diff --git a/internal/format/context.go b/internal/format/context.go index 207ca4a..7de916b 100644 --- a/internal/format/context.go +++ b/internal/format/context.go @@ -9,28 +9,24 @@ const ( completedChKey = "completedCh" ) +// RegisterFormatters is used to set a map of formatters in the provided context. func RegisterFormatters(ctx context.Context, formatters map[string]*Formatter) context.Context { return context.WithValue(ctx, formattersKey, formatters) } +// GetFormatters is used to retrieve a formatters map from the provided context. func GetFormatters(ctx context.Context) map[string]*Formatter { return ctx.Value(formattersKey).(map[string]*Formatter) } +// SetCompletedChannel is used to set a channel for indication processing completion in the provided context. func SetCompletedChannel(ctx context.Context, completedCh chan string) context.Context { return context.WithValue(ctx, completedChKey, completedCh) } +// MarkFormatComplete is used to indicate that all processing has finished for the provided path. +// This is done by adding the path to the completion channel which should have already been set using +// SetCompletedChannel. func MarkFormatComplete(ctx context.Context, path string) { ctx.Value(completedChKey).(chan string) <- path } - -func ForwardPath(ctx context.Context, path string, names []string) { - if len(names) == 0 { - return - } - formatters := GetFormatters(ctx) - for _, name := range names { - formatters[name].Put(path) - } -} diff --git a/internal/format/format.go b/internal/format/format.go index 404b433..fd75287 100644 --- a/internal/format/format.go +++ b/internal/format/format.go @@ -11,41 +11,50 @@ import ( ) const ( + // ErrFormatterNotFound is returned when the Command for a Formatter is not available. ErrFormatterNotFound = errors.ConstError("formatter not found") ) +// Formatter represents a command which should be applied to a filesystem. type Formatter struct { - Name string - Command string - Options []string + // Command is the command invoke when applying this Formatter. + Command string + // Options are an optional list of args to be passed to Command. + Options []string + // Includes is a list of glob patterns used to determine whether this Formatter should be applied against a path. Includes []string + // Excludes is an optional list of glob patterns used to exclude certain files from this Formatter. Excludes []string - Before []string - log *log.Logger + name string + log *log.Logger - // globs for matching against paths + // internal compiled versions of Includes and Excludes. includes []glob.Glob excludes []glob.Glob + // inbox is used to accept new paths for formatting. inbox chan string + // Entries from inbox are batched according to batchSize and stored in batch for processing when the batchSize has + // been reached or Close is invoked. batch []string batchSize int } func (f *Formatter) Init(name string) error { - f.Name = name + // capture the name from the config file + f.name = name // test if the formatter is available if err := exec.Command(f.Command, "--help").Run(); err != nil { return ErrFormatterNotFound } + // initialise internal state f.log = log.WithPrefix("format | " + name) - f.inbox = make(chan string, 1024) - f.batchSize = 1024 + f.inbox = make(chan string, f.batchSize) f.batch = make([]string, f.batchSize) f.batch = f.batch[:0] @@ -54,7 +63,7 @@ func (f *Formatter) Init(name string) error { for _, pattern := range f.Includes { g, err := glob.Compile("**/" + pattern) if err != nil { - return errors.Annotatef(err, "failed to compile include pattern '%v' for formatter '%v'", pattern, f.Name) + return errors.Annotatef(err, "failed to compile include pattern '%v' for formatter '%v'", pattern, f.name) } f.includes = append(f.includes, g) } @@ -64,7 +73,7 @@ func (f *Formatter) Init(name string) error { for _, pattern := range f.Excludes { g, err := glob.Compile("**/" + pattern) if err != nil { - return errors.Annotatef(err, "failed to compile exclude pattern '%v' for formatter '%v'", pattern, f.Name) + return errors.Annotatef(err, "failed to compile exclude pattern '%v' for formatter '%v'", pattern, f.name) } f.excludes = append(f.excludes, g) } @@ -73,6 +82,8 @@ func (f *Formatter) Init(name string) error { return nil } +// Wants is used to test if a Formatter wants path based on it's configured Includes and Excludes patterns. +// Returns true if the Formatter should be applied to path, false otherwise. func (f *Formatter) Wants(path string) bool { match := !PathMatches(path, f.excludes) && PathMatches(path, f.includes) if match { @@ -81,24 +92,31 @@ func (f *Formatter) Wants(path string) bool { return match } +// Put add path into this Formatter's inbox for processing. func (f *Formatter) Put(path string) { f.inbox <- path } +// Run is the main processing loop for this Formatter. +// It accepts a context which is used to lookup certain dependencies and for cancellation. func (f *Formatter) Run(ctx context.Context) (err error) { LOOP: + // keep processing until ctx has been cancelled or inbox has been closed for { select { + case <-ctx.Done(): + // ctx has been cancelled err = ctx.Err() break LOOP case path, ok := <-f.inbox: + // check if the inbox has been closed if !ok { break LOOP } - // add to the current batch + // add path to the current batch f.batch = append(f.batch, path) if len(f.batch) == f.batchSize { @@ -110,14 +128,17 @@ LOOP: } } + // check if LOOP was exited due to an error if err != nil { return } - // final flush + // processing any lingering batch return f.apply(ctx) } +// apply executes Command against the latest batch of paths. +// It accepts a context which is used to lookup certain dependencies and for cancellation. func (f *Formatter) apply(ctx context.Context) error { // empty check if len(f.batch) == 0 { @@ -132,6 +153,7 @@ func (f *Formatter) apply(ctx context.Context) error { args = append(args, path) } + // execute start := time.Now() cmd := exec.CommandContext(ctx, f.Command, args...) @@ -142,15 +164,9 @@ func (f *Formatter) apply(ctx context.Context) error { f.log.Infof("%v files processed in %v", len(f.batch), time.Now().Sub(start)) - // mark completed or forward on - if len(f.Before) == 0 { - for _, path := range f.batch { - MarkFormatComplete(ctx, path) - } - } else { - for _, path := range f.batch { - ForwardPath(ctx, path, f.Before) - } + // mark each path in this batch as completed + for _, path := range f.batch { + MarkFormatComplete(ctx, path) } // reset batch @@ -159,6 +175,7 @@ func (f *Formatter) apply(ctx context.Context) error { return nil } +// Close is used to indicate that a Formatter should process any remaining paths and then stop it's processing loop. func (f *Formatter) Close() { close(f.inbox) } diff --git a/internal/log/writer.go b/internal/log/writer.go deleted file mode 100644 index 25daa88..0000000 --- a/internal/log/writer.go +++ /dev/null @@ -1,21 +0,0 @@ -package log - -import ( - "bufio" - "bytes" - - "github.com/charmbracelet/log" -) - -type Writer struct { - Log *log.Logger -} - -func (l *Writer) Write(p []byte) (n int, err error) { - scanner := bufio.NewScanner(bytes.NewReader(p)) - for scanner.Scan() { - line := scanner.Text() - l.Log.Debug(line) - } - return len(p), nil -}