Compare commits
2 Commits
4c45d2aa7e
...
b6405d0714
Author | SHA1 | Date | |
---|---|---|---|
b6405d0714 | |||
f322e1ffb8 |
29
internal/cache/cache.go
vendored
29
internal/cache/cache.go
vendored
@ -8,6 +8,7 @@ import (
|
|||||||
"io/fs"
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/adrg/xdg"
|
"github.com/adrg/xdg"
|
||||||
"github.com/juju/errors"
|
"github.com/juju/errors"
|
||||||
@ -19,8 +20,19 @@ const (
|
|||||||
modifiedBucket = "modified"
|
modifiedBucket = "modified"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Entry represents a cache entry, indicating the last size and modified time for a file path.
|
||||||
|
type Entry struct {
|
||||||
|
Size int64
|
||||||
|
Modified time.Time
|
||||||
|
}
|
||||||
|
|
||||||
var db *bolt.DB
|
var db *bolt.DB
|
||||||
|
|
||||||
|
// Open creates an instance of bolt.DB for a given treeRoot path.
|
||||||
|
// If clean is true, Open will delete any existing data in the cache.
|
||||||
|
//
|
||||||
|
// The database will be located in `XDG_CACHE_DIR/treefmt/eval-cache/<id>.db`, where <id> is determined by hashing
|
||||||
|
// the treeRoot path. This associates a given treeRoot with a given instance of the cache.
|
||||||
func Open(treeRoot string, clean bool) (err error) {
|
func Open(treeRoot string, clean bool) (err error) {
|
||||||
// determine a unique and consistent db name for the tree root
|
// determine a unique and consistent db name for the tree root
|
||||||
h := sha1.New()
|
h := sha1.New()
|
||||||
@ -30,7 +42,7 @@ func Open(treeRoot string, clean bool) (err error) {
|
|||||||
name := base32.StdEncoding.EncodeToString(digest)
|
name := base32.StdEncoding.EncodeToString(digest)
|
||||||
path, err := xdg.CacheFile(fmt.Sprintf("treefmt/eval-cache/%v.db", name))
|
path, err := xdg.CacheFile(fmt.Sprintf("treefmt/eval-cache/%v.db", name))
|
||||||
|
|
||||||
// bust the cache if specified
|
// force a clean of the cache if specified
|
||||||
if clean {
|
if clean {
|
||||||
err := os.Remove(path)
|
err := os.Remove(path)
|
||||||
if errors.Is(err, os.ErrNotExist) {
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
@ -60,6 +72,7 @@ func Open(treeRoot string, clean bool) (err error) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Close closes any open instance of the cache.
|
||||||
func Close() error {
|
func Close() error {
|
||||||
if db == nil {
|
if db == nil {
|
||||||
return nil
|
return nil
|
||||||
@ -67,10 +80,11 @@ func Close() error {
|
|||||||
return db.Close()
|
return db.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
func getFileInfo(bucket *bolt.Bucket, path string) (*FileInfo, error) {
|
// getEntry is a helper for reading cache entries from bolt.
|
||||||
|
func getEntry(bucket *bolt.Bucket, path string) (*Entry, error) {
|
||||||
b := bucket.Get([]byte(path))
|
b := bucket.Get([]byte(path))
|
||||||
if b != nil {
|
if b != nil {
|
||||||
var cached FileInfo
|
var cached Entry
|
||||||
if err := msgpack.Unmarshal(b, &cached); err != nil {
|
if err := msgpack.Unmarshal(b, &cached); err != nil {
|
||||||
return nil, errors.Annotatef(err, "failed to unmarshal cache info for path '%v'", path)
|
return nil, errors.Annotatef(err, "failed to unmarshal cache info for path '%v'", path)
|
||||||
}
|
}
|
||||||
@ -80,6 +94,8 @@ func getFileInfo(bucket *bolt.Bucket, path string) (*FileInfo, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ChangeSet is used to walk a filesystem, starting at root, and outputting any new or changed paths using pathsCh.
|
||||||
|
// It determines if a path is new or has changed by comparing against cache entries.
|
||||||
func ChangeSet(ctx context.Context, root string, pathsCh chan<- string) error {
|
func ChangeSet(ctx context.Context, root string, pathsCh chan<- string) error {
|
||||||
return db.Update(func(tx *bolt.Tx) error {
|
return db.Update(func(tx *bolt.Tx) error {
|
||||||
bucket := tx.Bucket([]byte(modifiedBucket))
|
bucket := tx.Bucket([]byte(modifiedBucket))
|
||||||
@ -99,7 +115,7 @@ func ChangeSet(ctx context.Context, root string, pathsCh chan<- string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
cached, err := getFileInfo(bucket, path)
|
cached, err := getEntry(bucket, path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -118,6 +134,7 @@ func ChangeSet(ctx context.Context, root string, pathsCh chan<- string) error {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Update is used to record updated cache information for the specified list of paths.
|
||||||
func Update(paths []string) (int, error) {
|
func Update(paths []string) (int, error) {
|
||||||
if len(paths) == 0 {
|
if len(paths) == 0 {
|
||||||
return 0, nil
|
return 0, nil
|
||||||
@ -133,7 +150,7 @@ func Update(paths []string) (int, error) {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
cached, err := getFileInfo(bucket, path)
|
cached, err := getEntry(bucket, path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -150,7 +167,7 @@ func Update(paths []string) (int, error) {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
cacheInfo := FileInfo{
|
cacheInfo := Entry{
|
||||||
Size: pathInfo.Size(),
|
Size: pathInfo.Size(),
|
||||||
Modified: pathInfo.ModTime(),
|
Modified: pathInfo.ModTime(),
|
||||||
}
|
}
|
||||||
|
8
internal/cache/types.go
vendored
8
internal/cache/types.go
vendored
@ -1,8 +0,0 @@
|
|||||||
package cache
|
|
||||||
|
|
||||||
import "time"
|
|
||||||
|
|
||||||
type FileInfo struct {
|
|
||||||
Size int64
|
|
||||||
Modified time.Time
|
|
||||||
}
|
|
@ -2,10 +2,12 @@ package format
|
|||||||
|
|
||||||
import "github.com/BurntSushi/toml"
|
import "github.com/BurntSushi/toml"
|
||||||
|
|
||||||
|
// Config is used to represent the list of configured Formatters.
|
||||||
type Config struct {
|
type Config struct {
|
||||||
Formatters map[string]*Formatter `toml:"formatter"`
|
Formatters map[string]*Formatter `toml:"formatter"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ReadConfigFile reads from path and unmarshals toml into a Config instance.
|
||||||
func ReadConfigFile(path string) (cfg *Config, err error) {
|
func ReadConfigFile(path string) (cfg *Config, err error) {
|
||||||
_, err = toml.DecodeFile(path, &cfg)
|
_, err = toml.DecodeFile(path, &cfg)
|
||||||
return
|
return
|
||||||
|
@ -9,28 +9,24 @@ const (
|
|||||||
completedChKey = "completedCh"
|
completedChKey = "completedCh"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// RegisterFormatters is used to set a map of formatters in the provided context.
|
||||||
func RegisterFormatters(ctx context.Context, formatters map[string]*Formatter) context.Context {
|
func RegisterFormatters(ctx context.Context, formatters map[string]*Formatter) context.Context {
|
||||||
return context.WithValue(ctx, formattersKey, formatters)
|
return context.WithValue(ctx, formattersKey, formatters)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetFormatters is used to retrieve a formatters map from the provided context.
|
||||||
func GetFormatters(ctx context.Context) map[string]*Formatter {
|
func GetFormatters(ctx context.Context) map[string]*Formatter {
|
||||||
return ctx.Value(formattersKey).(map[string]*Formatter)
|
return ctx.Value(formattersKey).(map[string]*Formatter)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetCompletedChannel is used to set a channel for indication processing completion in the provided context.
|
||||||
func SetCompletedChannel(ctx context.Context, completedCh chan string) context.Context {
|
func SetCompletedChannel(ctx context.Context, completedCh chan string) context.Context {
|
||||||
return context.WithValue(ctx, completedChKey, completedCh)
|
return context.WithValue(ctx, completedChKey, completedCh)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// MarkFormatComplete is used to indicate that all processing has finished for the provided path.
|
||||||
|
// This is done by adding the path to the completion channel which should have already been set using
|
||||||
|
// SetCompletedChannel.
|
||||||
func MarkFormatComplete(ctx context.Context, path string) {
|
func MarkFormatComplete(ctx context.Context, path string) {
|
||||||
ctx.Value(completedChKey).(chan string) <- path
|
ctx.Value(completedChKey).(chan string) <- path
|
||||||
}
|
}
|
||||||
|
|
||||||
func ForwardPath(ctx context.Context, path string, names []string) {
|
|
||||||
if len(names) == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
formatters := GetFormatters(ctx)
|
|
||||||
for _, name := range names {
|
|
||||||
formatters[name].Put(path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
@ -11,41 +11,50 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
// ErrFormatterNotFound is returned when the Command for a Formatter is not available.
|
||||||
ErrFormatterNotFound = errors.ConstError("formatter not found")
|
ErrFormatterNotFound = errors.ConstError("formatter not found")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Formatter represents a command which should be applied to a filesystem.
|
||||||
type Formatter struct {
|
type Formatter struct {
|
||||||
Name string
|
// Command is the command invoke when applying this Formatter.
|
||||||
Command string
|
Command string
|
||||||
Options []string
|
// Options are an optional list of args to be passed to Command.
|
||||||
|
Options []string
|
||||||
|
// Includes is a list of glob patterns used to determine whether this Formatter should be applied against a path.
|
||||||
Includes []string
|
Includes []string
|
||||||
|
// Excludes is an optional list of glob patterns used to exclude certain files from this Formatter.
|
||||||
Excludes []string
|
Excludes []string
|
||||||
Before []string
|
|
||||||
|
|
||||||
log *log.Logger
|
name string
|
||||||
|
log *log.Logger
|
||||||
|
|
||||||
// globs for matching against paths
|
// internal compiled versions of Includes and Excludes.
|
||||||
includes []glob.Glob
|
includes []glob.Glob
|
||||||
excludes []glob.Glob
|
excludes []glob.Glob
|
||||||
|
|
||||||
|
// inbox is used to accept new paths for formatting.
|
||||||
inbox chan string
|
inbox chan string
|
||||||
|
|
||||||
|
// Entries from inbox are batched according to batchSize and stored in batch for processing when the batchSize has
|
||||||
|
// been reached or Close is invoked.
|
||||||
batch []string
|
batch []string
|
||||||
batchSize int
|
batchSize int
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *Formatter) Init(name string) error {
|
func (f *Formatter) Init(name string) error {
|
||||||
f.Name = name
|
// capture the name from the config file
|
||||||
|
f.name = name
|
||||||
|
|
||||||
// test if the formatter is available
|
// test if the formatter is available
|
||||||
if err := exec.Command(f.Command, "--help").Run(); err != nil {
|
if err := exec.Command(f.Command, "--help").Run(); err != nil {
|
||||||
return ErrFormatterNotFound
|
return ErrFormatterNotFound
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// initialise internal state
|
||||||
f.log = log.WithPrefix("format | " + name)
|
f.log = log.WithPrefix("format | " + name)
|
||||||
f.inbox = make(chan string, 1024)
|
|
||||||
|
|
||||||
f.batchSize = 1024
|
f.batchSize = 1024
|
||||||
|
f.inbox = make(chan string, f.batchSize)
|
||||||
f.batch = make([]string, f.batchSize)
|
f.batch = make([]string, f.batchSize)
|
||||||
f.batch = f.batch[:0]
|
f.batch = f.batch[:0]
|
||||||
|
|
||||||
@ -54,7 +63,7 @@ func (f *Formatter) Init(name string) error {
|
|||||||
for _, pattern := range f.Includes {
|
for _, pattern := range f.Includes {
|
||||||
g, err := glob.Compile("**/" + pattern)
|
g, err := glob.Compile("**/" + pattern)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Annotatef(err, "failed to compile include pattern '%v' for formatter '%v'", pattern, f.Name)
|
return errors.Annotatef(err, "failed to compile include pattern '%v' for formatter '%v'", pattern, f.name)
|
||||||
}
|
}
|
||||||
f.includes = append(f.includes, g)
|
f.includes = append(f.includes, g)
|
||||||
}
|
}
|
||||||
@ -64,7 +73,7 @@ func (f *Formatter) Init(name string) error {
|
|||||||
for _, pattern := range f.Excludes {
|
for _, pattern := range f.Excludes {
|
||||||
g, err := glob.Compile("**/" + pattern)
|
g, err := glob.Compile("**/" + pattern)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Annotatef(err, "failed to compile exclude pattern '%v' for formatter '%v'", pattern, f.Name)
|
return errors.Annotatef(err, "failed to compile exclude pattern '%v' for formatter '%v'", pattern, f.name)
|
||||||
}
|
}
|
||||||
f.excludes = append(f.excludes, g)
|
f.excludes = append(f.excludes, g)
|
||||||
}
|
}
|
||||||
@ -73,6 +82,8 @@ func (f *Formatter) Init(name string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Wants is used to test if a Formatter wants path based on it's configured Includes and Excludes patterns.
|
||||||
|
// Returns true if the Formatter should be applied to path, false otherwise.
|
||||||
func (f *Formatter) Wants(path string) bool {
|
func (f *Formatter) Wants(path string) bool {
|
||||||
match := !PathMatches(path, f.excludes) && PathMatches(path, f.includes)
|
match := !PathMatches(path, f.excludes) && PathMatches(path, f.includes)
|
||||||
if match {
|
if match {
|
||||||
@ -81,24 +92,31 @@ func (f *Formatter) Wants(path string) bool {
|
|||||||
return match
|
return match
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Put add path into this Formatter's inbox for processing.
|
||||||
func (f *Formatter) Put(path string) {
|
func (f *Formatter) Put(path string) {
|
||||||
f.inbox <- path
|
f.inbox <- path
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Run is the main processing loop for this Formatter.
|
||||||
|
// It accepts a context which is used to lookup certain dependencies and for cancellation.
|
||||||
func (f *Formatter) Run(ctx context.Context) (err error) {
|
func (f *Formatter) Run(ctx context.Context) (err error) {
|
||||||
LOOP:
|
LOOP:
|
||||||
|
// keep processing until ctx has been cancelled or inbox has been closed
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
|
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
|
// ctx has been cancelled
|
||||||
err = ctx.Err()
|
err = ctx.Err()
|
||||||
break LOOP
|
break LOOP
|
||||||
|
|
||||||
case path, ok := <-f.inbox:
|
case path, ok := <-f.inbox:
|
||||||
|
// check if the inbox has been closed
|
||||||
if !ok {
|
if !ok {
|
||||||
break LOOP
|
break LOOP
|
||||||
}
|
}
|
||||||
|
|
||||||
// add to the current batch
|
// add path to the current batch
|
||||||
f.batch = append(f.batch, path)
|
f.batch = append(f.batch, path)
|
||||||
|
|
||||||
if len(f.batch) == f.batchSize {
|
if len(f.batch) == f.batchSize {
|
||||||
@ -110,14 +128,17 @@ LOOP:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// check if LOOP was exited due to an error
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// final flush
|
// processing any lingering batch
|
||||||
return f.apply(ctx)
|
return f.apply(ctx)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// apply executes Command against the latest batch of paths.
|
||||||
|
// It accepts a context which is used to lookup certain dependencies and for cancellation.
|
||||||
func (f *Formatter) apply(ctx context.Context) error {
|
func (f *Formatter) apply(ctx context.Context) error {
|
||||||
// empty check
|
// empty check
|
||||||
if len(f.batch) == 0 {
|
if len(f.batch) == 0 {
|
||||||
@ -132,6 +153,7 @@ func (f *Formatter) apply(ctx context.Context) error {
|
|||||||
args = append(args, path)
|
args = append(args, path)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// execute
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
cmd := exec.CommandContext(ctx, f.Command, args...)
|
cmd := exec.CommandContext(ctx, f.Command, args...)
|
||||||
|
|
||||||
@ -142,15 +164,9 @@ func (f *Formatter) apply(ctx context.Context) error {
|
|||||||
|
|
||||||
f.log.Infof("%v files processed in %v", len(f.batch), time.Now().Sub(start))
|
f.log.Infof("%v files processed in %v", len(f.batch), time.Now().Sub(start))
|
||||||
|
|
||||||
// mark completed or forward on
|
// mark each path in this batch as completed
|
||||||
if len(f.Before) == 0 {
|
for _, path := range f.batch {
|
||||||
for _, path := range f.batch {
|
MarkFormatComplete(ctx, path)
|
||||||
MarkFormatComplete(ctx, path)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
for _, path := range f.batch {
|
|
||||||
ForwardPath(ctx, path, f.Before)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// reset batch
|
// reset batch
|
||||||
@ -159,6 +175,7 @@ func (f *Formatter) apply(ctx context.Context) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Close is used to indicate that a Formatter should process any remaining paths and then stop it's processing loop.
|
||||||
func (f *Formatter) Close() {
|
func (f *Formatter) Close() {
|
||||||
close(f.inbox)
|
close(f.inbox)
|
||||||
}
|
}
|
||||||
|
@ -1,21 +0,0 @@
|
|||||||
package log
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bufio"
|
|
||||||
"bytes"
|
|
||||||
|
|
||||||
"github.com/charmbracelet/log"
|
|
||||||
)
|
|
||||||
|
|
||||||
type Writer struct {
|
|
||||||
Log *log.Logger
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l *Writer) Write(p []byte) (n int, err error) {
|
|
||||||
scanner := bufio.NewScanner(bytes.NewReader(p))
|
|
||||||
for scanner.Scan() {
|
|
||||||
line := scanner.Text()
|
|
||||||
l.Log.Debug(line)
|
|
||||||
}
|
|
||||||
return len(p), nil
|
|
||||||
}
|
|
@ -23,7 +23,7 @@
|
|||||||
packages = with pkgs; [
|
packages = with pkgs; [
|
||||||
# golang
|
# golang
|
||||||
go
|
go
|
||||||
go-tools
|
gotools
|
||||||
delve
|
delve
|
||||||
golangci-lint
|
golangci-lint
|
||||||
|
|
||||||
|
Reference in New Issue
Block a user