15227100554

Committed 24 May 2025 12:35PM UTC coverage: 37.541% (+0.2%) from 37.358%

Build # 15227100554

Build Type

Pull #593

github

Committed by

brianmcgee

Commit Message

fix: force-remove db file when clearing cache

It also improves logging of cache file location when the db cannot be opened.

Fixes #592

Signed-off-by: Brian McGee <brian@bmcgee.ie>

Pull Request Pull Request #593: fix: force-remove db file when clearing cache

Run Details

0 of 24 new or added lines in 2 files covered. (0.0%)

3 existing lines in 1 file now uncovered.

690 of 1838 relevant lines covered (37.54%)

17.28 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/cmd/format/format.go

package format

import (
        "context"
        "errors"
        "fmt"
        "io"
        "os"
        "os/signal"
        "path/filepath"
        "runtime/pprof"
        "strings"
        "syscall"
        "time"

        "github.com/charmbracelet/log"
        "github.com/numtide/treefmt/v2/config"
        "github.com/numtide/treefmt/v2/format"
        "github.com/numtide/treefmt/v2/stats"
        "github.com/numtide/treefmt/v2/walk"
        "github.com/numtide/treefmt/v2/walk/cache"
        "github.com/spf13/cobra"
        "github.com/spf13/viper"
        bolt "go.etcd.io/bbolt"
)

const (
        BatchSize = 1024
)

var ErrFailOnChange = errors.New("unexpected changes detected, --fail-on-change is enabled")

func Run(v *viper.Viper, statz *stats.Stats, cmd *cobra.Command, paths []string) error {
        cmd.SilenceUsage = true

        cfg, err := config.FromViper(v)
        if err != nil {
                return fmt.Errorf("failed to load config: %w", err)
        }

        if cfg.CI {
                log.Info("ci mode enabled")

                startAfter := time.Now().
                        // truncate to second precision
                        Truncate(time.Second).
                        // add one second
                        Add(1 * time.Second).
                        // a little extra to ensure we don't start until the next second
                        Add(10 * time.Millisecond)

                log.Debugf("waiting until %v before continuing", startAfter)

                // Wait until we tick over into the next second before processing to ensure our EPOCH level modtime comparisons
                // for change detection are accurate.
                // This can fail in CI between checkout and running treefmt if everything happens too quickly.
                // For humans, the second level precision should not be a problem as they are unlikely to run treefmt in
                // sub-second succession.
                time.Sleep(time.Until(startAfter))
        }

        // cpu profiling
        if cfg.CPUProfile != "" {
                cpuProfile, err := os.Create(cfg.CPUProfile)
                if err != nil {
                        return fmt.Errorf("failed to open file for writing cpu profile: %w", err)
                } else if err = pprof.StartCPUProfile(cpuProfile); err != nil {
                        return fmt.Errorf("failed to start cpu profile: %w", err)
                }

                defer func() {
                        pprof.StopCPUProfile()

                        if err := cpuProfile.Close(); err != nil {
                                log.Errorf("failed to close cpu profile: %v", err)
                        }
                }()
        }

        // Remove the cache first before potentially opening a new one.
        if cfg.ClearCache {
                if err := cache.Remove(cfg.TreeRoot); err != nil {
                        return fmt.Errorf("failed to clear cache: %w", err)
                }
        }

        var db *bolt.DB

        // open the db unless --no-cache was specified
        if !cfg.NoCache {
                db, err = cache.Open(cfg.TreeRoot)
                if err != nil {
                        return fmt.Errorf("failed to open cache: %w", err)
                }

                // ensure db is closed after we're finished
                defer func() {
                        if closeErr := db.Close(); closeErr != nil {
                                log.Errorf("failed to close cache: %v", closeErr)
                        }
                }()
        }

        // create an overall app context
        ctx, cancel := context.WithCancel(context.Background())
        defer cancel()

        // listen for shutdown signal and cancel the context
        go func() {
                exit := make(chan os.Signal, 1)
                signal.Notify(exit, os.Interrupt, syscall.SIGTERM)
                <-exit
                cancel()
        }()

        // parse the walk type
        walkType, err := walk.TypeString(cfg.Walk)
        if err != nil {
                return fmt.Errorf("invalid walk type: %w", err)
        }

        if walkType == walk.Stdin && len(paths) != 1 {
                // check we have only received one path arg which we use for the file extension / matching to formatters
                return errors.New("exactly one path should be specified when using the --stdin flag")
        }

        if err = resolvePaths(paths, walkType, cfg.TreeRoot); err != nil {
                return err
        }

        // create a composite formatter which will handle applying the correct formatters to each file we traverse
        formatter, err := format.NewCompositeFormatter(cfg, statz, BatchSize)
        if err != nil {
                return fmt.Errorf("failed to create composite formatter: %w", err)
        }

        // create a new walker for traversing the paths
        walker, err := walk.NewCompositeReader(walkType, cfg.TreeRoot, paths, db, statz)
        if err != nil {
                return fmt.Errorf("failed to create walker: %w", err)
        }

        // start traversing
        files := make([]*walk.File, BatchSize)

        var (
                n                  int
                readErr, formatErr error
        )

        for {
                // read the next batch
                readCtx, cancelRead := context.WithTimeout(ctx, 1*time.Second)

                n, readErr = walker.Read(readCtx, files)
                log.Debugf("read %d files", n)

                // ensure context is cancelled to release resources
                cancelRead()

                // format any files that were read before processing the read error
                if formatErr = formatter.Apply(ctx, files[:n]); formatErr != nil {
                        break
                }

                // stop reading files if there was a read error
                if readErr != nil {
                        break
                }
        }

        // finalize formatting (there could be formatting tasks in-flight)
        formatCloseErr := formatter.Close(ctx)

        // close the walker, ensuring any pending file release hooks finish
        walkerCloseErr := walker.Close()

        // print stats to stderr
        if !cfg.Quiet {
                statz.PrintToStderr()
        }

        // process errors

        //nolint:gocritic
        if errors.Is(readErr, io.EOF) {
                // nothing more to read, reset the error and break out of the read loop
                log.Debugf("no more files to read")
        } else if errors.Is(readErr, context.DeadlineExceeded) {
                // the read timed-out
                return errors.New("timeout reading files")
        } else if readErr != nil {
                // something unexpected happened
                return fmt.Errorf("failed to read files: %w", readErr)
        }

        if formatErr != nil {
                return fmt.Errorf("failed to format files: %w", formatErr)
        }

        if formatCloseErr != nil {
                return fmt.Errorf("failed to finalise formatting: %w", formatCloseErr)
        }

        if walkerCloseErr != nil {
                return fmt.Errorf("failed to close walker: %w", walkerCloseErr)
        }

        if cfg.FailOnChange && statz.Value(stats.Changed) != 0 {
                // if fail on change has been enabled, check that no files were actually changed, throwing an error if so
                return ErrFailOnChange
        }

        return nil
}

// resolvePaths checks all paths are contained within the tree root and exist
// also "normalize" paths so they're relative to `cfg.TreeRoot`
// Symlinks are allowed in `paths` and we resolve them here, since
// the readers will ignore symlinks.
func resolvePaths(paths []string, walkType walk.Type, treeRoot string) error {
        // Note: `treeRoot` may itself be or contain a symlink (e.g. it is in
        // `$TMPDIR` on macOS or a user has set a symlink to shorten the repository
        // path for path length restrictions), so we resolve it here first.
        //
        // See: https://github.com/numtide/treefmt/issues/578
        treeRoot, err := resolvePath(walkType, treeRoot)
        if err != nil {
                return fmt.Errorf("error computing absolute path of %s: %w", treeRoot, err)
        }

        for i, path := range paths {
                absolutePath, err := resolvePath(walkType, path)
                if err != nil {
                        return fmt.Errorf("error computing absolute path of %s: %w", path, err)
                }

                relativePath, err := filepath.Rel(treeRoot, absolutePath)
                if err != nil {
                        return fmt.Errorf("error computing relative path from %s to %s: %w", treeRoot, absolutePath, err)
                }

                if strings.HasPrefix(relativePath, "..") {
                        return fmt.Errorf("path %s not inside the tree root %s", path, treeRoot)
                }

                paths[i] = relativePath
        }

        return nil
}

// Resolve a path to an absolute path, resolving symlinks if necessary.
func resolvePath(walkType walk.Type, path string) (string, error) {
        log.Debugf("Resolving path '%s': %v", path, walkType)

        absolutePath, err := filepath.Abs(path)
        if err != nil {
                return "", fmt.Errorf("error computing absolute path of %s: %w", path, err)
        }

        if walkType != walk.Stdin {
                realPath, err := filepath.EvalSymlinks(absolutePath)
                if err != nil {
                        return "", fmt.Errorf("path %s not found: %w", absolutePath, err)
                }

                absolutePath = realPath
        }

        return absolutePath, nil
}

1	package format
2
3	import (
4	"context"
5	"errors"
6	"fmt"
7	"io"
8	"os"
9	"os/signal"
10	"path/filepath"
11	"runtime/pprof"
12	"strings"
13	"syscall"
14	"time"
15
16	"github.com/charmbracelet/log"
17	"github.com/numtide/treefmt/v2/config"
18	"github.com/numtide/treefmt/v2/format"
19	"github.com/numtide/treefmt/v2/stats"
20	"github.com/numtide/treefmt/v2/walk"
21	"github.com/numtide/treefmt/v2/walk/cache"
22	"github.com/spf13/cobra"
23	"github.com/spf13/viper"
24	bolt "go.etcd.io/bbolt"
25	)
26
27	const (
28	BatchSize = 1024
29	)
30
31	var ErrFailOnChange = errors.New("unexpected changes detected, --fail-on-change is enabled")
32
33	func Run(v viper.Viper, statz stats.Stats, cmd *cobra.Command, paths []string) error {	×
34	cmd.SilenceUsage = true	×
35		×
36	cfg, err := config.FromViper(v)	×
37	if err != nil {	×
38	return fmt.Errorf("failed to load config: %w", err)	×
39	}	×
40
41	if cfg.CI {	×
42	log.Info("ci mode enabled")	×
43		×
44	startAfter := time.Now().	×
45	// truncate to second precision	×
46	Truncate(time.Second).	×
47	// add one second	×
48	Add(1 * time.Second).	×
49	// a little extra to ensure we don't start until the next second	×
50	Add(10 * time.Millisecond)	×
51		×
52	log.Debugf("waiting until %v before continuing", startAfter)	×
53		×
54	// Wait until we tick over into the next second before processing to ensure our EPOCH level modtime comparisons	×
55	// for change detection are accurate.	×
56	// This can fail in CI between checkout and running treefmt if everything happens too quickly.	×
57	// For humans, the second level precision should not be a problem as they are unlikely to run treefmt in	×
58	// sub-second succession.	×
59	time.Sleep(time.Until(startAfter))	×
60	}	×
61
62	// cpu profiling
63	if cfg.CPUProfile != "" {	×
64	cpuProfile, err := os.Create(cfg.CPUProfile)	×
65	if err != nil {	×
66	return fmt.Errorf("failed to open file for writing cpu profile: %w", err)	×
67	} else if err = pprof.StartCPUProfile(cpuProfile); err != nil {	×
68	return fmt.Errorf("failed to start cpu profile: %w", err)	×
69	}	×
70
71	defer func() {	×
72	pprof.StopCPUProfile()	×
73		×
74	if err := cpuProfile.Close(); err != nil {	×
75	log.Errorf("failed to close cpu profile: %v", err)	×
76	}	×
77	}()
78	}
79
80	// Remove the cache first before potentially opening a new one.
NEW 81	if cfg.ClearCache {	×
NEW 82	if err := cache.Remove(cfg.TreeRoot); err != nil {	×
NEW 83	return fmt.Errorf("failed to clear cache: %w", err)	×
NEW 84	}	×
85	}
86
87	var db *bolt.DB	×
88		×
89	// open the db unless --no-cache was specified	×
90	if !cfg.NoCache {	×
91	db, err = cache.Open(cfg.TreeRoot)	×
92	if err != nil {	×
93	return fmt.Errorf("failed to open cache: %w", err)	×
94	}	×
95
96	// ensure db is closed after we're finished
97	defer func() {	×
98	if closeErr := db.Close(); closeErr != nil {	×
99	log.Errorf("failed to close cache: %v", closeErr)	×
100	}	×
101	}()
102	}
103
104	// create an overall app context
105	ctx, cancel := context.WithCancel(context.Background())	×
106	defer cancel()	×
107		×
108	// listen for shutdown signal and cancel the context	×
109	go func() {	×
110	exit := make(chan os.Signal, 1)	×
111	signal.Notify(exit, os.Interrupt, syscall.SIGTERM)	×
112	<-exit	×
113	cancel()	×
114	}()	×
115
116	// parse the walk type
117	walkType, err := walk.TypeString(cfg.Walk)	×
118	if err != nil {	×
119	return fmt.Errorf("invalid walk type: %w", err)	×
120	}	×
121
122	if walkType == walk.Stdin && len(paths) != 1 {	×
123	// check we have only received one path arg which we use for the file extension / matching to formatters	×
124	return errors.New("exactly one path should be specified when using the --stdin flag")	×
125	}	×
126
127	if err = resolvePaths(paths, walkType, cfg.TreeRoot); err != nil {	×
128	return err	×
129	}	×
130
131	// create a composite formatter which will handle applying the correct formatters to each file we traverse
132	formatter, err := format.NewCompositeFormatter(cfg, statz, BatchSize)	×
133	if err != nil {	×
134	return fmt.Errorf("failed to create composite formatter: %w", err)	×
135	}	×
136
137	// create a new walker for traversing the paths
138	walker, err := walk.NewCompositeReader(walkType, cfg.TreeRoot, paths, db, statz)	×
139	if err != nil {	×
140	return fmt.Errorf("failed to create walker: %w", err)	×
141	}	×
142
143	// start traversing
144	files := make([]*walk.File, BatchSize)	×
145		×
146	var (	×
147	n int	×
148	readErr, formatErr error	×
149	)	×
150		×
151	for {	×
152	// read the next batch	×
153	readCtx, cancelRead := context.WithTimeout(ctx, 1*time.Second)	×
154		×
155	n, readErr = walker.Read(readCtx, files)	×
156	log.Debugf("read %d files", n)	×
157		×
158	// ensure context is cancelled to release resources	×
159	cancelRead()	×
160		×
161	// format any files that were read before processing the read error	×
162	if formatErr = formatter.Apply(ctx, files[:n]); formatErr != nil {	×
163	break	×
164	}
165
166	// stop reading files if there was a read error
167	if readErr != nil {	×
168	break	×
169	}
170	}
171
172	// finalize formatting (there could be formatting tasks in-flight)
173	formatCloseErr := formatter.Close(ctx)	×
174		×
175	// close the walker, ensuring any pending file release hooks finish	×
176	walkerCloseErr := walker.Close()	×
177		×
178	// print stats to stderr	×
179	if !cfg.Quiet {	×
180	statz.PrintToStderr()	×
181	}	×
182
183	// process errors
184
185	//nolint:gocritic
186	if errors.Is(readErr, io.EOF) {	×
187	// nothing more to read, reset the error and break out of the read loop	×
188	log.Debugf("no more files to read")	×
189	} else if errors.Is(readErr, context.DeadlineExceeded) {	×
190	// the read timed-out	×
191	return errors.New("timeout reading files")	×
192	} else if readErr != nil {	×
193	// something unexpected happened	×
194	return fmt.Errorf("failed to read files: %w", readErr)	×
195	}	×
196
197	if formatErr != nil {	×
198	return fmt.Errorf("failed to format files: %w", formatErr)	×
199	}	×
200
201	if formatCloseErr != nil {	×
202	return fmt.Errorf("failed to finalise formatting: %w", formatCloseErr)	×
203	}	×
204
205	if walkerCloseErr != nil {	×
206	return fmt.Errorf("failed to close walker: %w", walkerCloseErr)	×
207	}	×
208
209	if cfg.FailOnChange && statz.Value(stats.Changed) != 0 {	×
210	// if fail on change has been enabled, check that no files were actually changed, throwing an error if so	×
211	return ErrFailOnChange	×
212	}	×
213
214	return nil	×
215	}
216
217	// resolvePaths checks all paths are contained within the tree root and exist
218	// also "normalize" paths so they're relative to `cfg.TreeRoot`
219	// Symlinks are allowed in `paths` and we resolve them here, since
220	// the readers will ignore symlinks.
221	func resolvePaths(paths []string, walkType walk.Type, treeRoot string) error {	×
222	// Note: `treeRoot` may itself be or contain a symlink (e.g. it is in	×
223	// `$TMPDIR` on macOS or a user has set a symlink to shorten the repository	×
224	// path for path length restrictions), so we resolve it here first.	×
225	//	×
226	// See: https://github.com/numtide/treefmt/issues/578	×
227	treeRoot, err := resolvePath(walkType, treeRoot)	×
228	if err != nil {	×
229	return fmt.Errorf("error computing absolute path of %s: %w", treeRoot, err)	×
230	}	×
231
232	for i, path := range paths {	×
233	absolutePath, err := resolvePath(walkType, path)	×
234	if err != nil {	×
235	return fmt.Errorf("error computing absolute path of %s: %w", path, err)	×
236	}	×
237
238	relativePath, err := filepath.Rel(treeRoot, absolutePath)	×
239	if err != nil {	×
240	return fmt.Errorf("error computing relative path from %s to %s: %w", treeRoot, absolutePath, err)	×
241	}	×
242
243	if strings.HasPrefix(relativePath, "..") {	×
244	return fmt.Errorf("path %s not inside the tree root %s", path, treeRoot)	×
245	}	×
246
247	paths[i] = relativePath	×
248	}
249
250	return nil	×
251	}
252
253	// Resolve a path to an absolute path, resolving symlinks if necessary.
254	func resolvePath(walkType walk.Type, path string) (string, error) {	×
255	log.Debugf("Resolving path '%s': %v", path, walkType)	×
256		×
257	absolutePath, err := filepath.Abs(path)	×
258	if err != nil {	×
259	return "", fmt.Errorf("error computing absolute path of %s: %w", path, err)	×
260	}	×
261
262	if walkType != walk.Stdin {	×
263	realPath, err := filepath.EvalSymlinks(absolutePath)	×
264	if err != nil {	×
265	return "", fmt.Errorf("path %s not found: %w", absolutePath, err)	×
266	}	×
267
268	absolutePath = realPath	×
269	}
270
271	return absolutePath, nil	×
272	}

numtide / treefmt / 15227100554

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous