Build has been canceled!

15227078419

Committed 24 May 2025 12:32PM UTC coverage: 37.541% (+0.2%) from 37.358%

Build # 15227078419

Build Type

Pull #593

github

Committed by

brianmcgee

Commit Message

fix: force-remove db file when clearing cache

It also improves logging of cache file location when the db cannot be opened.

Fixes #592

Signed-off-by: Brian McGee <brian@bmcgee.ie>

Pull Request Pull Request #593: fix: force-remove db file when clearing cache

Run Details

0 of 25 new or added lines in 2 files covered. (0.0%)

42 existing lines in 2 files now uncovered.

690 of 1838 relevant lines covered (37.54%)

17.28 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/cmd/format/format.go

package format

import (
        "context"
        "errors"
        "fmt"
        "io"
        "os"
        "os/signal"
        "path/filepath"
        "runtime/pprof"
        "strings"
        "syscall"
        "time"

        "github.com/charmbracelet/log"
        "github.com/numtide/treefmt/v2/config"
        "github.com/numtide/treefmt/v2/format"
        "github.com/numtide/treefmt/v2/stats"
        "github.com/numtide/treefmt/v2/walk"
        "github.com/numtide/treefmt/v2/walk/cache"
        "github.com/spf13/cobra"
        "github.com/spf13/viper"
        bolt "go.etcd.io/bbolt"
)

const (
        BatchSize = 1024
)

var ErrFailOnChange = errors.New("unexpected changes detected, --fail-on-change is enabled")

func Run(v *viper.Viper, statz *stats.Stats, cmd *cobra.Command, paths []string) error {
        cmd.SilenceUsage = true

        cfg, err := config.FromViper(v)
        if err != nil {
                return fmt.Errorf("failed to load config: %w", err)
        }

        if cfg.CI {
                log.Info("ci mode enabled")

                startAfter := time.Now().
                        // truncate to second precision
                        Truncate(time.Second).
                        // add one second
                        Add(1 * time.Second).
                        // a little extra to ensure we don't start until the next second
                        Add(10 * time.Millisecond)

                log.Debugf("waiting until %v before continuing", startAfter)

                // Wait until we tick over into the next second before processing to ensure our EPOCH level modtime comparisons
                // for change detection are accurate.
                // This can fail in CI between checkout and running treefmt if everything happens too quickly.
                // For humans, the second level precision should not be a problem as they are unlikely to run treefmt in
                // sub-second succession.
                time.Sleep(time.Until(startAfter))
        }

        // cpu profiling
        if cfg.CPUProfile != "" {
                cpuProfile, err := os.Create(cfg.CPUProfile)
                if err != nil {
                        return fmt.Errorf("failed to open file for writing cpu profile: %w", err)
                } else if err = pprof.StartCPUProfile(cpuProfile); err != nil {
                        return fmt.Errorf("failed to start cpu profile: %w", err)
                }

                defer func() {
                        pprof.StopCPUProfile()

                        if err := cpuProfile.Close(); err != nil {
                                log.Errorf("failed to close cpu profile: %v", err)
                        }
                }()
        }

        // Clear the cache first before potentially opening a new one.
        // If a treefmt process is currently running with a db open at the same location, it will continue to function
        // as normal, however, when it exits the disk space its inode was referencing will be reclaimed.
        // This will not work on Windows if we ever support it.
        if cfg.ClearCache {
                if err := cache.Clear(cfg.TreeRoot); err != nil {
                        return fmt.Errorf("failed to clear cache: %w", err)
                }
        }

        var db *bolt.DB

        // open the db unless --no-cache was specified
        if !cfg.NoCache {
                db, err = cache.Open(cfg.TreeRoot)
                if err != nil {
                        return fmt.Errorf("failed to open cache: %w", err)
                }

                // ensure db is closed after we're finished
                defer func() {
                        if closeErr := db.Close(); closeErr != nil {
                                log.Errorf("failed to close cache: %v", closeErr)
                        }
                }()
        }

        // create an overall app context
        ctx, cancel := context.WithCancel(context.Background())
        defer cancel()

        // listen for shutdown signal and cancel the context
        go func() {
                exit := make(chan os.Signal, 1)
                signal.Notify(exit, os.Interrupt, syscall.SIGTERM)
                <-exit
                cancel()
        }()

        // parse the walk type
        walkType, err := walk.TypeString(cfg.Walk)
        if err != nil {
                return fmt.Errorf("invalid walk type: %w", err)
        }

        if walkType == walk.Stdin && len(paths) != 1 {
                // check we have only received one path arg which we use for the file extension / matching to formatters
                return errors.New("exactly one path should be specified when using the --stdin flag")
        }

        if err = resolvePaths(paths, walkType, cfg.TreeRoot); err != nil {
                return err
        }

        // create a composite formatter which will handle applying the correct formatters to each file we traverse
        formatter, err := format.NewCompositeFormatter(cfg, statz, BatchSize)
        if err != nil {
                return fmt.Errorf("failed to create composite formatter: %w", err)
        }

        // create a new walker for traversing the paths
        walker, err := walk.NewCompositeReader(walkType, cfg.TreeRoot, paths, db, statz)
        if err != nil {
                return fmt.Errorf("failed to create walker: %w", err)
        }

        // start traversing
        files := make([]*walk.File, BatchSize)

        var (
                n                  int
                readErr, formatErr error
        )

        for {
                // read the next batch
                readCtx, cancelRead := context.WithTimeout(ctx, 1*time.Second)

                n, readErr = walker.Read(readCtx, files)
                log.Debugf("read %d files", n)

                // ensure context is cancelled to release resources
                cancelRead()

                // format any files that were read before processing the read error
                if formatErr = formatter.Apply(ctx, files[:n]); formatErr != nil {
                        break
                }

                // stop reading files if there was a read error
                if readErr != nil {
                        break
                }
        }

        // finalize formatting (there could be formatting tasks in-flight)
        formatCloseErr := formatter.Close(ctx)

        // close the walker, ensuring any pending file release hooks finish
        walkerCloseErr := walker.Close()

        // print stats to stderr
        if !cfg.Quiet {
                statz.PrintToStderr()
        }

        // process errors

        //nolint:gocritic
        if errors.Is(readErr, io.EOF) {
                // nothing more to read, reset the error and break out of the read loop
                log.Debugf("no more files to read")
        } else if errors.Is(readErr, context.DeadlineExceeded) {
                // the read timed-out
                return errors.New("timeout reading files")
        } else if readErr != nil {
                // something unexpected happened
                return fmt.Errorf("failed to read files: %w", readErr)
        }

        if formatErr != nil {
                return fmt.Errorf("failed to format files: %w", formatErr)
        }

        if formatCloseErr != nil {
                return fmt.Errorf("failed to finalise formatting: %w", formatCloseErr)
        }

        if walkerCloseErr != nil {
                return fmt.Errorf("failed to close walker: %w", walkerCloseErr)
        }

        if cfg.FailOnChange && statz.Value(stats.Changed) != 0 {
                // if fail on change has been enabled, check that no files were actually changed, throwing an error if so
                return ErrFailOnChange
        }

        return nil
}

// resolvePaths checks all paths are contained within the tree root and exist
// also "normalize" paths so they're relative to `cfg.TreeRoot`
// Symlinks are allowed in `paths` and we resolve them here, since
// the readers will ignore symlinks.
func resolvePaths(paths []string, walkType walk.Type, treeRoot string) error {
        // Note: `treeRoot` may itself be or contain a symlink (e.g. it is in
        // `$TMPDIR` on macOS or a user has set a symlink to shorten the repository
        // path for path length restrictions), so we resolve it here first.
        //
        // See: https://github.com/numtide/treefmt/issues/578
        treeRoot, err := resolvePath(walkType, treeRoot)
        if err != nil {
                return fmt.Errorf("error computing absolute path of %s: %w", treeRoot, err)
        }

        for i, path := range paths {
                absolutePath, err := resolvePath(walkType, path)
                if err != nil {
                        return fmt.Errorf("error computing absolute path of %s: %w", path, err)
                }

                relativePath, err := filepath.Rel(treeRoot, absolutePath)
                if err != nil {
                        return fmt.Errorf("error computing relative path from %s to %s: %w", treeRoot, absolutePath, err)
                }

                if strings.HasPrefix(relativePath, "..") {
                        return fmt.Errorf("path %s not inside the tree root %s", path, treeRoot)
                }

                paths[i] = relativePath
        }

        return nil
}

// Resolve a path to an absolute path, resolving symlinks if necessary.
func resolvePath(walkType walk.Type, path string) (string, error) {
        log.Debugf("Resolving path '%s': %v", path, walkType)

        absolutePath, err := filepath.Abs(path)
        if err != nil {
                return "", fmt.Errorf("error computing absolute path of %s: %w", path, err)
        }

        if walkType != walk.Stdin {
                realPath, err := filepath.EvalSymlinks(absolutePath)
                if err != nil {
                        return "", fmt.Errorf("path %s not found: %w", absolutePath, err)
                }

                absolutePath = realPath
        }

        return absolutePath, nil
}

1	package format
2
3	import (
4	"context"
5	"errors"
6	"fmt"
7	"io"
8	"os"
9	"os/signal"
10	"path/filepath"
11	"runtime/pprof"
12	"strings"
13	"syscall"
14	"time"
15
16	"github.com/charmbracelet/log"
17	"github.com/numtide/treefmt/v2/config"
18	"github.com/numtide/treefmt/v2/format"
19	"github.com/numtide/treefmt/v2/stats"
20	"github.com/numtide/treefmt/v2/walk"
21	"github.com/numtide/treefmt/v2/walk/cache"
22	"github.com/spf13/cobra"
23	"github.com/spf13/viper"
24	bolt "go.etcd.io/bbolt"
25	)
26
27	const (
28	BatchSize = 1024
29	)
30
31	var ErrFailOnChange = errors.New("unexpected changes detected, --fail-on-change is enabled")
32
33	func Run(v viper.Viper, statz stats.Stats, cmd *cobra.Command, paths []string) error {	×
34	cmd.SilenceUsage = true	×
35		×
36	cfg, err := config.FromViper(v)	×
37	if err != nil {	×
38	return fmt.Errorf("failed to load config: %w", err)	×
39	}	×
40
41	if cfg.CI {	×
42	log.Info("ci mode enabled")	×
43		×
44	startAfter := time.Now().	×
45	// truncate to second precision	×
46	Truncate(time.Second).	×
47	// add one second	×
48	Add(1 * time.Second).	×
49	// a little extra to ensure we don't start until the next second	×
50	Add(10 * time.Millisecond)	×
51		×
52	log.Debugf("waiting until %v before continuing", startAfter)	×
53		×
54	// Wait until we tick over into the next second before processing to ensure our EPOCH level modtime comparisons	×
55	// for change detection are accurate.	×
56	// This can fail in CI between checkout and running treefmt if everything happens too quickly.	×
57	// For humans, the second level precision should not be a problem as they are unlikely to run treefmt in	×
58	// sub-second succession.	×
59	time.Sleep(time.Until(startAfter))	×
60	}	×
61
62	// cpu profiling
63	if cfg.CPUProfile != "" {	×
64	cpuProfile, err := os.Create(cfg.CPUProfile)	×
65	if err != nil {	×
66	return fmt.Errorf("failed to open file for writing cpu profile: %w", err)	×
67	} else if err = pprof.StartCPUProfile(cpuProfile); err != nil {	×
68	return fmt.Errorf("failed to start cpu profile: %w", err)	×
69	}	×
70
71	defer func() {	×
72	pprof.StopCPUProfile()	×
73		×
74	if err := cpuProfile.Close(); err != nil {	×
75	log.Errorf("failed to close cpu profile: %v", err)	×
76	}	×
77	}()
78	}
79
80	// Clear the cache first before potentially opening a new one.
81	// If a treefmt process is currently running with a db open at the same location, it will continue to function
82	// as normal, however, when it exits the disk space its inode was referencing will be reclaimed.
83	// This will not work on Windows if we ever support it.
NEW 84	if cfg.ClearCache {	×
NEW 85	if err := cache.Clear(cfg.TreeRoot); err != nil {	×
NEW 86	return fmt.Errorf("failed to clear cache: %w", err)	×
87	}	×
88	}
89
90	var db *bolt.DB	×
91		×
92	// open the db unless --no-cache was specified	×
93	if !cfg.NoCache {	×
94	db, err = cache.Open(cfg.TreeRoot)	×
UNCOV 95	if err != nil {	×
UNCOV 96	return fmt.Errorf("failed to open cache: %w", err)	×
97	}	×
98
99	// ensure db is closed after we're finished
100	defer func() {	×
UNCOV 101	if closeErr := db.Close(); closeErr != nil {	×
UNCOV 102	log.Errorf("failed to close cache: %v", closeErr)	×
UNCOV 103	}	×
104	}()
105	}
106
107	// create an overall app context
108	ctx, cancel := context.WithCancel(context.Background())	×
109	defer cancel()	×
110		×
111	// listen for shutdown signal and cancel the context	×
112	go func() {	×
113	exit := make(chan os.Signal, 1)	×
114	signal.Notify(exit, os.Interrupt, syscall.SIGTERM)	×
UNCOV 115	<-exit	×
UNCOV 116	cancel()	×
117	}()	×
118
119	// parse the walk type
120	walkType, err := walk.TypeString(cfg.Walk)	×
UNCOV 121	if err != nil {	×
122	return fmt.Errorf("invalid walk type: %w", err)	×
123	}	×
124
125	if walkType == walk.Stdin && len(paths) != 1 {	×
UNCOV 126	// check we have only received one path arg which we use for the file extension / matching to formatters	×
127	return errors.New("exactly one path should be specified when using the --stdin flag")	×
128	}	×
129
UNCOV 130	if err = resolvePaths(paths, walkType, cfg.TreeRoot); err != nil {	×
UNCOV 131	return err	×
132	}	×
133
134	// create a composite formatter which will handle applying the correct formatters to each file we traverse
135	formatter, err := format.NewCompositeFormatter(cfg, statz, BatchSize)	×
UNCOV 136	if err != nil {	×
UNCOV 137	return fmt.Errorf("failed to create composite formatter: %w", err)	×
138	}	×
139
140	// create a new walker for traversing the paths
141	walker, err := walk.NewCompositeReader(walkType, cfg.TreeRoot, paths, db, statz)	×
UNCOV 142	if err != nil {	×
UNCOV 143	return fmt.Errorf("failed to create walker: %w", err)	×
144	}	×
145
146	// start traversing
147	files := make([]*walk.File, BatchSize)	×
148		×
149	var (	×
150	n int	×
151	readErr, formatErr error	×
152	)	×
153		×
154	for {	×
155	// read the next batch	×
156	readCtx, cancelRead := context.WithTimeout(ctx, 1*time.Second)	×
157		×
158	n, readErr = walker.Read(readCtx, files)	×
159	log.Debugf("read %d files", n)	×
160		×
161	// ensure context is cancelled to release resources	×
162	cancelRead()	×
163		×
UNCOV 164	// format any files that were read before processing the read error	×
UNCOV 165	if formatErr = formatter.Apply(ctx, files[:n]); formatErr != nil {	×
UNCOV 166	break	×
167	}
168
169	// stop reading files if there was a read error
UNCOV 170	if readErr != nil {	×
UNCOV 171	break	×
172	}
173	}
174
175	// finalize formatting (there could be formatting tasks in-flight)
176	formatCloseErr := formatter.Close(ctx)	×
177		×
178	// close the walker, ensuring any pending file release hooks finish	×
179	walkerCloseErr := walker.Close()	×
180		×
181	// print stats to stderr	×
UNCOV 182	if !cfg.Quiet {	×
UNCOV 183	statz.PrintToStderr()	×
UNCOV 184	}	×
185
186	// process errors
187
188	//nolint:gocritic
189	if errors.Is(readErr, io.EOF) {	×
190	// nothing more to read, reset the error and break out of the read loop	×
191	log.Debugf("no more files to read")	×
192	} else if errors.Is(readErr, context.DeadlineExceeded) {	×
193	// the read timed-out	×
194	return errors.New("timeout reading files")	×
195	} else if readErr != nil {	×
UNCOV 196	// something unexpected happened	×
197	return fmt.Errorf("failed to read files: %w", readErr)	×
198	}	×
199
UNCOV 200	if formatErr != nil {	×
201	return fmt.Errorf("failed to format files: %w", formatErr)	×
202	}	×
203
UNCOV 204	if formatCloseErr != nil {	×
205	return fmt.Errorf("failed to finalise formatting: %w", formatCloseErr)	×
206	}	×
207
UNCOV 208	if walkerCloseErr != nil {	×
209	return fmt.Errorf("failed to close walker: %w", walkerCloseErr)	×
210	}	×
211
212	if cfg.FailOnChange && statz.Value(stats.Changed) != 0 {	×
UNCOV 213	// if fail on change has been enabled, check that no files were actually changed, throwing an error if so	×
214	return ErrFailOnChange	×
UNCOV 215	}	×
216
UNCOV 217	return nil	×
218	}
219
220	// resolvePaths checks all paths are contained within the tree root and exist
221	// also "normalize" paths so they're relative to `cfg.TreeRoot`
222	// Symlinks are allowed in `paths` and we resolve them here, since
223	// the readers will ignore symlinks.
224	func resolvePaths(paths []string, walkType walk.Type, treeRoot string) error {	×
225	// Note: `treeRoot` may itself be or contain a symlink (e.g. it is in	×
226	// `$TMPDIR` on macOS or a user has set a symlink to shorten the repository	×
227	// path for path length restrictions), so we resolve it here first.	×
228	//	×
229	// See: https://github.com/numtide/treefmt/issues/578	×
230	treeRoot, err := resolvePath(walkType, treeRoot)	×
UNCOV 231	if err != nil {	×
232	return fmt.Errorf("error computing absolute path of %s: %w", treeRoot, err)	×
233	}	×
234
235	for i, path := range paths {	×
236	absolutePath, err := resolvePath(walkType, path)	×
UNCOV 237	if err != nil {	×
238	return fmt.Errorf("error computing absolute path of %s: %w", path, err)	×
239	}	×
240
241	relativePath, err := filepath.Rel(treeRoot, absolutePath)	×
UNCOV 242	if err != nil {	×
243	return fmt.Errorf("error computing relative path from %s to %s: %w", treeRoot, absolutePath, err)	×
244	}	×
245
UNCOV 246	if strings.HasPrefix(relativePath, "..") {	×
247	return fmt.Errorf("path %s not inside the tree root %s", path, treeRoot)	×
UNCOV 248	}	×
249
250	paths[i] = relativePath	×
251	}
252
UNCOV 253	return nil	×
254	}
255
256	// Resolve a path to an absolute path, resolving symlinks if necessary.
257	func resolvePath(walkType walk.Type, path string) (string, error) {	×
258	log.Debugf("Resolving path '%s': %v", path, walkType)	×
259		×
260	absolutePath, err := filepath.Abs(path)	×
UNCOV 261	if err != nil {	×
262	return "", fmt.Errorf("error computing absolute path of %s: %w", path, err)	×
263	}	×
264
265	if walkType != walk.Stdin {	×
266	realPath, err := filepath.EvalSymlinks(absolutePath)	×
UNCOV 267	if err != nil {	×
268	return "", fmt.Errorf("path %s not found: %w", absolutePath, err)	×
UNCOV 269	}	×
270
271	absolutePath = realPath	×
272	}
273
UNCOV 274	return absolutePath, nil	×
275	}

numtide / treefmt / 15227078419

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous