16730909849

Committed 04 Aug 2025 06:16PM UTC coverage: 53.864%. First build

Build # 16730909849

Build Type

Pull #633

github

Committed by

web-flow

Commit Message

Merge dfcbe82ee into 10165a1bb

Pull Request Pull Request #633: Draft: Stuber query v2

Run Details

198 of 438 new or added lines in 5 files covered. (45.21%)

1366 of 2536 relevant lines covered (53.86%)

18.18 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

47.65

/internal/infra/storage/stubs.go

package storage

import (
        "context"
        "os"
        "path"
        "strings"
        "sync"
        "sync/atomic"

        "github.com/cockroachdb/errors"
        "github.com/google/uuid"
        "github.com/gripmock/stuber"
        "github.com/rs/zerolog"
        "github.com/samber/lo"

        "github.com/bavix/gripmock/v3/internal/infra/jsondecoder"
        "github.com/bavix/gripmock/v3/internal/infra/watcher"
        "github.com/bavix/gripmock/v3/internal/infra/yaml2json"
)

type Extender struct {
        storage      *stuber.Budgerigar
        convertor    *yaml2json.Convertor
        ch           chan struct{}
        watcher      *watcher.StubWatcher
        mapIDsByFile map[string]uuid.UUIDs
        muUniqueIDs  sync.Mutex
        uniqueIDs    map[uuid.UUID]struct{}
        loaded       atomic.Bool
}

func NewStub(
        storage *stuber.Budgerigar,
        convertor *yaml2json.Convertor,
        watcher *watcher.StubWatcher,
) *Extender {
        return &Extender{
                storage:      storage,
                convertor:    convertor,
                ch:           make(chan struct{}),
                watcher:      watcher,
                mapIDsByFile: make(map[string]uuid.UUIDs),
                uniqueIDs:    make(map[uuid.UUID]struct{}),
                loaded:       atomic.Bool{},
        }
}

func (s *Extender) Wait(ctx context.Context) {
        select {
        case <-ctx.Done():
                return
        case <-s.ch:
                s.loaded.Store(true)
        }
}

func (s *Extender) ReadFromPath(ctx context.Context, pathDir string) {
        if pathDir == "" {
                close(s.ch)

                return
        }

        zerolog.Ctx(ctx).Info().Msg("Loading stubs from directory (preserving API stubs)")

        s.readFromPath(ctx, pathDir)
        close(s.ch)

        // Only watch directories, not individual files
        if isDirectory(pathDir) {
                ch, err := s.watcher.Watch(ctx, pathDir)
                if err != nil {
                        return
                }

                for file := range ch {
                        zerolog.Ctx(ctx).
                                Debug().
                                Str("path", file).
                                Msg("Updating stub")

                        s.readByFile(ctx, file)
                }
        }
}

// readFromPath reads all the stubs from the given directory and its subdirectories,
// or from a single file if a file path is provided.
// The stub files can be in yaml or json format.
// If a file is in yaml format, it will be converted to json format.
//
//nolint:cyclop
func (s *Extender) readFromPath(ctx context.Context, pathDir string) {
        // Check if the path is a file or directory
        if !isDirectory(pathDir) {
                // It's a file, check if it's a stub file
                if strings.HasSuffix(pathDir, ".json") ||
                        strings.HasSuffix(pathDir, ".yaml") ||
                        strings.HasSuffix(pathDir, ".yml") {
                        s.readByFile(ctx, pathDir)
                }

                return
        }

        // It's a directory, read all files recursively
        files, err := os.ReadDir(pathDir)
        if err != nil {
                zerolog.Ctx(ctx).
                        Err(err).Str("path", pathDir).
                        Msg("read directory")

                return
        }

        for _, file := range files {
                if file.IsDir() {
                        s.readFromPath(ctx, path.Join(pathDir, file.Name()))

                        continue
                }

                // If the file is not a stub file, skip it.
                if !strings.HasSuffix(file.Name(), ".json") &&
                        !strings.HasSuffix(file.Name(), ".yaml") &&
                        !strings.HasSuffix(file.Name(), ".yml") {
                        continue
                }

                s.readByFile(ctx, path.Join(pathDir, file.Name()))
        }
}

//nolint:cyclop
func (s *Extender) readByFile(ctx context.Context, filePath string) {
        stubs, err := s.readStub(filePath)
        if err != nil {
                zerolog.Ctx(ctx).
                        Err(err).
                        Str("file", filePath).
                        Msg("failed to read file")

                // Remove existing stubs from this file if it was previously loaded
                if existingIDs, exists := s.mapIDsByFile[filePath]; exists {
                        s.storage.DeleteByID(existingIDs...)
                        delete(s.mapIDsByFile, filePath)
                }

                return
        }

        s.checkUniqIDs(ctx, filePath, stubs)

        existingIDs, exists := s.mapIDsByFile[filePath]
        if !exists {
                // First time loading this file - generate new IDs for stubs without them
                for _, stub := range stubs {
                        if stub.ID == uuid.Nil {
                                stub.ID = uuid.New()
                        }
                }

                s.mapIDsByFile[filePath] = s.storage.PutMany(stubs...)

                return
        }

        // File was previously loaded - handle ID reuse logic
        currentIDs := make(uuid.UUIDs, 0, len(stubs))
        for _, stub := range stubs {
                if stub.ID != uuid.Nil {
                        currentIDs = append(currentIDs, stub.ID)
                }
        }

        // Find IDs that are no longer used in this file
        unusedIDs := lo.Without(existingIDs, currentIDs...)
        newIDs := make(uuid.UUIDs, 0, len(stubs))

        // Generate IDs for stubs that don't have them, reusing unused IDs first
        for _, stub := range stubs {
                if stub.ID == uuid.Nil {
                        stub.ID, unusedIDs = genID(stub, unusedIDs)
                }

                newIDs = append(newIDs, stub.ID)
        }

        // Remove stubs that are no longer in the file
        if removedIDs := lo.Without(existingIDs, newIDs...); len(removedIDs) > 0 {
                s.storage.DeleteByID(removedIDs...)
        }

        // Add/update stubs and update file mapping
        if len(stubs) > 0 {
                s.mapIDsByFile[filePath] = s.storage.PutMany(stubs...)
        } else {
                delete(s.mapIDsByFile, filePath)
        }
}

// checkUniqIDs checks for unique IDs in the provided stubs.
// It logs a warning if a duplicate ID is found.
// If the Extender has already been loaded, it skips the check.
func (s *Extender) checkUniqIDs(ctx context.Context, filePath string, stubs []*stuber.Stub) {
        // If the Extender is already loaded, no need to check for unique IDs.
        if s.loaded.Load() {
                return
        }

        // The mutex is not needed now, but it may be useful in the future.
        // Lock the mutex to prevent concurrent access to the uniqIDs map.
        s.muUniqueIDs.Lock()
        defer s.muUniqueIDs.Unlock()

        // Iterate over each stub to verify uniqueness of IDs.
        for _, stub := range stubs {
                // Skip stubs without an ID.
                if stub.ID == uuid.Nil {
                        continue
                }

                // Check if the ID already exists in the uniqIDs map.
                if _, exists := s.uniqueIDs[stub.ID]; exists {
                        // Log a warning if a duplicate ID is found.
                        zerolog.Ctx(ctx).
                                Warn().
                                Str("file", filePath).
                                Str("id", stub.ID.String()).
                                Msg("duplicate stub ID")
                }

                // Mark the stub ID as seen by adding it to the uniqIDs map.
                s.uniqueIDs[stub.ID] = struct{}{}
        }
}

// genID generates a new ID for a stub if it does not already have one.
// It also returns the remaining free IDs after generating the new ID.
func genID(stub *stuber.Stub, freeIDs uuid.UUIDs) (uuid.UUID, uuid.UUIDs) {
        // If the stub already has an ID, return it.
        if stub.ID != uuid.Nil {
                return stub.ID, freeIDs
        }

        // If there are free IDs, use the first one.
        if len(freeIDs) > 0 {
                return freeIDs[0], freeIDs[1:]
        }

        // Otherwise, generate a new ID.
        return uuid.New(), nil
}

// readStub reads a stub file and returns a slice of stubs.
// The stub file can be in yaml or json format.
// If the file is in yaml format, it will be converted to json format.
func (s *Extender) readStub(path string) ([]*stuber.Stub, error) {
        file, err := os.ReadFile(path) //nolint:gosec
        if err != nil {
                return nil, errors.Wrapf(err, "failed to read file %s", path)
        }

        if strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml") {
                file, err = s.convertor.Execute(path, file)
                if err != nil {
                        return nil, errors.Wrapf(err, "failed to unmarshal file %s", path)
                }
        }

        var stubs []*stuber.Stub
        if err := jsondecoder.UnmarshalSlice(file, &stubs); err != nil {
                return nil, errors.Wrapf(err, "failed to unmarshal file %s: %v", path, string(file))
        }

        return stubs, nil
}

// isDirectory checks if the given path is a directory.
func isDirectory(path string) bool {
        info, err := os.Stat(path)
        if err != nil {
                return false
        }

        return info.IsDir()
}

1	package storage
2
3	import (
4	"context"
5	"os"
6	"path"
7	"strings"
8	"sync"
9	"sync/atomic"
10
11	"github.com/cockroachdb/errors"
12	"github.com/google/uuid"
13	"github.com/gripmock/stuber"
14	"github.com/rs/zerolog"
15	"github.com/samber/lo"
16
17	"github.com/bavix/gripmock/v3/internal/infra/jsondecoder"
18	"github.com/bavix/gripmock/v3/internal/infra/watcher"
19	"github.com/bavix/gripmock/v3/internal/infra/yaml2json"
20	)
21
22	type Extender struct {
23	storage *stuber.Budgerigar
24	convertor *yaml2json.Convertor
25	ch chan struct{}
26	watcher *watcher.StubWatcher
27	mapIDsByFile map[string]uuid.UUIDs
28	muUniqueIDs sync.Mutex
29	uniqueIDs map[uuid.UUID]struct{}
30	loaded atomic.Bool
31	}
32
33	func NewStub(
34	storage *stuber.Budgerigar,
35	convertor *yaml2json.Convertor,
36	watcher *watcher.StubWatcher,
37	) *Extender {	1✔
38	return &Extender{	1✔
39	storage: storage,	1✔
40	convertor: convertor,	1✔
41	ch: make(chan struct{}),	1✔
42	watcher: watcher,	1✔
43	mapIDsByFile: make(map[string]uuid.UUIDs),	1✔
44	uniqueIDs: make(map[uuid.UUID]struct{}),	1✔
45	loaded: atomic.Bool{},	1✔
46	}	1✔
47	}	1✔
48
49	func (s *Extender) Wait(ctx context.Context) {	2✔
50	select {	2✔
51	case <-ctx.Done():	×
52	return	×
53	case <-s.ch:	2✔
54	s.loaded.Store(true)	2✔
55	}
56	}
57
58	func (s *Extender) ReadFromPath(ctx context.Context, pathDir string) {	1✔
59	if pathDir == "" {	1✔
60	close(s.ch)	×
61		×
62	return	×
63	}	×
64
65	zerolog.Ctx(ctx).Info().Msg("Loading stubs from directory (preserving API stubs)")	1✔
66		1✔
67	s.readFromPath(ctx, pathDir)	1✔
68	close(s.ch)	1✔
69		1✔
70	// Only watch directories, not individual files	1✔
71	if isDirectory(pathDir) {	2✔
72	ch, err := s.watcher.Watch(ctx, pathDir)	1✔
73	if err != nil {	1✔
NEW 74	return	×
NEW 75	}	×
76
77	for file := range ch {	1✔
NEW 78	zerolog.Ctx(ctx).	×
NEW 79	Debug().	×
NEW 80	Str("path", file).	×
NEW 81	Msg("Updating stub")	×
82		×
NEW 83	s.readByFile(ctx, file)	×
NEW 84	}	×
85	}
86	}
87
88	// readFromPath reads all the stubs from the given directory and its subdirectories,
89	// or from a single file if a file path is provided.
90	// The stub files can be in yaml or json format.
91	// If a file is in yaml format, it will be converted to json format.
92	//
93	//nolint:cyclop
94	func (s *Extender) readFromPath(ctx context.Context, pathDir string) {	30✔
95	// Check if the path is a file or directory	30✔
96	if !isDirectory(pathDir) {	30✔
NEW 97	// It's a file, check if it's a stub file	×
NEW 98	if strings.HasSuffix(pathDir, ".json") \|\|	×
NEW 99	strings.HasSuffix(pathDir, ".yaml") \|\|	×
NEW 100	strings.HasSuffix(pathDir, ".yml") {	×
NEW 101	s.readByFile(ctx, pathDir)	×
NEW 102	}	×
103
NEW 104	return	×
105	}
106
107	// It's a directory, read all files recursively
108	files, err := os.ReadDir(pathDir)	30✔
109	if err != nil {	30✔
110	zerolog.Ctx(ctx).	×
111	Err(err).Str("path", pathDir).	×
112	Msg("read directory")	×
113		×
114	return	×
115	}	×
116
117	for _, file := range files {	223✔
118	if file.IsDir() {	222✔
119	s.readFromPath(ctx, path.Join(pathDir, file.Name()))	29✔
120		29✔
121	continue	29✔
122	}
123
124	// If the file is not a stub file, skip it.
125	if !strings.HasSuffix(file.Name(), ".json") &&	164✔
126	!strings.HasSuffix(file.Name(), ".yaml") &&	164✔
127	!strings.HasSuffix(file.Name(), ".yml") {	280✔
128	continue	116✔
129	}
130
131	s.readByFile(ctx, path.Join(pathDir, file.Name()))	48✔
132	}
133	}
134
135	//nolint:cyclop
136	func (s *Extender) readByFile(ctx context.Context, filePath string) {	48✔
137	stubs, err := s.readStub(filePath)	48✔
138	if err != nil {	48✔
139	zerolog.Ctx(ctx).	×
140	Err(err).	×
141	Str("file", filePath).	×
142	Msg("failed to read file")	×
143		×
NEW 144	// Remove existing stubs from this file if it was previously loaded	×
145	if existingIDs, exists := s.mapIDsByFile[filePath]; exists {	×
146	s.storage.DeleteByID(existingIDs...)	×
147	delete(s.mapIDsByFile, filePath)	×
148	}	×
149
150	return	×
151	}
152
153	s.checkUniqIDs(ctx, filePath, stubs)	48✔
154		48✔
155	existingIDs, exists := s.mapIDsByFile[filePath]	48✔
156	if !exists {	96✔
157	// First time loading this file - generate new IDs for stubs without them	48✔
158	for _, stub := range stubs {	119✔
159	if stub.ID == uuid.Nil {	142✔
160	stub.ID = uuid.New()	71✔
161	}	71✔
162	}
163
164	s.mapIDsByFile[filePath] = s.storage.PutMany(stubs...)	48✔
165		48✔
166	return	48✔
167	}
168
169	// File was previously loaded - handle ID reuse logic
170	currentIDs := make(uuid.UUIDs, 0, len(stubs))	×
171	for _, stub := range stubs {	×
172	if stub.ID != uuid.Nil {	×
173	currentIDs = append(currentIDs, stub.ID)	×
174	}	×
175	}
176
177	// Find IDs that are no longer used in this file
178	unusedIDs := lo.Without(existingIDs, currentIDs...)	×
179	newIDs := make(uuid.UUIDs, 0, len(stubs))	×
180		×
NEW 181	// Generate IDs for stubs that don't have them, reusing unused IDs first	×
182	for _, stub := range stubs {	×
183	if stub.ID == uuid.Nil {	×
184	stub.ID, unusedIDs = genID(stub, unusedIDs)	×
185	}	×
186
187	newIDs = append(newIDs, stub.ID)	×
188	}
189
190	// Remove stubs that are no longer in the file
191	if removedIDs := lo.Without(existingIDs, newIDs...); len(removedIDs) > 0 {	×
192	s.storage.DeleteByID(removedIDs...)	×
193	}	×
194
195	// Add/update stubs and update file mapping
196	if len(stubs) > 0 {	×
197	s.mapIDsByFile[filePath] = s.storage.PutMany(stubs...)	×
NEW 198	} else {	×
NEW 199	delete(s.mapIDsByFile, filePath)	×
200	}	×
201	}
202
203	// checkUniqIDs checks for unique IDs in the provided stubs.
204	// It logs a warning if a duplicate ID is found.
205	// If the Extender has already been loaded, it skips the check.
206	func (s Extender) checkUniqIDs(ctx context.Context, filePath string, stubs []stuber.Stub) {	48✔
207	// If the Extender is already loaded, no need to check for unique IDs.	48✔
208	if s.loaded.Load() {	48✔
209	return	×
210	}	×
211
212	// The mutex is not needed now, but it may be useful in the future.
213	// Lock the mutex to prevent concurrent access to the uniqIDs map.
214	s.muUniqueIDs.Lock()	48✔
215	defer s.muUniqueIDs.Unlock()	48✔
216		48✔
217	// Iterate over each stub to verify uniqueness of IDs.	48✔
218	for _, stub := range stubs {	119✔
219	// Skip stubs without an ID.	71✔
220	if stub.ID == uuid.Nil {	142✔
221	continue	71✔
222	}
223
224	// Check if the ID already exists in the uniqIDs map.
225	if _, exists := s.uniqueIDs[stub.ID]; exists {	×
226	// Log a warning if a duplicate ID is found.	×
227	zerolog.Ctx(ctx).	×
228	Warn().	×
229	Str("file", filePath).	×
230	Str("id", stub.ID.String()).	×
231	Msg("duplicate stub ID")	×
232	}	×
233
234	// Mark the stub ID as seen by adding it to the uniqIDs map.
235	s.uniqueIDs[stub.ID] = struct{}{}	×
236	}
237	}
238
239	// genID generates a new ID for a stub if it does not already have one.
240	// It also returns the remaining free IDs after generating the new ID.
241	func genID(stub *stuber.Stub, freeIDs uuid.UUIDs) (uuid.UUID, uuid.UUIDs) {	×
242	// If the stub already has an ID, return it.	×
243	if stub.ID != uuid.Nil {	×
244	return stub.ID, freeIDs	×
245	}	×
246
247	// If there are free IDs, use the first one.
248	if len(freeIDs) > 0 {	×
249	return freeIDs[0], freeIDs[1:]	×
250	}	×
251
252	// Otherwise, generate a new ID.
253	return uuid.New(), nil	×
254	}
255
256	// readStub reads a stub file and returns a slice of stubs.
257	// The stub file can be in yaml or json format.
258	// If the file is in yaml format, it will be converted to json format.
259	func (s Extender) readStub(path string) ([]stuber.Stub, error) {	48✔
260	file, err := os.ReadFile(path) //nolint:gosec	48✔
261	if err != nil {	48✔
262	return nil, errors.Wrapf(err, "failed to read file %s", path)	×
263	}	×
264
265	if strings.HasSuffix(path, ".yaml") \|\| strings.HasSuffix(path, ".yml") {	93✔
266	file, err = s.convertor.Execute(path, file)	45✔
267	if err != nil {	45✔
268	return nil, errors.Wrapf(err, "failed to unmarshal file %s", path)	×
269	}	×
270	}
271
272	var stubs []*stuber.Stub	48✔
273	if err := jsondecoder.UnmarshalSlice(file, &stubs); err != nil {	48✔
274	return nil, errors.Wrapf(err, "failed to unmarshal file %s: %v", path, string(file))	×
275	}	×
276
277	return stubs, nil	48✔
278	}
279
280	// isDirectory checks if the given path is a directory.
281	func isDirectory(path string) bool {	31✔
282	info, err := os.Stat(path)	31✔
283	if err != nil {	31✔
NEW 284	return false	×
NEW 285	}	×
286
287	return info.IsDir()	31✔
288	}

bavix / gripmock / 16730909849

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous