17455371724

Committed 04 Sep 2025 06:29AM UTC coverage: 64.156% (-0.8%) from 64.935%

Build # 17455371724

Build Type

push

github

Committed by

web-flow

Commit Message

feat: Improve refs garbage collection for redis Redis using TTL (#983)

* feat: add merge request handle to garbage collect closed mr refs

* refactor: simplify the code

we already have a deleteRef function

* fix: if no pipeline found check for merge results pipeline

* fix: add missing return case when no pipeline found

* feat: add ttl on redis field

only redis 7.4 support expire on hashmap thus i had to make a workaround using keys

* fix: continue if key has expired

* chore: better debug for garbage collection

* fix: debug log reporting wrong progress

* fix: a couple issue with the rebase

Run Details

42 of 138 new or added lines in 7 files covered. (30.43%)

1 existing line in 1 file now uncovered.

3671 of 5722 relevant lines covered (64.16%)

3.76 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

25.36

/pkg/controller/webhooks.go

package controller

import (
        "context"
        "fmt"
        "regexp"
        "strconv"
        "strings"

        log "github.com/sirupsen/logrus"
        goGitlab "gitlab.com/gitlab-org/api/client-go"

        "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/config"
        "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas"
)

func (c *Controller) processPipelineEvent(ctx context.Context, e goGitlab.PipelineEvent) {
        var (
                refKind schemas.RefKind
                refName = e.ObjectAttributes.Ref
        )

        // TODO: Perhaps it would be nice to match upon the regexp to validate
        // that it is actually a merge request ref
        if e.MergeRequest.IID != 0 {
                refKind = schemas.RefKindMergeRequest
                refName = strconv.Itoa(e.MergeRequest.IID)
        } else if e.ObjectAttributes.Tag {
                refKind = schemas.RefKindTag
        } else {
                refKind = schemas.RefKindBranch
        }

        c.triggerRefMetricsPull(ctx, schemas.NewRef(
                schemas.NewProject(e.Project.PathWithNamespace),
                refKind,
                refName,
        ))
}

func (c *Controller) processJobEvent(ctx context.Context, e goGitlab.JobEvent) {
        var (
                refKind schemas.RefKind
                refName = e.Ref
        )

        if e.Tag {
                refKind = schemas.RefKindTag
        } else {
                refKind = schemas.RefKindBranch
        }

        project, _, err := c.Gitlab.Projects.GetProject(e.ProjectID, nil)
        if err != nil {
                log.WithContext(ctx).
                        WithError(err).
                        Error("reading project from GitLab")

                return
        }

        c.triggerRefMetricsPull(ctx, schemas.NewRef(
                schemas.NewProject(project.PathWithNamespace),
                refKind,
                refName,
        ))
}

func (c *Controller) processPushEvent(ctx context.Context, e goGitlab.PushEvent) {
        if e.CheckoutSHA == "" {
                var (
                        refKind = schemas.RefKindBranch
                        refName string
                )

                // branch refs in push events have "refs/heads/" prefix
                if branch, found := strings.CutPrefix(e.Ref, "refs/heads/"); found {
                        refName = branch
                } else {
                        log.WithContext(ctx).
                                WithFields(log.Fields{
                                        "project-name": e.Project.Name,
                                        "ref":          e.Ref,
                                }).
                                Error("extracting branch name from ref")

                        return
                }

                _ = deleteRef(ctx, c.Store, schemas.NewRef(
                        schemas.NewProject(e.Project.PathWithNamespace),
                        refKind,
                        refName,
                ), "received branch deletion push event from webhook")
        }
}

func (c *Controller) processTagEvent(ctx context.Context, e goGitlab.TagEvent) {
        if e.CheckoutSHA == "" {
                var (
                        refKind = schemas.RefKindTag
                        refName string
                )

                // tags refs in tag events have "refs/tags/" prefix
                if tag, found := strings.CutPrefix(e.Ref, "refs/tags/"); found {
                        refName = tag
                } else {
                        log.WithContext(ctx).
                                WithFields(log.Fields{
                                        "project-name": e.Project.Name,
                                        "ref":          e.Ref,
                                }).
                                Error("extracting tag name from ref")

                        return
                }

                _ = deleteRef(ctx, c.Store, schemas.NewRef(
                        schemas.NewProject(e.Project.PathWithNamespace),
                        refKind,
                        refName,
                ), "received tag deletion tag event from webhook")
        }
}

func (c *Controller) processMergeEvent(ctx context.Context, e goGitlab.MergeEvent) {
        ref := schemas.NewRef(
                schemas.NewProject(e.Project.PathWithNamespace),
                schemas.RefKindMergeRequest,
                strconv.Itoa(e.ObjectAttributes.IID),
        )

        switch e.ObjectAttributes.Action {
        case "close":
                c.triggerRefDeletion(ctx, ref)
        case "merge":
                c.triggerRefDeletion(ctx, ref)
        default:
                log.
                        WithField("merge-request-event-type", e.ObjectAttributes.Action).
                        Debug("received a non supported merge-request event type as a webhook")
        }
}

func (c *Controller) triggerRefDeletion(ctx context.Context, ref schemas.Ref) {
        err := c.Store.DelRef(ctx, ref.Key())
        if err != nil {
                log.WithContext(ctx).
                        WithFields(log.Fields{
                                "project-name": ref.Project.Name,
                                "ref":          ref.Name,
                        }).
                        Error("failed deleting ref")
        }
}

func (c *Controller) triggerRefMetricsPull(ctx context.Context, ref schemas.Ref) {
        logFields := log.Fields{
                "project-name": ref.Project.Name,
                "ref":          ref.Name,
                "ref-kind":     ref.Kind,
        }

        refExists, err := c.Store.RefExists(ctx, ref.Key())
        if err != nil {
                log.WithContext(ctx).
                        WithFields(logFields).
                        WithError(err).
                        Error("reading ref from the store")

                return
        }

        // Let's try to see if the project is configured to export this ref
        if !refExists {
                p := schemas.NewProject(ref.Project.Name)

                projectExists, err := c.Store.ProjectExists(ctx, p.Key())
                if err != nil {
                        log.WithContext(ctx).
                                WithFields(logFields).
                                WithError(err).
                                Error("reading project from the store")

                        return
                }

                // Perhaps the project is discoverable through a wildcard
                if !projectExists && len(c.Config.Wildcards) > 0 {
                        for _, w := range c.Config.Wildcards {
                                // If in all our wildcards we have one which can potentially match the project ref
                                // received, we trigger a pull of the project
                                matches, err := isRefMatchingWilcard(w, ref)
                                if err != nil {
                                        log.WithContext(ctx).
                                                WithError(err).
                                                Warn("checking if the ref matches the wildcard config")

                                        continue
                                }

                                if matches {
                                        c.ScheduleTask(context.TODO(), schemas.TaskTypePullProject, ref.Project.Name, ref.Project.Name, w.Pull)
                                        log.WithFields(logFields).Info("project ref not currently exported but its configuration matches a wildcard, triggering a pull of the project")
                                } else {
                                        log.WithFields(logFields).Debug("project ref not matching wildcard, skipping..")
                                }
                        }

                        log.WithFields(logFields).Info("done looking up for wildcards matching the project ref")

                        return
                }

                if projectExists {
                        // If the project exists, we check that the ref matches it's configuration
                        if err := c.Store.GetProject(ctx, &p); err != nil {
                                log.WithContext(ctx).
                                        WithFields(logFields).
                                        WithError(err).
                                        Error("reading project from the store")

                                return
                        }

                        matches, err := isRefMatchingProjectPullRefs(p.Pull.Refs, ref)
                        if err != nil {
                                log.WithContext(ctx).
                                        WithError(err).
                                        Error("checking if the ref matches the project config")

                                return
                        }

                        if matches {
                                ref.Project = p

                                if err = c.Store.SetRef(ctx, ref); err != nil {
                                        log.WithContext(ctx).
                                                WithFields(logFields).
                                                WithError(err).
                                                Error("writing ref in the store")

                                        return
                                }

                                goto schedulePull
                        }
                }

                log.WithFields(logFields).Info("ref not configured in the exporter, ignoring pipeline webhook")

                return
        }

schedulePull:
        log.WithFields(logFields).Info("received a pipeline webhook from GitLab for a ref, triggering metrics pull")
        // TODO: When all the metrics will be sent over the webhook, we might be able to avoid redoing a pull
        // eg: 'coverage' is not in the pipeline payload yet, neither is 'artifacts' in the job one
        c.ScheduleTask(context.TODO(), schemas.TaskTypePullRefMetrics, string(ref.Key()), ref)
}

func (c *Controller) processDeploymentEvent(ctx context.Context, e goGitlab.DeploymentEvent) {
        c.triggerEnvironmentMetricsPull(
                ctx,
                schemas.Environment{
                        ProjectName: e.Project.PathWithNamespace,
                        Name:        e.Environment,
                },
        )
}

func (c *Controller) triggerEnvironmentMetricsPull(ctx context.Context, env schemas.Environment) {
        logFields := log.Fields{
                "project-name":     env.ProjectName,
                "environment-name": env.Name,
        }

        envExists, err := c.Store.EnvironmentExists(ctx, env.Key())
        if err != nil {
                log.WithContext(ctx).
                        WithFields(logFields).
                        WithError(err).
                        Error("reading environment from the store")

                return
        }

        if !envExists {
                p := schemas.NewProject(env.ProjectName)

                projectExists, err := c.Store.ProjectExists(ctx, p.Key())
                if err != nil {
                        log.WithContext(ctx).
                                WithFields(logFields).
                                WithError(err).
                                Error("reading project from the store")

                        return
                }

                // Perhaps the project is discoverable through a wildcard
                if !projectExists && len(c.Config.Wildcards) > 0 {
                        for _, w := range c.Config.Wildcards {
                                // If in all our wildcards we have one which can potentially match the env
                                // received, we trigger a pull of the project
                                matches, err := isEnvMatchingWilcard(w, env)
                                if err != nil {
                                        log.WithContext(ctx).
                                                WithError(err).
                                                Warn("checking if the env matches the wildcard config")

                                        continue
                                }

                                if matches {
                                        c.ScheduleTask(context.TODO(), schemas.TaskTypePullProject, env.ProjectName, env.ProjectName, w.Pull)
                                        log.WithFields(logFields).Info("project environment not currently exported but its configuration matches a wildcard, triggering a pull of the project")
                                } else {
                                        log.WithFields(logFields).Debug("project ref not matching wildcard, skipping..")
                                }
                        }

                        log.WithFields(logFields).Info("done looking up for wildcards matching the project ref")

                        return
                }

                if projectExists {
                        if err := c.Store.GetProject(ctx, &p); err != nil {
                                log.WithContext(ctx).
                                        WithFields(logFields).
                                        WithError(err).
                                        Error("reading project from the store")
                        }

                        matches, err := isEnvMatchingProjectPullEnvironments(p.Pull.Environments, env)
                        if err != nil {
                                log.WithContext(ctx).
                                        WithError(err).
                                        Error("checking if the env matches the project config")

                                return
                        }

                        if matches {
                                // As we do not get the environment ID within the deployment event, we need to query it back..
                                if err = c.UpdateEnvironment(ctx, &env); err != nil {
                                        log.WithContext(ctx).
                                                WithFields(logFields).
                                                WithError(err).
                                                Error("updating event from GitLab API")

                                        return
                                }

                                goto schedulePull
                        }
                }

                log.WithFields(logFields).
                        Info("environment not configured in the exporter, ignoring deployment webhook")

                return
        }

        // Need to refresh the env from the store in order to get at least it's ID
        if env.ID == 0 {
                if err = c.Store.GetEnvironment(ctx, &env); err != nil {
                        log.WithContext(ctx).
                                WithFields(logFields).
                                WithError(err).
                                Error("reading environment from the store")
                }
        }

schedulePull:
        log.WithFields(logFields).Info("received a deployment webhook from GitLab for an environment, triggering metrics pull")
        c.ScheduleTask(ctx, schemas.TaskTypePullEnvironmentMetrics, string(env.Key()), env)
}

func isRefMatchingProjectPullRefs(pprs config.ProjectPullRefs, ref schemas.Ref) (matches bool, err error) {
        // We check if the ref kind is enabled
        switch ref.Kind {
        case schemas.RefKindBranch:
                if !pprs.Branches.Enabled {
                        return
                }
        case schemas.RefKindTag:
                if !pprs.Tags.Enabled {
                        return
                }
        case schemas.RefKindMergeRequest:
                if !pprs.MergeRequests.Enabled {
                        return
                }
        default:
                return false, fmt.Errorf("invalid ref kind %v", ref.Kind)
        }

        // Then we check if it matches the regexp
        var re *regexp.Regexp

        if re, err = schemas.GetRefRegexp(pprs, ref.Kind); err != nil {
                return
        }

        return re.MatchString(ref.Name), nil
}

func isEnvMatchingProjectPullEnvironments(ppe config.ProjectPullEnvironments, env schemas.Environment) (matches bool, err error) {
        // We check if the environments pulling is enabled
        if !ppe.Enabled {
                return
        }

        // Then we check if it matches the regexp
        var re *regexp.Regexp

        if re, err = regexp.Compile(ppe.Regexp); err != nil {
                return
        }

        return re.MatchString(env.Name), nil
}

func isRefMatchingWilcard(w config.Wildcard, ref schemas.Ref) (matches bool, err error) {
        // Then we check if the owner matches the ref or is global
        if w.Owner.Kind != "" && !strings.Contains(ref.Project.Name, w.Owner.Name) {
                return
        }

        // Then we check if the ref matches the project pull parameters
        return isRefMatchingProjectPullRefs(w.Pull.Refs, ref)
}

func isEnvMatchingWilcard(w config.Wildcard, env schemas.Environment) (matches bool, err error) {
        // Then we check if the owner matches the ref or is global
        if w.Owner.Kind != "" && !strings.Contains(env.ProjectName, w.Owner.Name) {
                return
        }

        // Then we check if the ref matches the project pull parameters
        return isEnvMatchingProjectPullEnvironments(w.Pull.Environments, env)
}

1	package controller
2
3	import (
4	"context"
5	"fmt"
6	"regexp"
7	"strconv"
8	"strings"
9
10	log "github.com/sirupsen/logrus"
11	goGitlab "gitlab.com/gitlab-org/api/client-go"
12
13	"github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/config"
14	"github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas"
15	)
16
17	func (c *Controller) processPipelineEvent(ctx context.Context, e goGitlab.PipelineEvent) {	1✔
18	var (	1✔
19	refKind schemas.RefKind	1✔
20	refName = e.ObjectAttributes.Ref	1✔
21	)	1✔
22		1✔
23	// TODO: Perhaps it would be nice to match upon the regexp to validate	1✔
24	// that it is actually a merge request ref	1✔
25	if e.MergeRequest.IID != 0 {	1✔
26	refKind = schemas.RefKindMergeRequest	×
27	refName = strconv.Itoa(e.MergeRequest.IID)	×
28	} else if e.ObjectAttributes.Tag {	1✔
29	refKind = schemas.RefKindTag	×
30	} else {	1✔
31	refKind = schemas.RefKindBranch	1✔
32	}	1✔
33
34	c.triggerRefMetricsPull(ctx, schemas.NewRef(	1✔
35	schemas.NewProject(e.Project.PathWithNamespace),	1✔
36	refKind,	1✔
37	refName,	1✔
38	))	1✔
39	}
40
41	func (c *Controller) processJobEvent(ctx context.Context, e goGitlab.JobEvent) {	×
42	var (	×
43	refKind schemas.RefKind	×
44	refName = e.Ref	×
45	)	×
46		×
47	if e.Tag {	×
48	refKind = schemas.RefKindTag	×
49	} else {	×
50	refKind = schemas.RefKindBranch	×
51	}	×
52
53	project, _, err := c.Gitlab.Projects.GetProject(e.ProjectID, nil)	×
54	if err != nil {	×
55	log.WithContext(ctx).	×
56	WithError(err).	×
57	Error("reading project from GitLab")	×
58		×
59	return	×
60	}	×
61
62	c.triggerRefMetricsPull(ctx, schemas.NewRef(	×
63	schemas.NewProject(project.PathWithNamespace),	×
64	refKind,	×
65	refName,	×
66	))	×
67	}
68
69	func (c *Controller) processPushEvent(ctx context.Context, e goGitlab.PushEvent) {	×
70	if e.CheckoutSHA == "" {	×
71	var (	×
72	refKind = schemas.RefKindBranch	×
73	refName string	×
74	)	×
75		×
76	// branch refs in push events have "refs/heads/" prefix	×
77	if branch, found := strings.CutPrefix(e.Ref, "refs/heads/"); found {	×
78	refName = branch	×
79	} else {	×
80	log.WithContext(ctx).	×
81	WithFields(log.Fields{	×
82	"project-name": e.Project.Name,	×
83	"ref": e.Ref,	×
84	}).	×
85	Error("extracting branch name from ref")	×
86		×
87	return	×
88	}	×
89
90	_ = deleteRef(ctx, c.Store, schemas.NewRef(	×
91	schemas.NewProject(e.Project.PathWithNamespace),	×
92	refKind,	×
93	refName,	×
94	), "received branch deletion push event from webhook")	×
95	}
96	}
97
98	func (c *Controller) processTagEvent(ctx context.Context, e goGitlab.TagEvent) {	×
99	if e.CheckoutSHA == "" {	×
100	var (	×
101	refKind = schemas.RefKindTag	×
102	refName string	×
103	)	×
104		×
105	// tags refs in tag events have "refs/tags/" prefix	×
106	if tag, found := strings.CutPrefix(e.Ref, "refs/tags/"); found {	×
107	refName = tag	×
108	} else {	×
109	log.WithContext(ctx).	×
110	WithFields(log.Fields{	×
111	"project-name": e.Project.Name,	×
112	"ref": e.Ref,	×
113	}).	×
114	Error("extracting tag name from ref")	×
115		×
116	return	×
117	}	×
118
119	_ = deleteRef(ctx, c.Store, schemas.NewRef(	×
120	schemas.NewProject(e.Project.PathWithNamespace),	×
121	refKind,	×
122	refName,	×
123	), "received tag deletion tag event from webhook")	×
124	}
125	}
126
127	func (c *Controller) processMergeEvent(ctx context.Context, e goGitlab.MergeEvent) {	×
128	ref := schemas.NewRef(	×
129	schemas.NewProject(e.Project.PathWithNamespace),	×
130	schemas.RefKindMergeRequest,	×
131	strconv.Itoa(e.ObjectAttributes.IID),	×
132	)	×
133		×
134	switch e.ObjectAttributes.Action {	×
135	case "close":	×
NEW 136	c.triggerRefDeletion(ctx, ref)	×
137	case "merge":	×
NEW 138	c.triggerRefDeletion(ctx, ref)	×
139	default:	×
140	log.	×
141	WithField("merge-request-event-type", e.ObjectAttributes.Action).	×
142	Debug("received a non supported merge-request event type as a webhook")	×
143	}
144	}
145
NEW 146	func (c *Controller) triggerRefDeletion(ctx context.Context, ref schemas.Ref) {	×
NEW 147	err := c.Store.DelRef(ctx, ref.Key())	×
NEW 148	if err != nil {	×
NEW 149	log.WithContext(ctx).	×
NEW 150	WithFields(log.Fields{	×
NEW 151	"project-name": ref.Project.Name,	×
NEW 152	"ref": ref.Name,	×
NEW 153	}).	×
NEW 154	Error("failed deleting ref")	×
NEW 155	}	×
156	}
157
158	func (c *Controller) triggerRefMetricsPull(ctx context.Context, ref schemas.Ref) {	3✔
159	logFields := log.Fields{	3✔
160	"project-name": ref.Project.Name,	3✔
161	"ref": ref.Name,	3✔
162	"ref-kind": ref.Kind,	3✔
163	}	3✔
164		3✔
165	refExists, err := c.Store.RefExists(ctx, ref.Key())	3✔
166	if err != nil {	3✔
167	log.WithContext(ctx).	×
168	WithFields(logFields).	×
169	WithError(err).	×
170	Error("reading ref from the store")	×
171		×
172	return	×
173	}	×
174
175	// Let's try to see if the project is configured to export this ref
176	if !refExists {	5✔
177	p := schemas.NewProject(ref.Project.Name)	2✔
178		2✔
179	projectExists, err := c.Store.ProjectExists(ctx, p.Key())	2✔
180	if err != nil {	2✔
181	log.WithContext(ctx).	×
182	WithFields(logFields).	×
183	WithError(err).	×
184	Error("reading project from the store")	×
185		×
186	return	×
187	}	×
188
189	// Perhaps the project is discoverable through a wildcard
190	if !projectExists && len(c.Config.Wildcards) > 0 {	2✔
191	for _, w := range c.Config.Wildcards {	×
192	// If in all our wildcards we have one which can potentially match the project ref	×
193	// received, we trigger a pull of the project	×
194	matches, err := isRefMatchingWilcard(w, ref)	×
195	if err != nil {	×
196	log.WithContext(ctx).	×
197	WithError(err).	×
198	Warn("checking if the ref matches the wildcard config")	×
199		×
200	continue	×
201	}
202
203	if matches {	×
204	c.ScheduleTask(context.TODO(), schemas.TaskTypePullProject, ref.Project.Name, ref.Project.Name, w.Pull)	×
205	log.WithFields(logFields).Info("project ref not currently exported but its configuration matches a wildcard, triggering a pull of the project")	×
206	} else {	×
207	log.WithFields(logFields).Debug("project ref not matching wildcard, skipping..")	×
208	}	×
209	}
210
211	log.WithFields(logFields).Info("done looking up for wildcards matching the project ref")	×
212		×
213	return	×
214	}
215
216	if projectExists {	3✔
217	// If the project exists, we check that the ref matches it's configuration	1✔
218	if err := c.Store.GetProject(ctx, &p); err != nil {	1✔
219	log.WithContext(ctx).	×
220	WithFields(logFields).	×
221	WithError(err).	×
222	Error("reading project from the store")	×
223		×
224	return	×
225	}	×
226
227	matches, err := isRefMatchingProjectPullRefs(p.Pull.Refs, ref)	1✔
228	if err != nil {	2✔
229	log.WithContext(ctx).	1✔
230	WithError(err).	1✔
231	Error("checking if the ref matches the project config")	1✔
232		1✔
233	return	1✔
234	}	1✔
235
236	if matches {	×
237	ref.Project = p	×
238		×
239	if err = c.Store.SetRef(ctx, ref); err != nil {	×
240	log.WithContext(ctx).	×
241	WithFields(logFields).	×
242	WithError(err).	×
243	Error("writing ref in the store")	×
244		×
245	return	×
246	}	×
247
248	goto schedulePull	×
249	}
250	}
251
252	log.WithFields(logFields).Info("ref not configured in the exporter, ignoring pipeline webhook")	1✔
253		1✔
254	return	1✔
255	}
256
257	schedulePull:
258	log.WithFields(logFields).Info("received a pipeline webhook from GitLab for a ref, triggering metrics pull")	1✔
259	// TODO: When all the metrics will be sent over the webhook, we might be able to avoid redoing a pull	1✔
260	// eg: 'coverage' is not in the pipeline payload yet, neither is 'artifacts' in the job one	1✔
261	c.ScheduleTask(context.TODO(), schemas.TaskTypePullRefMetrics, string(ref.Key()), ref)	1✔
262	}
263
264	func (c *Controller) processDeploymentEvent(ctx context.Context, e goGitlab.DeploymentEvent) {	1✔
265	c.triggerEnvironmentMetricsPull(	1✔
266	ctx,	1✔
267	schemas.Environment{	1✔
268	ProjectName: e.Project.PathWithNamespace,	1✔
269	Name: e.Environment,	1✔
270	},	1✔
271	)	1✔
272	}	1✔
273
274	func (c *Controller) triggerEnvironmentMetricsPull(ctx context.Context, env schemas.Environment) {	3✔
275	logFields := log.Fields{	3✔
276	"project-name": env.ProjectName,	3✔
277	"environment-name": env.Name,	3✔
278	}	3✔
279		3✔
280	envExists, err := c.Store.EnvironmentExists(ctx, env.Key())	3✔
281	if err != nil {	3✔
282	log.WithContext(ctx).	×
283	WithFields(logFields).	×
284	WithError(err).	×
285	Error("reading environment from the store")	×
286		×
287	return	×
288	}	×
289
290	if !envExists {	4✔
291	p := schemas.NewProject(env.ProjectName)	1✔
292		1✔
293	projectExists, err := c.Store.ProjectExists(ctx, p.Key())	1✔
294	if err != nil {	1✔
295	log.WithContext(ctx).	×
296	WithFields(logFields).	×
297	WithError(err).	×
298	Error("reading project from the store")	×
299		×
300	return	×
301	}	×
302
303	// Perhaps the project is discoverable through a wildcard
304	if !projectExists && len(c.Config.Wildcards) > 0 {	1✔
305	for _, w := range c.Config.Wildcards {	×
306	// If in all our wildcards we have one which can potentially match the env	×
307	// received, we trigger a pull of the project	×
308	matches, err := isEnvMatchingWilcard(w, env)	×
309	if err != nil {	×
310	log.WithContext(ctx).	×
311	WithError(err).	×
312	Warn("checking if the env matches the wildcard config")	×
313		×
314	continue	×
315	}
316
317	if matches {	×
318	c.ScheduleTask(context.TODO(), schemas.TaskTypePullProject, env.ProjectName, env.ProjectName, w.Pull)	×
319	log.WithFields(logFields).Info("project environment not currently exported but its configuration matches a wildcard, triggering a pull of the project")	×
320	} else {	×
321	log.WithFields(logFields).Debug("project ref not matching wildcard, skipping..")	×
322	}	×
323	}
324
325	log.WithFields(logFields).Info("done looking up for wildcards matching the project ref")	×
326		×
327	return	×
328	}
329
330	if projectExists {	1✔
331	if err := c.Store.GetProject(ctx, &p); err != nil {	×
332	log.WithContext(ctx).	×
333	WithFields(logFields).	×
334	WithError(err).	×
335	Error("reading project from the store")	×
336	}	×
337
338	matches, err := isEnvMatchingProjectPullEnvironments(p.Pull.Environments, env)	×
339	if err != nil {	×
340	log.WithContext(ctx).	×
341	WithError(err).	×
342	Error("checking if the env matches the project config")	×
343		×
344	return	×
345	}	×
346
347	if matches {	×
348	// As we do not get the environment ID within the deployment event, we need to query it back..	×
349	if err = c.UpdateEnvironment(ctx, &env); err != nil {	×
350	log.WithContext(ctx).	×
351	WithFields(logFields).	×
352	WithError(err).	×
353	Error("updating event from GitLab API")	×
354		×
355	return	×
356	}	×
357
358	goto schedulePull	×
359	}
360	}
361
362	log.WithFields(logFields).	1✔
363	Info("environment not configured in the exporter, ignoring deployment webhook")	1✔
364		1✔
365	return	1✔
366	}
367
368	// Need to refresh the env from the store in order to get at least it's ID
369	if env.ID == 0 {	4✔
370	if err = c.Store.GetEnvironment(ctx, &env); err != nil {	2✔
371	log.WithContext(ctx).	×
372	WithFields(logFields).	×
373	WithError(err).	×
374	Error("reading environment from the store")	×
375	}	×
376	}
377
378	schedulePull:
379	log.WithFields(logFields).Info("received a deployment webhook from GitLab for an environment, triggering metrics pull")	2✔
380	c.ScheduleTask(ctx, schemas.TaskTypePullEnvironmentMetrics, string(env.Key()), env)	2✔
381	}
382
383	func isRefMatchingProjectPullRefs(pprs config.ProjectPullRefs, ref schemas.Ref) (matches bool, err error) {	1✔
384	// We check if the ref kind is enabled	1✔
385	switch ref.Kind {	1✔
386	case schemas.RefKindBranch:	×
387	if !pprs.Branches.Enabled {	×
388	return	×
389	}	×
390	case schemas.RefKindTag:	×
391	if !pprs.Tags.Enabled {	×
392	return	×
393	}	×
394	case schemas.RefKindMergeRequest:	×
395	if !pprs.MergeRequests.Enabled {	×
396	return	×
397	}	×
398	default:	1✔
399	return false, fmt.Errorf("invalid ref kind %v", ref.Kind)	1✔
400	}
401
402	// Then we check if it matches the regexp
403	var re *regexp.Regexp	×
404		×
405	if re, err = schemas.GetRefRegexp(pprs, ref.Kind); err != nil {	×
406	return	×
407	}	×
408
409	return re.MatchString(ref.Name), nil	×
410	}
411
412	func isEnvMatchingProjectPullEnvironments(ppe config.ProjectPullEnvironments, env schemas.Environment) (matches bool, err error) {	×
413	// We check if the environments pulling is enabled	×
414	if !ppe.Enabled {	×
415	return	×
416	}	×
417
418	// Then we check if it matches the regexp
419	var re *regexp.Regexp	×
420		×
421	if re, err = regexp.Compile(ppe.Regexp); err != nil {	×
422	return	×
423	}	×
424
425	return re.MatchString(env.Name), nil	×
426	}
427
428	func isRefMatchingWilcard(w config.Wildcard, ref schemas.Ref) (matches bool, err error) {	×
429	// Then we check if the owner matches the ref or is global	×
430	if w.Owner.Kind != "" && !strings.Contains(ref.Project.Name, w.Owner.Name) {	×
431	return	×
432	}	×
433
434	// Then we check if the ref matches the project pull parameters
435	return isRefMatchingProjectPullRefs(w.Pull.Refs, ref)	×
436	}
437
438	func isEnvMatchingWilcard(w config.Wildcard, env schemas.Environment) (matches bool, err error) {	×
439	// Then we check if the owner matches the ref or is global	×
440	if w.Owner.Kind != "" && !strings.Contains(env.ProjectName, w.Owner.Name) {	×
441	return	×
442	}	×
443
444	// Then we check if the ref matches the project pull parameters
445	return isEnvMatchingProjectPullEnvironments(w.Pull.Environments, env)	×
446	}

mvisonneau / gitlab-ci-pipelines-exporter / 17455371724

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous