11250208184

Committed 09 Oct 2024 07:25AM UTC coverage: 57.466% (-13.7%) from 71.177%

Build # 11250208184

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #380 from pace/sentry-tracing-poc

tracing: replace Jaeger with Sentry

Run Details

140 of 206 new or added lines in 19 files covered. (67.96%)

3 existing lines in 3 files now uncovered.

5515 of 9597 relevant lines covered (57.47%)

21.77 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

49.15

/backend/redis/redis.go

// Copyright © 2018 by PACE Telematics GmbH. All rights reserved.

// Package redis helps creating redis connection pools
package redis

import (
        "context"
        "time"

        "github.com/caarlos0/env/v10"
        "github.com/getsentry/sentry-go"
        "github.com/prometheus/client_golang/prometheus"
        "github.com/redis/go-redis/v9"

        "github.com/pace/bricks/maintenance/health/servicehealthcheck"
        "github.com/pace/bricks/maintenance/log"
)

type config struct {
        Addrs           []string      `env:"REDIS_HOSTS" envSeparator:"," envDefault:"redis:6379"`
        Password        string        `env:"REDIS_PASSWORD"`
        DB              int           `env:"REDIS_DB"`
        MaxRetries      int           `env:"REDIS_MAX_RETRIES"`
        MinRetryBackoff time.Duration `env:"REDIS_MIN_RETRY_BACKOFF"`
        MaxRetryBackoff time.Duration `env:"REDIS_MAX_RETRY_BACKOFF"`
        DialTimeout     time.Duration `env:"REDIS_DIAL_TIMEOUT"`
        ReadTimeout     time.Duration `env:"REDIS_READ_TIMEOUT"`
        WriteTimeout    time.Duration `env:"REDIS_WRITE_TIMEOUT"`
        PoolSize        int           `env:"REDIS_POOL_SIZE"`
        MinIdleConns    int           `env:"REDIS_MIN_IDLE_CONNS"`
        MaxConnAge      time.Duration `env:"REDIS_MAX_CONNAGE"`
        PoolTimeout     time.Duration `env:"REDIS_POOL_TIMEOUT"`
        IdleTimeout     time.Duration `env:"REDIS_IDLE_TIMEOUT"`
        // Name of the key that is written to check, if redis is healthy
        HealthCheckKey string `env:"REDIS_HEALTH_CHECK_KEY" envDefault:"healthy"`
        // Amount of time to cache the last health check result
        HealthCheckResultTTL time.Duration `env:"REDIS_HEALTH_CHECK_RESULT_TTL" envDefault:"10s"`
}

var (
        paceRedisCmdTotal = prometheus.NewCounterVec(
                prometheus.CounterOpts{
                        Name: "pace_redis_cmd_total",
                        Help: "Collects stats about the number of redis requests made",
                },
                []string{"method"},
        )
        paceRedisCmdFailed = prometheus.NewCounterVec(
                prometheus.CounterOpts{
                        Name: "pace_redis_cmd_failed",
                        Help: "Collects stats about the number of redis requests failed",
                },
                []string{"method"},
        )
        paceRedisCmdDurationSeconds = prometheus.NewHistogramVec(
                prometheus.HistogramOpts{
                        Name:    "pace_redis_cmd_duration_seconds",
                        Help:    "Collect performance metrics for each method",
                        Buckets: []float64{.1, .25, .5, 1, 2.5, 5, 10, 60},
                },
                []string{"method"},
        )
)

var cfg config

func init() {
        prometheus.MustRegister(paceRedisCmdTotal)
        prometheus.MustRegister(paceRedisCmdFailed)
        prometheus.MustRegister(paceRedisCmdDurationSeconds)

        // parse log config
        err := env.Parse(&cfg)
        if err != nil {
                log.Fatalf("Failed to parse redis environment: %v", err)
        }

        servicehealthcheck.RegisterHealthCheck("redis", &HealthCheck{
                Client: Client(),
        })
}

// Client with environment based configuration
func Client(overwriteOpts ...func(*redis.Options)) *redis.Client {
        opts := &redis.Options{
                Addr:            cfg.Addrs[0],
                Password:        cfg.Password,
                DB:              cfg.DB,
                MaxRetries:      cfg.MaxRetries,
                MinRetryBackoff: cfg.MinRetryBackoff,
                MaxRetryBackoff: cfg.MaxRetryBackoff,
                DialTimeout:     cfg.DialTimeout,
                ReadTimeout:     cfg.ReadTimeout,
                WriteTimeout:    cfg.WriteTimeout,
                PoolSize:        cfg.PoolSize,
                MinIdleConns:    cfg.MinIdleConns,
                ConnMaxLifetime: cfg.MaxConnAge,
                PoolTimeout:     cfg.PoolTimeout,
                ConnMaxIdleTime: cfg.IdleTimeout,
        }

        for _, o := range overwriteOpts {
                o(opts)
        }

        return CustomClient(opts)
}

// CustomClient with passed configuration
func CustomClient(opts *redis.Options) *redis.Client {
        log.Logger().Info().Str("addr", opts.Addr).
                Msg("Redis connection pool created")
        return redis.NewClient(opts)
}

// ClusterClient with environment based configuration
func ClusterClient() *redis.ClusterClient {
        return CustomClusterClient(&redis.ClusterOptions{
                Addrs:           cfg.Addrs,
                Password:        cfg.Password,
                MaxRetries:      cfg.MaxRetries,
                MinRetryBackoff: cfg.MinRetryBackoff,
                MaxRetryBackoff: cfg.MaxRetryBackoff,
                DialTimeout:     cfg.DialTimeout,
                ReadTimeout:     cfg.ReadTimeout,
                WriteTimeout:    cfg.WriteTimeout,
                PoolSize:        cfg.PoolSize,
                MinIdleConns:    cfg.MinIdleConns,
                ConnMaxLifetime: cfg.MaxConnAge,
                PoolTimeout:     cfg.PoolTimeout,
                ConnMaxIdleTime: cfg.IdleTimeout,
        })
}

// CustomClusterClient with passed configuration
func CustomClusterClient(opts *redis.ClusterOptions) *redis.ClusterClient {
        log.Logger().Info().Strs("addrs", opts.Addrs).
                Msg("Redis cluster connection pool created")
        return redis.NewClusterClient(opts)
}

// WithContext adds a logging and tracing wrapper to the passed client
func WithContext(ctx context.Context, c *redis.Client) *redis.Client {
        c.AddHook(&logtracer{})
        return c
}

// WithClusterContext adds a logging and tracing wrapper to the passed client
func WithClusterContext(ctx context.Context, c *redis.ClusterClient) *redis.ClusterClient {
        c.AddHook(&logtracer{})
        return c
}

type logtracer struct{}

type logtracerKey struct{}

type logtracerValues struct {
        startedAt time.Time
        span      *sentry.Span
}

func (lt *logtracer) DialHook(next redis.DialHook) redis.DialHook {
        return next
}

func (lt *logtracer) ProcessHook(next redis.ProcessHook) redis.ProcessHook {
        return func(ctx context.Context, cmd redis.Cmder) error {
                startedAt := time.Now()

                span := sentry.StartSpan(ctx, "db.redis", sentry.WithDescription(cmd.Name()))
                defer span.Finish()

                span.SetTag("db.system", "redis")

                span.SetData("cmd", cmd.Name())

                paceRedisCmdTotal.With(prometheus.Labels{
                        "method": cmd.Name(),
                }).Inc()

                ctx = context.WithValue(ctx, logtracerKey{}, &logtracerValues{
                        startedAt: startedAt,
                        span:      span,
                })

                _ = next(ctx, cmd)

                vals := ctx.Value(logtracerKey{}).(*logtracerValues)
                le := log.Ctx(ctx).Debug().Str("cmd", cmd.Name()).Str("sentry:category", "redis")

                // add error
                cmdErr := cmd.Err()
                if cmdErr != nil {
                        vals.span.SetData("error", cmdErr)
                        le = le.Err(cmdErr)
                        paceRedisCmdFailed.With(prometheus.Labels{
                                "method": cmd.Name(),
                        }).Inc()
                }

                // do log statement
                dur := float64(time.Since(vals.startedAt)) / float64(time.Millisecond)
                le.Float64("duration", dur).Msg("Redis query")

                paceRedisCmdDurationSeconds.With(prometheus.Labels{
                        "method": cmd.Name(),
                }).Observe(dur)

                return nil
        }
}

func (l *logtracer) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook {
        return next
}

1	// Copyright © 2018 by PACE Telematics GmbH. All rights reserved.
2
3	// Package redis helps creating redis connection pools
4	package redis
5
6	import (
7	"context"
8	"time"
9
10	"github.com/caarlos0/env/v10"
11	"github.com/getsentry/sentry-go"
12	"github.com/prometheus/client_golang/prometheus"
13	"github.com/redis/go-redis/v9"
14
15	"github.com/pace/bricks/maintenance/health/servicehealthcheck"
16	"github.com/pace/bricks/maintenance/log"
17	)
18
19	type config struct {
20	Addrs []string `env:"REDIS_HOSTS" envSeparator:"," envDefault:"redis:6379"`
21	Password string `env:"REDIS_PASSWORD"`
22	DB int `env:"REDIS_DB"`
23	MaxRetries int `env:"REDIS_MAX_RETRIES"`
24	MinRetryBackoff time.Duration `env:"REDIS_MIN_RETRY_BACKOFF"`
25	MaxRetryBackoff time.Duration `env:"REDIS_MAX_RETRY_BACKOFF"`
26	DialTimeout time.Duration `env:"REDIS_DIAL_TIMEOUT"`
27	ReadTimeout time.Duration `env:"REDIS_READ_TIMEOUT"`
28	WriteTimeout time.Duration `env:"REDIS_WRITE_TIMEOUT"`
29	PoolSize int `env:"REDIS_POOL_SIZE"`
30	MinIdleConns int `env:"REDIS_MIN_IDLE_CONNS"`
31	MaxConnAge time.Duration `env:"REDIS_MAX_CONNAGE"`
32	PoolTimeout time.Duration `env:"REDIS_POOL_TIMEOUT"`
33	IdleTimeout time.Duration `env:"REDIS_IDLE_TIMEOUT"`
34	// Name of the key that is written to check, if redis is healthy
35	HealthCheckKey string `env:"REDIS_HEALTH_CHECK_KEY" envDefault:"healthy"`
36	// Amount of time to cache the last health check result
37	HealthCheckResultTTL time.Duration `env:"REDIS_HEALTH_CHECK_RESULT_TTL" envDefault:"10s"`
38	}
39
40	var (
41	paceRedisCmdTotal = prometheus.NewCounterVec(
42	prometheus.CounterOpts{
43	Name: "pace_redis_cmd_total",
44	Help: "Collects stats about the number of redis requests made",
45	},
46	[]string{"method"},
47	)
48	paceRedisCmdFailed = prometheus.NewCounterVec(
49	prometheus.CounterOpts{
50	Name: "pace_redis_cmd_failed",
51	Help: "Collects stats about the number of redis requests failed",
52	},
53	[]string{"method"},
54	)
55	paceRedisCmdDurationSeconds = prometheus.NewHistogramVec(
56	prometheus.HistogramOpts{
57	Name: "pace_redis_cmd_duration_seconds",
58	Help: "Collect performance metrics for each method",
59	Buckets: []float64{.1, .25, .5, 1, 2.5, 5, 10, 60},
60	},
61	[]string{"method"},
62	)
63	)
64
65	var cfg config
66
67	func init() {	1✔
68	prometheus.MustRegister(paceRedisCmdTotal)	1✔
69	prometheus.MustRegister(paceRedisCmdFailed)	1✔
70	prometheus.MustRegister(paceRedisCmdDurationSeconds)	1✔
71		1✔
72	// parse log config	1✔
73	err := env.Parse(&cfg)	1✔
74	if err != nil {	1✔
75	log.Fatalf("Failed to parse redis environment: %v", err)	×
76	}	×
77
78	servicehealthcheck.RegisterHealthCheck("redis", &HealthCheck{	1✔
79	Client: Client(),	1✔
80	})	1✔
81	}
82
83	// Client with environment based configuration
84	func Client(overwriteOpts ...func(redis.Options)) redis.Client {	2✔
85	opts := &redis.Options{	2✔
86	Addr: cfg.Addrs[0],	2✔
87	Password: cfg.Password,	2✔
88	DB: cfg.DB,	2✔
89	MaxRetries: cfg.MaxRetries,	2✔
90	MinRetryBackoff: cfg.MinRetryBackoff,	2✔
91	MaxRetryBackoff: cfg.MaxRetryBackoff,	2✔
92	DialTimeout: cfg.DialTimeout,	2✔
93	ReadTimeout: cfg.ReadTimeout,	2✔
94	WriteTimeout: cfg.WriteTimeout,	2✔
95	PoolSize: cfg.PoolSize,	2✔
96	MinIdleConns: cfg.MinIdleConns,	2✔
97	ConnMaxLifetime: cfg.MaxConnAge,	2✔
98	PoolTimeout: cfg.PoolTimeout,	2✔
99	ConnMaxIdleTime: cfg.IdleTimeout,	2✔
100	}	2✔
101		2✔
102	for _, o := range overwriteOpts {	2✔
103	o(opts)	×
104	}	×
105
106	return CustomClient(opts)	2✔
107	}
108
109	// CustomClient with passed configuration
110	func CustomClient(opts redis.Options) redis.Client {	3✔
111	log.Logger().Info().Str("addr", opts.Addr).	3✔
112	Msg("Redis connection pool created")	3✔
113	return redis.NewClient(opts)	3✔
114	}	3✔
115
116	// ClusterClient with environment based configuration
117	func ClusterClient() *redis.ClusterClient {	1✔
118	return CustomClusterClient(&redis.ClusterOptions{	1✔
119	Addrs: cfg.Addrs,	1✔
120	Password: cfg.Password,	1✔
121	MaxRetries: cfg.MaxRetries,	1✔
122	MinRetryBackoff: cfg.MinRetryBackoff,	1✔
123	MaxRetryBackoff: cfg.MaxRetryBackoff,	1✔
124	DialTimeout: cfg.DialTimeout,	1✔
125	ReadTimeout: cfg.ReadTimeout,	1✔
126	WriteTimeout: cfg.WriteTimeout,	1✔
127	PoolSize: cfg.PoolSize,	1✔
128	MinIdleConns: cfg.MinIdleConns,	1✔
129	ConnMaxLifetime: cfg.MaxConnAge,	1✔
130	PoolTimeout: cfg.PoolTimeout,	1✔
131	ConnMaxIdleTime: cfg.IdleTimeout,	1✔
132	})	1✔
133	}	1✔
134
135	// CustomClusterClient with passed configuration
136	func CustomClusterClient(opts redis.ClusterOptions) redis.ClusterClient {	1✔
137	log.Logger().Info().Strs("addrs", opts.Addrs).	1✔
138	Msg("Redis cluster connection pool created")	1✔
139	return redis.NewClusterClient(opts)	1✔
140	}	1✔
141
142	// WithContext adds a logging and tracing wrapper to the passed client
143	func WithContext(ctx context.Context, c redis.Client) redis.Client {	×
144	c.AddHook(&logtracer{})	×
145	return c	×
146	}	×
147
148	// WithClusterContext adds a logging and tracing wrapper to the passed client
149	func WithClusterContext(ctx context.Context, c redis.ClusterClient) redis.ClusterClient {	×
150	c.AddHook(&logtracer{})	×
151	return c	×
152	}	×
153
154	type logtracer struct{}
155
156	type logtracerKey struct{}
157
158	type logtracerValues struct {
159	startedAt time.Time
160	span *sentry.Span
161	}
162
163	func (lt *logtracer) DialHook(next redis.DialHook) redis.DialHook {	×
164	return next	×
165	}	×
166
167	func (lt *logtracer) ProcessHook(next redis.ProcessHook) redis.ProcessHook {	×
168	return func(ctx context.Context, cmd redis.Cmder) error {	×
169	startedAt := time.Now()	×
170		×
NEW 171	span := sentry.StartSpan(ctx, "db.redis", sentry.WithDescription(cmd.Name()))	×
172	defer span.Finish()	×
173		×
174	span.SetTag("db.system", "redis")	×
175		×
NEW 176	span.SetData("cmd", cmd.Name())	×
NEW 177		×
178	paceRedisCmdTotal.With(prometheus.Labels{	×
179	"method": cmd.Name(),	×
180	}).Inc()	×
181		×
182	ctx = context.WithValue(ctx, logtracerKey{}, &logtracerValues{	×
183	startedAt: startedAt,	×
184	span: span,	×
185	})	×
186		×
187	_ = next(ctx, cmd)	×
188		×
189	vals := ctx.Value(logtracerKey{}).(*logtracerValues)	×
190	le := log.Ctx(ctx).Debug().Str("cmd", cmd.Name()).Str("sentry:category", "redis")	×
191		×
192	// add error	×
193	cmdErr := cmd.Err()	×
194	if cmdErr != nil {	×
NEW 195	vals.span.SetData("error", cmdErr)	×
196	le = le.Err(cmdErr)	×
197	paceRedisCmdFailed.With(prometheus.Labels{	×
198	"method": cmd.Name(),	×
199	}).Inc()	×
200	}	×
201
202	// do log statement
203	dur := float64(time.Since(vals.startedAt)) / float64(time.Millisecond)	×
204	le.Float64("duration", dur).Msg("Redis query")	×
205		×
206	paceRedisCmdDurationSeconds.With(prometheus.Labels{	×
207	"method": cmd.Name(),	×
208	}).Observe(dur)	×
209		×
210	return nil	×
211	}
212	}
213
214	func (l *logtracer) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook {	×
215	return next	×
216	}	×

pace / bricks / 11250208184

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous