12058212013

Committed 27 Nov 2024 08:45PM UTC coverage: 57.508% (+0.04%) from 57.466%

Build # 12058212013

Build Type

Pull #387

github

Committed by

monstermunchkin

Commit Message

pkg/context: Add deprecation notice

Pull Request Pull Request #387: tracing: Fix bugs and clean up code

Run Details

19 of 29 new or added lines in 7 files covered. (65.52%)

2 existing lines in 1 file now uncovered.

5519 of 9597 relevant lines covered (57.51%)

21.8 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/pkg/routine/instance.go

// Copyright © 2020 by PACE Telematics GmbH. All rights reserved.

package routine

import (
        "context"
        "fmt"
        "time"

        "github.com/getsentry/sentry-go"
        "github.com/pace/bricks/maintenance/errors"
        "github.com/pace/bricks/pkg/lock/redis"

        exponential "github.com/jpillora/backoff"
)

type routineThatKeepsRunningOneInstance struct {
        Name    string
        Routine func(context.Context)

        lockTTL       time.Duration
        retryInterval time.Duration
        backoff       combinedExponentialBackoff
        num           int64
}

func (r *routineThatKeepsRunningOneInstance) Run(ctx context.Context) {
        // The retry interval is used if we did not get the lock because some
        // other caller got it. The exponential backoff is used if we encounter
        // problems with obtaining the lock, like the Redis not being available.
        // The retry interval is also used if the routine returned regularly, to
        // avoid uncontrollably short restart cycles. If the routine panicked we
        // use exponential backoff as well.
        r.lockTTL = cfg.RedisLockTTL
        r.retryInterval = r.lockTTL / 5
        r.backoff = combinedExponentialBackoff{
                "lock":    &exponential.Backoff{Min: r.retryInterval, Max: 10 * time.Minute},
                "routine": &exponential.Backoff{Min: r.retryInterval, Max: 10 * time.Minute},
        }

        r.num = ctx.Value(ctxNumKey{}).(int64)
        var tryAgainIn time.Duration // zero on first run
        for {
                select {
                case <-ctx.Done():
                        return
                case <-time.After(tryAgainIn):
                }
                // Make sure to cancel the singleRunCtx so that the lock is released
                // after the routine returned.
                singleRunCtx, cancel := context.WithCancel(ctx)
                tryAgainIn = r.singleRun(singleRunCtx)
                cancel()
        }
}

// Performs a single run. That is, to try to obtain the lock and run the routine
// until it returns. Return the backoff duration after which another single run
// should be performed.
func (r *routineThatKeepsRunningOneInstance) singleRun(ctx context.Context) time.Duration {
        l := redis.NewLock("routine:lock:"+r.Name, redis.SetTTL(r.lockTTL))
        lockCtx, cancel, err := l.AcquireAndKeepUp(ctx)
        if err != nil {
                go errors.Handle(ctx, err) // report error to Sentry, non-blocking
                return r.backoff.Duration("lock")
        }
        if lockCtx != nil {
                defer cancel()
                routinePanicked := true
                func() {
                        defer errors.HandleWithCtx(ctx, fmt.Sprintf("routine %d", r.num)) // handle panics

                        span := sentry.StartSpan(lockCtx, "function", sentry.WithDescription(fmt.Sprintf("routine %d", r.num)))
                        defer span.Finish()

                        r.Routine(span.Context())
                        routinePanicked = false
                }()
                if routinePanicked {
                        return r.backoff.Duration("routine")
                }
        }
        r.backoff.ResetAll()
        return r.retryInterval
}

1	// Copyright © 2020 by PACE Telematics GmbH. All rights reserved.
2
3	package routine
4
5	import (
6	"context"
7	"fmt"
8	"time"
9
10	"github.com/getsentry/sentry-go"
11	"github.com/pace/bricks/maintenance/errors"
12	"github.com/pace/bricks/pkg/lock/redis"
13
14	exponential "github.com/jpillora/backoff"
15	)
16
17	type routineThatKeepsRunningOneInstance struct {
18	Name string
19	Routine func(context.Context)
20
21	lockTTL time.Duration
22	retryInterval time.Duration
23	backoff combinedExponentialBackoff
24	num int64
25	}
26
27	func (r *routineThatKeepsRunningOneInstance) Run(ctx context.Context) {	×
28	// The retry interval is used if we did not get the lock because some	×
29	// other caller got it. The exponential backoff is used if we encounter	×
30	// problems with obtaining the lock, like the Redis not being available.	×
31	// The retry interval is also used if the routine returned regularly, to	×
32	// avoid uncontrollably short restart cycles. If the routine panicked we	×
33	// use exponential backoff as well.	×
34	r.lockTTL = cfg.RedisLockTTL	×
35	r.retryInterval = r.lockTTL / 5	×
36	r.backoff = combinedExponentialBackoff{	×
37	"lock": &exponential.Backoff{Min: r.retryInterval, Max: 10 * time.Minute},	×
38	"routine": &exponential.Backoff{Min: r.retryInterval, Max: 10 * time.Minute},	×
39	}	×
40		×
41	r.num = ctx.Value(ctxNumKey{}).(int64)	×
42	var tryAgainIn time.Duration // zero on first run	×
43	for {	×
44	select {	×
45	case <-ctx.Done():	×
46	return	×
47	case <-time.After(tryAgainIn):	×
48	}
49	// Make sure to cancel the singleRunCtx so that the lock is released
50	// after the routine returned.
51	singleRunCtx, cancel := context.WithCancel(ctx)	×
52	tryAgainIn = r.singleRun(singleRunCtx)	×
53	cancel()	×
54	}
55	}
56
57	// Performs a single run. That is, to try to obtain the lock and run the routine
58	// until it returns. Return the backoff duration after which another single run
59	// should be performed.
60	func (r *routineThatKeepsRunningOneInstance) singleRun(ctx context.Context) time.Duration {	×
61	l := redis.NewLock("routine:lock:"+r.Name, redis.SetTTL(r.lockTTL))	×
62	lockCtx, cancel, err := l.AcquireAndKeepUp(ctx)	×
63	if err != nil {	×
64	go errors.Handle(ctx, err) // report error to Sentry, non-blocking	×
65	return r.backoff.Duration("lock")	×
66	}	×
67	if lockCtx != nil {	×
68	defer cancel()	×
69	routinePanicked := true	×
70	func() {	×
71	defer errors.HandleWithCtx(ctx, fmt.Sprintf("routine %d", r.num)) // handle panics	×
NEW 72		×
NEW 73	span := sentry.StartSpan(lockCtx, "function", sentry.WithDescription(fmt.Sprintf("routine %d", r.num)))	×
NEW 74	defer span.Finish()	×
NEW 75		×
NEW 76	r.Routine(span.Context())	×
77	routinePanicked = false	×
78	}()	×
79	if routinePanicked {	×
80	return r.backoff.Duration("routine")	×
81	}	×
82	}
83	r.backoff.ResetAll()	×
84	return r.retryInterval	×
85	}

pace / bricks / 12058212013

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous