19555540738

Committed 21 Nov 2025 12:18AM UTC coverage: 44.461% (-0.5%) from 44.974%

Build # 19555540738

Build Type

push

github

Committed by

rom8726

Commit Message

Add threshold configuration options and improve chaos testing verdict logic.

Run Details

0 of 64 new or added lines in 1 file covered. (0.0%)

2 existing lines in 1 file now uncovered.

2051 of 4613 relevant lines covered (44.46%)

731.91 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/testing/testing.go

package testing

import (
        "context"
        "fmt"

        "github.com/rom8726/chaoskit"
)

// TestingT is an interface that matches testing.T and similar types
type TestingT interface {
        Errorf(format string, args ...interface{})
        FailNow()
        Helper()
}

// ChaosTestOption configures chaos testing
type ChaosTestOption func(*chaosTestConfig)

type chaosTestConfig struct {
        repeat         int
        failurePolicy  chaoskit.FailurePolicy
        executorOpts   []chaoskit.ExecutorOption
        skipReport     bool
        reportToStderr bool
        thresholds     *chaoskit.SuccessThresholds
        skipVerdict    bool
}

// WithRepeat sets the number of times to repeat the test scenario
func WithRepeat(n int) ChaosTestOption {
        return func(c *chaosTestConfig) {
                c.repeat = n
        }
}

// WithFailurePolicy sets how to handle failures (FailFast or ContinueOnFailure)
func WithFailurePolicy(policy chaoskit.FailurePolicy) ChaosTestOption {
        return func(c *chaosTestConfig) {
                c.failurePolicy = policy
        }
}

// WithExecutorOptions passes options to the underlying executor
func WithExecutorOptions(opts ...chaoskit.ExecutorOption) ChaosTestOption {
        return func(c *chaosTestConfig) {
                c.executorOpts = append(c.executorOpts, opts...)
        }
}

// WithoutReport skips printing the report after execution
func WithoutReport() ChaosTestOption {
        return func(c *chaosTestConfig) {
                c.skipReport = true
        }
}

// WithReportToStderr prints the report to stderr instead of stdout
func WithReportToStderr() ChaosTestOption {
        return func(c *chaosTestConfig) {
                c.reportToStderr = true
        }
}

// WithThresholds sets custom success thresholds for verdict calculation
func WithThresholds(thresholds *chaoskit.SuccessThresholds) ChaosTestOption {
        return func(c *chaosTestConfig) {
                c.thresholds = thresholds
        }
}

// WithDefaultThresholds uses default success thresholds (95% success rate)
func WithDefaultThresholds() ChaosTestOption {
        return func(c *chaosTestConfig) {
                c.thresholds = chaoskit.DefaultThresholds()
        }
}

// WithStrictThresholds uses strict success thresholds (100% success rate)
func WithStrictThresholds() ChaosTestOption {
        return func(c *chaosTestConfig) {
                c.thresholds = chaoskit.StrictThresholds()
        }
}

// WithRelaxedThresholds uses relaxed success thresholds (80% success rate)
func WithRelaxedThresholds() ChaosTestOption {
        return func(c *chaosTestConfig) {
                c.thresholds = chaoskit.RelaxedThresholds()
        }
}

// WithoutVerdict skips verdict calculation (only basic report)
func WithoutVerdict() ChaosTestOption {
        return func(c *chaosTestConfig) {
                c.skipVerdict = true
        }
}

// RunChaos creates a chaos test function that uses the full ChaosKit framework.
// It creates a scenario using ScenarioBuilder, runs it with an Executor, and validates results.
//
// The builderFn receives a pre-initialized ScenarioBuilder with the target already set.
// You should add steps, injectors, and validators to the builder.
//
// Usage:
//
//        func TestWithChaos(t *testing.T) {
//            target := &MyTarget{}
//
//            chaoskit.RunChaos(t, "name", target, func(s *chaoskit.ScenarioBuilder) *chaoskit.ScenarioBuilder {
//                return s.
//                    Step("step1", func(ctx context.Context, target chaoskit.Target) error {
//                        // Your test logic
//                        return DoSomething()
//                    }).
//                    Inject("delay", injectors.RandomDelay(10*time.Millisecond, 50*time.Millisecond)).
//                    Assert("goroutines", validators.GoroutineLimit(100))
//            },
//                WithRepeat(10),
//                WithDefaultThresholds(), // Enables verdict with 95% success rate
//            )()
//        }
func RunChaos(
        t TestingT,
        name string,
        target chaoskit.Target,
        builderFn func(*chaoskit.ScenarioBuilder) *chaoskit.ScenarioBuilder,
        opts ...ChaosTestOption,
) func() {
        return func() {
                t.Helper()

                // Apply options
                config := &chaosTestConfig{
                        repeat:        1,
                        failurePolicy: chaoskit.FailFast,
                        thresholds:    chaoskit.DefaultThresholds(), // Use default thresholds
                }
                for _, opt := range opts {
                        opt(config)
                }

                // Create scenario builder
                builder := chaoskit.NewScenario(name).WithTarget(target)

                // Let user configure the scenario
                builder = builderFn(builder)

                // Set repeat count
                builder = builder.Repeat(config.repeat)

                // Build scenario
                scenario := builder.Build()

                // Create executor with options
                executorOpts := append(
                        []chaoskit.ExecutorOption{chaoskit.WithFailurePolicy(config.failurePolicy)},
                        config.executorOpts...,
                )
                executor := chaoskit.NewExecutor(executorOpts...)

                // Run scenario
                ctx := context.Background()
                if err := executor.Run(ctx, scenario); err != nil {
                        t.Errorf("chaos test execution failed: %v", err)

                        // Print report on failure
                        if !config.skipReport {
                                printReport(t, executor, config)
                        }

                        t.FailNow()
                        return
                }

                // Calculate verdict and print report
                if !config.skipReport || !config.skipVerdict {
                        verdict := evaluateVerdict(t, executor, config)

                        // Fail test if verdict is FAIL
                        if verdict == chaoskit.VerdictFail {
                                t.Errorf("chaos test verdict: FAIL")
                                t.FailNow()
                        }
                }
        }
}

// printReport prints the test report
func printReport(t TestingT, executor *chaoskit.Executor, config *chaosTestConfig) {
        if config.skipVerdict {
                // Print simple report
                report := executor.Reporter().GenerateReport()
                if logger, ok := t.(interface{ Logf(string, ...interface{}) }); ok {
                        logger.Logf("\n%s", report)
                }
        } else {
                // Print detailed report with verdict
                report, err := executor.Reporter().GetVerdict(config.thresholds)
                if err != nil {
                        if logger, ok := t.(interface{ Logf(string, ...interface{}) }); ok {
                                logger.Logf("\nFailed to generate verdict: %v", err)
                                logger.Logf("\n%s", executor.Reporter().GenerateReport())
                        }
                        return
                }

                textReport := executor.Reporter().GenerateTextReport(report)
                if logger, ok := t.(interface{ Logf(string, ...interface{}) }); ok {
                        logger.Logf("\n%s", textReport)
                }
        }
}

// evaluateVerdict evaluates the verdict and returns it
func evaluateVerdict(t TestingT, executor *chaoskit.Executor, config *chaosTestConfig) chaoskit.Verdict {
        if config.skipVerdict {
                return chaoskit.VerdictPass
        }

        // Get verdict
        report, err := executor.Reporter().GetVerdict(config.thresholds)
        if err != nil {
                t.Errorf("failed to generate verdict: %v", err)
                return chaoskit.VerdictFail
        }

        // Print report
        if !config.skipReport {
                textReport := executor.Reporter().GenerateTextReport(report)
                if logger, ok := t.(interface{ Logf(string, ...interface{}) }); ok {
                        logger.Logf("\n%s", textReport)
                }
        }

        return report.Verdict
}

// RunChaosSimple is a simplified version that takes steps, injectors, and validators directly.
// This is useful when you don't need the full builder flexibility.
//
// Usage:
//
//        func TestSimpleChaos(t *testing.T) {
//            target := &MyTarget{}
//            steps := []chaoskit.StepFunc{
//                func(ctx context.Context, target chaoskit.Target) error {
//                    return DoSomething()
//                },
//            }
//            injectors := []chaoskit.Injector{
//                injectors.RandomDelay(10*time.Millisecond, 50*time.Millisecond),
//            }
//            validators := []chaoskit.Validator{
//                validators.GoroutineLimit(100),
//            }
//
//            chaoskit.RunChaosSimple(t, "name", target, steps, injectors, validators,
//                WithRepeat(10),
//                WithDefaultThresholds(),
//            )()
//        }
func RunChaosSimple(
        t TestingT,
        name string,
        target chaoskit.Target,
        steps []func(context.Context, chaoskit.Target) error,
        injectors []chaoskit.Injector,
        validators []chaoskit.Validator,
        opts ...ChaosTestOption,
) func() {
        return RunChaos(t, name, target, func(s *chaoskit.ScenarioBuilder) *chaoskit.ScenarioBuilder {
                // Add steps
                for i, stepFn := range steps {
                        s = s.Step(fmt.Sprintf("step-%d", i+1), stepFn)
                }

                // Add injectors
                for i, inj := range injectors {
                        s = s.Inject(fmt.Sprintf("injector-%d", i+1), inj)
                }

                // Add validators
                for i, val := range validators {
                        s = s.Assert(fmt.Sprintf("validator-%d", i+1), val)
                }

                return s
        }, opts...)
}

1	package testing
2
3	import (
4	"context"
5	"fmt"
6
7	"github.com/rom8726/chaoskit"
8	)
9
10	// TestingT is an interface that matches testing.T and similar types
11	type TestingT interface {
12	Errorf(format string, args ...interface{})
13	FailNow()
14	Helper()
15	}
16
17	// ChaosTestOption configures chaos testing
18	type ChaosTestOption func(*chaosTestConfig)
19
20	type chaosTestConfig struct {
21	repeat int
22	failurePolicy chaoskit.FailurePolicy
23	executorOpts []chaoskit.ExecutorOption
24	skipReport bool
25	reportToStderr bool
26	thresholds *chaoskit.SuccessThresholds
27	skipVerdict bool
28	}
29
30	// WithRepeat sets the number of times to repeat the test scenario
31	func WithRepeat(n int) ChaosTestOption {	×
32	return func(c *chaosTestConfig) {	×
33	c.repeat = n	×
34	}	×
35	}
36
37	// WithFailurePolicy sets how to handle failures (FailFast or ContinueOnFailure)
38	func WithFailurePolicy(policy chaoskit.FailurePolicy) ChaosTestOption {	×
39	return func(c *chaosTestConfig) {	×
40	c.failurePolicy = policy	×
41	}	×
42	}
43
44	// WithExecutorOptions passes options to the underlying executor
45	func WithExecutorOptions(opts ...chaoskit.ExecutorOption) ChaosTestOption {	×
46	return func(c *chaosTestConfig) {	×
47	c.executorOpts = append(c.executorOpts, opts...)	×
48	}	×
49	}
50
51	// WithoutReport skips printing the report after execution
52	func WithoutReport() ChaosTestOption {	×
53	return func(c *chaosTestConfig) {	×
54	c.skipReport = true	×
55	}	×
56	}
57
58	// WithReportToStderr prints the report to stderr instead of stdout
59	func WithReportToStderr() ChaosTestOption {	×
60	return func(c *chaosTestConfig) {	×
61	c.reportToStderr = true	×
62	}	×
63	}
64
65	// WithThresholds sets custom success thresholds for verdict calculation
NEW 66	func WithThresholds(thresholds *chaoskit.SuccessThresholds) ChaosTestOption {	×
NEW 67	return func(c *chaosTestConfig) {	×
NEW 68	c.thresholds = thresholds	×
NEW 69	}	×
70	}
71
72	// WithDefaultThresholds uses default success thresholds (95% success rate)
NEW 73	func WithDefaultThresholds() ChaosTestOption {	×
NEW 74	return func(c *chaosTestConfig) {	×
NEW 75	c.thresholds = chaoskit.DefaultThresholds()	×
NEW 76	}	×
77	}
78
79	// WithStrictThresholds uses strict success thresholds (100% success rate)
NEW 80	func WithStrictThresholds() ChaosTestOption {	×
NEW 81	return func(c *chaosTestConfig) {	×
NEW 82	c.thresholds = chaoskit.StrictThresholds()	×
NEW 83	}	×
84	}
85
86	// WithRelaxedThresholds uses relaxed success thresholds (80% success rate)
NEW 87	func WithRelaxedThresholds() ChaosTestOption {	×
NEW 88	return func(c *chaosTestConfig) {	×
NEW 89	c.thresholds = chaoskit.RelaxedThresholds()	×
NEW 90	}	×
91	}
92
93	// WithoutVerdict skips verdict calculation (only basic report)
NEW 94	func WithoutVerdict() ChaosTestOption {	×
NEW 95	return func(c *chaosTestConfig) {	×
NEW 96	c.skipVerdict = true	×
NEW 97	}	×
98	}
99
100	// RunChaos creates a chaos test function that uses the full ChaosKit framework.
101	// It creates a scenario using ScenarioBuilder, runs it with an Executor, and validates results.
102	//
103	// The builderFn receives a pre-initialized ScenarioBuilder with the target already set.
104	// You should add steps, injectors, and validators to the builder.
105	//
106	// Usage:
107	//
108	// func TestWithChaos(t *testing.T) {
109	// target := &MyTarget{}
110	//
111	// chaoskit.RunChaos(t, "name", target, func(s chaoskit.ScenarioBuilder) chaoskit.ScenarioBuilder {
112	// return s.
113	// Step("step1", func(ctx context.Context, target chaoskit.Target) error {
114	// // Your test logic
115	// return DoSomething()
116	// }).
117	// Inject("delay", injectors.RandomDelay(10time.Millisecond, 50time.Millisecond)).
118	// Assert("goroutines", validators.GoroutineLimit(100))
119	// },
120	// WithRepeat(10),
121	// WithDefaultThresholds(), // Enables verdict with 95% success rate
122	// )()
123	// }
124	func RunChaos(
125	t TestingT,
126	name string,
127	target chaoskit.Target,
128	builderFn func(chaoskit.ScenarioBuilder) chaoskit.ScenarioBuilder,
129	opts ...ChaosTestOption,
130	) func() {	×
131	return func() {	×
132	t.Helper()	×
133		×
134	// Apply options	×
135	config := &chaosTestConfig{	×
136	repeat: 1,	×
137	failurePolicy: chaoskit.FailFast,	×
NEW 138	thresholds: chaoskit.DefaultThresholds(), // Use default thresholds	×
139	}	×
140	for _, opt := range opts {	×
141	opt(config)	×
142	}	×
143
144	// Create scenario builder
145	builder := chaoskit.NewScenario(name).WithTarget(target)	×
146		×
147	// Let user configure the scenario	×
148	builder = builderFn(builder)	×
149		×
150	// Set repeat count	×
151	builder = builder.Repeat(config.repeat)	×
152		×
153	// Build scenario	×
154	scenario := builder.Build()	×
155		×
156	// Create executor with options	×
157	executorOpts := append(	×
158	[]chaoskit.ExecutorOption{chaoskit.WithFailurePolicy(config.failurePolicy)},	×
159	config.executorOpts...,	×
160	)	×
161	executor := chaoskit.NewExecutor(executorOpts...)	×
162		×
163	// Run scenario	×
164	ctx := context.Background()	×
165	if err := executor.Run(ctx, scenario); err != nil {	×
NEW 166	t.Errorf("chaos test execution failed: %v", err)	×
167		×
168	// Print report on failure	×
169	if !config.skipReport {	×
NEW 170	printReport(t, executor, config)	×
171	}	×
172
173	t.FailNow()	×
174	return	×
175	}
176
177	// Calculate verdict and print report
NEW 178	if !config.skipReport \|\| !config.skipVerdict {	×
NEW 179	verdict := evaluateVerdict(t, executor, config)	×
NEW 180		×
NEW 181	// Fail test if verdict is FAIL	×
NEW 182	if verdict == chaoskit.VerdictFail {	×
NEW 183	t.Errorf("chaos test verdict: FAIL")	×
NEW 184	t.FailNow()	×
NEW 185	}	×
186	}
187	}
188	}
189
190	// printReport prints the test report
NEW 191	func printReport(t TestingT, executor chaoskit.Executor, config chaosTestConfig) {	×
NEW 192	if config.skipVerdict {	×
NEW 193	// Print simple report	×
NEW 194	report := executor.Reporter().GenerateReport()	×
NEW 195	if logger, ok := t.(interface{ Logf(string, ...interface{}) }); ok {	×
NEW 196	logger.Logf("\n%s", report)	×
NEW 197	}	×
NEW 198	} else {	×
NEW 199	// Print detailed report with verdict	×
NEW 200	report, err := executor.Reporter().GetVerdict(config.thresholds)	×
NEW 201	if err != nil {	×
202	if logger, ok := t.(interface{ Logf(string, ...interface{}) }); ok {	×
NEW 203	logger.Logf("\nFailed to generate verdict: %v", err)	×
NEW 204	logger.Logf("\n%s", executor.Reporter().GenerateReport())	×
UNCOV 205	}	×
NEW 206	return	×
207	}
208
NEW 209	textReport := executor.Reporter().GenerateTextReport(report)	×
NEW 210	if logger, ok := t.(interface{ Logf(string, ...interface{}) }); ok {	×
NEW 211	logger.Logf("\n%s", textReport)	×
UNCOV 212	}	×
213	}
214	}
215
216	// evaluateVerdict evaluates the verdict and returns it
NEW 217	func evaluateVerdict(t TestingT, executor chaoskit.Executor, config chaosTestConfig) chaoskit.Verdict {	×
NEW 218	if config.skipVerdict {	×
NEW 219	return chaoskit.VerdictPass	×
NEW 220	}	×
221
222	// Get verdict
NEW 223	report, err := executor.Reporter().GetVerdict(config.thresholds)	×
NEW 224	if err != nil {	×
NEW 225	t.Errorf("failed to generate verdict: %v", err)	×
NEW 226	return chaoskit.VerdictFail	×
NEW 227	}	×
228
229	// Print report
NEW 230	if !config.skipReport {	×
NEW 231	textReport := executor.Reporter().GenerateTextReport(report)	×
NEW 232	if logger, ok := t.(interface{ Logf(string, ...interface{}) }); ok {	×
NEW 233	logger.Logf("\n%s", textReport)	×
NEW 234	}	×
235	}
236
NEW 237	return report.Verdict	×
238	}
239
240	// RunChaosSimple is a simplified version that takes steps, injectors, and validators directly.
241	// This is useful when you don't need the full builder flexibility.
242	//
243	// Usage:
244	//
245	// func TestSimpleChaos(t *testing.T) {
246	// target := &MyTarget{}
247	// steps := []chaoskit.StepFunc{
248	// func(ctx context.Context, target chaoskit.Target) error {
249	// return DoSomething()
250	// },
251	// }
252	// injectors := []chaoskit.Injector{
253	// injectors.RandomDelay(10time.Millisecond, 50time.Millisecond),
254	// }
255	// validators := []chaoskit.Validator{
256	// validators.GoroutineLimit(100),
257	// }
258	//
259	// chaoskit.RunChaosSimple(t, "name", target, steps, injectors, validators,
260	// WithRepeat(10),
261	// WithDefaultThresholds(),
262	// )()
263	// }
264	func RunChaosSimple(
265	t TestingT,
266	name string,
267	target chaoskit.Target,
268	steps []func(context.Context, chaoskit.Target) error,
269	injectors []chaoskit.Injector,
270	validators []chaoskit.Validator,
271	opts ...ChaosTestOption,
272	) func() {	×
NEW 273	return RunChaos(t, name, target, func(s chaoskit.ScenarioBuilder) chaoskit.ScenarioBuilder {	×
274	// Add steps	×
275	for i, stepFn := range steps {	×
276	s = s.Step(fmt.Sprintf("step-%d", i+1), stepFn)	×
277	}	×
278
279	// Add injectors
280	for i, inj := range injectors {	×
281	s = s.Inject(fmt.Sprintf("injector-%d", i+1), inj)	×
282	}	×
283
284	// Add validators
285	for i, val := range validators {	×
286	s = s.Assert(fmt.Sprintf("validator-%d", i+1), val)	×
287	}	×
288
289	return s	×
290	}, opts...)
291	}

rom8726 / chaoskit / 19555540738

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous