Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 02ec880

Browse files
deansheatherdeansheather
committed
feat: add load testing harness
1 parent dde9a43 commit 02ec880

File tree

7 files changed

+1036
-0
lines changed

7 files changed

+1036
-0
lines changed

‎loadtest/harness/harness.go

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
package harness
2+
3+
import (
4+
"context"
5+
"sync"
6+
7+
"github.com/hashicorp/go-multierror"
8+
"golang.org/x/xerrors"
9+
)
10+
11+
// ExecutionStrategy defines how a TestHarness should execute a set of runs. It
12+
// essentially defines the concurrency model for a given testing session.
13+
type ExecutionStrategy interface {
14+
// Execute runs the given runs in whatever way the strategy wants. An error
15+
// may only be returned if the strategy has a failure itself, not if any of
16+
// the runs fail.
17+
Execute(ctx context.Context, runs []*TestRun) error
18+
}
19+
20+
// TestHarness runs a bunch of registered test runs using the given
21+
// ExecutionStrategy.
22+
type TestHarness struct {
23+
execStrat ExecutionStrategy
24+
25+
mut *sync.Mutex
26+
runIDs map[string]struct{}
27+
runs []*TestRun
28+
started bool
29+
done chan struct{}
30+
}
31+
32+
// NewTestHarness creates a new TestHarness with the given ExecutionStrategy.
33+
func NewTestHarness(execStrat ExecutionStrategy) *TestHarness {
34+
return &TestHarness{
35+
execStrat: execStrat,
36+
mut: new(sync.Mutex),
37+
runIDs: map[string]struct{}{},
38+
runs: []*TestRun{},
39+
done: make(chan struct{}),
40+
}
41+
}
42+
43+
// Run runs the registered tests using the given ExecutionStrategy. The provided
44+
// context can be used to cancel or set a deadline for the test run. Blocks
45+
// until the tests have finished and returns the test execution error (not
46+
// individual run errors).
47+
//
48+
// Panics if called more than once.
49+
func (h *TestHarness) Run(ctx context.Context) (err error) {
50+
h.mut.Lock()
51+
if h.started {
52+
h.mut.Unlock()
53+
panic("harness is already started")
54+
}
55+
h.started = true
56+
h.mut.Unlock()
57+
58+
defer close(h.done)
59+
defer func() {
60+
e := recover()
61+
if e != nil {
62+
err = xerrors.Errorf("execution strategy panicked: %w", e)
63+
}
64+
}()
65+
66+
err = h.execStrat.Execute(ctx, h.runs)
67+
//nolint:revive // we use named returns because we mutate it in a defer
68+
return
69+
}
70+
71+
// Cleanup should be called after the test run has finished and results have
72+
// been collected.
73+
func (h *TestHarness) Cleanup(ctx context.Context) (err error) {
74+
h.mut.Lock()
75+
defer h.mut.Unlock()
76+
if !h.started {
77+
panic("harness has not started")
78+
}
79+
select {
80+
case <-h.done:
81+
default:
82+
panic("harness has not finished")
83+
}
84+
85+
defer func() {
86+
e := recover()
87+
if e != nil {
88+
err = multierror.Append(err, xerrors.Errorf("panic in cleanup: %w", e))
89+
}
90+
}()
91+
92+
for _, run := range h.runs {
93+
e := run.Cleanup(ctx)
94+
if e != nil {
95+
err = multierror.Append(err, xerrors.Errorf("cleanup for %s failed: %w", run.FullID(), e))
96+
}
97+
}
98+
99+
//nolint:revive // we use named returns because we mutate it in a defer
100+
return
101+
}

‎loadtest/harness/harness_test.go

Lines changed: 262 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,262 @@
1+
package harness_test
2+
3+
import (
4+
"context"
5+
"io"
6+
"testing"
7+
"time"
8+
9+
"github.com/stretchr/testify/assert"
10+
"github.com/stretchr/testify/require"
11+
"golang.org/x/xerrors"
12+
13+
"github.com/coder/coder/loadtest/harness"
14+
)
15+
16+
const testPanicMessage = "expected test panic"
17+
18+
type panickingExecutionStrategy struct{}
19+
20+
var _ harness.ExecutionStrategy = panickingExecutionStrategy{}
21+
22+
func (panickingExecutionStrategy) Execute(_ context.Context, _ []*harness.TestRun) error {
23+
panic(testPanicMessage)
24+
}
25+
26+
type erroringExecutionStrategy struct {
27+
err error
28+
}
29+
30+
var _ harness.ExecutionStrategy = erroringExecutionStrategy{}
31+
32+
func (e erroringExecutionStrategy) Execute(_ context.Context, _ []*harness.TestRun) error {
33+
return e.err
34+
}
35+
36+
func Test_TestHarness(t *testing.T) {
37+
t.Parallel()
38+
39+
t.Run("OK", func(t *testing.T) {
40+
t.Parallel()
41+
42+
expectedErr := xerrors.New("expected error")
43+
44+
h := harness.NewTestHarness(harness.LinearExecutionStrategy{})
45+
r1 := h.AddRun("test", "1", fakeTestFns(nil, nil))
46+
r2 := h.AddRun("test", "2", fakeTestFns(expectedErr, nil))
47+
48+
err := h.Run(context.Background())
49+
require.NoError(t, err)
50+
51+
res := h.Results()
52+
require.Equal(t, 2, res.TotalRuns)
53+
require.Equal(t, 1, res.TotalPass)
54+
require.Equal(t, 1, res.TotalFail)
55+
require.Equal(t, map[string]harness.RunResult{
56+
r1.FullID(): r1.Result(),
57+
r2.FullID(): r2.Result(),
58+
}, res.Runs)
59+
60+
err = h.Cleanup(context.Background())
61+
require.NoError(t, err)
62+
})
63+
64+
t.Run("CatchesExecutionError", func(t *testing.T) {
65+
t.Parallel()
66+
67+
expectedErr := xerrors.New("expected error")
68+
69+
h := harness.NewTestHarness(erroringExecutionStrategy{err: expectedErr})
70+
_ = h.AddRun("test", "1", fakeTestFns(nil, nil))
71+
72+
err := h.Run(context.Background())
73+
require.Error(t, err)
74+
require.ErrorIs(t, err, expectedErr)
75+
})
76+
77+
t.Run("CatchesExecutionPanic", func(t *testing.T) {
78+
t.Parallel()
79+
80+
h := harness.NewTestHarness(panickingExecutionStrategy{})
81+
_ = h.AddRun("test", "1", fakeTestFns(nil, nil))
82+
83+
err := h.Run(context.Background())
84+
require.Error(t, err)
85+
require.ErrorContains(t, err, "panic")
86+
require.ErrorContains(t, err, testPanicMessage)
87+
})
88+
89+
t.Run("Cleanup", func(t *testing.T) {
90+
t.Parallel()
91+
92+
t.Run("Error", func(t *testing.T) {
93+
t.Parallel()
94+
95+
expectedErr := xerrors.New("expected error")
96+
97+
h := harness.NewTestHarness(harness.LinearExecutionStrategy{})
98+
_ = h.AddRun("test", "1", fakeTestFns(nil, expectedErr))
99+
100+
err := h.Run(context.Background())
101+
require.NoError(t, err)
102+
103+
err = h.Cleanup(context.Background())
104+
require.Error(t, err)
105+
require.ErrorContains(t, err, expectedErr.Error())
106+
})
107+
108+
t.Run("Panic", func(t *testing.T) {
109+
t.Parallel()
110+
111+
h := harness.NewTestHarness(harness.LinearExecutionStrategy{})
112+
_ = h.AddRun("test", "1", testFns{
113+
RunFn: func(_ context.Context, _ string, _ io.Writer) error {
114+
return nil
115+
},
116+
CleanupFn: func(_ context.Context, _ string) error {
117+
panic(testPanicMessage)
118+
},
119+
})
120+
121+
err := h.Run(context.Background())
122+
require.NoError(t, err)
123+
124+
err = h.Cleanup(context.Background())
125+
require.Error(t, err)
126+
require.ErrorContains(t, err, "panic")
127+
require.ErrorContains(t, err, testPanicMessage)
128+
})
129+
})
130+
131+
t.Run("Panics", func(t *testing.T) {
132+
t.Parallel()
133+
134+
t.Run("RegisterAfterStart", func(t *testing.T) {
135+
t.Parallel()
136+
137+
h := harness.NewTestHarness(harness.LinearExecutionStrategy{})
138+
_ = h.Run(context.Background())
139+
140+
require.Panics(t, func() {
141+
_ = h.AddRun("test", "1", fakeTestFns(nil, nil))
142+
})
143+
})
144+
145+
t.Run("DuplicateTestID", func(t *testing.T) {
146+
t.Parallel()
147+
148+
h := harness.NewTestHarness(harness.LinearExecutionStrategy{})
149+
150+
name, id := "test", "1"
151+
_ = h.AddRun(name, id, fakeTestFns(nil, nil))
152+
153+
require.Panics(t, func() {
154+
_ = h.AddRun(name, id, fakeTestFns(nil, nil))
155+
})
156+
})
157+
158+
t.Run("StartedTwice", func(t *testing.T) {
159+
t.Parallel()
160+
161+
h := harness.NewTestHarness(harness.LinearExecutionStrategy{})
162+
h.Run(context.Background())
163+
164+
require.Panics(t, func() {
165+
h.Run(context.Background())
166+
})
167+
})
168+
169+
t.Run("ResultsBeforeStart", func(t *testing.T) {
170+
t.Parallel()
171+
172+
h := harness.NewTestHarness(harness.LinearExecutionStrategy{})
173+
174+
require.Panics(t, func() {
175+
h.Results()
176+
})
177+
})
178+
179+
t.Run("ResultsBeforeFinish", func(t *testing.T) {
180+
t.Parallel()
181+
182+
var (
183+
endRun = make(chan struct{})
184+
testsEnded = make(chan struct{})
185+
)
186+
h := harness.NewTestHarness(harness.LinearExecutionStrategy{})
187+
_ = h.AddRun("test", "1", testFns{
188+
RunFn: func(_ context.Context, _ string, _ io.Writer) error {
189+
<-endRun
190+
return nil
191+
},
192+
})
193+
go func() {
194+
defer close(testsEnded)
195+
err := h.Run(context.Background())
196+
assert.NoError(t, err)
197+
}()
198+
199+
time.Sleep(100 * time.Millisecond)
200+
require.Panics(t, func() {
201+
h.Results()
202+
})
203+
204+
close(endRun)
205+
<-testsEnded
206+
_ = h.Results()
207+
})
208+
209+
t.Run("CleanupBeforeStart", func(t *testing.T) {
210+
t.Parallel()
211+
212+
h := harness.NewTestHarness(harness.LinearExecutionStrategy{})
213+
214+
require.Panics(t, func() {
215+
h.Cleanup(context.Background())
216+
})
217+
})
218+
219+
t.Run("ClenaupBeforeFinish", func(t *testing.T) {
220+
t.Parallel()
221+
222+
var (
223+
endRun = make(chan struct{})
224+
testsEnded = make(chan struct{})
225+
)
226+
h := harness.NewTestHarness(harness.LinearExecutionStrategy{})
227+
_ = h.AddRun("test", "1", testFns{
228+
RunFn: func(_ context.Context, _ string, _ io.Writer) error {
229+
<-endRun
230+
return nil
231+
},
232+
})
233+
go func() {
234+
defer close(testsEnded)
235+
err := h.Run(context.Background())
236+
assert.NoError(t, err)
237+
}()
238+
239+
time.Sleep(100 * time.Millisecond)
240+
require.Panics(t, func() {
241+
h.Cleanup(context.Background())
242+
})
243+
244+
close(endRun)
245+
<-testsEnded
246+
247+
err := h.Cleanup(context.Background())
248+
require.NoError(t, err)
249+
})
250+
})
251+
}
252+
253+
func fakeTestFns(err, cleanupErr error) testFns {
254+
return testFns{
255+
RunFn: func(_ context.Context, _ string, _ io.Writer) error {
256+
return err
257+
},
258+
CleanupFn: func(_ context.Context, _ string) error {
259+
return cleanupErr
260+
},
261+
}
262+
}

0 commit comments

Comments
 (0)