Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 70a9d04

Browse files
nikita-tkachenko-datadognikita-tkachenko-datadog
authored
Implement fail-fast tests ordering for JUnit 5 (#8055)
1 parent 5b58772 commit 70a9d04

File tree

50 files changed

+654
-78
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+654
-78
lines changed

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationApiImpl.java

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector;
1616
import datadog.trace.api.civisibility.telemetry.tag.CoverageEnabled;
1717
import datadog.trace.api.civisibility.telemetry.tag.EarlyFlakeDetectionEnabled;
18+
import datadog.trace.api.civisibility.telemetry.tag.FlakyTestRetriesEnabled;
1819
import datadog.trace.api.civisibility.telemetry.tag.ItrEnabled;
1920
import datadog.trace.api.civisibility.telemetry.tag.ItrSkipEnabled;
2021
import datadog.trace.api.civisibility.telemetry.tag.RequireGit;
@@ -132,6 +133,7 @@ public CiVisibilitySettings getSettings(TracerEnvironment tracerEnvironment) thr
132133
settings.getEarlyFlakeDetectionSettings().isEnabled()
133134
? EarlyFlakeDetectionEnabled.TRUE
134135
: null,
136+
settings.isFlakyTestRetriesEnabled() ? FlakyTestRetriesEnabled.TRUE : null,
135137
settings.isGitUploadRequired() ? RequireGit.TRUE : null);
136138

137139
return settings;
@@ -185,6 +187,14 @@ public SkippableTests getSkippableTests(TracerEnvironment tracerEnvironment) thr
185187
@Override
186188
public Map<String, Collection<TestIdentifier>> getFlakyTestsByModule(
187189
TracerEnvironment tracerEnvironment) throws IOException {
190+
OkHttpUtils.CustomListener telemetryListener =
191+
new TelemetryListener.Builder(metricCollector)
192+
.requestCount(CiVisibilityCountMetric.FLAKY_TESTS_REQUEST)
193+
.requestErrors(CiVisibilityCountMetric.FLAKY_TESTS_REQUEST_ERRORS)
194+
.requestDuration(CiVisibilityDistributionMetric.FLAKY_TESTS_REQUEST_MS)
195+
.responseBytes(CiVisibilityDistributionMetric.FLAKY_TESTS_RESPONSE_BYTES)
196+
.build();
197+
188198
String uuid = uuidGenerator.get();
189199
EnvelopeDto<TracerEnvironment> request =
190200
new EnvelopeDto<>(
@@ -196,11 +206,12 @@ public Map<String, Collection<TestIdentifier>> getFlakyTestsByModule(
196206
FLAKY_TESTS_URI,
197207
requestBody,
198208
is -> testIdentifiersResponseAdapter.fromJson(Okio.buffer(Okio.source(is))).data,
199-
null,
209+
telemetryListener,
200210
false);
201211

202212
LOGGER.debug("Received {} flaky tests in total", response.size());
203213

214+
int flakyTestsCount = 0;
204215
Map<String, Collection<TestIdentifier>> testIdentifiers = new HashMap<>();
205216
for (DataDto<TestIdentifierJson> dataDto : response) {
206217
TestIdentifierJson testIdentifierJson = dataDto.getAttributes();
@@ -209,7 +220,10 @@ public Map<String, Collection<TestIdentifier>> getFlakyTestsByModule(
209220
testIdentifiers
210221
.computeIfAbsent(moduleName, k -> new HashSet<>())
211222
.add(testIdentifierJson.toTestIdentifier());
223+
flakyTestsCount++;
212224
}
225+
226+
metricCollector.add(CiVisibilityDistributionMetric.FLAKY_TESTS_RESPONSE_TESTS, flakyTestsCount);
213227
return testIdentifiers;
214228
}
215229

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettings.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,6 @@ public Collection<TestIdentifier> getKnownTests() {
114114

115115
@Nullable
116116
public Collection<TestIdentifier> getFlakyTests() {
117-
// backend does not store module info for flaky tests yet
118117
return flakyTests;
119118
}
120119

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettingsFactoryImpl.java

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package datadog.trace.civisibility.config;
22

33
import datadog.trace.api.Config;
4+
import datadog.trace.api.civisibility.CIConstants;
45
import datadog.trace.api.civisibility.CiVisibilityWellKnownTags;
56
import datadog.trace.api.civisibility.config.TestIdentifier;
67
import datadog.trace.api.civisibility.config.TestMetadata;
@@ -139,11 +140,17 @@ private TracerEnvironment buildTracerEnvironment(
139140

140141
Map<String, Collection<TestIdentifier>> flakyTestsByModule =
141142
flakyTestRetriesEnabled && config.isCiVisibilityFlakyRetryOnlyKnownFlakes()
143+
|| CIConstants.FAIL_FAST_TEST_ORDER.equalsIgnoreCase(
144+
config.getCiVisibilityTestOrder())
142145
? getFlakyTestsByModule(tracerEnvironment)
143146
: null;
144147

145148
Map<String, Collection<TestIdentifier>> knownTestsByModule =
146-
earlyFlakeDetectionEnabled ? getKnownTestsByModule(tracerEnvironment) : null;
149+
earlyFlakeDetectionEnabled
150+
|| CIConstants.FAIL_FAST_TEST_ORDER.equalsIgnoreCase(
151+
config.getCiVisibilityTestOrder())
152+
? getKnownTestsByModule(tracerEnvironment)
153+
: null;
147154

148155
Set<String> moduleNames = new HashSet<>(Collections.singleton(DEFAULT_SETTINGS));
149156
moduleNames.addAll(skippableTestIdentifiers.keySet());

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/TestIdentifierSerializer.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,10 @@ public static void serialize(Serializer serializer, TestIdentifier testIdentifie
1313
}
1414

1515
public static TestIdentifier deserialize(ByteBuffer buffer) {
16+
String suiteName = Serializer.readString(buffer);
1617
return new TestIdentifier(
17-
Serializer.readString(buffer),
18+
// suite name repeats a lot; interning it to save memory
19+
suiteName != null ? suiteName.intern() : null,
1820
Serializer.readString(buffer),
1921
Serializer.readString(buffer));
2022
}

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/AbstractTestSession.java

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,14 @@
55
import datadog.trace.api.Config;
66
import datadog.trace.api.DDTraceId;
77
import datadog.trace.api.IdGenerationStrategy;
8+
import datadog.trace.api.civisibility.CIConstants;
89
import datadog.trace.api.civisibility.telemetry.CiVisibilityCountMetric;
910
import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector;
1011
import datadog.trace.api.civisibility.telemetry.TagValue;
12+
import datadog.trace.api.civisibility.telemetry.tag.AgentlessLogSubmissionEnabled;
1113
import datadog.trace.api.civisibility.telemetry.tag.AutoInjected;
1214
import datadog.trace.api.civisibility.telemetry.tag.EventType;
15+
import datadog.trace.api.civisibility.telemetry.tag.FailFastTestOrderEnabled;
1316
import datadog.trace.api.civisibility.telemetry.tag.HasCodeowner;
1417
import datadog.trace.api.civisibility.telemetry.tag.IsHeadless;
1518
import datadog.trace.api.civisibility.telemetry.tag.IsUnsupportedCI;
@@ -109,7 +112,11 @@ public AbstractTestSession(
109112
CiVisibilityCountMetric.TEST_SESSION,
110113
1,
111114
ciProvider,
112-
config.isCiVisibilityAutoInjected() ? AutoInjected.TRUE : null);
115+
config.isCiVisibilityAutoInjected() ? AutoInjected.TRUE : null,
116+
config.isAgentlessLogSubmissionEnabled() ? AgentlessLogSubmissionEnabled.TRUE : null,
117+
CIConstants.FAIL_FAST_TEST_ORDER.equalsIgnoreCase(config.getCiVisibilityTestOrder())
118+
? FailFastTestOrderEnabled.TRUE
119+
: null);
113120

114121
if (instrumentationType == InstrumentationType.MANUAL_API) {
115122
metricCollector.add(CiVisibilityCountMetric.MANUAL_API_EVENTS, 1, EventType.SESSION);

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestFrameworkModule.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ TestSuiteImpl testSuiteStart(
2525
*/
2626
boolean isNew(TestIdentifier test);
2727

28+
boolean isFlaky(TestIdentifier test);
29+
2830
/**
2931
* Checks if a given test should be skipped with Intelligent Test Runner or not
3032
*

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/ProxyTestModule.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,11 @@ public boolean isNew(TestIdentifier test) {
8787
return executionStrategy.isNew(test);
8888
}
8989

90+
@Override
91+
public boolean isFlaky(TestIdentifier test) {
92+
return executionStrategy.isFlaky(test);
93+
}
94+
9095
@Override
9196
public boolean shouldBeSkipped(TestIdentifier test) {
9297
return executionStrategy.shouldBeSkipped(test);

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestModule.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,11 @@ public boolean isNew(TestIdentifier test) {
7272
return executionStrategy.isNew(test);
7373
}
7474

75+
@Override
76+
public boolean isFlaky(TestIdentifier test) {
77+
return executionStrategy.isFlaky(test);
78+
}
79+
7580
@Override
7681
public boolean shouldBeSkipped(TestIdentifier test) {
7782
return executionStrategy.shouldBeSkipped(test);

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,16 @@ public TestRetryPolicy retryPolicy(TestIdentifier test) {
272272
return testModule.retryPolicy(test);
273273
}
274274

275+
@Override
276+
public boolean isNew(TestIdentifier test) {
277+
return testModule.isNew(test);
278+
}
279+
280+
@Override
281+
public boolean isFlaky(TestIdentifier test) {
282+
return testModule.isFlaky(test);
283+
}
284+
275285
@Override
276286
public void close() {
277287
testModule.end(null);

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,11 @@ public boolean isNew(TestIdentifier test) {
4343
return knownTests != null && !knownTests.contains(test.withoutParameters());
4444
}
4545

46+
public boolean isFlaky(TestIdentifier test) {
47+
Collection<TestIdentifier> flakyTests = executionSettings.getFlakyTests();
48+
return flakyTests != null && flakyTests.contains(test.withoutParameters());
49+
}
50+
4651
public boolean shouldBeSkipped(TestIdentifier test) {
4752
if (test == null) {
4853
return false;

‎dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy

Lines changed: 41 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import datadog.communication.serialization.GrowableBuffer
55
import datadog.communication.serialization.msgpack.MsgPackWriter
66
import datadog.trace.agent.test.AgentTestRunner
77
import datadog.trace.api.Config
8+
import datadog.trace.api.civisibility.CIConstants
89
import datadog.trace.api.civisibility.DDTest
910
import datadog.trace.api.civisibility.DDTestSuite
1011
import datadog.trace.api.civisibility.InstrumentationBridge
@@ -124,7 +125,7 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
124125
skippableTestsWithMetadata,
125126
[:],
126127
flakyTests,
127-
earlyFlakinessDetectionEnabled ? knownTests : null)
128+
earlyFlakinessDetectionEnabled || CIConstants.FAIL_FAST_TEST_ORDER.equalsIgnoreCase(Config.get().ciVisibilityTestOrder) ? knownTests : null)
128129
}
129130
}
130131

@@ -224,12 +225,22 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
224225

225226
def givenFlakyTests(List<TestIdentifier> tests) {
226227
flakyTests.addAll(tests)
227-
flakyRetryEnabled = true
228228
}
229229

230230
def givenKnownTests(List<TestIdentifier> tests) {
231231
knownTests.addAll(tests)
232-
earlyFlakinessDetectionEnabled = true
232+
}
233+
234+
def givenFlakyRetryEnabled(boolean flakyRetryEnabled) {
235+
this.flakyRetryEnabled = flakyRetryEnabled
236+
}
237+
238+
def givenEarlyFlakinessDetectionEnabled(boolean earlyFlakinessDetectionEnabled) {
239+
this.earlyFlakinessDetectionEnabled = earlyFlakinessDetectionEnabled
240+
}
241+
242+
def givenTestsOrder(String testsOrder) {
243+
injectSysConfig(CiVisibilityConfig.CIVISIBILITY_TEST_ORDER, testsOrder)
233244
}
234245

235246
@Override
@@ -274,6 +285,33 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
274285
return CiVisibilityTestUtils.assertData(testcaseName, events, coverages, additionalReplacements)
275286
}
276287

288+
def assertTestsOrder(List<TestIdentifier> expectedOrder) {
289+
TEST_WRITER.waitForTraces(expectedOrder.size() + 1)
290+
def traces = TEST_WRITER.toList()
291+
def events = getEventsAsJson(traces)
292+
def identifiers = getTestIdentifiers(events)
293+
if (identifiers != expectedOrder) {
294+
throw new AssertionError("Expected order: $expectedOrder, but got: $identifiers")
295+
}
296+
return true
297+
}
298+
299+
def getTestIdentifiers(List<Map> events) {
300+
events.sort(Comparator.comparing { it['content']['start'] as Long })
301+
def testIdentifiers = []
302+
for (Map event : events) {
303+
if (event['content']['meta']['test.name']) {
304+
testIdentifiers.add(test(event['content']['meta']['test.suite'] as String, event['content']['meta']['test.name'] as String))
305+
}
306+
}
307+
return testIdentifiers
308+
}
309+
310+
def test(String suite, String name, String parameters = null) {
311+
312+
return new TestIdentifier(suite, name, parameters)
313+
}
314+
277315
def getEventsAsJson(List<List<DDSpan>> traces) {
278316
return getSpansAsJson(new CiTestCycleMapperV1(Config.get().getCiVisibilityWellKnownTags(), false), traces)
279317
}

‎dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/groovy/CucumberTest.groovy

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
5353
}
5454

5555
def "test flaky retries #testcaseName"() {
56+
givenFlakyRetryEnabled(true)
5657
givenFlakyTests(retriedTests)
5758

5859
runFeatures(features)
@@ -71,6 +72,7 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
7172
}
7273

7374
def "test early flakiness detection #testcaseName"() {
75+
givenEarlyFlakinessDetectionEnabled(true)
7476
givenKnownTests(knownTestsList)
7577

7678
runFeatures(features)

‎dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ class MUnitTest extends CiVisibilityInstrumentationTest {
3131
}
3232

3333
def "test flaky retries #testcaseName"() {
34+
givenFlakyRetryEnabled(true)
3435
givenFlakyTests(retriedTests)
3536

3637
runTests(tests)
@@ -45,6 +46,7 @@ class MUnitTest extends CiVisibilityInstrumentationTest {
4546
}
4647

4748
def "test early flakiness detection #testcaseName"() {
49+
givenEarlyFlakinessDetectionEnabled(true)
4850
givenKnownTests(knownTestsList)
4951

5052
runTests(tests)

‎dd-java-agent/instrumentation/junit-4.10/src/test/groovy/JUnit4Test.groovy

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@ class JUnit4Test extends CiVisibilityInstrumentationTest {
9090
}
9191

9292
def "test flaky retries #testcaseName"() {
93+
givenFlakyRetryEnabled(true)
9394
givenFlakyTests(retriedTests)
9495

9596
runTests(tests)
@@ -110,6 +111,7 @@ class JUnit4Test extends CiVisibilityInstrumentationTest {
110111
}
111112

112113
def "test early flakiness detection #testcaseName"() {
114+
givenEarlyFlakinessDetectionEnabled(true)
113115
givenKnownTests(knownTestsList)
114116

115117
runTests(tests)

‎dd-java-agent/instrumentation/junit-5.3/build.gradle

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,9 @@ dependencies {
3333

3434
// versions used below are not the minimum ones that we support,
3535
// but the tests need to use them in order to be compliant with Spock 2.x
36-
testImplementation group: 'org.junit.platform', name: 'junit-platform-launcher', version: '1.8.2'
37-
testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.8.2'
38-
testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: '5.8.2'
36+
testImplementation group: 'org.junit.platform', name: 'junit-platform-launcher', version: '1.9.2'
37+
testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.9.2'
38+
testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: '5.9.2'
3939

4040
latestDepTestImplementation group: 'org.junit.platform', name: 'junit-platform-launcher', version: '+'
4141
latestDepTestImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '+'
@@ -44,8 +44,8 @@ dependencies {
4444

4545
configurations.matching({ it.name.startsWith('test') }).each({
4646
it.resolutionStrategy {
47-
force group: 'org.junit.platform', name: 'junit-platform-launcher', version: '1.8.2'
48-
force group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.8.2'
49-
force group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: '5.8.2'
47+
force group: 'org.junit.platform', name: 'junit-platform-launcher', version: '1.9.2'
48+
force group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.9.2'
49+
force group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: '5.9.2'
5050
}
5151
})

‎dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/build.gradle

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ addTestSuiteForDir('latestDepTest', 'test')
1313

1414
dependencies {
1515
implementation project(':dd-java-agent:instrumentation:junit-5.3')
16+
1617
compileOnly group: 'io.cucumber', name: 'cucumber-junit-platform-engine', version: '5.4.0'
1718
compileOnly group: 'io.cucumber', name: 'cucumber-java', version: '5.4.0'
1819

‎dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5CucumberInstrumentation.java

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -7,17 +7,12 @@
77
import com.google.auto.service.AutoService;
88
import datadog.trace.agent.tooling.Instrumenter;
99
import datadog.trace.agent.tooling.InstrumenterModule;
10-
import datadog.trace.bootstrap.ContextStore;
11-
import datadog.trace.bootstrap.InstrumentationContext;
1210
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
1311
import io.cucumber.junit.platform.engine.CucumberTestEngine;
14-
import java.util.Collections;
15-
import java.util.Map;
1612
import net.bytebuddy.asm.Advice;
1713
import net.bytebuddy.matcher.ElementMatcher;
1814
import org.junit.platform.engine.EngineExecutionListener;
1915
import org.junit.platform.engine.ExecutionRequest;
20-
import org.junit.platform.engine.TestDescriptor;
2116
import org.junit.platform.engine.TestEngine;
2217
import org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService;
2318

@@ -51,11 +46,6 @@ public String[] helperClassNames() {
5146
};
5247
}
5348

54-
@Override
55-
public Map<String, String> contextStore() {
56-
return Collections.singletonMap("org.junit.platform.engine.TestDescriptor", "java.lang.Object");
57-
}
58-
5949
@Override
6050
public void methodAdvice(MethodTransformer transformer) {
6151
transformer.applyAdvice(
@@ -87,12 +77,6 @@ public static void addTracingListener(
8777
return;
8878
}
8979

90-
ContextStore<TestDescriptor, Object> contextStore =
91-
InstrumentationContext.get(TestDescriptor.class, Object.class);
92-
TestEventsHandlerHolder.setContextStores(
93-
(ContextStore) contextStore, (ContextStore) contextStore);
94-
TestEventsHandlerHolder.start();
95-
9680
CucumberTracingListener tracingListener = new CucumberTracingListener(testEngine);
9781
EngineExecutionListener originalListener = executionRequest.getEngineExecutionListener();
9882
EngineExecutionListener compositeListener =

0 commit comments

Comments
 (0)