add maven-surefire instrumentation for junit4 class ordering

DataDog · daniel-mohedano · Apr 4, 2025 · Mar 28, 2025 · Mar 28, 2025 · Mar 31, 2025
commit 1d3c3944dffcfecaf62c4b9522a8106e02fc7f54
@@ -388,11 +388,7 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
     TEST_WRITER.waitForTraces(expectedOrder.size() + 1)
     def traces = TEST_WRITER.toList()
     def events = getEventsAsJson(traces)
-    def identifiers = getTestIdentifiers(events)
-    if (identifiers != expectedOrder) {
-      throw new AssertionError("Expected order: $expectedOrder, but got: $identifiers")
-    }
-    return true
+    return CiVisibilityTestUtils.assertTestsOrder(events, expectedOrder)
   }
 
   def assertCapabilities(Collection<LibraryCapability> capabilities, int expectedTraceCount) {
@@ -411,17 +407,6 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
     return true
   }
 
-  def getTestIdentifiers(List<Map> events) {
-    events.sort(Comparator.comparing { it['content']['start'] as Long })
-    def testIdentifiers = []
-    for (Map event : events) {
-      if (event['content']['meta']['test.name']) {
-        testIdentifiers.add(test(event['content']['meta']['test.suite'] as String, event['content']['meta']['test.name'] as String))
-      }
-    }
-    return testIdentifiers
-  }
-
   def test(String suite, String name) {
     return new TestFQN(suite, name)
   }

@@ -1,6 +1,6 @@
 package datadog.trace.civisibility
 
-
+import datadog.trace.api.civisibility.config.TestFQN
 import spock.lang.Specification
 
 abstract class CiVisibilitySmokeTest extends Specification {
@@ -16,6 +16,14 @@ abstract class CiVisibilitySmokeTest extends Specification {
     }
   }
 
+  protected test(String suiteName, String testName) {
+    return new TestFQN(suiteName, testName)
+  }
+
+  protected verifyTestOrder(List<Map<String, Object>> events, List<TestFQN> expectedOrder) {
+    CiVisibilityTestUtils.assertTestsOrder(events, expectedOrder)
+  }
+
   /**
    * This is a basic sanity check for telemetry metrics.
    * It only checks that the reported number of events created and finished is as expected.

@@ -9,6 +9,7 @@ import com.jayway.jsonpath.ReadContext
 import com.jayway.jsonpath.WriteContext
 import datadog.trace.api.DDSpanTypes
 import datadog.trace.api.civisibility.config.LibraryCapability
+import datadog.trace.api.civisibility.config.TestFQN
 import datadog.trace.core.DDSpan
 import freemarker.core.Environment
 import freemarker.core.InvalidReferenceException
@@ -136,6 +137,25 @@ abstract class CiVisibilityTestUtils {
     return replacementMap
   }
 
+  static boolean assertTestsOrder(List<Map<?, ?>> events, List<TestFQN> expectedOrder) {
+    def identifiers = getTestIdentifiers(events)
+    if (identifiers != expectedOrder) {
+      throw new AssertionError("Expected order: $expectedOrder, but got: $identifiers")
+    }
+    return true
+  }
+
+  static List<TestFQN> getTestIdentifiers(List<Map<?,?>> events) {
+    events.sort(Comparator.comparing { it['content']['start'] as Long })
+    def testIdentifiers = []
+    for (Map event : events) {
+      if (event['content']['meta']['test.name']) {
+        testIdentifiers.add(new TestFQN(event['content']['meta']['test.suite'] as String, event['content']['meta']['test.name'] as String))
+      }
+    }
+    return testIdentifiers
  }
+
   static List<Map<?, ?>> removeTags(List<Map<?, ?>> events, List<String> tags) {
     def filteredEvents = []
 

@@ -43,11 +43,11 @@ public String instrumentedType() {
   @Override
   public String[] helperClassNames() {
     return new String[] {
-        parentPackageName + ".SkippedByDatadog",
-        parentPackageName + ".TracingListener",
-        parentPackageName + ".JUnit4Utils",
-        parentPackageName + ".TestEventsHandlerHolder",
-        packageName + ".FailFastDescriptionComparator",
+      parentPackageName + ".SkippedByDatadog",
+      parentPackageName + ".TracingListener",
+      parentPackageName + ".JUnit4Utils",
+      parentPackageName + ".TestEventsHandlerHolder",
+      packageName + ".FailFastDescriptionComparator",
     };
   }
 

@@ -51,11 +51,11 @@ public ElementMatcher<TypeDescription> hierarchyMatcher() {
   @Override
   public String[] helperClassNames() {
     return new String[] {
-        parentPackageName + ".SkippedByDatadog",
-        parentPackageName + ".TracingListener",
-        parentPackageName + ".JUnit4Utils",
-        parentPackageName + ".TestEventsHandlerHolder",
-        packageName + ".FailFastDescriptionComparator",
+      parentPackageName + ".SkippedByDatadog",
+      parentPackageName + ".TracingListener",
+      parentPackageName + ".JUnit4Utils",
+      parentPackageName + ".TestEventsHandlerHolder",
+      packageName + ".FailFastDescriptionComparator",
     };
   }
 

@@ -51,8 +51,16 @@ class JUnit413Test extends CiVisibilityInstrumentationTest {
 
     where:
     testcaseName       | tests         | knownTestsList                                      | expectedOrder
-    "sorting-methods"  | [TestSorter]  | [test("org.example.TestSorter", "test_succeed_1")]  | [test("org.example.TestSorter", "test_succeed_2"), test("org.example.TestSorter", "test_succeed_3"), test("org.example.TestSorter", "test_succeed_1")]
-    "ordering-methods" | [TestOrderer] | [test("org.example.TestOrderer", "test_succeed_2")] | [test("org.example.TestOrderer", "test_succeed_3"), test("org.example.TestOrderer", "test_succeed_1"), test("org.example.TestOrderer", "test_succeed_2")]
+    "sorting-methods"  | [TestSorter]  | [test("org.example.TestSorter", "test_succeed_1")]  | [
+      test("org.example.TestSorter", "test_succeed_2"),
+      test("org.example.TestSorter", "test_succeed_3"),
+      test("org.example.TestSorter", "test_succeed_1")
+    ]
+    "ordering-methods" | [TestOrderer] | [test("org.example.TestOrderer", "test_succeed_2")] | [
+      test("org.example.TestOrderer", "test_succeed_3"),
+      test("org.example.TestOrderer", "test_succeed_1"),
+      test("org.example.TestOrderer", "test_succeed_2")
+    ]
   }
 
   def "test flaky tests ordering #testcaseName"() {
@@ -65,8 +73,19 @@ class JUnit413Test extends CiVisibilityInstrumentationTest {
 
     where:
     testcaseName       | tests         | flakyTestList                                                                                          | expectedOrder
-    "sorting-methods"  | [TestSorter]  | [test("org.example.TestSorter", "test_succeed_2"), test("org.example.TestSorter", "test_succeed_3")]   | [test("org.example.TestSorter", "test_succeed_2"), test("org.example.TestSorter", "test_succeed_3"), test("org.example.TestSorter", "test_succeed_1")]
-    "ordering-methods" | [TestOrderer] | [test("org.example.TestOrderer", "test_succeed_1"), test("org.example.TestOrderer", "test_succeed_3")] | [test("org.example.TestOrderer", "test_succeed_3"), test("org.example.TestOrderer", "test_succeed_1"), test("org.example.TestOrderer", "test_succeed_2")]
+    "sorting-methods"  | [TestSorter]  | [test("org.example.TestSorter", "test_succeed_2"), test("org.example.TestSorter", "test_succeed_3")]   | [
+      test("org.example.TestSorter", "test_succeed_2"),
+      test("org.example.TestSorter", "test_succeed_3"),
+      test("org.example.TestSorter", "test_succeed_1")
+    ]
+    "ordering-methods" | [TestOrderer] | [
+      test("org.example.TestOrderer", "test_succeed_1"),
+      test("org.example.TestOrderer", "test_succeed_3")
+    ] | [
+      test("org.example.TestOrderer", "test_succeed_3"),
+      test("org.example.TestOrderer", "test_succeed_1"),
+      test("org.example.TestOrderer", "test_succeed_2")
+    ]
   }
 
   def "test capabilities tagging #testcaseName"() {

@@ -8,7 +8,6 @@
 import datadog.trace.bootstrap.ContextStore;
 import java.lang.reflect.Method;
 import java.util.List;
-import junit.runner.Version;
 import org.junit.Ignore;
 import org.junit.runner.Description;
 import org.junit.runner.notification.Failure;

@@ -353,11 +353,16 @@ public static TestFrameworkInstrumentation runnerToFramework(Runner runner) {
     }
   }
 
-  public static TestFrameworkInstrumentation frameworkForClass(Class<?> testClass) {
+  public static TestFrameworkInstrumentation classToFramework(Class<?> testClass) {
     Runner runner = Request.aClass(testClass).getRunner();
     return runnerToFramework(runner);
   }
 
+  public static List<Description> getClassChildren(Class<?> testClass) {
+    Runner runner = Request.aClass(testClass).getRunner();
+    return runner.getDescription().getChildren();
+  }
+
   public static String getVersion() {
     return Version.id();
   }

@@ -7,7 +7,6 @@ import datadog.trace.civisibility.diff.FileDiff
 import datadog.trace.civisibility.diff.LineDiff
 import datadog.trace.instrumentation.junit4.JUnit4Utils
 import datadog.trace.instrumentation.junit4.TestEventsHandlerHolder
-import junit.runner.Version
 import org.example.*
 import org.junit.jupiter.api.Assumptions
 import org.junit.runner.JUnitCore

@@ -0,0 +1,20 @@
+apply from: "$rootDir/gradle/java.gradle"
+
+muzzle {
+  pass {
+    group = 'org.apache.maven.surefire'
+    module = 'surefire-junit4'
+    versions = '[3.0.0,)'
+  }
+  pass {
+    group = 'org.apache.maven.surefire'
+    module = 'surefire-junit47'
+    versions = '[3.0.0,)'
+  }
+}
+
+dependencies {
+  compileOnly group: 'org.apache.maven.surefire', name: 'surefire-junit4', version: '3.0.0'
+  compileOnly group: 'org.apache.maven.surefire', name: 'surefire-junit47', version: '3.0.0' // parallel provider
+  compileOnly project(":dd-java-agent:instrumentation:junit-4.10")
+}