update junit5 instrumentations

DataDog · daniel-mohedano · Mar 7, 2025 · Feb 25, 2025 · Feb 25, 2025 · Feb 25, 2025
commit 8129852c4ebf1a73be1e4f5514194ad9601fc8cc
@@ -1,7 +1,12 @@
+import datadog.trace.agent.test.asserts.ListWriterAssert
+import datadog.trace.api.DDSpanTypes
+import datadog.trace.api.DDTags
 import datadog.trace.api.DisableTestTrace
 import datadog.trace.api.civisibility.config.TestFQN
 import datadog.trace.api.civisibility.config.TestIdentifier
 import datadog.trace.civisibility.CiVisibilityInstrumentationTest
+import datadog.trace.civisibility.CiVisibilityTestUtils
+import datadog.trace.civisibility.config.LibraryCapabilityUtils
 import datadog.trace.instrumentation.junit5.TestEventsHandlerHolder
 import io.cucumber.core.api.TypeRegistry
 import io.cucumber.core.options.Constants
@@ -210,6 +215,29 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
     ]
   }
 
+  def "test capabilities tagging #testcaseName"() {
+    def notPresentTags = new HashSet<>(LibraryCapabilityUtils.CAPABILITY_TAG_MAP.values())
+    notPresentTags.removeAll(presentTags)
+
+    runFeatures(["org/example/cucumber/calculator/basic_arithmetic.feature"], false, true)
+
+    ListWriterAssert.assertTraces(TEST_WRITER, 4, true, new CiVisibilityTestUtils.SortTracesByType(), {
+      trace(1) {
+        span(0) {
+          spanType DDSpanTypes.TEST_SESSION_END
+          tags(false) {
+            arePresent(presentTags)
+            areNotPresent(notPresentTags)
+          }
+        }
+      }
+    })
+
+    where:
+    testcaseName                 | presentTags
+    "test-capabilities-base"     | [DDTags.LIBRARY_CAPABILITIES_TIA, DDTags.LIBRARY_CAPABILITIES_ATR, DDTags.LIBRARY_CAPABILITIES_EFD, DDTags.LIBRARY_CAPABILITIES_QUARANTINE, DDTags.LIBRARY_CAPABILITIES_DISABLED, DDTags.LIBRARY_CAPABILITIES_ATTEMPT_TO_FIX]
+  }
+
   private String parameterizedTestNameSuffix() {
     // older releases report different example names
     version() == "5.4.0" ? "Example #1" : "Example #1.1"

@@ -1,8 +1,15 @@
+import datadog.trace.agent.test.asserts.ListWriterAssert
+import datadog.trace.api.DDSpanTypes
+import datadog.trace.api.DDTags
 import datadog.trace.api.DisableTestTrace
 import datadog.trace.api.civisibility.CIConstants
 import datadog.trace.civisibility.CiVisibilityInstrumentationTest
+import datadog.trace.civisibility.CiVisibilityTestUtils
+import datadog.trace.civisibility.config.LibraryCapabilityUtils
+import datadog.trace.instrumentation.junit5.JUnitPlatformUtils
 import datadog.trace.instrumentation.junit5.TestEventsHandlerHolder
 import org.example.*
+import org.junit.jupiter.api.Assumptions
 import org.junit.jupiter.api.ClassOrderer
 import org.junit.jupiter.api.MethodOrderer
 import org.junit.jupiter.engine.Constants
@@ -82,6 +89,31 @@ class JUnit58Test extends CiVisibilityInstrumentationTest {
     ]
   }
 
+  def "test capabilities tagging #testcaseName"() {
+    Assumptions.assumeTrue(assumption)
+
+    def notPresentTags = new HashSet<>(LibraryCapabilityUtils.CAPABILITY_TAG_MAP.values())
+    notPresentTags.removeAll(presentTags)
+
+    runTests([TestSucceed], true)
+
+    ListWriterAssert.assertTraces(TEST_WRITER, 5, true, new CiVisibilityTestUtils.SortTracesByType(), {
+      trace(1) {
+        span(0) {
+          spanType DDSpanTypes.TEST_SESSION_END
+          tags(false) {
+            arePresent(presentTags)
+            areNotPresent(notPresentTags)
+          }
+        }
+      }
+    })
+
+    where:
+    testcaseName                 | presentTags                                                                                                                                                                                                                                                                                                                 | assumption
+    "test-capabilities-ordering" | [DDTags.LIBRARY_CAPABILITIES_TIA, DDTags.LIBRARY_CAPABILITIES_ATR, DDTags.LIBRARY_CAPABILITIES_EFD, DDTags.LIBRARY_CAPABILITIES_IMPACTED_TESTS, DDTags.LIBRARY_CAPABILITIES_QUARANTINE, DDTags.LIBRARY_CAPABILITIES_DISABLED, DDTags.LIBRARY_CAPABILITIES_ATTEMPT_TO_FIX, DDTags.LIBRARY_CAPABILITIES_FAIL_FAST_TEST_ORDER] | JUnitPlatformUtils.isJunitTestOrderingSupported(instrumentedLibraryVersion())
+  }
+
   private static void runTests(List<Class<?>> tests, boolean expectSuccess = true) {
     DiscoverySelector[] selectors = new DiscoverySelector[tests.size()]
     for (i in 0..<tests.size()) {

@@ -22,7 +22,7 @@ public class SpockTracingListener implements EngineExecutionListener {
 
   public SpockTracingListener(TestEngine testEngine) {
     testFramework = testEngine.getId();
-    testFrameworkVersion = testEngine.getVersion().orElse(null);
+    testFrameworkVersion = JUnitPlatformUtils.getFrameworkVersion(testEngine);
   }
 
   @Override

@@ -1,9 +1,15 @@
+import datadog.trace.agent.test.asserts.ListWriterAssert
+import datadog.trace.api.DDSpanTypes
+import datadog.trace.api.DDTags
 import datadog.trace.api.DisableTestTrace
 import datadog.trace.api.civisibility.config.TestFQN
 import datadog.trace.api.civisibility.config.TestIdentifier
 import datadog.trace.civisibility.CiVisibilityInstrumentationTest
+import datadog.trace.civisibility.CiVisibilityTestUtils
+import datadog.trace.civisibility.config.LibraryCapabilityUtils
 import datadog.trace.civisibility.diff.FileDiff
 import datadog.trace.civisibility.diff.LineDiff
+import datadog.trace.instrumentation.junit5.JUnitPlatformUtils
 import datadog.trace.instrumentation.junit5.TestEventsHandlerHolder
 import org.example.TestFailedParameterizedSpock
 import org.example.TestFailedSpock
@@ -18,6 +24,7 @@ import org.example.TestSucceedSpockSlow
 import org.example.TestSucceedSpockUnskippable
 import org.example.TestSucceedSpockUnskippableSuite
 import org.example.TestSucceedSpockVerySlow
+import org.junit.jupiter.api.Assumptions
 import org.junit.platform.engine.DiscoverySelector
 import org.junit.platform.engine.TestExecutionResult
 import org.junit.platform.launcher.TestExecutionListener
@@ -210,6 +217,29 @@ class SpockTest extends CiVisibilityInstrumentationTest {
     "test-attempt-to-fix-disabled-succeeded"    | true    | [TestSucceedSpock] | [new TestFQN("org.example.TestSucceedSpock", "test success")] | []                                                            | [new TestFQN("org.example.TestSucceedSpock", "test success")]
   }
 
+  def "test capabilities tagging #testcaseName"() {
+    def notPresentTags = new HashSet<>(LibraryCapabilityUtils.CAPABILITY_TAG_MAP.values())
+    notPresentTags.removeAll(presentTags)
+
+    runTests([TestSucceedSpock], true)
+
+    ListWriterAssert.assertTraces(TEST_WRITER, 4, true, new CiVisibilityTestUtils.SortTracesByType(), {
+      trace(1) {
+        span(0) {
+          spanType DDSpanTypes.TEST_SESSION_END
+          tags(false) {
+            arePresent(presentTags)
+            areNotPresent(notPresentTags)
+          }
+        }
+      }
+    })
+
+    where:
+    testcaseName                 | presentTags
+    "test-capabilities-base"     | [DDTags.LIBRARY_CAPABILITIES_TIA, DDTags.LIBRARY_CAPABILITIES_ATR, DDTags.LIBRARY_CAPABILITIES_EFD, DDTags.LIBRARY_CAPABILITIES_IMPACTED_TESTS, DDTags.LIBRARY_CAPABILITIES_QUARANTINE, DDTags.LIBRARY_CAPABILITIES_DISABLED, DDTags.LIBRARY_CAPABILITIES_ATTEMPT_TO_FIX]
+  }
+
   private static void runTests(List<Class<?>> classes, boolean expectSuccess = true) {
     DiscoverySelector[] selectors = new DiscoverySelector[classes.size()]
     for (i in 0..<classes.size()) {

@@ -2,16 +2,20 @@
 
 import static datadog.json.JsonMapper.toJson;
 
+import datadog.trace.api.civisibility.config.LibraryCapability;
 import datadog.trace.api.civisibility.config.TestIdentifier;
 import datadog.trace.api.civisibility.config.TestSourceData;
 import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation;
 import datadog.trace.bootstrap.instrumentation.api.AgentScope;
 import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
 import datadog.trace.bootstrap.instrumentation.api.AgentTracer;
 import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes;
+import datadog.trace.util.ComparableVersion;
 import datadog.trace.util.MethodHandles;
 import java.lang.invoke.MethodHandle;
 import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.ListIterator;
 import javax.annotation.Nonnull;
@@ -44,6 +48,8 @@ public abstract class JUnitPlatformUtils {
   public static final String ENGINE_ID_CUCUMBER = "cucumber";
   public static final String ENGINE_ID_SPOCK = "spock";
 
+  public static final ComparableVersion junitV58 = new ComparableVersion("5.8");
+
   private JUnitPlatformUtils() {}
 
   private static final MethodHandles METHOD_HANDLES =
@@ -245,4 +251,65 @@ public static TestFrameworkInstrumentation engineIdToFramework(String engineId)
       return TestFrameworkInstrumentation.JUNIT5;
     }
   }
+
+  // only used in junit5 and spock, cucumber has its own utils method
+  @Nullable
+  public static String getFrameworkVersion(TestEngine testEngine) {
+    return testEngine.getVersion().orElse(null);
+  }
+
+  public static boolean isJunitTestOrderingSupported(String version) {
+    return version != null && junitV58.compareTo(new ComparableVersion(version)) <= 0;
+  }
+
+  private static List<LibraryCapability> junitCapabilities(String version) {
+    List<LibraryCapability> baseCapabilities =
+        new ArrayList<>(
+            Arrays.asList(
+                LibraryCapability.TIA,
+                LibraryCapability.ATR,
+                LibraryCapability.EFD,
+                LibraryCapability.IMPACTED,
+                LibraryCapability.QUARANTINE,
+                LibraryCapability.DISABLED,
+                LibraryCapability.ATTEMPT_TO_FIX));
+
+    if (isJunitTestOrderingSupported(version)) {
+      baseCapabilities.add(LibraryCapability.FAIL_FAST);
+    }
+
+    return baseCapabilities;
+  }
+
+  private static List<LibraryCapability> spockCapabilities() {
+    return Arrays.asList(
+        LibraryCapability.TIA,
+        LibraryCapability.ATR,
+        LibraryCapability.EFD,
+        LibraryCapability.IMPACTED,
+        LibraryCapability.QUARANTINE,
+        LibraryCapability.DISABLED,
+        LibraryCapability.ATTEMPT_TO_FIX);
+  }
+
+  private static List<LibraryCapability> cucumberCapabilities() {
+    return Arrays.asList(
+        LibraryCapability.TIA,
+        LibraryCapability.ATR,
+        LibraryCapability.EFD,
+        LibraryCapability.QUARANTINE,
+        LibraryCapability.DISABLED,
+        LibraryCapability.ATTEMPT_TO_FIX);
+  }
+
+  public static List<LibraryCapability> availableCapabilities(TestEngine testEngine) {
+    TestFrameworkInstrumentation framework = engineToFramework(testEngine);
+    if (framework.equals(TestFrameworkInstrumentation.CUCUMBER)) {
+      return cucumberCapabilities();
+    } else if (framework.equals(TestFrameworkInstrumentation.SPOCK)) {
+      return spockCapabilities();
+    } else {
+      return junitCapabilities(getFrameworkVersion(testEngine));
+    }
+  }
 }
@@ -63,7 +63,10 @@ public static synchronized void start(
     if (handler == null) {
       handler =
           InstrumentationBridge.createTestEventsHandler(
-              framework.name().toLowerCase(), suiteStore, testStore);
+              framework.name().toLowerCase(),
+              suiteStore,
+              testStore,
+              JUnitPlatformUtils.availableCapabilities(testEngine));
       HANDLERS.put(framework, handler);
     }
   }

@@ -23,7 +23,7 @@ public class TracingListener implements EngineExecutionListener {
   public TracingListener(TestEngine testEngine) {
     String engineId = testEngine.getId();
     testFramework = engineId == null || engineId.startsWith("junit") ? "junit5" : engineId;
-    testFrameworkVersion = testEngine.getVersion().orElse(null);
+    testFrameworkVersion = JUnitPlatformUtils.getFrameworkVersion(testEngine);
   }
 
   @Override

@@ -1,9 +1,15 @@
+import datadog.trace.agent.test.asserts.ListWriterAssert
+import datadog.trace.api.DDSpanTypes
+import datadog.trace.api.DDTags
 import datadog.trace.api.DisableTestTrace
 import datadog.trace.api.civisibility.config.TestFQN
 import datadog.trace.api.civisibility.config.TestIdentifier
 import datadog.trace.civisibility.CiVisibilityInstrumentationTest
+import datadog.trace.civisibility.CiVisibilityTestUtils
+import datadog.trace.civisibility.config.LibraryCapabilityUtils
 import datadog.trace.civisibility.diff.FileDiff
 import datadog.trace.civisibility.diff.LineDiff
+import datadog.trace.instrumentation.junit5.JUnitPlatformUtils
 import datadog.trace.instrumentation.junit5.TestEventsHandlerHolder
 import org.example.TestAssumption
 import org.example.TestAssumptionAndSucceed
@@ -35,6 +41,7 @@ import org.example.TestSucceedVerySlow
 import org.example.TestSucceedWithCategories
 import org.example.TestSuiteSetUpAssumption
 import org.example.TestTemplate
+import org.junit.jupiter.api.Assumptions
 import org.junit.jupiter.engine.JupiterTestEngine
 import org.junit.platform.engine.DiscoverySelector
 import org.junit.platform.engine.TestExecutionResult
@@ -243,6 +250,31 @@ class JUnit5Test extends CiVisibilityInstrumentationTest {
     "test-attempt-to-fix-disabled-succeeded"    | true    | [TestSucceed] | [new TestFQN("org.example.TestSucceed", "test_succeed")] | []                                                       | [new TestFQN("org.example.TestSucceed", "test_succeed")]
   }
 
+  def "test capabilities tagging #testcaseName"() {
+    Assumptions.assumeTrue(assumption)
+
+    def notPresentTags = new HashSet<>(LibraryCapabilityUtils.CAPABILITY_TAG_MAP.values())
+    notPresentTags.removeAll(presentTags)
+
+    runTests([TestSucceed], true)
+
+    ListWriterAssert.assertTraces(TEST_WRITER, 4, true, new CiVisibilityTestUtils.SortTracesByType(), {
+      trace(1) {
+        span(0) {
+          spanType DDSpanTypes.TEST_SESSION_END
+          tags(false) {
+            arePresent(presentTags)
+            areNotPresent(notPresentTags)
+          }
+        }
+      }
+    })
+
+    where:
+    testcaseName                 | presentTags                                                                                                                                                                                                                                                                | assumption
+    "test-capabilities-base"     | [DDTags.LIBRARY_CAPABILITIES_TIA, DDTags.LIBRARY_CAPABILITIES_ATR, DDTags.LIBRARY_CAPABILITIES_EFD, DDTags.LIBRARY_CAPABILITIES_IMPACTED_TESTS, DDTags.LIBRARY_CAPABILITIES_QUARANTINE, DDTags.LIBRARY_CAPABILITIES_DISABLED, DDTags.LIBRARY_CAPABILITIES_ATTEMPT_TO_FIX]  | !JUnitPlatformUtils.isJunitTestOrderingSupported(instrumentedLibraryVersion())
+  }
+
   protected void runTests(List<Class<?>> tests, boolean expectSuccess = true) {
     DiscoverySelector[] selectors = new DiscoverySelector[tests.size()]
     for (i in 0..<tests.size()) {