10000 add LLM obs configs · DataDog/dd-trace-java@3863387 · GitHub
[go: up one dir, main page]
Skip to content

Commit 3863387

Browse files
gary-huanggary-huang
committed
add LLM obs configs
1 parent 2b24697 commit 3863387

File tree

7 files changed

+129
-1
lines changed

7 files changed

+129
-1
lines changed

‎dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/AgentInstaller.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -311,6 +311,9 @@ public static Set<InstrumenterModule.TargetSystem> getEnabledSystems() {
311311
if (cfg.isUsmEnabled()) {
312312
enabledSystems.add(InstrumenterModule.TargetSystem.USM);
313313
}
314+
if (cfg.isLlmObsEnabled()) {
315+
enabledSystems.add(InstrumenterModule.TargetSystem.LLMOBS);
316+
}
314317
return enabledSystems;
315318
}
316319

‎dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/InstrumenterModule.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,8 @@ public enum TargetSystem {
4949
APPSEC,
5050
IAST,
5151
CIVISIBILITY,
52-
USM
52+
USM,
53+
LLMOBS,
5354
}
5455

5556
private static final Logger log = LoggerFactory.getLogger(InstrumenterModule.class);

‎dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,9 @@ public final class ConfigDefaults {
135135

136136
static final boolean DEFAULT_IAST_STACK_TRACE_ENABLED = true;
137137

138+
static final boolean DEFAULT_LLM_OBS_ENABLED = false;
139+
static final boolean DEFAULT_LLM_OBS_AGENTLESS_ENABLED = false;
140+
138141
static final boolean DEFAULT_USM_ENABLED = false;
139142

140143
static final boolean DEFAULT_CIVISIBILITY_ENABLED = false;

‎dd-trace-api/src/main/java/datadog/trace/api/config/LlmObsConfig.java

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
package datadog.trace.api.config;
2+
3+
/**
4+
* Constant with names of configuration options for LLM Observability. (EXPERIMENTAL AND SUBJECT TO
5+
* CHANGE)
6+
*/
7+
public final class LlmObsConfig {
8+
9+
public static final String LLM_OBS_ENABLED = "llmobs.enabled";
10+
11+
public static final String LLM_OBS_ML_APP = "llmobs.ml.app";
12+
13+
public static final String LLM_OBS_AGENTLESS_ENABLED = "llmobs.agentless.enabled";
14+
15+
private LlmObsConfig() {}
16+
}

‎internal-api/src/main/java/datadog/trace/api/Config.java

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import static datadog.trace.api.config.GeneralConfig.SERVICE_NAME;
1414
import static datadog.trace.api.config.IastConfig.*;
1515
import static datadog.trace.api.config.JmxFetchConfig.*;
16+
import static datadog.trace.api.config.LlmObsConfig.*;
1617
import static datadog.trace.api.config.ProfilingConfig.*;
1718
import static datadog.trace.api.config.RemoteConfigConfig.*;
1819
import static datadog.trace.api.config.TraceInstrumentationConfig.*;
@@ -307,6 +308,9 @@ public static String getHostName() {
307308
private final boolean iastStackTraceEnabled;
308309
private final boolean iastExperimentalPropagationEnabled;
309310

311+
private final boolean llmObsAgentlessEnabled;
312+
private final String llmObsMlApp;
313+
310314
private final boolean ciVisibilityTraceSanitationEnabled;
311315
private final boolean ciVisibilityAgentlessEnabled;
312316
private final String ciVisibilityAgentlessUrl;
@@ -1319,6 +1323,10 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment())
13191323
iastExperimentalPropagationEnabled =
13201324
configProvider.getBoolean(IAST_EXPERIMENTAL_PROPAGATION_ENABLED, false);
13211325

1326+
llmObsAgentlessEnabled =
1327+
configProvider.getBoolean(LLM_OBS_AGENTLESS_ENABLED, DEFAULT_LLM_OBS_AGENTLESS_ENABLED);
1328+
llmObsMlApp = configProvider.getString(LLM_OBS_ML_APP);
1329+
13221330
ciVisibilityTraceSanitationEnabled =
13231331
configProvider.getBoolean(CIVISIBILITY_TRACE_SANITATION_ENABLED, true);
13241332

@@ -1754,6 +1762,18 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment())
17541762
configProvider.getLong(
17551763
TRACE_POST_PROCESSING_TIMEOUT, ConfigDefaults.DEFAULT_TRACE_POST_PROCESSING_TIMEOUT);
17561764

1765+
if (isLlmObsEnabled()) {
1766+
if (llmObsMlApp == null || llmObsMlApp.isEmpty()) {
1767+
throw new IllegalArgumentException("Attempt to enable LLM Observability without ML app defined."
1768+
+ "Please ensure that the name of the ML app is provided through properties or env variable");
1769+
}
1770+
if (llmObsAgentlessEnabled && (apiKey == null || apiKey.isEmpty())) {
1771+
throw new FatalAgentMisconfigurationError(
1772+
"Attempt to start LLM Observability in Agentless mode without API key. "
1773+
+ "Please ensure that either an API key is configured, or the tracer is set up to work with the Agent");
1774+
}
1775+
}
1776+
17571777
if (isCiVisibilityEnabled()
17581778
&& ciVisibilityAgentlessEnabled
17591779
&& (apiKey == null || apiKey.isEmpty())) {
@@ -2606,6 +2626,18 @@ public boolean isIastExperimentalPropagationEnabled() {
26062626
return iastExperimentalPropagationEnabled;
26072627
}
26082628

2629+
public boolean isLlmObsEnabled() {
2630+
return instrumenterConfig.isLlmObsEnabled();
2631+
}
2632+
2633+
public boolean isLlmObsAgentlessEnabled() {
2634+
return llmObsAgentlessEnabled;
2635+
}
2636+
2637+
public String getLlmObsMlApp() {
2638+
return llmObsMlApp;
2639+
}
2640+
26092641
public boolean isCiVisibilityEnabled() {
26102642
return instrumenterConfig.isCiVisibilityEnabled();
26112643
}

‎internal-api/src/main/java/datadog/trace/api/InstrumenterConfig.java

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import static datadog.trace.api.ConfigDefaults.DEFAULT_CODE_ORIGIN_FOR_SPANS_ENABLED;
66
import static datadog.trace.api.ConfigDefaults.DEFAULT_IAST_ENABLED;
77
import static datadog.trace.api.ConfigDefaults.DEFAULT_INTEGRATIONS_ENABLED;
8+
import static datadog.trace.api.ConfigDefaults.DEFAULT_LLM_OBS_ENABLED;
89
import static datadog.trace.api.ConfigDefaults.DEFAULT_MEASURE_METHODS;
910
import static datadog.trace.api.ConfigDefaults.DEFAULT_RESOLVER_RESET_INTERVAL;
1011
import static datadog.trace.api.ConfigDefaults.DEFAULT_RUNTIME_CONTEXT_FIELD_INJECTION;
@@ -26,6 +27,7 @@
2627
import static datadog.trace.api.config.GeneralConfig.TRACE_TRIAGE;
2728
import static datadog.trace.api.config.GeneralConfig.TRIAGE_REPORT_TRIGGER;
2829
import static datadog.trace.api.config.IastConfig.IAST_ENABLED;
30+
import static datadog.trace.api.config.LlmObsConfig.LLM_OBS_ENABLED;
2931
import static datadog.trace.api.config.ProfilingConfig.PROFILING_DIRECT_ALLOCATION_ENABLED;
3032
import static datadog.trace.api.config.ProfilingConfig.PROFILING_DIRECT_ALLOCATION_ENABLED_DEFAULT;
3133
import static datadog.trace.api.config.ProfilingConfig.PROFILING_ENABLED;
@@ -111,6 +113,7 @@ public class InstrumenterConfig {
111113
private final boolean iastFullyDisabled;
112114
private final boolean usmEnabled;
113115
private final boolean telemetryEnabled;
116+
private final boolean llmObsEnabled;
114117

115118
private final String traceExtensionsPath;
116119

@@ -199,6 +202,7 @@ private InstrumenterConfig() {
199202
iastFullyDisabled = iastEnabled != null && !iastEnabled;
200203
usmEnabled = configProvider.getBoolean(USM_ENABLED, DEFAULT_USM_ENABLED);
201204
telemetryEnabled = configProvider.getBoolean(TELEMETRY_ENABLED, DEFAULT_TELEMETRY_ENABLED);
205+
llmObsEnabled = configProvider.getBoolean(LLM_OBS_ENABLED, DEFAULT_LLM_OBS_ENABLED);
202206
} else {
203207
// disable these features in native-image
204208
ciVisibilityEnabled = false;
@@ -207,6 +211,7 @@ private InstrumenterConfig() {
207211
iastFullyDisabled = true;
208212
telemetryEnabled = false;
209213
usmEnabled = false;
214+
llmObsEnabled = false;
210215
}
211216

212217
traceExtensionsPath = configProvider.getString(TRACE_EXTENSIONS_PATH);
@@ -355,6 +360,10 @@ public boolean isIastFullyDisabled() {
355360
return iastFullyDisabled;
356361
}
357362

363+
public boolean isLlmObsEnabled() {
364+
return llmObsEnabled;
365+
}
366+
358367
public boolean isUsmEnabled() {
359368
return usmEnabled;
360369
}

‎internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,9 @@ import static datadog.trace.api.config.JmxFetchConfig.JMX_FETCH_REFRESH_BEANS_PE
6363
import static datadog.trace.api.config.JmxFetchConfig.JMX_FETCH_STATSD_HOST
6464
import static datadog.trace.api.config.JmxFetchConfig.JMX_FETCH_STATSD_PORT
6565
import static datadog.trace.api.config.JmxFetchConfig.JMX_TAGS
66+
import static datadog.trace.api.con 10670 fig.LlmObsConfig.LLM_OBS_AGENTLESS_ENABLED
67+
import static datadog.trace.api.config.LlmObsConfig.LLM_OBS_ML_APP
68+
import static datadog.trace.api.config.LlmObsConfig.LLM_OBS_ENABLED
6669
import static datadog.trace.api.config.ProfilingConfig.PROFILING_AGENTLESS
6770
import static datadog.trace.api.config.ProfilingConfig.PROFILING_API_KEY_FILE_OLD
6871
import static datadog.trace.api.config.ProfilingConfig.PROFILING_API_KEY_FILE_VERY_OLD
@@ -2208,6 +2211,67 @@ class ConfigTest extends DDSpecification {
22082211
!hostname.trim().isEmpty()
22092212
}
22102213

2214+
def "config instantiation should fail if llm obs is enabled and ml app is not set"() {
2215+
setup:
2216+
Properties properties = new Properties()
2217+
properties.setProperty(LLM_OBS_ENABLED, "true")
2218+
2219+
when:
2220+
new Config(ConfigProvider.withPropertiesOverride(properties))
2221+
2222+
then:
2223+
thrown IllegalArgumentException
2224+
}
2225+
2226+
def "config instantiation should NOT fail if llm obs is enabled (agentless disabled) and ml app is set"() {
2227+
setup:
2228+
Properties properties = new Properties()
2229+
properties.setProperty(LLM_OBS_ENABLED, "true")
2230+
properties.setProperty(LLM_OBS_AGENTLESS_ENABLED, "false")
2231+
properties.setProperty(LLM_OBS_ML_APP, "test-ml-app")
2232+
2233+
when:
2234+
def config = new Config(ConfigProvider.withPropertiesOverride(properties))
2235+
2236+
then:
2237+
noExceptionThrown()
2238+
config.isLlmObsEnabled()
2239+
!config.isLlmObsAgentlessEnabled()
2240+
config.llmObsMlApp == "test-ml-app"
2241+
}
2242+
2243+
def "config instantiation should fail if llm obs is in agentless mode and API key is not set"() {
2244+
setup:
2245+
Properties properties = new Properties()
2246+
properties.setProperty(LLM_OBS_ENABLED, "true")
2247+
properties.setProperty(LLM_OBS_AGENTLESS_ENABLED, "true")
2248+
properties.setProperty(LLM_OBS_ML_APP, "test-ml-app")
2249+
2250+
when:
2251+
new Config(ConfigProvider.withPropertiesOverride(properties))
2252+
2253+
then:
2254+
thrown FatalAgentMisconfigurationError
2255+
}
2256+
2257+
def "config instantiation should NOT fail if llm obs is enabled (agentless enabled) and API key & ml app are set"() {
2258+
setup:
2259+
Properties properties = new Properties()
2260+
properties.setProperty(LLM_OBS_ENABLED, "true")
2261+
properties.setProperty(LLM_OBS_AGENTLESS_ENABLED, "true")
2262+
properties.setProperty(LLM_OBS_ML_APP, "test-ml-app")
2263+
properties.setProperty(API_KEY, "123456789")
2264+
2265+
when:
2266+
def config = new Config(ConfigProvider.withPropertiesOverride(properties))
2267+
2268+
then:
2269+
noExceptionThrown()
2270+
config.isLlmObsEnabled()
2271+
config.isLlmObsAgentlessEnabled()
2272+
config.llmObsMlApp == "test-ml-app"
2273+
}
2274+
22112275
def "config instantiation should fail if CI visibility agentless mode is enabled and API key is not set"() {
22122276
setup:
22132277
Properties properties = new Properties()

0 commit comments

Comments
 (0)
0