10000 Update OpenTelemetryChatClient/EmbeddingGenerator to 1.33 (#6366) · dotnet/extensions@fb929d8 · GitHub
[go: up one dir, main page]
Skip to content

Commit fb929d8

Browse files
stephentoubjeffhandley
stephentoub
authored and
jeffhandley
committed
Update OpenTelemetryChatClient/EmbeddingGenerator to 1.33 (#6366)
Also adds the Enable SensitiveData property to OpenTelemetryEmbeddingGenerators. This was missed when adding it to OpenTelemetryChatClient.
1 parent 7314312 commit fb929d8

File tree

6 files changed

+63
-33
lines changed

6 files changed

+63
-33
lines changed

‎src/Libraries/Microsoft.Extensions.AI/ChatCompletion/FunctionInvokingChatClient.cs

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
using Microsoft.Shared.Diagnostics;
1616

1717
#pragma warning disable CA2213 // Disposable fields should be disposed
18-
#pragma warning disable CA1002 // Do not expose generic lists
1918
#pragma warning disable EA0002 // Use 'System.TimeProvider' to make the code easier to test
2019
#pragma warning disable SA1202 // 'protected' members should come before 'private' members
2120
#pragma warning disable S107 // Methods should not have too many parameters
@@ -214,7 +213,7 @@ public override async Task<ChatResponse> GetResponseAsync(
214213

215214
// A single request into this GetResponseAsync may result in multiple requests to the inner client.
216215
// Create an activity to group them together for better observability.
217-
using Activity? activity = _activitySource?.StartActivity(nameof(FunctionInvokingChatClient));
216+
using Activity? activity = _activitySource?.StartActivity($"{nameof(FunctionInvokingChatClient)}.{nameof(GetResponseAsync)}");
218217

219218
// Copy the original messages in order to avoid enumerating the original messages multiple times.
220219
// The IEnumerable can represent an arbitrary amount of work.
@@ -309,7 +308,7 @@ public override async IAsyncEnumerable<ChatResponseUpdate> GetStreamingResponseA
309308

310309
// A single request into this GetStreamingResponseAsync may result in multiple requests to the inner client.
311310
// Create an activity to group them together for better observability.
312-
using Activity? activity = _activitySource?.StartActivity(nameof(FunctionInvokingChatClient));
311+
using Activity? activity = _activitySource?.StartActivity($"{nameof(FunctionInvokingChatClient)}.{nameof(GetStreamingResponseAsync)}");
313312
UsageDetails? totalUsage = activity is { IsAllDataRequested: true } ? new() : null; // tracked usage across all turns, to be used for activity purposes
314313

315314
// Copy the original messages in order to avoid enumerating the original messages multiple times.
@@ -795,7 +794,16 @@ FunctionResultContent CreateFunctionResultContent(FunctionInvocationResult resul
795794
{
796795
_ = Throw.IfNull(context);
797796

798-
using Activity? activity = _activitySource?.StartActivity(context.Function.Name);
797+
using Activity? activity = _activitySource?.StartActivity(
798+
$"execute_tool {context.Function.Name}",
799+
ActivityKind.Internal,
800+
default(ActivityContext),
801+
[
802+
new(OpenTelemetryConsts.GenAI.Operation.Name, "execute_tool"),
803+
new(OpenTelemetryConsts.GenAI.Tool.Call.Id, context.CallContent.CallId),
804+
new(OpenTelemetryConsts.GenAI.Tool.Name, context.Function.Name),
805+
new(OpenTelemetryConsts.GenAI.Tool.Description, context.Function.Description),
806+
]);
799807

800808
long startingTimestamp = 0;
801809
if (_logger.IsEnabled(LogLevel.Debug))

‎src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ namespace Microsoft.Extensions.AI;
2525

2626
/// <summary>Represents a delegating chat client that implements the OpenTelemetry Semantic Conventions for Generative AI systems.</summary>
2727
/// <remarks>
28-
/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.32, defined at <see href="https://opentelemetry.io/docs/specs/semconv/gen-ai/" />.
28+
/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.33, defined at <see href="https://opentelemetry.io/docs/specs/semconv/gen-ai/" />.
2929
/// The specification is still experimental and subject to change; as such, the telemetry output by this client is also subject to change.
3030
/// </remarks>
3131
public sealed partial class OpenTelemetryChatClient : DelegatingChatClient

‎src/Libraries/Microsoft.Extensions.AI/Embeddings/OpenTelemetryEmbeddingGenerator.cs

Lines changed: 28 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ namespace Microsoft.Extensions.AI;
1919

2020
/// <summary>Represents a delegating embedding generator that implements the OpenTelemetry Semantic Conventions for Generative AI systems.</summary>
2121
/// <remarks>
22-
/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.32, defined at <see href="https://opentelemetry.io/docs/specs/semconv/gen-ai/" />.
22+
/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.33, defined at <see href="https://opentelemetry.io/docs/specs/semconv/gen-ai/" />.
2323
/// The specification is still experimental and subject to change; as such, the telemetry output by this client is also subject to change.
2424
/// </remarks>
2525
/// <typeparam name="TInput">The type of input used to produce embeddings.</typeparam>
@@ -86,6 +86,20 @@ public OpenTelemetryEmbeddingGenerator(IEmbeddingGenerator<TInput, TEmbedding> i
8686
);
8787
}
8888

89+
/// <summary>
90+
/// Gets or sets a value indicating whether potentially sensitive information should be included in telemetry.
91+
/// </summary>
92+
/// <value>
93+
/// <see langword="true"/> if potentially sensitive information should be included in telemetry;
94+
/// <see langword="false"/> if telemetry shouldn't include raw inputs and outputs.
95+
/// The default value is <see langword="false"/>.
96+
/// </value>
97+
/// <remarks>
98+
/// By default, telemetry includes metadata, such as token counts, but not raw inputs
99+
/// and outputs or additional options data.
100+
/// </remarks>
101+
public bool EnableSensitiveData { get; set; }
102+
89103
/// <inheritdoc/>
90104
public override object? GetService(Type serviceType, object? serviceKey = null) =>
91105
serviceType == typeof(ActivitySource) ? _activitySource :
@@ -163,20 +177,19 @@ protected override void Dispose(bool disposing)
163177
_ = activity.AddTag(OpenTelemetryConsts.GenAI.Request.EmbeddingDimensions, dimensionsValue);
164178
}
165179

166-
if (options is not null &&
167-
_system is not null)
180+
// Log all additional request options as per-provider tags. This is non-normative, but it covers cases where
181+
// there's a per-provider specification in a best-effort manner (e.g. gen_ai.openai.request.service_tier),
182+
// and more generally cases where there's additional useful information to be logged.
183+
// Since AdditionalProperties has undefined meaning, we treat it as potentially sensitive data.
184+
if (EnableSensitiveData &&
185+
_system is not null &&
186+
options?.AdditionalProperties is { } props)
168187
{
169-
// Log all additional request options as per-provider tags. This is non-normative, but it covers cases where
170-
// there's a per-provider specification in a best-effort manner (e.g. gen_ai.openai.request.service_tier),
171-
// and more generally cases where there's additional useful information to be logged.
172-
if (options.AdditionalProperties is { } props)
188+
foreach (KeyValuePair<string, object?> prop in props)
173189
{
174-
foreach (KeyValuePair<string, object?> prop in props)
175-
{
176-
_ = activity.AddTag(
177-
OpenTelemetryConsts.GenAI.Request.PerProvider(_system, JsonNamingPolicy.SnakeCaseLower.ConvertName(prop.Key)),
178-
prop.Value);
179-
}
190+
_ = activity.AddTag(
191+
OpenTelemetryConsts.GenAI.Request.PerProvider(_system, JsonNamingPolicy.SnakeCaseLower.ConvertName(prop.Key)),
192+
prop.Value);
180193
}
181194
}
182195
}
@@ -247,7 +260,8 @@ private void TraceResponse(
247260
// Log all additional response properties as per-provider tags. This is non-normative, but it covers cases where
248261
// there's a per-provider specification in a best-effort manner (e.g. gen_ai.openai.response.system_fingerprint),
249262
// and more generally cases where there's additional useful information to be logged.
250-
if (_system is not null &&
263+
if (EnableSensitiveData &&
264+
_system is not null &&
251265
embeddings?.AdditionalProperties is { } props)
252266
{
253267
foreach (KeyValuePair<string, object?> prop in props)

‎src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,14 @@ public static class Token
103103

104104
public static class Tool
105105
{
106+
public const string Name = "gen_ai.tool.name";
107+
public const string Description = "gen_ai.tool.description";
106108
public const string Message = "gen_ai.tool.message";
109+
110+
public static class Call
111+
{
112+
public const string Id = "gen_ai.tool.call.id";
113+
}
107114
}
108115

109116
public static class User

‎test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/FunctionInvokingChatClientTests.cs

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -532,11 +532,11 @@ public async Task FunctionInvocationTrackedWithActivity(bool enableTelemetry)
532532
Func<ChatClientBuilder, ChatClientBuilder> configure = b => b.Use(c =>
533533
new FunctionInvokingChatClient(new OpenTelemetryChatClient(c, sourceName: sourceName)));
534534

535-
await InvokeAsync(() => InvokeAndAssertAsync(options, plan, configurePipeline: configure));
535+
await InvokeAsync(() => InvokeAndAssertAsync(options, plan, configurePipeline: configure), streaming: false);
536536

537-
await InvokeAsync(() => InvokeAndAssertStreamingAsync(options, plan, configurePipeline: configure));
537+
await InvokeAsync(() => InvokeAndAssertStreamingAsync(options, plan, configurePipeline: configure), streaming: true);
538538

539-
async Task InvokeAsync(Func<Task> work)
539+
async Task InvokeAsync(Func<Task> work, bool streaming)
540540
{
541541
var activities = new List<Activity>();
542542
using TracerProvider? tracerProvider = enableTelemetry ?
@@ -552,9 +552,9 @@ async Task InvokeAsync(Func<Task> work)
552552
{
553553
Assert.Collection(activities,
554554
activity => Assert.Equal("chat", activity.DisplayName),
555-
activity => Assert.Equal("Func1", activity.DisplayName),
555+
activity => Assert.Equal("execute_tool Func1", activity.DisplayName),
556556
activity => Assert.Equal("chat", activity.DisplayName),
557-
activity => Assert.Equal(nameof(FunctionInvokingChatClient), activity.DisplayName));
557+
activity => Assert.Equal(streaming ? "FunctionInvokingChatClient.GetStreamingResponseAsync" : "FunctionInvokingChatClient.GetResponseAsync", activity.DisplayName));
558558

559559
for (int i = 0; i < activities.Count - 1; i++)
560560
{

‎test/Libraries/Microsoft.Extensions.AI.Tests/Embeddings/OpenTelemetryEmbeddingGeneratorTests.cs

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,10 @@ namespace Microsoft.Extensions.AI;
1616
public class OpenTelemetryEmbeddingGeneratorTests
1717
{
1818
[Theory]
19-
[InlineData(null)]
20-
[InlineData("replacementmodel")]
21-
public async Task ExpectedInformationLogged_Async(string? perRequestModelId)
19+
[InlineData(null, false)]
20+
[InlineData("replacementmodel", false)]
21+
[InlineData("replacementmodel", true)]
22+
public async Task ExpectedInformationLogged_Async(string? perRequestModelId, bool enableSensitiveData)
2223
{
2324
var sourceName = Guid.NewGuid().ToString();
2425
var activities = new List<Activity>();
@@ -45,7 +46,7 @@ public async Task ExpectedInformationLogged_Async(string? perRequestModelId)
4546
AdditionalProperties = new()
4647
{
4748
["system_fingerprint"] = "abcdefgh",
48-
["AndSomethingElse"] = "value2",
49+
["AndSomethingElse"] = "value3",
4950
}
5051
};
5152
},
@@ -56,7 +57,7 @@ public async Task ExpectedInformationLogged_Async(string? perRequestModelId)
5657

5758
using var generator = innerGenerator
5859
.AsBuilder()
59-
.UseOpenTelemetry(loggerFactory, sourceName)
60+
.UseOpenTelemetry(loggerFactory, sourceName, configure: g => g.EnableSensitiveData = enableSensitiveData)
6061
.Build();
6162

6263
var options = new EmbeddingGenerationOptions
@@ -85,12 +86,12 @@ public async Task ExpectedInformationLogged_Async(string? perRequestModelId)
8586

8687
Assert.Equal(expectedModelName, activity.GetTagItem("gen_ai.request.model"));
8788
Assert.Equal(1234, activity.GetTagItem("gen_ai.request.embedding.dimensions"));
88-
Assert.Equal("value1", activity.GetTagItem("gen_ai.testservice.request.service_tier"));
89-
Assert.Equal("value2", activity.GetTagItem("gen_ai.testservice.request.something_else"));
89+
Assert.Equal(enableSensitiveData ? "value1" : null, activity.GetTagItem("gen_ai.testservice.request.service_tier"));
90+
Assert.Equal(enableSensitiveData ? "value2" : null, activity.GetTagItem("gen_ai.testservice.request.something_else"));
9091

9192
Assert.Equal(10, activity.GetTagItem("gen_ai.response.input_tokens"));
92-
Assert.Equal("abcdefgh", activity.GetTagItem("gen_ai.testservice.response.system_fingerprint"));
93-
Assert.Equal("value2", activity.GetTagItem("gen_ai.testservice.response.and_something_else"));
93+
Assert.Equal(enableSensitiveData ? "abcdefgh" : null, activity.GetTagItem("gen_ai.testservice.response.system_fingerprint"));
94+
Assert.Equal(enableSensitiveData ? "value3" : null, activity.GetTagItem("gen_ai.testservice.response.and_something_else"));
9495

9596
Assert.True(activity.Duration.TotalMilliseconds > 0);
9697
}

0 commit comments

Comments
 (0)
0