8000 Add connection pool metrics. Fixes #491 · mysql-net/MySqlConnector@d02539b · GitHub
[go: up one dir, main page]
Skip to content

Commit d02539b

Browse files
bgraingerbgrainger
committed
Add connection pool metrics. Fixes #491
This is based on the specification at https://github.com/open-telemetry/semantic-conventions/blob/main/specification/metrics/semantic_conventions/database-metrics.md. Signed-off-by: Bradley Grainger <bgrainger@gmail.com>
1 parent f53882e commit d02539b

15 files changed

+955
-16
lines changed

‎src/MySqlConnector/Core/ConnectionPool.cs

Lines changed: 60 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
using System.Collections.Concurrent;
22
using System.Diagnostics;
3+
using System.Diagnostics.Metrics;
34
using System.Net;
45
using System.Security.Authentication;
56
using Microsoft.Extensions.Logging;
@@ -19,6 +20,8 @@ internal sealed class ConnectionPool : IDisposable
1920

2021
public SslProtocols SslProtocols { get; set; }
2122

23+
public void AddPendingRequestCount(int delta) => s_pendingRequestsCounter.Add(delta, PoolNameTagList);
24+
2225
public async ValueTask<ServerSession> GetSessionAsync(MySqlConnection connection, int startTickCount, Activity? activity, IOBehavior ioBehavior, CancellationToken cancellationToken)
2326
{
2427
cancellationToken.ThrowIfCancellationRequested();
@@ -50,12 +53,14 @@ public async ValueTask<ServerSession> GetSessionAsync(MySqlConnection connection
5053
{
5154
if (m_sessions.Count > 0)
5255
{
56+
// NOTE: s_connectionsUsageCounter updated outside lock below
5357
session = m_sessions.First!.Value;
5458
m_sessions.RemoveFirst();
5559
}
5660
}
5761
if (session is not null)
5862
{
63+
s_connectionsUsageCounter.Add(-1, IdleStateTagList);
5964
Log.FoundExistingSession(m_logger, Id);
6065
bool reuseSession;
6166

@@ -89,8 +94,12 @@ public async ValueTask<ServerSession> GetSessionAsync(MySqlConnection connection
8994
m_leasedSessions.Add(session.Id, session);
9095
leasedSessionsCountPooled = m_leasedSessions.Count;
9196
}
97+
s_connectionsUsageCounter.Add(1, UsedStateTagList);
9298
ActivitySourceHelper.CopyTags(session.ActivityTags, activity);
9399
Log.ReturningPooledSession(m_logger, Id, session.Id, leasedSessionsCountPooled);
100+
101+
session.LastLeasedTicks = unchecked((uint) Environment.TickCount);
102+
s_waitTimeHistory.Record(unchecked(session.LastLeasedTicks - (uint) startTickCount), PoolNameTagList);
94103
return session;
95104
}
96105
}
@@ -105,7 +114,11 @@ public async ValueTask<ServerSession> GetSessionAsync(MySqlConnection connection
105114
m_leasedSessions.Add(session.Id, session);
106115
leasedSessionsCountNew = m_leasedSessions.Count;
107116
}
117+
s_connectionsUsageCounter.Add(1, UsedStateTagList);
108118
Log.ReturningNewSession(m_logger, Id, session.Id, leasedSessionsCountNew);
119+
120+
session.LastLeasedTicks = unchecked((uint) Environment.TickCount);
121+
s_createTimeHistory.Record(unchecked(session.LastLeasedTicks - (uint) startTickCount), PoolNameTagList);
109122
return session;
110123
}
111124
catch (Exception ex)
@@ -157,12 +170,14 @@ public async ValueTask ReturnAsync(IOBehavior ioBehavior, ServerSession session)
157170
{
158171
lock (m_leasedSessions)
159172
m_leasedSessions.Remove(session.Id);
173+
s_connectionsUsageCounter.Add(-1, UsedStateTagList);
160174
session.OwningConnection = null;
161175
var sessionHealth = GetSessionHealth(session);
162176
if (sessionHealth == 0)
163177
{
164178
lock (m_sessions)
165179
m_sessions.AddFirst(session);
180+
s_connectionsUsageCounter.Add(1, IdleStateTagList);
166181
}
167182
else
168183
{
@@ -236,6 +251,10 @@ public void Dispose()
236251
reaperWaitHandle.WaitOne();
237252
}
238253
#endif
254+
255+
s_minIdleConnectionsCounter.Add(-ConnectionSettings.MinimumPoolSize, PoolNameTagList);
256+
s_maxIdleConnectionsCounter.Add(-ConnectionSettings.MaximumPoolSize, PoolNameTagList);
257+
s_maxConnectionsCounter.Add(-ConnectionSettings.MaximumPoolSize, PoolNameTagList);
239258
}
240259

241260
/// <summary>
@@ -319,12 +338,14 @@ private async Task CleanPoolAsync(IOBehavior ioBehavior, Func<ServerSession, boo
319338
{
320339
if (m_sessions.Count > 0)
321340
{
341+
// NOTE: s_connectionsUsageCounter updated outside lock below
322342
session = m_sessions.Last!.Value;
323343
m_sessions.RemoveLast();
324344
}
325345
}
326346
if (session is null)
327347
return;
348+
s_connectionsUsageCounter.Add(-1, IdleStateTagList);
328349

329350
if (shouldCleanFn(session))
330351
{
@@ -337,6 +358,7 @@ private async Task CleanPoolAsync(IOBehavior ioBehavior, Func<ServerSession, boo
337358
// session should not be cleaned; put it back in the queue and stop iterating
338359
lock (m_sessions)
339360
m_sessions.AddLast(session);
361+
s_connectionsUsageCounter.Add(1, IdleStateTagList);
340362
return;
341363
}
342364
}
@@ -382,6 +404,7 @@ private async Task CreateMinimumPooledSessions(MySqlConnection connection, IOBeh
382404
AdjustHostConnectionCount(session, 1);
383405
lock (m_sessions)
384406
m_sessions.AddFirst(session);
407+
s_connectionsUsageCounter.Add(1, IdleStateTagList);
385408
}
386409
finally
387410
{
@@ -587,8 +610,22 @@ private ConnectionPool(MySqlConnectorLoggingConfiguration loggingConfiguration,
587610
cs.LoadBalance == MySqlLoadBalance.LeastConnections ? new LeastConnectionsLoadBalancer(m_hostSessions!) :
588611
(ILoadBalancer) new RoundRobinLoadBalancer();
589612

613+
// create tag lists for reporting pool metrics
614+
var connectionString = cs.ConnectionStringBuilder.GetConnectionString(includePassword: false);
615+
m_stateTagList = new KeyValuePair<string, object?>[3]
616+
{
617+
new("state", "idle"),
618+
new("pool.name", Name ?? connectionString),
619+
new("state", "used"),
620+
};
621+
622+
// set pool size counters
623+
s_minIdleConnectionsCounter.Add(ConnectionSettings.MinimumPoolSize, PoolNameTagList);
624+
s_maxIdleConnectionsCounter.Add(ConnectionSettings.MaximumPoolSize, PoolNameTagList);
625+
s_maxConnectionsCounter.Add(ConnectionSettings.MaximumPoolSize, PoolNameTagList);
626+
590627
Id = Interlocked.Increment(ref s_poolId);
591-
Log.CreatingNewConnectionPool(m_logger, Id, cs.ConnectionStringBuilder.GetConnectionString(includePassword: false));
628+
Log.CreatingNewConnectionPool(m_logger, Id, connectionString);
592629
}
593630

594631
private void StartReaperTask()
@@ -734,6 +771,13 @@ private void AdjustHostConnectionCount(ServerSession session, int delta)
734771
}
735772
}
736773

774+
// Provides a slice of m_stateTagList that contains either the 'idle' or 'used' state tag along with the pool name.
775+
private ReadOnlySpan<KeyValuePair<string, object?>> IdleStateTagList => m_stateTagList.AsSpan(0, 2);
776+
private ReadOnlySpan<KeyValuePair<string, object?>> UsedStateTagList => m_stateTagList.AsSpan(1, 2);
777+
778+
// A slice of m_stateTagList that contains only the pool name tag.
779+
public ReadOnlySpan<KeyValuePair<string, object?>> PoolNameTagList => m_stateTagList.AsSpan(1, 1);
780+
737781
private sealed class LeastConnectionsLoadBalancer : ILoadBalancer
738782
{
739783
public LeastConnectionsLoadBalancer(Dictionary<string, int> hostSessions) => m_hostSessions = hostSessions;
@@ -768,6 +812,20 @@ static ConnectionPool()
768812
private static void OnAppDomainShutDown(object? sender, EventArgs e) =>
769813
ClearPoolsAsync(IOBehavior.Synchronous, CancellationToken.None).GetAwaiter().GetResult();
770814

815+
private static readonly UpDownCounter<int> s_connectionsUsageCounter = ActivitySourceHelper.Meter.CreateUpDownCounter<int>("db.client.connections.usage",
816+
unit: "{connection}", description: "The number of connections that are currently in the state described by the state tag.");
817+
private static readonly UpDownCounter<int> s_maxIdleConnectionsCounter = ActivitySourceHelper.Meter.CreateUpDownCounter<int>("db.client.connections.idle.max",
818+
unit: "{connection}", description: "The maximum number of idle open connections allowed.");
819+
private static readonly UpDownCounter<int> s_minIdleConnectionsCounter = ActivitySourceHelper.Meter.CreateUpDownCounter<int>("db.client.connections.idle.min",
820+
unit: "{connection}", description: "The minimum number of idle open connections allowed.");
821+
private static readonly UpDownCounter<int> s_maxConnectionsCounter = ActivitySourceHelper.Meter.CreateUpDownCounter<int>("db.client.connections.max",
822+
unit: "{connection}", description: "The maximum number of open connections allowed.");
823+
private static readonly UpDownCounter<int> s_pendingRequestsCounter = ActivitySourceHelper.Meter.CreateUpDownCounter<int>("db.client.connections.pending_requests",
824+
unit: "{request}", description: "The number of pending requests for an open connection, cumulative for the entire pool.");
825+
private static readonly Histogram<float> s_createTimeHistory = ActivitySourceHelper.Meter.CreateHistogram<float>("db.client.connections.create_time",
826+
unit: "ms", description: "The time it took to create a new connection.");
827+
private static readonly Histogram<float> s_waitTimeHistory = ActivitySourceHelper.Meter.CreateHistogram<float>("db.client.connections.wait_time",
828+
unit: "ms", description: "The time it took to obtain an open connection from the pool.");
771829
private static readonly ConcurrentDictionary<string, ConnectionPool?> s_pools = new();
772830
private static readonly Action<ILogger, int, string, Exception?> s_createdNewSession = LoggerMessage.Define<int, string>(
773831
LogLevel.Debug, new EventId(EventIds.PoolCreatedNewSession, nameof(EventIds.PoolCreatedNewSession)),
@@ -780,6 +838,7 @@ private static void OnAppDomainShutDown(object? sender, EventArgs e) =>
780838

781839
private readonly ILogger m_logger;
782840
private readonly ILogger m_connectionLogger;
841+
private readonly KeyValuePair<string, object?>[] m_stateTagList;
783842
private readonly SemaphoreSlim m_cleanSemaphore;
784843
private readonly SemaphoreSlim m_sessionSemaphore;
785844
private readonly LinkedList<ServerSession> m_sessions;

‎src/MySqlConnector/Core/ServerSession.cs

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using System.Buffers.Text;
22
using System.ComponentModel;
33
using System.Diagnostics;
4+
using System.Diagnostics.Metrics;
45
using System.Globalization;
56
using System.IO.Pipes;
67
using System.Net;
@@ -56,6 +57,7 @@ public ServerSession(ILogger logger, ConnectionPool? pool, int poolGeneration, i
5657
public uint CreatedTicks { get; }
5758
public ConnectionPool? Pool { get; }
5859
public int PoolGeneration { get; }
60+
public uint LastLeasedTicks { get; set; }
5961
public uint LastReturnedTicks { get; private set; }
6062
public string? DatabaseOverride { get; set; }
6163
public string HostName { get; private set; }
@@ -75,7 +77,11 @@ public ValueTask ReturnToPoolAsync(IOBehavior ioBehavior, MySqlConnection? ownin
7577
{
7678
Log.ReturningToPool(m_logger, Id, Pool?.Id ?? 0);
7779
LastReturnedTicks = unchecked((uint) Environment.TickCount);
78-
return Pool is null ? default : Pool.ReturnAsync(ioBehavior, this);
80+
if (Pool is null)
81+
return default;
82+
s_useTimeHistory.Record(unchecked(LastReturnedTicks - LastLeasedTicks), Pool.PoolNameTagList);
83+
LastLeasedTicks = 0;
84+
return Pool.ReturnAsync(ioBehavior, this);
7985
}
8086

8187
public bool IsConnected
@@ -1953,6 +1959,8 @@ protected override void OnStatementBegin(int index)
19531959
[LoggerMessage(EventIds.ExpectedSessionState6, LogLevel.Error, "Session {SessionId} should have state {ExpectedState1} or {ExpectedState2} or {ExpectedState3} or {ExpectedState4} or {ExpectedState5} or {ExpectedState6} but was {SessionState}")]
19541960
private static partial void ExpectedSessionState6(ILogger logger, string sessionId, State expectedState1, State expectedState2, State expectedState3, State expectedState4, State expectedState5, State expectedState6, State sessionState);
19551961

1962+
private static readonly Histogram<float> s_useTimeHistory = ActivitySourceHelper.Meter.CreateHistogram<float>("db.client.connections.use_time",
1963+
unit: "ms", description: "The time between borrowing a connection and returning it to the pool.");
19561964
private static ReadOnlySpan<byte> BeginCertificateBytes => "-----BEGIN CERTIFICATE-----"u8;
19571965
private static readonly PayloadData s_setNamesUtf8NoAttributesPayload = QueryPayload.Create(false, "SET NAMES utf8;"u8);
19581966
private static readonly PayloadData s_setNamesUtf8mb4NoAttributesPayload = QueryPayload.Create(false, "SET NAMES utf8mb4;"u8);

‎src/MySqlConnector/MySqlConnection.cs

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -390,6 +390,8 @@ private async ValueTask<bool> PingAsync(IOBehavior ioBehavior, CancellationToken
390390

391391
internal async Task OpenAsync(IOBehavior? ioBehavior, CancellationToken cancellationToken)
392392
{
393+
var openStartTickCount = Environment.TickCount;
394+
393395
VerifyNotDisposed();
394396
cancellationToken.ThrowIfCancellationRequested();
395397
if (State != ConnectionState.Closed)
@@ -398,8 +400,6 @@ internal async Task OpenAsync(IOBehavior? ioBehavior, CancellationToken cancella
398400
using var activity = ActivitySourceHelper.StartActivity(ActivitySourceHelper.OpenActivityName);
399401
try
400402
{
401-
var openStartTickCount = Environment.TickCount;
402-
403403
SetState(ConnectionState.Connecting);
404404

405405
var pool = m_dataSource?.Pool ??
@@ -896,6 +896,7 @@ internal void FinishQuerying(bool hasWarnings)
896896

897897
private async ValueTask<ServerSession> CreateSessionAsync(ConnectionPool? pool, int startTickCount, Activity? activity, IOBehavior? ioBehavior, CancellationToken cancellationToken)
898898
{
899+
pool?.AddPendingRequestCount(1);
899900
var connectionSettings = GetInitializedConnectionSettings();
900901
var actualIOBehavior = ioBehavior ?? (connectionSettings.ForceSynchronous ? IOBehavior.Synchronous : IOBehavior.Asynchronous);
901902

@@ -949,6 +950,7 @@ private async ValueTask<ServerSession> CreateSessionAsync(ConnectionPool? pool,
949950
}
950951
finally
951952
{
953+
pool?.AddPendingRequestCount(-1);
952954
linkedSource?.Dispose();
953955
timeoutSource?.Dispose();
954956
}

‎src/MySqlConnector/MySqlConnector.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
<PackageReference Include="System.Threading.Tasks.Extensions" Version="4.5.4" />
3232
</ItemGroup>
3333

34-
<ItemGroup Condition=" '$(TargetFrameworkIdentifier)' != '.NETCoreApp' ">
34+
<ItemGroup Condition=" '$(TargetFrameworkIdentifier)' != '.NETCoreApp' OR '$(TargetFramework)' == 'net6.0' ">
3535
<PackageReference Include="System.Diagnostics.DiagnosticSource" Version="7.0.0" />
3636
</ItemGroup>
3737

‎src/MySqlConnector/Utilities/ActivitySourceHelper.cs

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using System.Diagnostics;
2+
using System.Diagnostics.Metrics;
23
using System.Globalization;
34
using System.Reflection;
45

@@ -64,12 +65,10 @@ public static void CopyTags(IEnumerable<KeyValuePair<string, object?>> tags, Act
6465
}
6566
}
6667

67-
private static ActivitySource ActivitySource { get; } = CreateActivitySource();
68+
public static Meter Meter { get; } = new("MySqlConnector", GetVersion());
6869

69-
private static ActivitySource CreateActivitySource()
70-
{
71-
var assembly = typeof(ActivitySourceHelper).Assembly;
72-
var version = assembly.GetCustomAttribute<AssemblyFileVersionAttribute>()!.Version;
73-
return new("MySqlConnector", version);
74-
}
70+
private static ActivitySource ActivitySource { get; } = new("MySqlConnector", GetVersion());
71+
72+
private static Version GetVersion() =>
73+
typeof(ActivitySourceHelper).Assembly.GetCustomAttribute<AssemblyFileVersionAttribute>()!.Version;
7574
}

0 commit comments

Comments
 (0)
0