8000 Add connection pool metrics. Fixes #491 · mysql-net/MySqlConnector@ec5f314 · GitHub
[go: up one dir, main page]
Skip to content

Commit ec5f314

Browse files
bgraingerbgrainger
committed
Add connection pool metrics. Fixes #491
This is based on the specification at https://github.com/open-telemetry/semantic-conventions/blob/main/specification/metrics/semantic_conventions/database-metrics.md. Signed-off-by: Bradley Grainger <bgrainger@gmail.com>
1 parent 7a5993d commit ec5f314

14 files changed

+656
-16
lines changed

‎src/MySqlConnector/Core/ConnectionPool.cs

Lines changed: 60 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
using System.Collections.Concurrent;
22
using System.Diagnostics;
3+
using System.Diagnostics.Metrics;
34
using System.Net;
45
using System.Security.Authentication;
56
using Microsoft.Extensions.Logging;
@@ -19,6 +20,8 @@ internal sealed class ConnectionPool : IDisposable
1920

2021
public SslProtocols SslProtocols { get; set; }
2122

23+
public void AddPendingRequestCount(int delta) => s_pendingRequestsCounter.Add(delta, PoolNameTagList);
24+
2225
public async ValueTask<ServerSession> GetSessionAsync(MySqlConnection connection, int startTickCount, int timeoutMilliseconds, Activity? activity, IOBehavior ioBehavior, CancellationToken cancellationToken)
2326
{
2427
cancellationToken.ThrowIfCancellationRequested();
@@ -50,12 +53,14 @@ public async ValueTask<ServerSession> GetSessionAsync(MySqlConnection connection
5053
{
5154
if (m_sessions.Count > 0)
5255
{
56+
// NOTE: s_connectionsUsageCounter updated outside lock below
5357
session = m_sessions.First!.Value;
5458
m_sessions.RemoveFirst();
5559
}
5660
}
5761
if (session is not null)
5862
{
63+
s_connectionsUsageCounter.Add(-1, IdleStateTagList);
5964
Log.FoundExistingSession(m_logger, Id);
6065
bool reuseSession;
6166

@@ -96,8 +101,12 @@ public async ValueTask<ServerSession> GetSessionAsync(MySqlConnection connection
96101
m_leasedSessions.Add(session.Id, session);
97102
leasedSessionsCountPooled = m_leasedSessions.Count;
98103
}
104+
s_connectionsUsageCounter.Add(1, UsedStateTagList);
99105
ActivitySourceHelper.CopyTags(session.ActivityTags, activity);
100106
Log.ReturningPooledSession(m_logger, Id, session.Id, leasedSessionsCountPooled);
107+
108+
session.LastLeasedTicks = unchecked((uint) Environment.TickCount);
109+
s_waitTimeHistory.Record(unchecked(session.LastLeasedTicks - (uint) startTickCount), PoolNameTagList);
101110
return session;
102111
}
103112
}
@@ -112,7 +121,11 @@ public async ValueTask<ServerSession> GetSessionAsync(MySqlConnection connection
112121
m_leasedSessions.Add(session.Id, session);
113122
leasedSessionsCountNew = m_leasedSessions.Count;
114123
}
124+
s_connectionsUsageCounter.Add(1, UsedStateTagList);
115125
Log.ReturningNewSession(m_logger, Id, session.Id, leasedSessionsCountNew);
126+
127+
session.LastLeasedTicks = unchecked((uint) Environment.TickCount);
128+
s_createTimeHistory.Record(unchecked(session.LastLeasedTicks - (uint) startTickCount), PoolNameTagList);
116129
return session;
117130
}
118131
catch (Exception ex)
@@ -164,12 +177,14 @@ public async ValueTask ReturnAsync(IOBehavior ioBehavior, ServerSession session)
164177
{
165178
lock (m_leasedSessions)
166179
m_leasedSessions.Remove(session.Id);
180+
s_connectionsUsageCounter.Add(-1, UsedStateTagList);
167181
session.OwningConnection = null;
168182
var sessionHealth = GetSessionHealth(session);
169183
if (sessionHealth == 0)
170184
{
171185
lock (m_sessions)
172186
m_sessions.AddFirst(session);
187+
s_connectionsUsageCounter.Add(1, IdleStateTagList);
173188
}
174189
else
175190
{
@@ -243,6 +258,10 @@ public void Dispose()
243258
reaperWaitHandle.WaitOne();
244259
}
245260
#endif
261+
262+
s_minIdleConnectionsCounter.Add(-ConnectionSettings.MinimumPoolSize, PoolNameTagList);
263+
s_maxIdleConnectionsCounter.Add(-ConnectionSettings.MaximumPoolSize, PoolNameTagList);
264+
s_maxConnectionsCounter 10000 .Add(-ConnectionSettings.MaximumPoolSize, PoolNameTagList);
246265
}
247266

248267
/// <summary>
@@ -326,12 +345,14 @@ private async Task CleanPoolAsync(IOBehavior ioBehavior, Func<ServerSession, boo
326345
{
327346
if (m_sessions.Count > 0)
328347
{
348+
// NOTE: s_connectionsUsageCounter updated outside lock below
329349
session = m_sessions.Last!.Value;
330350
m_sessions.RemoveLast();
331351
}
332352
}
333353
if (session is null)
334354
return;
355+
s_connectionsUsageCounter.Add(-1, IdleStateTagList);
335356

336357
if (shouldCleanFn(session))
337358
{
@@ -344,6 +365,7 @@ private async Task CleanPoolAsync(IOBehavior ioBehavior, Func<ServerSession, boo
344365
// session should not be cleaned; put it back in the queue and stop iterating
345366
lock (m_sessions)
346367
m_sessions.AddLast(session);
368+
s_connectionsUsageCounter.Add(1, IdleStateTagList);
347369
return;
348370
}
349371
}
@@ -389,6 +411,7 @@ private async Task CreateMinimumPooledSessions(MySqlConnection connection, IOBeh
389411
AdjustHostConnectionCount(session, 1);
390412
lock (m_sessions)
391413
m_sessions.AddFirst(session);
414+
s_connectionsUsageCounter.Add(1, IdleStateTagList);
392415
}
393416
finally
394417
{
@@ -594,8 +617,22 @@ private ConnectionPool(MySqlConnectorLoggingConfiguration loggingConfiguration,
594617
cs.LoadBalance == MySqlLoadBalance.LeastConnections ? new LeastConnectionsLoadBalancer(m_hostSessions!) :
595618
(ILoadBalancer) new RoundRobinLoadBalancer();
596619

620+
// create tag lists for reporting pool metrics
621+
var connectionString = cs.ConnectionStringBuilder.GetConnectionString(includePassword: false);
622+
m_stateTagList =
623+
[
624+
new("state", "idle"),
625+
new("pool.name", Name ?? connectionString),
626+
new("state", "used"),
627+
];
628+
629+
// set pool size counters
630+
s_minIdleConnectionsCounter.Add(ConnectionSettings.MinimumPoolSize, PoolNameTagList);
631+
s_maxIdleConnectionsCounter.Add(ConnectionSettings.MaximumPoolSize, PoolNameTagList);
632+
s_maxConnectionsCounter.Add(ConnectionSettings.MaximumPoolSize, PoolNameTagList);
633+
597634
Id = Interlocked.Increment(ref s_poolId);
598-
Log.CreatingNewConnectionPool(m_logger, Id, cs.ConnectionStringBuilder.GetConnectionString(includePassword: false));
635+
Log.CreatingNewConnectionPool(m_logger, Id, connectionString);
599636
}
600637

601638
private void StartReaperTask()
@@ -741,6 +778,13 @@ private void AdjustHostConnectionCount(ServerSession session, int delta)
741778
}
742779
}
743780

781+
// Provides a slice of m_stateTagList that contains either the 'idle' or 'used' state tag along with the pool name.
782+
private ReadOnlySpan<KeyValuePair<string, object?>> IdleStateTagList => m_stateTagList.AsSpan(0, 2);
783+
private ReadOnlySpan<KeyValuePair<string, object?>> UsedStateTagList => m_stateTagList.AsSpan(1, 2);
784+
785+
// A slice of m_stateTagList that contains only the pool name tag.
786+
public ReadOnlySpan<KeyValuePair<string, object?>> PoolNameTagList => m_stateTagList.AsSpan(1, 1);
787+
744788
private sealed class LeastConnectionsLoadBalancer : ILoadBalancer
745789
{
746790
public LeastConnectionsLoadBalancer(Dictionary<string, int> hostSessions) => m_hostSessions = hostSessions;
@@ -775,6 +819,20 @@ static ConnectionPool()
775819
private static void OnAppDomainShutDown(object? sender, EventArgs e) =>
776820
ClearPoolsAsync(IOBehavior.Synchronous, CancellationToken.None).GetAwaiter().GetResult();
777821

822+
private static readonly UpDownCounter<int> s_connectionsUsageCounter = ActivitySourceHelper.Meter.CreateUpDownCounter<int>("db.client.connections.usage",
823+
unit: "{connection}", description: "The number of connections that are currently in the state described by the state tag.");
824+
private static readonly UpDownCounter<int> s_maxIdleConnectionsCounter = ActivitySourceHelper.Meter.CreateUpDownCounter<int>("db.client.connections.idle.max",
825+
unit: "{connection}", description: "The maximum number of idle open connections allowed.");
826+
private static readonly UpDownCounter<int> s_minIdleConnectionsCounter = ActivitySourceHelper.Meter.CreateUpDownCounter<int>("db.client.connections.idle.min",
827+
unit: "{connection}", description: "The minimum number of idle open connections allowed.");
828+
private static readonly UpDownCounter<int> s_maxConnectionsCounter = ActivitySourceHelper.Meter.CreateUpDownCounter<int>("db.client.connections.max",
829+
unit: "{connection}", description: "The maximum number of open connections allowed.");
830+
private static readonly UpDownCounter<int> s_pendingRequestsCounter = ActivitySourceHelper.Meter.CreateUpDownCounter<int>("db.client.connections.pending_requests",
831+
unit: "{request}", description: "The number of pending requests for an open connection, cumulative for the entire pool.");
832+
private static readonly Histogram<float> s_createTimeHistory = ActivitySourceHelper.Meter.CreateHistogram<float>("db.client.connections.create_time",
833+
unit: "ms", description: "The time it took to create a new connection.");
834+
private static readonly Histogram<float> s_waitTimeHistory = ActivitySourceHelper.Meter.CreateHistogram<float>("db.client.connections.wait_time",
835+
unit: "ms", description: "The time it took to obtain an open connection from the pool.");
778836
private static readonly ConcurrentDictionary<string, ConnectionPool?> s_pools = new();
779837
private static readonly Action<ILogger, int, string, Exception?> s_createdNewSession = LoggerMessage.Define<int, string>(
780838
LogLevel.Debug, new EventId(EventIds.PoolCreatedNewSession, nameof(EventIds.PoolCreatedNewSession)),
@@ -787,6 +845,7 @@ private static void OnAppDomainShutDown(object? sender, EventArgs e) =>
787845

788846
private readonly ILogger m_logger;
789847
private readonly ILogger m_connectionLogger;
848+
private readonly KeyValuePair<string, object?>[] m_stateTagList;
790849
private readonly SemaphoreSlim m_cleanSemaphore;
791850
private readonly SemaphoreSlim m_sessionSemaphore;
792851
private readonly LinkedList<ServerSession> m_sessions;

‎src/MySqlConnector/Core/ServerSession.cs

Lines changed: 9 additions & 1 deletion
< 10000 td data-grid-cell-id="diff-22f5eec5ea345b8f0fd8c96f47bb2a7e59d35071c7ad2525d6f251a5ece19d5c-75-77-0" data-selected="false" role="gridcell" style="background-color:var(--bgColor-default);text-align:center" tabindex="-1" valign="top" class="focusable-grid-cell diff-line-number position-relative diff-line-number-neutral left-side">75
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using System.Buffers.Text;
22
using System.ComponentModel;
33
using System.Diagnostics;
4+
using System.Diagnostics.Metrics;
45
using System.Globalization;
56
using System.IO.Pipes;
67
using System.Net;
@@ -56,6 +57,7 @@ public ServerSession(ILogger logger, ConnectionPool? pool, int poolGeneration, i
5657
public uint CreatedTicks { get; }
5758
public ConnectionPool? Pool { get; }
5859
public int PoolGeneration { get; }
60+
public uint LastLeasedTicks { get; set; }
5961
public uint LastReturnedTicks { get; private set; }
6062
public string? DatabaseOverride { get; set; }
6163
public string HostName { get; private set; }
@@ -75,7 +77,11 @@ public ValueTask ReturnToPoolAsync(IOBehavior ioBehavior, MySqlConnection? ownin
77
{
7678
Log.ReturningToPool(m_logger, Id, Pool?.Id ?? 0);
7779
LastReturnedTicks = unchecked((uint) Environment.TickCount);
78-
return Pool is null ? default : Pool.ReturnAsync(ioBehavior, this);
80+
if (Pool is null)
81+
return default;
82+
s_useTimeHistory.Record(unchecked(LastReturnedTicks - LastLeasedTicks), Pool.PoolNameTagList);
83+
LastLeasedTicks = 0;
84+
return Pool.ReturnAsync(ioBehavior, this);
7985
}
8086

8187
public bool IsConnected
@@ -1915,6 +1921,8 @@ protected override void OnStatementBegin(int index)
19151921
[LoggerMessage(EventIds.ExpectedSessionState6, LogLevel.Error, "Session {SessionId} should have state {ExpectedState1} or {ExpectedState2} or {ExpectedState3} or {ExpectedState4} or {ExpectedState5} or {ExpectedState6} but was {SessionState}")]
19161922
private static partial void ExpectedSessionState6(ILogger logger, string sessionId, State expectedState1, State expectedState2, State expectedState3, State expectedState4, State expectedState5, State expectedState6, State sessionState);
19171923

1924+
private static readonly Histogram<float> s_useTimeHistory = ActivitySourceHelper.Meter.CreateHistogram<float>("db.client.connections.use_time",
1925+
unit: "ms", description: "The time between borrowing a connection and returning it to the pool.");
19181926
private static readonly PayloadData s_setNamesUtf8NoAttributesPayload = QueryPayload.Create(false, "SET NAMES utf8;"u8);
19191927
private static readonly PayloadData s_setNamesUtf8mb4NoAttributesPayload = QueryPayload.Create(false, "SET NAMES utf8mb4;"u8);
19201928
private static readonly PayloadData s_setNamesUtf8WithAttributesPayload = QueryPayload.Create(true, "SET NAMES utf8;"u8);

‎src/MySqlConnector/MySqlConnection.cs

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -380,6 +380,8 @@ private async ValueTask<bool> PingAsync(IOBehavior ioBehavior, CancellationToken
380380

381381
internal async Task OpenAsync(IOBehavior? ioBehavior, CancellationToken cancellationToken)
382382
{
383+
var openStartTickCount = Environment.TickCount;
384+
383385
VerifyNotDisposed();
384386
cancellationToken.ThrowIfCancellationRequested();
385387
if (State != ConnectionState.Closed)
@@ -388,8 +390,6 @@ internal async Task OpenAsync(IOBehavior? ioBehavior, CancellationToken cancella
388390
using var activity = ActivitySourceHelper.StartActivity(ActivitySourceHelper.OpenActivityName);
389391
try
390392
{
391-
var openStartTickCount = Environment.TickCount;
392-
393393
SetState(ConnectionState.Connecting);
394394

395395
var pool = m_dataSource?.Pool ??
@@ -884,6 +884,7 @@ internal void FinishQuerying(bool hasWarnings)
884884

885885
private async ValueTask<ServerSession> CreateSessionAsync(ConnectionPool? pool, int startTickCount, Activity? activity, IOBehavior? ioBehavior, CancellationToken cancellationToken)
886886
{
887+
pool?.AddPendingRequestCount(1);
887888
var connectionSettings = GetInitializedConnectionSettings();
888889
var actualIOBehavior = ioBehavior ?? (connectionSettings.ForceSynchronous ? IOBehavior.Synchronous : IOBehavior.Asynchronous);
889890

@@ -937,6 +938,7 @@ private async ValueTask<ServerSession> CreateSessionAsync(ConnectionPool? pool,
937938
}
938939
finally
939940
{
941+
pool?.AddPendingRequestCount(-1);
940942
linkedSource?.Dispose();
941943
timeoutSource?.Dispose();
942944
}

‎src/MySqlConnector/MySqlConnector.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
<PackageReference Include="System.Threading.Tasks.Extensions" />
3030
</ItemGroup>
3131

32-
<ItemGroup Condition=" '$(TargetFrameworkIdentifier)' != '.NETCoreApp' ">
32+
<ItemGroup Condition=" '$(TargetFrameworkIdentifier)' != '.NETCoreApp' OR '$(TargetFramework)' == 'net6.0' ">
3333
<PackageReference Include="System.Diagnostics.DiagnosticSource" />
3434
</ItemGroup>
3535

‎src/MySqlConnector/Utilities/ActivitySourceHelper.cs

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using System.Diagnostics;
2+
using System.Diagnostics.Metrics;
23
using System.Globalization;
34
using System.Reflection;
45

@@ -55,12 +56,10 @@ public static void CopyTags(IEnumerable<KeyValuePair<string, object?>> tags, Act
5556
}
5657
}
5758

58-
private static ActivitySource ActivitySource { get; } = CreateActivitySource();
59+
public static Meter Meter { get; } = new("MySqlConnector", GetVersion());
5960

60-
private static ActivitySource CreateActivitySource()
61-
{
62-
var assembly = typeof(ActivitySourceHelper).Assembly;
63-
var version = assembly.GetCustomAttribute<AssemblyFileVersionAttribute>()!.Version;
64-
return new("MySqlConnector", version);
65-
}
61+
private static ActivitySource ActivitySource { get; } = new("MySqlConnector", GetVersion());
62+
63+
private static string GetVersion() =>
64+
typeof(ActivitySourceHelper).Assembly.GetCustomAttribute<AssemblyFileVersionAttribute>()!.Version;
6665
}

‎tests/MySqlConnector.Tests/ConnectionTests.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,7 @@ public void PingWhenClosed()
179179
[Fact]
180180
public void ConnectionTimeout()
181181
{
182-
m_server.BlockOnConnect = true;
182+
m_server.ConnectDelay = TimeSpan.FromSeconds(10);
183183
var csb = new MySqlConnectionStringBuilder(m_csb.ConnectionString);
184184
csb.ConnectionTimeout = 4;
185185
using var connection = new MySqlConnection(csb.ConnectionString);

‎tests/MySqlConnector.Tests/FakeMySqlServer.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ public void Stop()
4949

5050
public bool SuppressAuthPluginNameTerminatingNull { get; set; }
5151
public bool SendIncompletePostHandshakeResponse { get; set; }
52-
public bool BlockOnConnect { get; set; }
52+
public TimeSpan? ConnectDelay { get; set; }
5353
public TimeSpan? ResetDelay { get; set; }
5454

5555
internal void CancelQuery(int connectionId)

‎tests/MySqlConnector.Tests/FakeMySqlServerConnection.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@ public async Task RunAsync(TcpClient client, CancellationToken token)
2525
using (client)
2626
using (var stream = client.GetStream())
2727
{
28-
if (m_server.BlockOnConnect)
29-
Thread.Sleep(TimeSpan.FromSeconds(10));
28+
if (m_server.ConnectDelay is { } connectDelay)
29+
await Task.Delay(connectDelay);
3030

3131
await SendAsync(stream, 0, WriteInitialHandshake);
3232
await ReadPayloadAsync(stream, token); // handshake response

0 commit comments

Comments
 (0)
0