8000 feature: configurable cache expiration (#55) · OddDotNet/OddDotNet@652dc24 · GitHub
[go: up one dir, main page]
Skip to content

Commit 652dc24

Browse files
DoubleTKDoubleTK
authored
feature: configurable cache expiration (#55)
* Added CacheCleanupBackgroundService.cs. Added ODD_CACHE_EXPIRATION EnvVar. ADDED ODD_CACHE_CLEANUP_INTERVAL EnvVar. * Test updates.
1 parent d26f97b commit 652dc24

File tree

7 files changed

+176
-19
lines changed

7 files changed

+176
-19
lines changed

‎src/OddDotNet/CacheCleanupBackgroundService.cs‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
using Microsoft.Extensions.Options;
2+
3+
namespace OddDotNet;
4+
5+
public class CacheCleanupBackgroundService : BackgroundService
6+
{
7+
private readonly IServiceProvider _services;
8+
private readonly ILogger<CacheCleanupBackgroundService> _logger;
9+
10+
public CacheCleanupBackgroundService(IServiceProvider services, ILogger<CacheCleanupBackgroundService> logger)
11+
{
12+
_services = services;
13+
_logger = logger;
14+
}
15+
16+
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
17+
{
18+
using var scope = _services.CreateScope();
19+
var signalLists = scope.ServiceProvider.GetServices<ISignalList>().ToList();
20+
var oddSettings = scope.ServiceProvider.GetRequiredService<IOptions<OddSettings>>().Value;
21+
22+
while (!stoppingToken.IsCancellationRequested)
23+
{
24+
_logger.LogDebug("Pruning spans");
25+
foreach (var signalList in signalLists)
26+
signalList.Prune();
27+
28+
await Task.Delay(TimeSpan.FromMilliseconds(oddSettings.Cache.CleanupInterval), stoppingToken);
29+
}
30+
}
31+
}

‎src/OddDotNet/ISignalList.cs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,14 @@
22

33
namespace OddDotNet;
44

5-
public interface ISignalList<TSignal> where TSignal : class
5+
public interface ISignalList<TSignal> : ISignalList where TSignal : class
66
{
77
void Add(TSignal signal);
88
IAsyncEnumerable<TSignal> QueryAsync(IQueryRequest<TSignal> request, CancellationToken cancellationToken = default);
99
void Reset(IResetRequest<TSignal> request);
10+
}
11+
12+
public interface ISignalList
13+
{
14+
void Prune();
1015
}

‎src/OddDotNet/OddSettings.cs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
namespace OddDotNet;
2+
3+
public sealed record OddSettings
4+
{
5+
public const string CacheExpirationEnvVarName = "ODD_CACHE_EXPIRATION";
6+
public const string CacheCleanupIntervalEnvVarName = "ODD_CACHE_CLEANUP_INTERVAL";
7+
public CacheSettings Cache { get; init; } = new();
8+
}
9+
10+
public sealed record CacheSettings
11+
{
12+
public uint Expiration { get; set; } = 30000;
13+
public uint CleanupInterval { get; set; } = 1000;
14+
}

‎src/OddDotNet/Program.cs‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,24 @@
1212
builder.Services.AddSwaggerGen();
1313
builder.Services.AddHealthChecks();
1414

15-
builder.Services.AddScoped<ISignalList<FlatSpan>, SpanSignalList>();
15+
builder.Services.AddScoped<ISignalList<FlatSpan>, SpanSignalList>().AddScoped<ISignalList, SpanSignalList>();
1616
builder.Services.AddScoped<IChannelManager<FlatSpan>, SpanChannelManager>();
1717
builder.Services.AddSingleton(TimeProvider.System);
18+
builder.Services.AddHostedService<CacheCleanupBackgroundService>();
19+
20+
builder.Services.Configure<OddSettings>(options =>
21+
{
22+
// We allow single and double underscores ('_', '__'). Single underscores
23+
// don't automatically map, so check for them explicitly.
24+
// eg. ODD__CACHE__EXPIRATION and ODD_CACHE_EXPIRATION are both valid
25+
var cacheExpirationEnvVarValue = Environment.GetEnvironmentVariable(OddSettings.CacheExpirationEnvVarName);
26+
if (!string.IsNullOrEmpty(cacheExpirationEnvVarValue))
27+
options.Cache.Expiration = uint.Parse(cacheExpirationEnvVarValue);
28+
29+
var cacheCleanupIntervalEnvVarValue = Environment.GetEnvironmentVariable(OddSettings.CacheCleanupIntervalEnvVarName);
30+
if (!string.IsNullOrEmpty(cacheCleanupIntervalEnvVarValue))
31+
options.Cache.CleanupInterval = uint.Parse(cacheCleanupIntervalEnvVarValue);
32+
});
1833

1934
var app = builder.Build();
2035

‎src/OddDotNet/SpanSignalList.cs‎

Lines changed: 24 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
using System.Threading.Channels;
33
using Google.Protobuf;
44
using Google.Protobuf.Collections;
5+
using Microsoft.Extensions.Options;
56
using OddDotNet.Proto.Trace.V1;
67
using OpenTelemetry.Proto.Common.V1;
78
using OpenTelemetry.Proto.Trace.V1;
@@ -14,26 +15,26 @@ public class SpanSignalList : ISignalList<FlatSpan>
1415
private readonly IChannelManager<FlatSpan> _channels;
1516
private readonly TimeProvider _timeProvider;
1617
private readonly ILogger<SpanSignalList> _logger;
18+
private readonly OddSettings _oddSettings;
1719

1820
private static readonly List<Expirable<FlatSpan>> Spans = [];
1921

2022
private static readonly object Lock = new();
21-
public SpanSignalList(IChannelManager<FlatSpan> channels, TimeProvider timeProvider, ILogger<SpanSignalList> logger)
23+
public SpanSignalList(IChannelManager<FlatSpan> channels, TimeProvider timeProvider, ILogger<SpanSignalList> logger, IOptions<OddSettings> oddSettings)
2224
{
2325
_channels = channels;
2426
_timeProvider = timeProvider;
2527
_logger = logger;
28+
_oddSettings = oddSettings.Value;
2629
}
2730

2831
public void Add(FlatSpan signal)
2932
{
3033
lock (Lock)
3134
{
32-
PruneExpiredSpans();
3335

34-
// Add the new span with 30 second expire
35-
// TODO make this configurable
36-
DateTimeOffset expiresAt = _timeProvider.GetUtcNow().AddSeconds(30);
36+
// Add the new span with configured expiration
37+
DateTimeOffset expiresAt = _timeProvider.GetUtcNow().AddMilliseconds(_oddSettings.Cache.Expiration);
3738
Spans.Add(new Expirable<FlatSpan>(signal, expiresAt));
3839

3940
// Notify any listening channels
@@ -54,8 +55,6 @@ public async IAsyncEnumerable<FlatSpan> QueryAsync(IQueryRequest<FlatSpan> reque
5455
// Create the channel and populate it with the current contents of the span list
5556
lock (Lock)
5657
{
57-
PruneExpiredSpans();
58-
5958
foreach (var expirableSpan in Spans)
6059
{
6160
channel.Writer.TryWrite(expirableSpan.Signal);
@@ -67,15 +66,15 @@ public async IAsyncEnumerable<FlatSpan> QueryAsync(IQueryRequest<FlatSpan> reque
6766

6867
while (currentCount < takeCount && !cts.IsCancellationRequested)
6968
{
70-
FlatSpan? span = null;
69+
FlatSpan? span;
7170
try
7271
{
7372
await channel.Reader.WaitToReadAsync(cts.Token);
7473
span = await channel.Reader.ReadAsync(cts.Token);
7574
}
7675
catch (OperationCanceledException ex)
7776
{
78-
_logger.LogWarning(ex, "The query operation was cancelled");
77+
_logger.LogDebug(ex, "The query operation was cancelled");
7978
break;
8079
}
8180

@@ -103,6 +102,17 @@ public void Reset(IResetRequest<FlatSpan> request)
103102
}
104103
}
105104

105+
public void Prune()
106+
{
107+
_logger.LogDebug("Prune");
108+
lock (Lock)
109+
{
110+
DateTimeOffset currentTime = _timeProvider.GetUtcNow();
111+
int numRemoved = Spans.RemoveAll(expirable => expirable.ExpireAt < currentTime);
112+
_logger.LogDebug("Removed {numRemoved} spans", numRemoved);
113+
}
114+
}
115+
106116
private static CancellationTokenSource GetQueryTimeout(SpanQueryRequest spanRequest)
107117
{
108118
var defaultTimeout = new CancellationTokenSource(TimeSpan.FromMilliseconds(int.MaxValue));
@@ -112,11 +122,11 @@ private static CancellationTokenSource GetQueryTimeout(SpanQueryRequest spanRequ
112122
: new CancellationTokenSource(TimeSpan.FromMilliseconds(spanRequest.Duration.Milliseconds));
113123
}
114124

115-
private void PruneExpiredSpans()
116-
{
117-
DateTimeOffset currentTime = _timeProvider.GetUtcNow();
118-
Spans.RemoveAll(expirable => expirable.ExpireAt < currentTime);
119-
}
125+
// private void PruneExpiredSpans()
126+
// {
127+
// DateTimeOffset currentTime = _timeProvider.GetUtcNow();
128+
// Spans.RemoveAll(expirable => expirable.ExpireAt < currentTime);
129+
// }
120130

121131
private static int GetTakeCount(SpanQueryRequest spanQueryRequest) => spanQueryRequest.Take.ValueCase switch
122132
{

‎tests/OddDotNet.Aspire.Tests/SpanCacheTests.cs‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
using Grpc.Net.Client;
2+
using Microsoft.Extensions.DependencyInjection;
3+
using OddDotNet.Proto.Trace.V1;
4+
using OpenTelemetry.Proto.Collector.Trace.V1;
5+
6+
namespace OddDotNet.Aspire.Tests;
7+
8+
public class SpanCacheTests : IAsyncLifetime
9+
{
10+
#pragma warning disable CS8618
11+
private TraceService.TraceServiceClient _traceServiceClient;
12+
private SpanQueryService.SpanQueryServiceClient _spanQueryServiceClient;
13+
private DistributedApplication _app;
14+
#pragma warning disable CS8618
15+
[Fact]
16+
public async Task RemoveSpansAfterConfiguredTimeout()
17+
{
18+
// Arrange
19+
var exportRequest = TestHelpers.CreateExportTraceServiceRequest();
20+
var traceId = exportRequest.ResourceSpans[0].ScopeSpans[0].Spans[0].TraceId;
21+
var take = new Take { TakeFirst = new() };
22+
var duration = new Duration { Milliseconds = 1000 };
23+
var traceIdFilter = new WhereSpanFilter
24+
{
25+
SpanProperty = new WhereSpanPropertyFilter
26+
{
27+
3044 TraceId = new ByteStringProperty
28+
{
29+
Compare = traceId,
30+
CompareAs = ByteStringCompareAsType.Equals
31+
}
32+
}
33+
};
34+
var spanQueryRequest = new SpanQueryRequest { Take = take, Duration = duration, Filters = { traceIdFilter } };
35+
36+
// ACT
37+
await _traceServiceClient.ExportAsync(exportRequest);
38+
var response = await _spanQueryServiceClient.QueryAsync(spanQueryRequest);
39+
40+
// The first response should contain the span since it hasn't yet been deleted
41+
Assert.NotEmpty(response.Spans);
42+
43+
// Give the background service time to clear the cache
44+
await Task.Delay(1000);
45+
46+
// The second response should be empty as the span should have been cleared from cache.
47+
response = await _spanQueryServiceClient.QueryAsync(spanQueryRequest);
48+
Assert.Empty(response.Spans);
49+
}
50+
51+
// Need a separate "AspireFixture" here as we need to modify the env vars of the project before starting.
52+
public async Task InitializeAsync()
53+
{
54+
const string oddResource = "odd";
55+
var builder = await DistributedApplicationTestingBuilder.CreateAsync<Projects.OddDotNet_Aspire_AppHost>();
56+
builder
57+
.CreateResourceBuilder(builder
58+
.Resources
59+
.First(resource => resource.Name == oddResource))
60+
.WithAnnotation(new EnvironmentCallbackAnnotation("ODD_CACHE_EXPIRATION", () => "250"))
61+
.WithAnnotation(new EnvironmentCallbackAnnotation("ODD_CACHE_CLEANUP_INTERVAL", () => "500"));
62+
_app = await builder.BuildAsync();
63+
64+
var resourceNotificationService = _app.Services.GetRequiredService<ResourceNotificationService>();
65+
await _app.StartAsync();
66+
67+
await resourceNotificationService.WaitForResourceAsync(oddResource).WaitAsync(TimeSpan.FromSeconds(30));
68+
69+
var endpoint = _app.GetEndpoint(oddResource, "grpc");
70+
var traceServiceChannel = GrpcChannel.ForAddress(endpoint.AbsoluteUri);
71+
_traceServiceClient = new TraceService.TraceServiceClient(traceServiceChannel);
72+
73+
var spanQueryChannel = GrpcChannel.ForAddress(endpoint.AbsoluteUri);
74+
_spanQueryServiceClient = new SpanQueryService.SpanQueryServiceClient(spanQueryChannel);
75+
}
76+
77+
public async Task DisposeAsync()
78+
{
79+
await _app.StopAsync();
80+
await _app.DisposeAsync();
81+
}
82+
}

‎tests/OddDotNet.Aspire.Tests/SpanQueryServiceTests.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -108,10 +108,10 @@ public async Task ReturnAllSpansWithinTimeframe(int takeDuration, int expectedCo
108108

109109
var spanQueryRequest = new SpanQueryRequest { Take = take, Duration = duration};
110110

111-
// Start the query waiting for 3 seconds, and send spans at 500, 1000, 2000 ms
111+
// Start the query waiting for 3 seconds, and send spans at delayed intervals
112112
var responseTask = _spanQueryServiceClient.QueryAsync(spanQueryRequest);
113-
var exportFirst = ExportDelayedTrace(request, TimeSpan.FromMilliseconds(500));
114-
var exportSecond = ExportDelayedTrace(request, TimeSpan.FromMilliseconds(1000));
113+
var exportFirst = ExportDelayedTrace(request, TimeSpan.FromMilliseconds(250));
114+
var exportSecond = ExportDelayedTrace(request, TimeSpan.FromMilliseconds(500));
115115
var exportThird = ExportDelayedTrace(request, TimeSpan.FromMilliseconds(2000));
116116

117117
await Task.WhenAll(responseTask.ResponseAsync, exportFirst, exportSecond, exportThird);

0 commit comments

Comments
 (0)
0