From 070147eabd643e521c513978f90eab9675a1d6e4 Mon Sep 17 00:00:00 2001 From: Damian Hickey Date: Sun, 20 May 2018 19:53:38 +0200 Subject: [PATCH 1/7] Add a deterministic guid generator and tests --- .../DeterministicGuidGeneratorTests.cs | 29 ++++++ .../DeterministicGuidGenerator.cs | 93 +++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 src/SqlStreamStore.Tests/Infrastructure/DeterministicGuidGeneratorTests.cs create mode 100644 src/SqlStreamStore/Infrastructure/DeterministicGuidGenerator.cs diff --git a/src/SqlStreamStore.Tests/Infrastructure/DeterministicGuidGeneratorTests.cs b/src/SqlStreamStore.Tests/Infrastructure/DeterministicGuidGeneratorTests.cs new file mode 100644 index 000000000..fd5994fe5 --- /dev/null +++ b/src/SqlStreamStore.Tests/Infrastructure/DeterministicGuidGeneratorTests.cs @@ -0,0 +1,29 @@ +namespace SqlStreamStore.Infrastructure +{ + using System; + using Shouldly; + using Xunit; + + public class DeterministicGuidGeneratorTests + { + [Fact] + public void Given_same_input_should_generate_same_Guid() + { + var generator = new DeterministicGuidGenerator(Guid.NewGuid()); + var guid1 = generator.Create("some-data"); + var guid2 = generator.Create("some-data"); + + guid2.ShouldBe(guid1); + } + + [Fact] + public void Given_different_input_should_generate_different_Guid() + { + var generator = new DeterministicGuidGenerator(Guid.NewGuid()); + var guid1 = generator.Create("some-data"); + var guid2 = generator.Create("other-data"); + + guid2.ShouldNotBe(guid1); + } + } +} \ No newline at end of file diff --git a/src/SqlStreamStore/Infrastructure/DeterministicGuidGenerator.cs b/src/SqlStreamStore/Infrastructure/DeterministicGuidGenerator.cs new file mode 100644 index 000000000..41e55664d --- /dev/null +++ b/src/SqlStreamStore/Infrastructure/DeterministicGuidGenerator.cs @@ -0,0 +1,93 @@ +namespace SqlStreamStore.Infrastructure +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Security.Cryptography; + using System.Text; + + // Adapted from https://github.com/LogosBible/Logos.Utility/blob/master/src/Logos.Utility/GuidUtility.cs + // MIT Licence + + /// + /// A helper utility to generate deterministed GUIDS. + /// + public class DeterministicGuidGenerator + { + private readonly byte[] _namespaceBytes; + + /// + /// Initializes a new instance of + /// + /// + /// A namespace that ensures that the GUID generated with this instance + /// do not collided with other generators. Your application should define + /// it's namespace as a constant. + /// + public DeterministicGuidGenerator(Guid guidNameSpace) + { + _namespaceBytes = guidNameSpace.ToByteArray(); + SwapByteOrder(_namespaceBytes); + } + + /// + /// Creates a deterministic GUID. + /// + /// + /// A source to generate the GUID from. + /// + /// + /// A deterministically generated GUID. + /// + public Guid Create(string source) + { + return Create(Encoding.UTF8.GetBytes(source)); + } + + /// + /// Creates a deterministic GUID. + /// + /// + /// A source to generate the GUID from. + /// + /// + /// A deterministically generated GUID. + /// + public Guid Create(IEnumerable source) + { + byte[] hash; + byte[] inputBuffer = source.ToArray(); + using (var algorithm = SHA1.Create()) + { + algorithm.TransformBlock(_namespaceBytes, 0, _namespaceBytes.Length, null, 0); + algorithm.TransformFinalBlock(inputBuffer, 0, inputBuffer.Length); + + hash = algorithm.Hash; + } + + var newGuid = new byte[16]; + Array.Copy(hash, 0, newGuid, 0, 16); + + newGuid[6] = (byte)((newGuid[6] & 0x0F) | (5 << 4)); + newGuid[8] = (byte)((newGuid[8] & 0x3F) | 0x80); + + SwapByteOrder(newGuid); + return new Guid(newGuid); + } + + private static void SwapByteOrder(byte[] guid) + { + SwapBytes(guid, 0, 3); + SwapBytes(guid, 1, 2); + SwapBytes(guid, 4, 5); + SwapBytes(guid, 6, 7); + } + + private static void SwapBytes(byte[] guid, int left, int right) + { + var temp = guid[left]; + guid[left] = guid[right]; + guid[right] = temp; + } + } +} From a16c2827fb8ab24e68b1355233f763cc9a3d0ff0 Mon Sep 17 00:00:00 2001 From: Damian Hickey Date: Sun, 20 May 2018 20:30:50 +0200 Subject: [PATCH 2/7] Set LangVersion to latest in all projects --- .../SqlStreamStore.MsSql.Tests.csproj | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/SqlStreamStore.MsSql.Tests/SqlStreamStore.MsSql.Tests.csproj b/src/SqlStreamStore.MsSql.Tests/SqlStreamStore.MsSql.Tests.csproj index 8711b8b70..506c88f9c 100644 --- a/src/SqlStreamStore.MsSql.Tests/SqlStreamStore.MsSql.Tests.csproj +++ b/src/SqlStreamStore.MsSql.Tests/SqlStreamStore.MsSql.Tests.csproj @@ -11,9 +11,6 @@ - - - From d205bbb893f42d166d57370514a7b76e45f3d6bb Mon Sep 17 00:00:00 2001 From: Damian Hickey Date: Sun, 20 May 2018 20:37:41 +0200 Subject: [PATCH 3/7] Add failing test for handling SetStreamMetadata idempotently --- ...reamStoreAcceptanceTests.StreamMetadata.cs | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/SqlStreamStore.AcceptanceTests/SqlStreamStoreAcceptanceTests.StreamMetadata.cs b/src/SqlStreamStore.AcceptanceTests/SqlStreamStoreAcceptanceTests.StreamMetadata.cs index fe635e043..a951af69d 100644 --- a/src/SqlStreamStore.AcceptanceTests/SqlStreamStoreAcceptanceTests.StreamMetadata.cs +++ b/src/SqlStreamStore.AcceptanceTests/SqlStreamStoreAcceptanceTests.StreamMetadata.cs @@ -279,5 +279,26 @@ await store } } } + + + [Fact, Trait("Category", "StreamMetadata")] + public async Task When_set_metadata_with_same_data_then_should_handle_idempotently() + { + using (var fixture = GetFixture()) + { + using (var store = await fixture.GetStreamStore()) + { + const string streamId = "stream-1"; + await store + .SetStreamMetadata(streamId, maxCount: 2, maxAge: 30, metadataJson: "meta"); + await store + .SetStreamMetadata(streamId, maxCount: 2, maxAge: 30, metadataJson: "meta"); + + var metadata = await store.GetStreamMetadata(streamId); + + metadata.MetadataStreamVersion.ShouldBe(0); + } + } + } } } From 30ad68f7503c23d3a41d4115a6a6518050a2782a Mon Sep 17 00:00:00 2001 From: Damian Hickey Date: Sun, 20 May 2018 20:41:49 +0200 Subject: [PATCH 4/7] The metadata message id generator with a fixed guid namespace. Should be used by all storage implementations. --- .../MetadataMessageIdGenerator.cs | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 src/SqlStreamStore/Infrastructure/MetadataMessageIdGenerator.cs diff --git a/src/SqlStreamStore/Infrastructure/MetadataMessageIdGenerator.cs b/src/SqlStreamStore/Infrastructure/MetadataMessageIdGenerator.cs new file mode 100644 index 000000000..2487e73ed --- /dev/null +++ b/src/SqlStreamStore/Infrastructure/MetadataMessageIdGenerator.cs @@ -0,0 +1,32 @@ +namespace SqlStreamStore.Infrastructure +{ + using System; + + /// + /// A deterministic GUID generator for metadata messages. + /// + public static class MetadataMessageIdGenerator + { + private static readonly DeterministicGuidGenerator s_deterministicGuidGenerator; + + static MetadataMessageIdGenerator() + { + s_deterministicGuidGenerator + = new DeterministicGuidGenerator(Guid.Parse("8D1E0B02-0D78-408E-8211-F899BE6F8AA2")); + } + + /// + /// Create a GUID for metadata message Ids. + /// + /// + /// The metadata message uses as input into the generation algorithim. + /// + /// + /// A deterministically generated GUID. + /// + public static Guid Create(string message) + { + return s_deterministicGuidGenerator.Create(message); + } + } +} \ No newline at end of file From 276f90d8271e2562db27a0cd03c6b96e3e8e38f6 Mon Sep 17 00:00:00 2001 From: Damian Hickey Date: Sun, 20 May 2018 20:43:43 +0200 Subject: [PATCH 5/7] InMemoryStreamStore: use MetadataMessageIdGenerator to generate the message id. The underlying append operation handles the operation idempotently --- src/SqlStreamStore/InMemory/InMemoryStreamStore.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/SqlStreamStore/InMemory/InMemoryStreamStore.cs b/src/SqlStreamStore/InMemory/InMemoryStreamStore.cs index b51ff77b3..7e4a0c6a3 100644 --- a/src/SqlStreamStore/InMemory/InMemoryStreamStore.cs +++ b/src/SqlStreamStore/InMemory/InMemoryStreamStore.cs @@ -225,9 +225,10 @@ protected override async Task SetStreamMetadataInternal MetaJson = metadataJson }; var json = SimpleJson.SerializeObject(metadataMessage); - var newmessage = new NewStreamMessage(Guid.NewGuid(), "$stream-metadata", json); + var messageId = MetadataMessageIdGenerator.Create(json); + var newStreamMessage = new NewStreamMessage(messageId, "$stream-metadata", json); - var result = AppendToStreamInternal(metaStreamId, expectedStreamMetadataVersion, new[] { newmessage }); + var result = AppendToStreamInternal(metaStreamId, expectedStreamMetadataVersion, new[] { newStreamMessage }); await CheckStreamMaxCount(streamId, metadataMessage.MaxCount, cancellationToken); From 6c75a1ad9e9105d06e22384adacddfd0667e0bda Mon Sep 17 00:00:00 2001 From: Damian Hickey Date: Sun, 20 May 2018 20:48:06 +0200 Subject: [PATCH 6/7] MsSql: use MetadataMessageIdGenerator to generate the message id when setting stream metadata. Need to pass the existing outer transaction to subsequent sql commands. --- .../MsSqlStreamStore.AppendStream.cs | 3 +++ .../MsSqlStreamStore.ReadStream.cs | 10 ++++++---- .../MsSqlStreamStore.StreamMetadata.cs | 7 ++++--- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/SqlStreamStore.MsSql/MsSqlStreamStore.AppendStream.cs b/src/SqlStreamStore.MsSql/MsSqlStreamStore.AppendStream.cs index 0c47dd7cb..448ca0a70 100644 --- a/src/SqlStreamStore.MsSql/MsSqlStreamStore.AppendStream.cs +++ b/src/SqlStreamStore.MsSql/MsSqlStreamStore.AppendStream.cs @@ -193,6 +193,7 @@ private async Task AppendToStreamExpectedVersionAny( false, null, connection, + transaction, cancellationToken) .NotOnCapturedContext(); @@ -288,6 +289,7 @@ private async Task AppendToStreamExpectedVersionNoStream( false, null, connection, + transaction, cancellationToken) .NotOnCapturedContext(); @@ -387,6 +389,7 @@ private async Task AppendToStreamExpectedVersion( false, null, connection, + transaction, cancellationToken); if(messages.Length > page.Messages.Length) diff --git a/src/SqlStreamStore.MsSql/MsSqlStreamStore.ReadStream.cs b/src/SqlStreamStore.MsSql/MsSqlStreamStore.ReadStream.cs index 5687497ff..728207103 100644 --- a/src/SqlStreamStore.MsSql/MsSqlStreamStore.ReadStream.cs +++ b/src/SqlStreamStore.MsSql/MsSqlStreamStore.ReadStream.cs @@ -24,7 +24,7 @@ protected override async Task ReadStreamForwardsInternal( await connection.OpenAsync(cancellationToken).NotOnCapturedContext(); var streamIdInfo = new StreamIdInfo(streamId); return await ReadStreamInternal(streamIdInfo.SqlStreamId, start, count, ReadDirection.Forward, - prefetch, readNext, connection, cancellationToken); + prefetch, readNext, connection, null, cancellationToken); } } @@ -41,7 +41,7 @@ protected override async Task ReadStreamBackwardsInternal( await connection.OpenAsync(cancellationToken).NotOnCapturedContext(); var streamIdInfo = new StreamIdInfo(streamId); return await ReadStreamInternal(streamIdInfo.SqlStreamId, start, count, ReadDirection.Backward, - prefetch, readNext, connection, cancellationToken); + prefetch, readNext, connection, null, cancellationToken); } } @@ -52,7 +52,9 @@ private async Task ReadStreamInternal( ReadDirection direction, bool prefetch, ReadNextStreamPage readNext, - SqlConnection connection, CancellationToken cancellationToken) + SqlConnection connection, + SqlTransaction transaction, + CancellationToken cancellationToken) { // If the count is int.MaxValue, TSql will see it as a negative number. // Users shouldn't be using int.MaxValue in the first place anyway. @@ -87,7 +89,7 @@ private async Task ReadStreamInternal( }; } - using(var command = new SqlCommand(commandText, connection)) + using (var command = new SqlCommand(commandText, connection, transaction)) { command.Parameters.AddWithValue("streamId", sqlStreamId.Id); command.Parameters.AddWithValue("count", count + 1); //Read extra row to see if at end or not diff --git a/src/SqlStreamStore.MsSql/MsSqlStreamStore.StreamMetadata.cs b/src/SqlStreamStore.MsSql/MsSqlStreamStore.StreamMetadata.cs index dd807d8b5..231dbc4b2 100644 --- a/src/SqlStreamStore.MsSql/MsSqlStreamStore.StreamMetadata.cs +++ b/src/SqlStreamStore.MsSql/MsSqlStreamStore.StreamMetadata.cs @@ -1,6 +1,5 @@ namespace SqlStreamStore { - using System; using System.Threading; using System.Threading.Tasks; using SqlStreamStore.Streams; @@ -27,6 +26,7 @@ protected override async Task GetStreamMetadataInternal( true, null, connection, + null, cancellationToken); } @@ -70,14 +70,15 @@ protected override async Task SetStreamMetadataInternal MetaJson = metadataJson }; var json = SimpleJson.SerializeObject(metadataMessage); - var newmessage = new NewStreamMessage(Guid.NewGuid(), "$stream-metadata", json); + var messageId = MetadataMessageIdGenerator.Create(json); + var message = new NewStreamMessage(messageId, "$stream-metadata", json); result = await AppendToStreamInternal( connection, transaction, streamIdInfo.MetadataSqlStreamId, expectedStreamMetadataVersion, - new[] { newmessage }, + new[] { message }, cancellationToken); transaction.Commit(); From d5985336f154886c45f12cca8492d5cd5b11bc4f Mon Sep 17 00:00:00 2001 From: Damian Hickey Date: Mon, 21 May 2018 13:33:29 +0200 Subject: [PATCH 7/7] Just take an array --- .../Infrastructure/DeterministicGuidGenerator.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/SqlStreamStore/Infrastructure/DeterministicGuidGenerator.cs b/src/SqlStreamStore/Infrastructure/DeterministicGuidGenerator.cs index 41e55664d..6ca498f3c 100644 --- a/src/SqlStreamStore/Infrastructure/DeterministicGuidGenerator.cs +++ b/src/SqlStreamStore/Infrastructure/DeterministicGuidGenerator.cs @@ -53,14 +53,13 @@ public Guid Create(string source) /// /// A deterministically generated GUID. /// - public Guid Create(IEnumerable source) + public Guid Create(byte[] source) { byte[] hash; - byte[] inputBuffer = source.ToArray(); using (var algorithm = SHA1.Create()) { algorithm.TransformBlock(_namespaceBytes, 0, _namespaceBytes.Length, null, 0); - algorithm.TransformFinalBlock(inputBuffer, 0, inputBuffer.Length); + algorithm.TransformFinalBlock(source, 0, source.Length); hash = algorithm.Hash; }