From d8fa57d4ffe8b1a40b0a6dc2a8727d24464af4b6 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Sun, 16 Nov 2025 23:28:45 +0000 Subject: [PATCH 1/8] chore(deps): update tunit to 1.2.3 (#3861) Co-authored-by: Renovate Bot --- Directory.Packages.props | 6 +++--- .../TUnit.AspNet.FSharp/TestProject/TestProject.fsproj | 4 ++-- .../content/TUnit.AspNet/TestProject/TestProject.csproj | 2 +- .../ExampleNamespace.TestProject.csproj | 2 +- .../content/TUnit.Aspire.Test/ExampleNamespace.csproj | 2 +- TUnit.Templates/content/TUnit.FSharp/TestProject.fsproj | 4 ++-- TUnit.Templates/content/TUnit.Playwright/TestProject.csproj | 2 +- TUnit.Templates/content/TUnit.VB/TestProject.vbproj | 2 +- TUnit.Templates/content/TUnit/TestProject.csproj | 2 +- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index f7ebc259bd..62e3564e91 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -83,9 +83,9 @@ - - - + + + diff --git a/TUnit.Templates/content/TUnit.AspNet.FSharp/TestProject/TestProject.fsproj b/TUnit.Templates/content/TUnit.AspNet.FSharp/TestProject/TestProject.fsproj index 966217ce72..d2e029d0c6 100644 --- a/TUnit.Templates/content/TUnit.AspNet.FSharp/TestProject/TestProject.fsproj +++ b/TUnit.Templates/content/TUnit.AspNet.FSharp/TestProject/TestProject.fsproj @@ -10,8 +10,8 @@ - - + + diff --git a/TUnit.Templates/content/TUnit.AspNet/TestProject/TestProject.csproj b/TUnit.Templates/content/TUnit.AspNet/TestProject/TestProject.csproj index d71fa475b1..905a5dd8b7 100644 --- a/TUnit.Templates/content/TUnit.AspNet/TestProject/TestProject.csproj +++ b/TUnit.Templates/content/TUnit.AspNet/TestProject/TestProject.csproj @@ -9,7 +9,7 @@ - + diff --git a/TUnit.Templates/content/TUnit.Aspire.Starter/ExampleNamespace.TestProject/ExampleNamespace.TestProject.csproj b/TUnit.Templates/content/TUnit.Aspire.Starter/ExampleNamespace.TestProject/ExampleNamespace.TestProject.csproj index 29270c3a6f..107780bda0 100644 --- a/TUnit.Templates/content/TUnit.Aspire.Starter/ExampleNamespace.TestProject/ExampleNamespace.TestProject.csproj +++ b/TUnit.Templates/content/TUnit.Aspire.Starter/ExampleNamespace.TestProject/ExampleNamespace.TestProject.csproj @@ -11,7 +11,7 @@ - + diff --git a/TUnit.Templates/content/TUnit.Aspire.Test/ExampleNamespace.csproj b/TUnit.Templates/content/TUnit.Aspire.Test/ExampleNamespace.csproj index 2750f80878..bbe3ede435 100644 --- a/TUnit.Templates/content/TUnit.Aspire.Test/ExampleNamespace.csproj +++ b/TUnit.Templates/content/TUnit.Aspire.Test/ExampleNamespace.csproj @@ -10,7 +10,7 @@ - + diff --git a/TUnit.Templates/content/TUnit.FSharp/TestProject.fsproj b/TUnit.Templates/content/TUnit.FSharp/TestProject.fsproj index 77c58677aa..1751db5e8c 100644 --- a/TUnit.Templates/content/TUnit.FSharp/TestProject.fsproj +++ b/TUnit.Templates/content/TUnit.FSharp/TestProject.fsproj @@ -10,8 +10,8 @@ - - + + diff --git a/TUnit.Templates/content/TUnit.Playwright/TestProject.csproj b/TUnit.Templates/content/TUnit.Playwright/TestProject.csproj index 5e661fc20e..ee11fb3565 100644 --- a/TUnit.Templates/content/TUnit.Playwright/TestProject.csproj +++ b/TUnit.Templates/content/TUnit.Playwright/TestProject.csproj @@ -8,7 +8,7 @@ - + diff --git a/TUnit.Templates/content/TUnit.VB/TestProject.vbproj b/TUnit.Templates/content/TUnit.VB/TestProject.vbproj index 16d50df436..548b07f51c 100644 --- a/TUnit.Templates/content/TUnit.VB/TestProject.vbproj +++ b/TUnit.Templates/content/TUnit.VB/TestProject.vbproj @@ -8,6 +8,6 @@ - + diff --git a/TUnit.Templates/content/TUnit/TestProject.csproj b/TUnit.Templates/content/TUnit/TestProject.csproj index f7c72ad4f5..b49bdc3b89 100644 --- a/TUnit.Templates/content/TUnit/TestProject.csproj +++ b/TUnit.Templates/content/TUnit/TestProject.csproj @@ -8,7 +8,7 @@ - + \ No newline at end of file From 2461771b8016f147653f4b9f9186f8128fce79ec Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Mon, 17 Nov 2025 00:29:23 +0000 Subject: [PATCH 2/8] chore: update benchmark results (#3862) --- docs/docs/benchmarks/AsyncTests.md | 18 +- docs/docs/benchmarks/BuildTime.md | 14 +- docs/docs/benchmarks/DataDrivenTests.md | 18 +- docs/docs/benchmarks/MassiveParallelTests.md | 18 +- docs/docs/benchmarks/MatrixTests.md | 18 +- docs/docs/benchmarks/ScaleTests.md | 18 +- docs/docs/benchmarks/SetupTeardownTests.md | 18 +- docs/docs/benchmarks/index.md | 6 +- docs/static/benchmarks/AsyncTests.json | 46 +-- docs/static/benchmarks/BuildTime.json | 36 +-- docs/static/benchmarks/DataDrivenTests.json | 46 +-- .../benchmarks/MassiveParallelTests.json | 46 +-- docs/static/benchmarks/MatrixTests.json | 46 +-- docs/static/benchmarks/ScaleTests.json | 46 +-- .../static/benchmarks/SetupTeardownTests.json | 38 +-- docs/static/benchmarks/historical.json | 4 + docs/static/benchmarks/latest.json | 294 +++++++++--------- docs/static/benchmarks/summary.json | 2 +- 18 files changed, 368 insertions(+), 364 deletions(-) diff --git a/docs/docs/benchmarks/AsyncTests.md b/docs/docs/benchmarks/AsyncTests.md index 152246dad5..65c0986e55 100644 --- a/docs/docs/benchmarks/AsyncTests.md +++ b/docs/docs/benchmarks/AsyncTests.md @@ -7,7 +7,7 @@ sidebar_position: 2 # AsyncTests Benchmark :::info Last Updated -This benchmark was automatically generated on **2025-11-16** from the latest CI run. +This benchmark was automatically generated on **2025-11-17** from the latest CI run. **Environment:** Ubuntu Latest • .NET SDK 10.0.100 ::: @@ -16,11 +16,11 @@ This benchmark was automatically generated on **2025-11-16** from the latest CI | Framework | Version | Mean | Median | StdDev | |-----------|---------|------|--------|--------| -| **TUnit** | 1.2.0 | 565.8 ms | 565.3 ms | 3.67 ms | -| NUnit | 4.4.0 | 647.2 ms | 646.3 ms | 5.71 ms | -| MSTest | 4.0.2 | 620.4 ms | 618.3 ms | 8.09 ms | -| xUnit3 | 3.2.0 | 712.3 ms | 712.0 ms | 11.12 ms | -| **TUnit (AOT)** | 1.2.0 | 125.3 ms | 125.3 ms | 0.28 ms | +| **TUnit** | 1.2.3 | 572.1 ms | 571.8 ms | 2.01 ms | +| NUnit | 4.4.0 | 696.5 ms | 695.9 ms | 6.14 ms | +| MSTest | 4.0.2 | 666.7 ms | 666.2 ms | 5.78 ms | +| xUnit3 | 3.2.0 | 742.0 ms | 742.6 ms | 4.27 ms | +| **TUnit (AOT)** | 1.2.3 | 124.2 ms | 124.1 ms | 0.46 ms | ## 📈 Visual Comparison @@ -58,8 +58,8 @@ This benchmark was automatically generated on **2025-11-16** from the latest CI xychart-beta title "AsyncTests Performance Comparison" x-axis ["TUnit", "NUnit", "MSTest", "xUnit3", "TUnit_AOT"] - y-axis "Time (ms)" 0 --> 855 - bar [565.8, 647.2, 620.4, 712.3, 125.3] + y-axis "Time (ms)" 0 --> 891 + bar [572.1, 696.5, 666.7, 742, 124.2] ``` ## 🎯 Key Insights @@ -72,4 +72,4 @@ This benchmark compares TUnit's performance against NUnit, MSTest, xUnit3 using View the [benchmarks overview](/docs/benchmarks) for methodology details and environment information. ::: -*Last generated: 2025-11-16T00:30:01.239Z* +*Last generated: 2025-11-17T00:29:04.102Z* diff --git a/docs/docs/benchmarks/BuildTime.md b/docs/docs/benchmarks/BuildTime.md index 20967452e6..79260c2c6a 100644 --- a/docs/docs/benchmarks/BuildTime.md +++ b/docs/docs/benchmarks/BuildTime.md @@ -7,7 +7,7 @@ sidebar_position: 8 # Build Performance Benchmark :::info Last Updated -This benchmark was automatically generated on **2025-11-16** from the latest CI run. +This benchmark was automatically generated on **2025-11-17** from the latest CI run. **Environment:** Ubuntu Latest • .NET SDK 10.0.100 ::: @@ -18,10 +18,10 @@ Compilation time comparison across frameworks: | Framework | Version | Mean | Median | StdDev | |-----------|---------|------|--------|--------| -| **TUnit** | 1.2.0 | 2.003 s | 1.999 s | 0.0153 s | -| Build_NUnit | 4.4.0 | 1.612 s | 1.610 s | 0.0296 s | -| Build_MSTest | 4.0.2 | 1.681 s | 1.680 s | 0.0164 s | -| Build_xUnit3 | 3.2.0 | 1.585 s | 1.582 s | 0.0210 s | +| **TUnit** | 1.2.3 | 2.046 s | 2.041 s | 0.0255 s | +| Build_NUnit | 4.4.0 | 1.657 s | 1.655 s | 0.0286 s | +| Build_MSTest | 4.0.2 | 1.743 s | 1.744 s | 0.0212 s | +| Build_xUnit3 | 3.2.0 | 1.640 s | 1.638 s | 0.0151 s | ## 📈 Visual Comparison @@ -60,7 +60,7 @@ xychart-beta title "Build Time Comparison" x-axis ["Build_TUnit", "Build_NUnit", "Build_MSTest", "Build_xUnit3"] y-axis "Time (s)" 0 --> 3 - bar [2.003, 1.612, 1.681, 1.585] + bar [2.046, 1.657, 1.743, 1.64] ``` --- @@ -69,4 +69,4 @@ xychart-beta View the [benchmarks overview](/docs/benchmarks) for methodology details and environment information. ::: -*Last generated: 2025-11-16T00:30:01.241Z* +*Last generated: 2025-11-17T00:29:04.104Z* diff --git a/docs/docs/benchmarks/DataDrivenTests.md b/docs/docs/benchmarks/DataDrivenTests.md index 5cc4ffb92f..c7beba2cee 100644 --- a/docs/docs/benchmarks/DataDrivenTests.md +++ b/docs/docs/benchmarks/DataDrivenTests.md @@ -7,7 +7,7 @@ sidebar_position: 3 # DataDrivenTests Benchmark :::info Last Updated -This benchmark was automatically generated on **2025-11-16** from the latest CI run. +This benchmark was automatically generated on **2025-11-17** from the latest CI run. **Environment:** Ubuntu Latest • .NET SDK 10.0.100 ::: @@ -16,11 +16,11 @@ This benchmark was automatically generated on **2025-11-16** from the latest CI | Framework | Version | Mean | Median | StdDev | |-----------|---------|------|--------|--------| -| **TUnit** | 1.2.0 | 472.15 ms | 472.07 ms | 5.427 ms | -| NUnit | 4.4.0 | 573.40 ms | 571.78 ms | 8.309 ms | -| MSTest | 4.0.2 | 592.88 ms | 592.10 ms | 9.922 ms | -| xUnit3 | 3.2.0 | 591.10 ms | 591.00 ms | 10.996 ms | -| **TUnit (AOT)** | 1.2.0 | 25.00 ms | 25.00 ms | 0.200 ms | +| **TUnit** | 1.2.3 | 507.19 ms | 507.75 ms | 5.647 ms | +| NUnit | 4.4.0 | 537.32 ms | 536.59 ms | 8.966 ms | +| MSTest | 4.0.2 | 504.77 ms | 506.26 ms | 9.497 ms | +| xUnit3 | 3.2.0 | 586.00 ms | 585.07 ms | 9.163 ms | +| **TUnit (AOT)** | 1.2.3 | 24.73 ms | 24.76 ms | 0.142 ms | ## 📈 Visual Comparison @@ -58,8 +58,8 @@ This benchmark was automatically generated on **2025-11-16** from the latest CI xychart-beta title "DataDrivenTests Performance Comparison" x-axis ["TUnit", "NUnit", "MSTest", "xUnit3", "TUnit_AOT"] - y-axis "Time (ms)" 0 --> 712 - bar [472.15, 573.4, 592.88, 591.1, 25] + y-axis "Time (ms)" 0 --> 704 + bar [507.19, 537.32, 504.77, 586, 24.73] ``` ## 🎯 Key Insights @@ -72,4 +72,4 @@ This benchmark compares TUnit's performance against NUnit, MSTest, xUnit3 using View the [benchmarks overview](/docs/benchmarks) for methodology details and environment information. ::: -*Last generated: 2025-11-16T00:30:01.239Z* +*Last generated: 2025-11-17T00:29:04.102Z* diff --git a/docs/docs/benchmarks/MassiveParallelTests.md b/docs/docs/benchmarks/MassiveParallelTests.md index cc4af7feab..f06a769496 100644 --- a/docs/docs/benchmarks/MassiveParallelTests.md +++ b/docs/docs/benchmarks/MassiveParallelTests.md @@ -7,7 +7,7 @@ sidebar_position: 4 # MassiveParallelTests Benchmark :::info Last Updated -This benchmark was automatically generated on **2025-11-16** from the latest CI run. +This benchmark was automatically generated on **2025-11-17** from the latest CI run. **Environment:** Ubuntu Latest • .NET SDK 10.0.100 ::: @@ -16,11 +16,11 @@ This benchmark was automatically generated on **2025-11-16** from the latest CI | Framework | Version | Mean | Median | StdDev | |-----------|---------|------|--------|--------| -| **TUnit** | 1.2.0 | 567.7 ms | 568.3 ms | 5.65 ms | -| NUnit | 4.4.0 | 1,165.6 ms | 1,165.9 ms | 5.31 ms | -| MSTest | 4.0.2 | 2,964.1 ms | 2,966.5 ms | 9.82 ms | -| xUnit3 | 3.2.0 | 3,035.9 ms | 3,036.5 ms | 6.47 ms | -| **TUnit (AOT)** | 1.2.0 | 128.8 ms | 128.8 ms | 0.37 ms | +| **TUnit** | 1.2.3 | 580.3 ms | 579.7 ms | 3.05 ms | +| NUnit | 4.4.0 | 1,199.9 ms | 1,199.0 ms | 7.89 ms | +| MSTest | 4.0.2 | 3,002.4 ms | 3,000.7 ms | 8.30 ms | +| xUnit3 | 3.2.0 | 3,080.0 ms | 3,078.2 ms | 7.99 ms | +| **TUnit (AOT)** | 1.2.3 | 130.5 ms | 130.4 ms | 0.45 ms | ## 📈 Visual Comparison @@ -58,8 +58,8 @@ This benchmark was automatically generated on **2025-11-16** from the latest CI xychart-beta title "MassiveParallelTests Performance Comparison" x-axis ["TUnit", "NUnit", "MSTest", "xUnit3", "TUnit_AOT"] - y-axis "Time (ms)" 0 --> 3644 - bar [567.7, 1165.6, 2964.1, 3035.9, 128.8] + y-axis "Time (ms)" 0 --> 3696 + bar [580.3, 1199.9, 3002.4, 3080, 130.5] ``` ## 🎯 Key Insights @@ -72,4 +72,4 @@ This benchmark compares TUnit's performance against NUnit, MSTest, xUnit3 using View the [benchmarks overview](/docs/benchmarks) for methodology details and environment information. ::: -*Last generated: 2025-11-16T00:30:01.240Z* +*Last generated: 2025-11-17T00:29:04.102Z* diff --git a/docs/docs/benchmarks/MatrixTests.md b/docs/docs/benchmarks/MatrixTests.md index 4fb0d2f8a5..ac198f558f 100644 --- a/docs/docs/benchmarks/MatrixTests.md +++ b/docs/docs/benchmarks/MatrixTests.md @@ -7,7 +7,7 @@ sidebar_position: 5 # MatrixTests Benchmark :::info Last Updated -This benchmark was automatically generated on **2025-11-16** from the latest CI run. +This benchmark was automatically generated on **2025-11-17** from the latest CI run. **Environment:** Ubuntu Latest • .NET SDK 10.0.100 ::: @@ -16,11 +16,11 @@ This benchmark was automatically generated on **2025-11-16** from the latest CI | Framework | Version | Mean | Median | StdDev | |-----------|---------|------|--------|--------| -| **TUnit** | 1.2.0 | 587.68 ms | 588.39 ms | 5.809 ms | -| NUnit | 4.4.0 | 1,560.52 ms | 1,557.34 ms | 9.306 ms | -| MSTest | 4.0.2 | 1,522.76 ms | 1,519.51 ms | 9.857 ms | -| xUnit3 | 3.2.0 | 1,618.75 ms | 1,615.45 ms | 8.652 ms | -| **TUnit (AOT)** | 1.2.0 | 80.22 ms | 80.11 ms | 0.343 ms | +| **TUnit** | 1.2.3 | 567.17 ms | 565.30 ms | 5.174 ms | +| NUnit | 4.4.0 | 1,552.19 ms | 1,551.63 ms | 4.500 ms | +| MSTest | 4.0.2 | 1,523.45 ms | 1,522.39 ms | 14.764 ms | +| xUnit3 | 3.2.0 | 1,607.93 ms | 1,606.94 ms | 15.902 ms | +| **TUnit (AOT)** | 1.2.3 | 79.51 ms | 79.48 ms | 0.251 ms | ## 📈 Visual Comparison @@ -58,8 +58,8 @@ This benchmark was automatically generated on **2025-11-16** from the latest CI xychart-beta title "MatrixTests Performance Comparison" x-axis ["TUnit", "NUnit", "MSTest", "xUnit3", "TUnit_AOT"] - y-axis "Time (ms)" 0 --> 1943 - bar [587.68, 1560.52, 1522.76, 1618.75, 80.22] + y-axis "Time (ms)" 0 --> 1930 + bar [567.17, 1552.19, 1523.45, 1607.93, 79.51] ``` ## 🎯 Key Insights @@ -72,4 +72,4 @@ This benchmark compares TUnit's performance against NUnit, MSTest, xUnit3 using View the [benchmarks overview](/docs/benchmarks) for methodology details and environment information. ::: -*Last generated: 2025-11-16T00:30:01.240Z* +*Last generated: 2025-11-17T00:29:04.103Z* diff --git a/docs/docs/benchmarks/ScaleTests.md b/docs/docs/benchmarks/ScaleTests.md index d582c6831d..d229307af8 100644 --- a/docs/docs/benchmarks/ScaleTests.md +++ b/docs/docs/benchmarks/ScaleTests.md @@ -7,7 +7,7 @@ sidebar_position: 6 # ScaleTests Benchmark :::info Last Updated -This benchmark was automatically generated on **2025-11-16** from the latest CI run. +This benchmark was automatically generated on **2025-11-17** from the latest CI run. **Environment:** Ubuntu Latest • .NET SDK 10.0.100 ::: @@ -16,11 +16,11 @@ This benchmark was automatically generated on **2025-11-16** from the latest CI | Framework | Version | Mean | Median | StdDev | |-----------|---------|------|--------|--------| -| **TUnit** | 1.2.0 | 526.00 ms | 526.16 ms | 5.220 ms | -| NUnit | 4.4.0 | 619.91 ms | 616.01 ms | 28.759 ms | -| MSTest | 4.0.2 | 615.62 ms | 614.08 ms | 21.339 ms | -| xUnit3 | 3.2.0 | 614.08 ms | 614.08 ms | 16.235 ms | -| **TUnit (AOT)** | 1.2.0 | 46.63 ms | 46.83 ms | 3.501 ms | +| **TUnit** | 1.2.3 | 524.89 ms | 524.09 ms | 5.578 ms | +| NUnit | 4.4.0 | 577.30 ms | 578.87 ms | 6.118 ms | +| MSTest | 4.0.2 | 501.88 ms | 501.83 ms | 13.036 ms | +| xUnit3 | 3.2.0 | 576.33 ms | 573.62 ms | 9.545 ms | +| **TUnit (AOT)** | 1.2.3 | 45.60 ms | 45.71 ms | 3.635 ms | ## 📈 Visual Comparison @@ -58,8 +58,8 @@ This benchmark was automatically generated on **2025-11-16** from the latest CI xychart-beta title "ScaleTests Performance Comparison" x-axis ["TUnit", "NUnit", "MSTest", "xUnit3", "TUnit_AOT"] - y-axis "Time (ms)" 0 --> 744 - bar [526, 619.91, 615.62, 614.08, 46.63] + y-axis "Time (ms)" 0 --> 693 + bar [524.89, 577.3, 501.88, 576.33, 45.6] ``` ## 🎯 Key Insights @@ -72,4 +72,4 @@ This benchmark compares TUnit's performance against NUnit, MSTest, xUnit3 using View the [benchmarks overview](/docs/benchmarks) for methodology details and environment information. ::: -*Last generated: 2025-11-16T00:30:01.240Z* +*Last generated: 2025-11-17T00:29:04.103Z* diff --git a/docs/docs/benchmarks/SetupTeardownTests.md b/docs/docs/benchmarks/SetupTeardownTests.md index 37623791c6..fefdeeedc8 100644 --- a/docs/docs/benchmarks/SetupTeardownTests.md +++ b/docs/docs/benchmarks/SetupTeardownTests.md @@ -7,7 +7,7 @@ sidebar_position: 7 # SetupTeardownTests Benchmark :::info Last Updated -This benchmark was automatically generated on **2025-11-16** from the latest CI run. +This benchmark was automatically generated on **2025-11-17** from the latest CI run. **Environment:** Ubuntu Latest • .NET SDK 10.0.100 ::: @@ -16,11 +16,11 @@ This benchmark was automatically generated on **2025-11-16** from the latest CI | Framework | Version | Mean | Median | StdDev | |-----------|---------|------|--------|--------| -| **TUnit** | 1.2.0 | 567.7 ms | 566.9 ms | 7.78 ms | -| NUnit | 4.4.0 | 1,135.4 ms | 1,134.9 ms | 10.64 ms | -| MSTest | 4.0.2 | 1,114.5 ms | 1,114.7 ms | 8.36 ms | -| xUnit3 | 3.2.0 | 1,194.5 ms | 1,193.0 ms | 8.58 ms | -| **TUnit (AOT)** | 1.2.0 | NA | NA | NA | +| **TUnit** | 1.2.3 | 585.9 ms | 585.4 ms | 3.06 ms | +| NUnit | 4.4.0 | 1,159.2 ms | 1,161.2 ms | 4.69 ms | +| MSTest | 4.0.2 | 1,139.4 ms | 1,140.1 ms | 9.12 ms | +| xUnit3 | 3.2.0 | 1,216.8 ms | 1,216.2 ms | 5.74 ms | +| **TUnit (AOT)** | 1.2.3 | NA | NA | NA | ## 📈 Visual Comparison @@ -58,8 +58,8 @@ This benchmark was automatically generated on **2025-11-16** from the latest CI xychart-beta title "SetupTeardownTests Performance Comparison" x-axis ["TUnit", "NUnit", "MSTest", "xUnit3", "TUnit_AOT"] - y-axis "Time (ms)" 0 --> 1434 - bar [567.7, 1135.4, 1114.5, 1194.5, 0] + y-axis "Time (ms)" 0 --> 1461 + bar [585.9, 1159.2, 1139.4, 1216.8, 0] ``` ## 🎯 Key Insights @@ -72,4 +72,4 @@ This benchmark compares TUnit's performance against NUnit, MSTest, xUnit3 using View the [benchmarks overview](/docs/benchmarks) for methodology details and environment information. ::: -*Last generated: 2025-11-16T00:30:01.241Z* +*Last generated: 2025-11-17T00:29:04.103Z* diff --git a/docs/docs/benchmarks/index.md b/docs/docs/benchmarks/index.md index 931f739ce4..575817362c 100644 --- a/docs/docs/benchmarks/index.md +++ b/docs/docs/benchmarks/index.md @@ -7,7 +7,7 @@ sidebar_position: 1 # Performance Benchmarks :::info Last Updated -These benchmarks were automatically generated on **2025-11-16** from the latest CI run. +These benchmarks were automatically generated on **2025-11-17** from the latest CI run. **Environment:** Ubuntu Latest • .NET SDK 10.0.100 ::: @@ -37,7 +37,7 @@ These benchmarks compare TUnit against the most popular .NET testing frameworks: | Framework | Version Tested | |-----------|----------------| -| **TUnit** | 1.2.0 | +| **TUnit** | 1.2.3 | | **xUnit v3** | 3.2.0 | | **NUnit** | 4.4.0 | | **MSTest** | 4.0.2 | @@ -80,4 +80,4 @@ These benchmarks run automatically daily via [GitHub Actions](https://github.com Each benchmark runs multiple iterations with statistical analysis to ensure accuracy. Results may vary based on hardware and test characteristics. ::: -*Last generated: 2025-11-16T00:30:01.241Z* +*Last generated: 2025-11-17T00:29:04.104Z* diff --git a/docs/static/benchmarks/AsyncTests.json b/docs/static/benchmarks/AsyncTests.json index 80d258bc04..5a0c8447db 100644 --- a/docs/static/benchmarks/AsyncTests.json +++ b/docs/static/benchmarks/AsyncTests.json @@ -1,5 +1,5 @@ { - "timestamp": "2025-11-16T00:30:01.239Z", + "timestamp": "2025-11-17T00:29:04.102Z", "category": "AsyncTests", "environment": { "benchmarkDotNetVersion": "BenchmarkDotNet v0.15.7, Linux Ubuntu 24.04.3 LTS (Noble Numbat)", @@ -9,43 +9,43 @@ "results": [ { "Method": "TUnit", - "Version": "1.2.0", - "Mean": "565.8 ms", - "Error": "3.92 ms", - "StdDev": "3.67 ms", - "Median": "565.3 ms" + "Version": "1.2.3", + "Mean": "572.1 ms", + "Error": "2.41 ms", + "StdDev": "2.01 ms", + "Median": "571.8 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "647.2 ms", - "Error": "6.44 ms", - "StdDev": "5.71 ms", - "Median": "646.3 ms" + "Mean": "696.5 ms", + "Error": "7.35 ms", + "StdDev": "6.14 ms", + "Median": "695.9 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "620.4 ms", - "Error": "9.12 ms", - "StdDev": "8.09 ms", - "Median": "618.3 ms" + "Mean": "666.7 ms", + "Error": "6.92 ms", + "StdDev": "5.78 ms", + "Median": "666.2 ms" }, { "Method": "xUnit3", "Version": "3.2.0", - "Mean": "712.3 ms", - "Error": "11.89 ms", - "StdDev": "11.12 ms", - "Median": "712.0 ms" + "Mean": "742.0 ms", + "Error": "4.82 ms", + "StdDev": "4.27 ms", + "Median": "742.6 ms" }, { "Method": "TUnit_AOT", - "Version": "1.2.0", - "Mean": "125.3 ms", - "Error": "0.31 ms", - "StdDev": "0.28 ms", - "Median": "125.3 ms" + "Version": "1.2.3", + "Mean": "124.2 ms", + "Error": "0.49 ms", + "StdDev": "0.46 ms", + "Median": "124.1 ms" } ] } \ No newline at end of file diff --git a/docs/static/benchmarks/BuildTime.json b/docs/static/benchmarks/BuildTime.json index 5ceec4bc66..f9edf1e8b3 100644 --- a/docs/static/benchmarks/BuildTime.json +++ b/docs/static/benchmarks/BuildTime.json @@ -1,5 +1,5 @@ { - "timestamp": "2025-11-16T00:30:01.241Z", + "timestamp": "2025-11-17T00:29:04.104Z", "category": "BuildTime", "environment": { "benchmarkDotNetVersion": "BenchmarkDotNet v0.15.7, Linux Ubuntu 24.04.3 LTS (Noble Numbat)", @@ -9,35 +9,35 @@ "results": [ { "Method": "Build_TUnit", - "Version": "1.2.0", - "Mean": "2.003 s", - "Error": "0.0164 s", - "StdDev": "0.0153 s", - "Median": "1.999 s" + "Version": "1.2.3", + "Mean": "2.046 s", + "Error": "0.0272 s", + "StdDev": "0.0255 s", + "Median": "2.041 s" }, { "Method": "Build_NUnit", "Version": "4.4.0", - "Mean": "1.612 s", - "Error": "0.0302 s", - "StdDev": "0.0296 s", - "Median": "1.610 s" + "Mean": "1.657 s", + "Error": "0.0306 s", + "StdDev": "0.0286 s", + "Median": "1.655 s" }, { "Method": "Build_MSTest", "Version": "4.0.2", - "Mean": "1.681 s", - "Error": "0.0176 s", - "StdDev": "0.0164 s", - "Median": "1.680 s" + "Mean": "1.743 s", + "Error": "0.0239 s", + "StdDev": "0.0212 s", + "Median": "1.744 s" }, { "Method": "Build_xUnit3", "Version": "3.2.0", - "Mean": "1.585 s", - "Error": "0.0224 s", - "StdDev": "0.0210 s", - "Median": "1.582 s" + "Mean": "1.640 s", + "Error": "0.0162 s", + "StdDev": "0.0151 s", + "Median": "1.638 s" } ] } \ No newline at end of file diff --git a/docs/static/benchmarks/DataDrivenTests.json b/docs/static/benchmarks/DataDrivenTests.json index 66bb4e6cac..55272c5497 100644 --- a/docs/static/benchmarks/DataDrivenTests.json +++ b/docs/static/benchmarks/DataDrivenTests.json @@ -1,5 +1,5 @@ { - "timestamp": "2025-11-16T00:30:01.239Z", + "timestamp": "2025-11-17T00:29:04.102Z", "category": "DataDrivenTests", "environment": { "benchmarkDotNetVersion": "BenchmarkDotNet v0.15.7, Linux Ubuntu 24.04.3 LTS (Noble Numbat)", @@ -9,43 +9,43 @@ "results": [ { "Method": "TUnit", - "Version": "1.2.0", - "Mean": "472.15 ms", - "Error": "6.122 ms", - "StdDev": "5.427 ms", - "Median": "472.07 ms" + "Version": "1.2.3", + "Mean": "507.19 ms", + "Error": "6.037 ms", + "StdDev": "5.647 ms", + "Median": "507.75 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "573.40 ms", - "Error": "9.950 ms", - "StdDev": "8.309 ms", - "Median": "571.78 ms" + "Mean": "537.32 ms", + "Error": "10.114 ms", + "StdDev": "8.966 ms", + "Median": "536.59 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "592.88 ms", - "Error": "10.607 ms", - "StdDev": "9.922 ms", - "Median": "592.10 ms" + "Mean": "504.77 ms", + "Error": "9.670 ms", + "StdDev": "9.497 ms", + "Median": "506.26 ms" }, { "Method": "xUnit3", "Version": "3.2.0", - "Mean": "591.10 ms", - "Error": "11.196 ms", - "StdDev": "10.996 ms", - "Median": "591.00 ms" + "Mean": "586.00 ms", + "Error": "10.336 ms", + "StdDev": "9.163 ms", + "Median": "585.07 ms" }, { "Method": "TUnit_AOT", - "Version": "1.2.0", - "Mean": "25.00 ms", - "Error": "0.213 ms", - "StdDev": "0.200 ms", - "Median": "25.00 ms" + "Version": "1.2.3", + "Mean": "24.73 ms", + "Error": "0.152 ms", + "StdDev": "0.142 ms", + "Median": "24.76 ms" } ] } \ No newline at end of file diff --git a/docs/static/benchmarks/MassiveParallelTests.json b/docs/static/benchmarks/MassiveParallelTests.json index d50d6f1e5f..48a451c377 100644 --- a/docs/static/benchmarks/MassiveParallelTests.json +++ b/docs/static/benchmarks/MassiveParallelTests.json @@ -1,5 +1,5 @@ { - "timestamp": "2025-11-16T00:30:01.240Z", + "timestamp": "2025-11-17T00:29:04.103Z", "category": "MassiveParallelTests", "environment": { "benchmarkDotNetVersion": "BenchmarkDotNet v0.15.7, Linux Ubuntu 24.04.3 LTS (Noble Numbat)", @@ -9,43 +9,43 @@ "results": [ { "Method": "TUnit", - "Version": "1.2.0", - "Mean": "567.7 ms", - "Error": "6.04 ms", - "StdDev": "5.65 ms", - "Median": "568.3 ms" + "Version": "1.2.3", + "Mean": "580.3 ms", + "Error": "3.44 ms", + "StdDev": "3.05 ms", + "Median": "579.7 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "1,165.6 ms", - "Error": "6.36 ms", - "StdDev": "5.31 ms", - "Median": "1,165.9 ms" + "Mean": "1,199.9 ms", + "Error": "8.90 ms", + "StdDev": "7.89 ms", + "Median": "1,199.0 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "2,964.1 ms", - "Error": "10.50 ms", - "StdDev": "9.82 ms", - "Median": "2,966.5 ms" + "Mean": "3,002.4 ms", + "Error": "9.36 ms", + "StdDev": "8.30 ms", + "Median": "3,000.7 ms" }, { "Method": "xUnit3", "Version": "3.2.0", - "Mean": "3,035.9 ms", - "Error": "7.74 ms", - "StdDev": "6.47 ms", - "Median": "3,036.5 ms" + "Mean": "3,080.0 ms", + "Error": "8.54 ms", + "StdDev": "7.99 ms", + "Median": "3,078.2 ms" }, { "Method": "TUnit_AOT", - "Version": "1.2.0", - "Mean": "128.8 ms", - "Error": "0.39 ms", - "StdDev": "0.37 ms", - "Median": "128.8 ms" + "Version": "1.2.3", + "Mean": "130.5 ms", + "Error": "0.48 ms", + "StdDev": "0.45 ms", + "Median": "130.4 ms" } ] } \ No newline at end of file diff --git a/docs/static/benchmarks/MatrixTests.json b/docs/static/benchmarks/MatrixTests.json index 8108fb81dd..40e8dd9c42 100644 --- a/docs/static/benchmarks/MatrixTests.json +++ b/docs/static/benchmarks/MatrixTests.json @@ -1,5 +1,5 @@ { - "timestamp": "2025-11-16T00:30:01.240Z", + "timestamp": "2025-11-17T00:29:04.103Z", "category": "MatrixTests", "environment": { "benchmarkDotNetVersion": "BenchmarkDotNet v0.15.7, Linux Ubuntu 24.04.3 LTS (Noble Numbat)", @@ -9,43 +9,43 @@ "results": [ { "Method": "TUnit", - "Version": "1.2.0", - "Mean": "587.68 ms", - "Error": "6.553 ms", - "StdDev": "5.809 ms", - "Median": "588.39 ms" + "Version": "1.2.3", + "Mean": "567.17 ms", + "Error": "5.532 ms", + "StdDev": "5.174 ms", + "Median": "565.30 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "1,560.52 ms", - "Error": "10.497 ms", - "StdDev": "9.306 ms", - "Median": "1,557.34 ms" + "Mean": "1,552.19 ms", + "Error": "5.388 ms", + "StdDev": "4.500 ms", + "Median": "1,551.63 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "1,522.76 ms", - "Error": "11.804 ms", - "StdDev": "9.857 ms", - "Median": "1,519.51 ms" + "Mean": "1,523.45 ms", + "Error": "17.681 ms", + "StdDev": "14.764 ms", + "Median": "1,522.39 ms" }, { "Method": "xUnit3", "Version": "3.2.0", - "Mean": "1,618.75 ms", - "Error": "10.361 ms", - "StdDev": "8.652 ms", - "Median": "1,615.45 ms" + "Mean": "1,607.93 ms", + "Error": "17.000 ms", + "StdDev": "15.902 ms", + "Median": "1,606.94 ms" }, { "Method": "TUnit_AOT", - "Version": "1.2.0", - "Mean": "80.22 ms", - "Error": "0.387 ms", - "StdDev": "0.343 ms", - "Median": "80.11 ms" + "Version": "1.2.3", + "Mean": "79.51 ms", + "Error": "0.269 ms", + "StdDev": "0.251 ms", + "Median": "79.48 ms" } ] } \ No newline at end of file diff --git a/docs/static/benchmarks/ScaleTests.json b/docs/static/benchmarks/ScaleTests.json index be54358cac..8b93e3c061 100644 --- a/docs/static/benchmarks/ScaleTests.json +++ b/docs/static/benchmarks/ScaleTests.json @@ -1,5 +1,5 @@ { - "timestamp": "2025-11-16T00:30:01.240Z", + "timestamp": "2025-11-17T00:29:04.103Z", "category": "ScaleTests", "environment": { "benchmarkDotNetVersion": "BenchmarkDotNet v0.15.7, Linux Ubuntu 24.04.3 LTS (Noble Numbat)", @@ -9,43 +9,43 @@ "results": [ { "Method": "TUnit", - "Version": "1.2.0", - "Mean": "526.00 ms", - "Error": "5.581 ms", - "StdDev": "5.220 ms", - "Median": "526.16 ms" + "Version": "1.2.3", + "Mean": "524.89 ms", + "Error": "5.963 ms", + "StdDev": "5.578 ms", + "Median": "524.09 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "619.91 ms", - "Error": "12.303 ms", - "StdDev": "28.759 ms", - "Median": "616.01 ms" + "Mean": "577.30 ms", + "Error": "7.326 ms", + "StdDev": "6.118 ms", + "Median": "578.87 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "615.62 ms", - "Error": "11.361 ms", - "StdDev": "21.339 ms", - "Median": "614.08 ms" + "Mean": "501.88 ms", + "Error": "9.090 ms", + "StdDev": "13.036 ms", + "Median": "501.83 ms" }, { "Method": "xUnit3", "Version": "3.2.0", - "Mean": "614.08 ms", - "Error": "12.161 ms", - "StdDev": "16.235 ms", - "Median": "614.08 ms" + "Mean": "576.33 ms", + "Error": "10.205 ms", + "StdDev": "9.545 ms", + "Median": "573.62 ms" }, { "Method": "TUnit_AOT", - "Version": "1.2.0", - "Mean": "46.63 ms", - "Error": "1.187 ms", - "StdDev": "3.501 ms", - "Median": "46.83 ms" + "Version": "1.2.3", + "Mean": "45.60 ms", + "Error": "1.239 ms", + "StdDev": "3.635 ms", + "Median": "45.71 ms" } ] } \ No newline at end of file diff --git a/docs/static/benchmarks/SetupTeardownTests.json b/docs/static/benchmarks/SetupTeardownTests.json index 0c437005a5..ed69b9c0e9 100644 --- a/docs/static/benchmarks/SetupTeardownTests.json +++ b/docs/static/benchmarks/SetupTeardownTests.json @@ -1,5 +1,5 @@ { - "timestamp": "2025-11-16T00:30:01.241Z", + "timestamp": "2025-11-17T00:29:04.104Z", "category": "SetupTeardownTests", "environment": { "benchmarkDotNetVersion": "BenchmarkDotNet v0.15.7, Linux Ubuntu 24.04.3 LTS (Noble Numbat)", @@ -9,39 +9,39 @@ "results": [ { "Method": "TUnit", - "Version": "1.2.0", - "Mean": "567.7 ms", - "Error": "8.32 ms", - "StdDev": "7.78 ms", - "Median": "566.9 ms" + "Version": "1.2.3", + "Mean": "585.9 ms", + "Error": "3.28 ms", + "StdDev": "3.06 ms", + "Median": "585.4 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "1,135.4 ms", - "Error": "12.01 ms", - "StdDev": "10.64 ms", - "Median": "1,134.9 ms" + "Mean": "1,159.2 ms", + "Error": "5.29 ms", + "StdDev": "4.69 ms", + "Median": "1,161.2 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "1,114.5 ms", - "Error": "8.93 ms", - "StdDev": "8.36 ms", - "Median": "1,114.7 ms" + "Mean": "1,139.4 ms", + "Error": "9.75 ms", + "StdDev": "9.12 ms", + "Median": "1,140.1 ms" }, { "Method": "xUnit3", "Version": "3.2.0", - "Mean": "1,194.5 ms", - "Error": "9.17 ms", - "StdDev": "8.58 ms", - "Median": "1,193.0 ms" + "Mean": "1,216.8 ms", + "Error": "6.48 ms", + "StdDev": "5.74 ms", + "Median": "1,216.2 ms" }, { "Method": "TUnit_AOT", - "Version": "1.2.0", + "Version": "1.2.3", "Mean": "NA", "Error": "NA", "StdDev": "NA", diff --git a/docs/static/benchmarks/historical.json b/docs/static/benchmarks/historical.json index 51c8b30c35..d7b44bbe49 100644 --- a/docs/static/benchmarks/historical.json +++ b/docs/static/benchmarks/historical.json @@ -46,5 +46,9 @@ { "date": "2025-11-16", "environment": "Ubuntu" + }, + { + "date": "2025-11-17", + "environment": "Ubuntu" } ] \ No newline at end of file diff --git a/docs/static/benchmarks/latest.json b/docs/static/benchmarks/latest.json index ba84d8ce5b..51a9a35af6 100644 --- a/docs/static/benchmarks/latest.json +++ b/docs/static/benchmarks/latest.json @@ -1,5 +1,5 @@ { - "timestamp": "2025-11-16T00:30:01.241Z", + "timestamp": "2025-11-17T00:29:04.104Z", "environment": { "benchmarkDotNetVersion": "BenchmarkDotNet v0.15.7, Linux Ubuntu 24.04.3 LTS (Noble Numbat)", "sdk": ".NET SDK 10.0.100", @@ -9,249 +9,249 @@ "AsyncTests": [ { "Method": "TUnit", - "Version": "1.2.0", - "Mean": "565.8 ms", - "Error": "3.92 ms", - "StdDev": "3.67 ms", - "Median": "565.3 ms" + "Version": "1.2.3", + "Mean": "572.1 ms", + "Error": "2.41 ms", + "StdDev": "2.01 ms", + "Median": "571.8 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "647.2 ms", - "Error": "6.44 ms", - "StdDev": "5.71 ms", - "Median": "646.3 ms" + "Mean": "696.5 ms", + "Error": "7.35 ms", + "StdDev": "6.14 ms", + "Median": "695.9 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "620.4 ms", - "Error": "9.12 ms", - "StdDev": "8.09 ms", - "Median": "618.3 ms" + "Mean": "666.7 ms", + "Error": "6.92 ms", + "StdDev": "5.78 ms", + "Median": "666.2 ms" }, { "Method": "xUnit3", "Version": "3.2.0", - "Mean": "712.3 ms", - "Error": "11.89 ms", - "StdDev": "11.12 ms", - "Median": "712.0 ms" + "Mean": "742.0 ms", + "Error": "4.82 ms", + "StdDev": "4.27 ms", + "Median": "742.6 ms" }, { "Method": "TUnit_AOT", - "Version": "1.2.0", - "Mean": "125.3 ms", - "Error": "0.31 ms", - "StdDev": "0.28 ms", - "Median": "125.3 ms" + "Version": "1.2.3", + "Mean": "124.2 ms", + "Error": "0.49 ms", + "StdDev": "0.46 ms", + "Median": "124.1 ms" } ], "DataDrivenTests": [ { "Method": "TUnit", - "Version": "1.2.0", - "Mean": "472.15 ms", - "Error": "6.122 ms", - "StdDev": "5.427 ms", - "Median": "472.07 ms" + "Version": "1.2.3", + "Mean": "507.19 ms", + "Error": "6.037 ms", + "StdDev": "5.647 ms", + "Median": "507.75 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "573.40 ms", - "Error": "9.950 ms", - "StdDev": "8.309 ms", - "Median": "571.78 ms" + "Mean": "537.32 ms", + "Error": "10.114 ms", + "StdDev": "8.966 ms", + "Median": "536.59 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "592.88 ms", - "Error": "10.607 ms", - "StdDev": "9.922 ms", - "Median": "592.10 ms" + "Mean": "504.77 ms", + "Error": "9.670 ms", + "StdDev": "9.497 ms", + "Median": "506.26 ms" }, { "Method": "xUnit3", "Version": "3.2.0", - "Mean": "591.10 ms", - "Error": "11.196 ms", - "StdDev": "10.996 ms", - "Median": "591.00 ms" + "Mean": "586.00 ms", + "Error": "10.336 ms", + "StdDev": "9.163 ms", + "Median": "585.07 ms" }, { "Method": "TUnit_AOT", - "Version": "1.2.0", - "Mean": "25.00 ms", - "Error": "0.213 ms", - "StdDev": "0.200 ms", - "Median": "25.00 ms" + "Version": "1.2.3", + "Mean": "24.73 ms", + "Error": "0.152 ms", + "StdDev": "0.142 ms", + "Median": "24.76 ms" } ], "MassiveParallelTests": [ { "Method": "TUnit", - "Version": "1.2.0", - "Mean": "567.7 ms", - "Error": "6.04 ms", - "StdDev": "5.65 ms", - "Median": "568.3 ms" + "Version": "1.2.3", + "Mean": "580.3 ms", + "Error": "3.44 ms", + "StdDev": "3.05 ms", + "Median": "579.7 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "1,165.6 ms", - "Error": "6.36 ms", - "StdDev": "5.31 ms", - "Median": "1,165.9 ms" + "Mean": "1,199.9 ms", + "Error": "8.90 ms", + "StdDev": "7.89 ms", + "Median": "1,199.0 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "2,964.1 ms", - "Error": "10.50 ms", - "StdDev": "9.82 ms", - "Median": "2,966.5 ms" + "Mean": "3,002.4 ms", + "Error": "9.36 ms", + "StdDev": "8.30 ms", + "Median": "3,000.7 ms" }, { "Method": "xUnit3", "Version": "3.2.0", - "Mean": "3,035.9 ms", - "Error": "7.74 ms", - "StdDev": "6.47 ms", - "Median": "3,036.5 ms" + "Mean": "3,080.0 ms", + "Error": "8.54 ms", + "StdDev": "7.99 ms", + "Median": "3,078.2 ms" }, { "Method": "TUnit_AOT", - "Version": "1.2.0", - "Mean": "128.8 ms", - "Error": "0.39 ms", - "StdDev": "0.37 ms", - "Median": "128.8 ms" + "Version": "1.2.3", + "Mean": "130.5 ms", + "Error": "0.48 ms", + "StdDev": "0.45 ms", + "Median": "130.4 ms" } ], "MatrixTests": [ { "Method": "TUnit", - "Version": "1.2.0", - "Mean": "587.68 ms", - "Error": "6.553 ms", - "StdDev": "5.809 ms", - "Median": "588.39 ms" + "Version": "1.2.3", + "Mean": "567.17 ms", + "Error": "5.532 ms", + "StdDev": "5.174 ms", + "Median": "565.30 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "1,560.52 ms", - "Error": "10.497 ms", - "StdDev": "9.306 ms", - "Median": "1,557.34 ms" + "Mean": "1,552.19 ms", + "Error": "5.388 ms", + "StdDev": "4.500 ms", + "Median": "1,551.63 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "1,522.76 ms", - "Error": "11.804 ms", - "StdDev": "9.857 ms", - "Median": "1,519.51 ms" + "Mean": "1,523.45 ms", + "Error": "17.681 ms", + "StdDev": "14.764 ms", + "Median": "1,522.39 ms" }, { "Method": "xUnit3", "Version": "3.2.0", - "Mean": "1,618.75 ms", - "Error": "10.361 ms", - "StdDev": "8.652 ms", - "Median": "1,615.45 ms" + "Mean": "1,607.93 ms", + "Error": "17.000 ms", + "StdDev": "15.902 ms", + "Median": "1,606.94 ms" }, { "Method": "TUnit_AOT", - "Version": "1.2.0", - "Mean": "80.22 ms", - "Error": "0.387 ms", - "StdDev": "0.343 ms", - "Median": "80.11 ms" + "Version": "1.2.3", + "Mean": "79.51 ms", + "Error": "0.269 ms", + "StdDev": "0.251 ms", + "Median": "79.48 ms" } ], "ScaleTests": [ { "Method": "TUnit", - "Version": "1.2.0", - "Mean": "526.00 ms", - "Error": "5.581 ms", - "StdDev": "5.220 ms", - "Median": "526.16 ms" + "Version": "1.2.3", + "Mean": "524.89 ms", + "Error": "5.963 ms", + "StdDev": "5.578 ms", + "Median": "524.09 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "619.91 ms", - "Error": "12.303 ms", - "StdDev": "28.759 ms", - "Median": "616.01 ms" + "Mean": "577.30 ms", + "Error": "7.326 ms", + "StdDev": "6.118 ms", + "Median": "578.87 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "615.62 ms", - "Error": "11.361 ms", - "StdDev": "21.339 ms", - "Median": "614.08 ms" + "Mean": "501.88 ms", + "Error": "9.090 ms", + "StdDev": "13.036 ms", + "Median": "501.83 ms" }, { "Method": "xUnit3", "Version": "3.2.0", - "Mean": "614.08 ms", - "Error": "12.161 ms", - "StdDev": "16.235 ms", - "Median": "614.08 ms" + "Mean": "576.33 ms", + "Error": "10.205 ms", + "StdDev": "9.545 ms", + "Median": "573.62 ms" }, { "Method": "TUnit_AOT", - "Version": "1.2.0", - "Mean": "46.63 ms", - "Error": "1.187 ms", - "StdDev": "3.501 ms", - "Median": "46.83 ms" + "Version": "1.2.3", + "Mean": "45.60 ms", + "Error": "1.239 ms", + "StdDev": "3.635 ms", + "Median": "45.71 ms" } ], "SetupTeardownTests": [ { "Method": "TUnit", - "Version": "1.2.0", - "Mean": "567.7 ms", - "Error": "8.32 ms", - "StdDev": "7.78 ms", - "Median": "566.9 ms" + "Version": "1.2.3", + "Mean": "585.9 ms", + "Error": "3.28 ms", + "StdDev": "3.06 ms", + "Median": "585.4 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "1,135.4 ms", - "Error": "12.01 ms", - "StdDev": "10.64 ms", - "Median": "1,134.9 ms" + "Mean": "1,159.2 ms", + "Error": "5.29 ms", + "StdDev": "4.69 ms", + "Median": "1,161.2 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "1,114.5 ms", - "Error": "8.93 ms", - "StdDev": "8.36 ms", - "Median": "1,114.7 ms" + "Mean": "1,139.4 ms", + "Error": "9.75 ms", + "StdDev": "9.12 ms", + "Median": "1,140.1 ms" }, { "Method": "xUnit3", "Version": "3.2.0", - "Mean": "1,194.5 ms", - "Error": "9.17 ms", - "StdDev": "8.58 ms", - "Median": "1,193.0 ms" + "Mean": "1,216.8 ms", + "Error": "6.48 ms", + "StdDev": "5.74 ms", + "Median": "1,216.2 ms" }, { "Method": "TUnit_AOT", - "Version": "1.2.0", + "Version": "1.2.3", "Mean": "NA", "Error": "NA", "StdDev": "NA", @@ -263,35 +263,35 @@ "BuildTime": [ { "Method": "Build_TUnit", - "Version": "1.2.0", - "Mean": "2.003 s", - "Error": "0.0164 s", - "StdDev": "0.0153 s", - "Median": "1.999 s" + "Version": "1.2.3", + "Mean": "2.046 s", + "Error": "0.0272 s", + "StdDev": "0.0255 s", + "Median": "2.041 s" }, { "Method": "Build_NUnit", "Version": "4.4.0", - "Mean": "1.612 s", - "Error": "0.0302 s", - "StdDev": "0.0296 s", - "Median": "1.610 s" + "Mean": "1.657 s", + "Error": "0.0306 s", + "StdDev": "0.0286 s", + "Median": "1.655 s" }, { "Method": "Build_MSTest", "Version": "4.0.2", - "Mean": "1.681 s", - "Error": "0.0176 s", - "StdDev": "0.0164 s", - "Median": "1.680 s" + "Mean": "1.743 s", + "Error": "0.0239 s", + "StdDev": "0.0212 s", + "Median": "1.744 s" }, { "Method": "Build_xUnit3", "Version": "3.2.0", - "Mean": "1.585 s", - "Error": "0.0224 s", - "StdDev": "0.0210 s", - "Median": "1.582 s" + "Mean": "1.640 s", + "Error": "0.0162 s", + "StdDev": "0.0151 s", + "Median": "1.638 s" } ] }, @@ -299,6 +299,6 @@ "runtimeCategories": 6, "buildCategories": 1, "totalBenchmarks": 7, - "lastUpdated": "2025-11-16T00:30:01.238Z" + "lastUpdated": "2025-11-17T00:29:04.101Z" } } \ No newline at end of file diff --git a/docs/static/benchmarks/summary.json b/docs/static/benchmarks/summary.json index f51fb29b33..465cf055e5 100644 --- a/docs/static/benchmarks/summary.json +++ b/docs/static/benchmarks/summary.json @@ -10,6 +10,6 @@ "build": [ "BuildTime" ], - "timestamp": "2025-11-16", + "timestamp": "2025-11-17", "environment": "Ubuntu Latest • .NET SDK 10.0.100" } \ No newline at end of file From bf31b5de63ae85b830c6ed1d62455c777faae097 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Tue, 18 Nov 2025 00:28:33 +0000 Subject: [PATCH 3/8] chore: update benchmark results (#3864) --- docs/docs/benchmarks/AsyncTests.md | 18 +- docs/docs/benchmarks/BuildTime.md | 14 +- docs/docs/benchmarks/DataDrivenTests.md | 18 +- docs/docs/benchmarks/MassiveParallelTests.md | 18 +- docs/docs/benchmarks/MatrixTests.md | 18 +- docs/docs/benchmarks/ScaleTests.md | 18 +- docs/docs/benchmarks/SetupTeardownTests.md | 16 +- docs/docs/benchmarks/index.md | 4 +- docs/static/benchmarks/AsyncTests.json | 42 +-- docs/static/benchmarks/BuildTime.json | 34 +-- docs/static/benchmarks/DataDrivenTests.json | 42 +-- .../benchmarks/MassiveParallelTests.json | 42 +-- docs/static/benchmarks/MatrixTests.json | 42 +-- docs/static/benchmarks/ScaleTests.json | 42 +-- .../static/benchmarks/SetupTeardownTests.json | 34 +-- docs/static/benchmarks/historical.json | 4 + docs/static/benchmarks/latest.json | 268 +++++++++--------- docs/static/benchmarks/summary.json | 2 +- 18 files changed, 340 insertions(+), 336 deletions(-) diff --git a/docs/docs/benchmarks/AsyncTests.md b/docs/docs/benchmarks/AsyncTests.md index 65c0986e55..d3f9769eb3 100644 --- a/docs/docs/benchmarks/AsyncTests.md +++ b/docs/docs/benchmarks/AsyncTests.md @@ -7,7 +7,7 @@ sidebar_position: 2 # AsyncTests Benchmark :::info Last Updated -This benchmark was automatically generated on **2025-11-17** from the latest CI run. +This benchmark was automatically generated on **2025-11-18** from the latest CI run. **Environment:** Ubuntu Latest • .NET SDK 10.0.100 ::: @@ -16,11 +16,11 @@ This benchmark was automatically generated on **2025-11-17** from the latest CI | Framework | Version | Mean | Median | StdDev | |-----------|---------|------|--------|--------| -| **TUnit** | 1.2.3 | 572.1 ms | 571.8 ms | 2.01 ms | -| NUnit | 4.4.0 | 696.5 ms | 695.9 ms | 6.14 ms | -| MSTest | 4.0.2 | 666.7 ms | 666.2 ms | 5.78 ms | -| xUnit3 | 3.2.0 | 742.0 ms | 742.6 ms | 4.27 ms | -| **TUnit (AOT)** | 1.2.3 | 124.2 ms | 124.1 ms | 0.46 ms | +| **TUnit** | 1.2.3 | 556.5 ms | 556.3 ms | 2.03 ms | +| NUnit | 4.4.0 | 663.3 ms | 662.2 ms | 9.02 ms | +| MSTest | 4.0.2 | 635.0 ms | 632.0 ms | 11.80 ms | +| xUnit3 | 3.2.0 | 715.8 ms | 716.7 ms | 8.28 ms | +| **TUnit (AOT)** | 1.2.3 | 124.5 ms | 124.5 ms | 0.26 ms | ## 📈 Visual Comparison @@ -58,8 +58,8 @@ This benchmark was automatically generated on **2025-11-17** from the latest CI xychart-beta title "AsyncTests Performance Comparison" x-axis ["TUnit", "NUnit", "MSTest", "xUnit3", "TUnit_AOT"] - y-axis "Time (ms)" 0 --> 891 - bar [572.1, 696.5, 666.7, 742, 124.2] + y-axis "Time (ms)" 0 --> 859 + bar [556.5, 663.3, 635, 715.8, 124.5] ``` ## 🎯 Key Insights @@ -72,4 +72,4 @@ This benchmark compares TUnit's performance against NUnit, MSTest, xUnit3 using View the [benchmarks overview](/docs/benchmarks) for methodology details and environment information. ::: -*Last generated: 2025-11-17T00:29:04.102Z* +*Last generated: 2025-11-18T00:28:14.218Z* diff --git a/docs/docs/benchmarks/BuildTime.md b/docs/docs/benchmarks/BuildTime.md index 79260c2c6a..d06fcc5888 100644 --- a/docs/docs/benchmarks/BuildTime.md +++ b/docs/docs/benchmarks/BuildTime.md @@ -7,7 +7,7 @@ sidebar_position: 8 # Build Performance Benchmark :::info Last Updated -This benchmark was automatically generated on **2025-11-17** from the latest CI run. +This benchmark was automatically generated on **2025-11-18** from the latest CI run. **Environment:** Ubuntu Latest • .NET SDK 10.0.100 ::: @@ -18,10 +18,10 @@ Compilation time comparison across frameworks: | Framework | Version | Mean | Median | StdDev | |-----------|---------|------|--------|--------| -| **TUnit** | 1.2.3 | 2.046 s | 2.041 s | 0.0255 s | -| Build_NUnit | 4.4.0 | 1.657 s | 1.655 s | 0.0286 s | -| Build_MSTest | 4.0.2 | 1.743 s | 1.744 s | 0.0212 s | -| Build_xUnit3 | 3.2.0 | 1.640 s | 1.638 s | 0.0151 s | +| **TUnit** | 1.2.3 | 1.947 s | 1.953 s | 0.0381 s | +| Build_NUnit | 4.4.0 | 1.556 s | 1.557 s | 0.0171 s | +| Build_MSTest | 4.0.2 | 1.621 s | 1.624 s | 0.0156 s | +| Build_xUnit3 | 3.2.0 | 1.523 s | 1.527 s | 0.0162 s | ## 📈 Visual Comparison @@ -60,7 +60,7 @@ xychart-beta title "Build Time Comparison" x-axis ["Build_TUnit", "Build_NUnit", "Build_MSTest", "Build_xUnit3"] y-axis "Time (s)" 0 --> 3 - bar [2.046, 1.657, 1.743, 1.64] + bar [1.947, 1.556, 1.621, 1.523] ``` --- @@ -69,4 +69,4 @@ xychart-beta View the [benchmarks overview](/docs/benchmarks) for methodology details and environment information. ::: -*Last generated: 2025-11-17T00:29:04.104Z* +*Last generated: 2025-11-18T00:28:14.220Z* diff --git a/docs/docs/benchmarks/DataDrivenTests.md b/docs/docs/benchmarks/DataDrivenTests.md index c7beba2cee..06066bfb23 100644 --- a/docs/docs/benchmarks/DataDrivenTests.md +++ b/docs/docs/benchmarks/DataDrivenTests.md @@ -7,7 +7,7 @@ sidebar_position: 3 # DataDrivenTests Benchmark :::info Last Updated -This benchmark was automatically generated on **2025-11-17** from the latest CI run. +This benchmark was automatically generated on **2025-11-18** from the latest CI run. **Environment:** Ubuntu Latest • .NET SDK 10.0.100 ::: @@ -16,11 +16,11 @@ This benchmark was automatically generated on **2025-11-17** from the latest CI | Framework | Version | Mean | Median | StdDev | |-----------|---------|------|--------|--------| -| **TUnit** | 1.2.3 | 507.19 ms | 507.75 ms | 5.647 ms | -| NUnit | 4.4.0 | 537.32 ms | 536.59 ms | 8.966 ms | -| MSTest | 4.0.2 | 504.77 ms | 506.26 ms | 9.497 ms | -| xUnit3 | 3.2.0 | 586.00 ms | 585.07 ms | 9.163 ms | -| **TUnit (AOT)** | 1.2.3 | 24.73 ms | 24.76 ms | 0.142 ms | +| **TUnit** | 1.2.3 | 493.26 ms | 492.16 ms | 3.175 ms | +| NUnit | 4.4.0 | 601.85 ms | 601.22 ms | 13.698 ms | +| MSTest | 4.0.2 | 617.89 ms | 622.69 ms | 12.013 ms | +| xUnit3 | 3.2.0 | 610.52 ms | 609.06 ms | 11.016 ms | +| **TUnit (AOT)** | 1.2.3 | 24.33 ms | 24.31 ms | 0.233 ms | ## 📈 Visual Comparison @@ -58,8 +58,8 @@ This benchmark was automatically generated on **2025-11-17** from the latest CI xychart-beta title "DataDrivenTests Performance Comparison" x-axis ["TUnit", "NUnit", "MSTest", "xUnit3", "TUnit_AOT"] - y-axis "Time (ms)" 0 --> 704 - bar [507.19, 537.32, 504.77, 586, 24.73] + y-axis "Time (ms)" 0 --> 742 + bar [493.26, 601.85, 617.89, 610.52, 24.33] ``` ## 🎯 Key Insights @@ -72,4 +72,4 @@ This benchmark compares TUnit's performance against NUnit, MSTest, xUnit3 using View the [benchmarks overview](/docs/benchmarks) for methodology details and environment information. ::: -*Last generated: 2025-11-17T00:29:04.102Z* +*Last generated: 2025-11-18T00:28:14.218Z* diff --git a/docs/docs/benchmarks/MassiveParallelTests.md b/docs/docs/benchmarks/MassiveParallelTests.md index f06a769496..5808198b81 100644 --- a/docs/docs/benchmarks/MassiveParallelTests.md +++ b/docs/docs/benchmarks/MassiveParallelTests.md @@ -7,7 +7,7 @@ sidebar_position: 4 # MassiveParallelTests Benchmark :::info Last Updated -This benchmark was automatically generated on **2025-11-17** from the latest CI run. +This benchmark was automatically generated on **2025-11-18** from the latest CI run. **Environment:** Ubuntu Latest • .NET SDK 10.0.100 ::: @@ -16,11 +16,11 @@ This benchmark was automatically generated on **2025-11-17** from the latest CI | Framework | Version | Mean | Median | StdDev | |-----------|---------|------|--------|--------| -| **TUnit** | 1.2.3 | 580.3 ms | 579.7 ms | 3.05 ms | -| NUnit | 4.4.0 | 1,199.9 ms | 1,199.0 ms | 7.89 ms | -| MSTest | 4.0.2 | 3,002.4 ms | 3,000.7 ms | 8.30 ms | -| xUnit3 | 3.2.0 | 3,080.0 ms | 3,078.2 ms | 7.99 ms | -| **TUnit (AOT)** | 1.2.3 | 130.5 ms | 130.4 ms | 0.45 ms | +| **TUnit** | 1.2.3 | 595.5 ms | 595.7 ms | 2.63 ms | +| NUnit | 4.4.0 | 1,164.1 ms | 1,163.4 ms | 6.01 ms | +| MSTest | 4.0.2 | 2,947.1 ms | 2,948.2 ms | 6.02 ms | +| xUnit3 | 3.2.0 | 3,048.5 ms | 3,046.6 ms | 14.96 ms | +| **TUnit (AOT)** | 1.2.3 | 130.9 ms | 130.9 ms | 0.41 ms | ## 📈 Visual Comparison @@ -58,8 +58,8 @@ This benchmark was automatically generated on **2025-11-17** from the latest CI xychart-beta title "MassiveParallelTests Performance Comparison" x-axis ["TUnit", "NUnit", "MSTest", "xUnit3", "TUnit_AOT"] - y-axis "Time (ms)" 0 --> 3696 - bar [580.3, 1199.9, 3002.4, 3080, 130.5] + y-axis "Time (ms)" 0 --> 3659 + bar [595.5, 1164.1, 2947.1, 3048.5, 130.9] ``` ## 🎯 Key Insights @@ -72,4 +72,4 @@ This benchmark compares TUnit's performance against NUnit, MSTest, xUnit3 using View the [benchmarks overview](/docs/benchmarks) for methodology details and environment information. ::: -*Last generated: 2025-11-17T00:29:04.102Z* +*Last generated: 2025-11-18T00:28:14.219Z* diff --git a/docs/docs/benchmarks/MatrixTests.md b/docs/docs/benchmarks/MatrixTests.md index ac198f558f..7807cc0d66 100644 --- a/docs/docs/benchmarks/MatrixTests.md +++ b/docs/docs/benchmarks/MatrixTests.md @@ -7,7 +7,7 @@ sidebar_position: 5 # MatrixTests Benchmark :::info Last Updated -This benchmark was automatically generated on **2025-11-17** from the latest CI run. +This benchmark was automatically generated on **2025-11-18** from the latest CI run. **Environment:** Ubuntu Latest • .NET SDK 10.0.100 ::: @@ -16,11 +16,11 @@ This benchmark was automatically generated on **2025-11-17** from the latest CI | Framework | Version | Mean | Median | StdDev | |-----------|---------|------|--------|--------| -| **TUnit** | 1.2.3 | 567.17 ms | 565.30 ms | 5.174 ms | -| NUnit | 4.4.0 | 1,552.19 ms | 1,551.63 ms | 4.500 ms | -| MSTest | 4.0.2 | 1,523.45 ms | 1,522.39 ms | 14.764 ms | -| xUnit3 | 3.2.0 | 1,607.93 ms | 1,606.94 ms | 15.902 ms | -| **TUnit (AOT)** | 1.2.3 | 79.51 ms | 79.48 ms | 0.251 ms | +| **TUnit** | 1.2.3 | 561.43 ms | 562.61 ms | 4.718 ms | +| NUnit | 4.4.0 | 1,556.36 ms | 1,557.95 ms | 7.815 ms | +| MSTest | 4.0.2 | 1,517.35 ms | 1,517.64 ms | 12.824 ms | +| xUnit3 | 3.2.0 | 1,602.60 ms | 1,603.72 ms | 11.920 ms | +| **TUnit (AOT)** | 1.2.3 | 79.00 ms | 79.03 ms | 0.223 ms | ## 📈 Visual Comparison @@ -58,8 +58,8 @@ This benchmark was automatically generated on **2025-11-17** from the latest CI xychart-beta title "MatrixTests Performance Comparison" x-axis ["TUnit", "NUnit", "MSTest", "xUnit3", "TUnit_AOT"] - y-axis "Time (ms)" 0 --> 1930 - bar [567.17, 1552.19, 1523.45, 1607.93, 79.51] + y-axis "Time (ms)" 0 --> 1924 + bar [561.43, 1556.36, 1517.35, 1602.6, 79] ``` ## 🎯 Key Insights @@ -72,4 +72,4 @@ This benchmark compares TUnit's performance against NUnit, MSTest, xUnit3 using View the [benchmarks overview](/docs/benchmarks) for methodology details and environment information. ::: -*Last generated: 2025-11-17T00:29:04.103Z* +*Last generated: 2025-11-18T00:28:14.219Z* diff --git a/docs/docs/benchmarks/ScaleTests.md b/docs/docs/benchmarks/ScaleTests.md index d229307af8..368e584fb2 100644 --- a/docs/docs/benchmarks/ScaleTests.md +++ b/docs/docs/benchmarks/ScaleTests.md @@ -7,7 +7,7 @@ sidebar_position: 6 # ScaleTests Benchmark :::info Last Updated -This benchmark was automatically generated on **2025-11-17** from the latest CI run. +This benchmark was automatically generated on **2025-11-18** from the latest CI run. **Environment:** Ubuntu Latest • .NET SDK 10.0.100 ::: @@ -16,11 +16,11 @@ This benchmark was automatically generated on **2025-11-17** from the latest CI | Framework | Version | Mean | Median | StdDev | |-----------|---------|------|--------|--------| -| **TUnit** | 1.2.3 | 524.89 ms | 524.09 ms | 5.578 ms | -| NUnit | 4.4.0 | 577.30 ms | 578.87 ms | 6.118 ms | -| MSTest | 4.0.2 | 501.88 ms | 501.83 ms | 13.036 ms | -| xUnit3 | 3.2.0 | 576.33 ms | 573.62 ms | 9.545 ms | -| **TUnit (AOT)** | 1.2.3 | 45.60 ms | 45.71 ms | 3.635 ms | +| **TUnit** | 1.2.3 | 541.57 ms | 540.90 ms | 3.848 ms | +| NUnit | 4.4.0 | 591.49 ms | 591.89 ms | 9.280 ms | +| MSTest | 4.0.2 | 512.41 ms | 507.96 ms | 11.331 ms | +| xUnit3 | 3.2.0 | 596.95 ms | 593.96 ms | 9.270 ms | +| **TUnit (AOT)** | 1.2.3 | 43.84 ms | 43.96 ms | 3.376 ms | ## 📈 Visual Comparison @@ -58,8 +58,8 @@ This benchmark was automatically generated on **2025-11-17** from the latest CI xychart-beta title "ScaleTests Performance Comparison" x-axis ["TUnit", "NUnit", "MSTest", "xUnit3", "TUnit_AOT"] - y-axis "Time (ms)" 0 --> 693 - bar [524.89, 577.3, 501.88, 576.33, 45.6] + y-axis "Time (ms)" 0 --> 717 + bar [541.57, 591.49, 512.41, 596.95, 43.84] ``` ## 🎯 Key Insights @@ -72,4 +72,4 @@ This benchmark compares TUnit's performance against NUnit, MSTest, xUnit3 using View the [benchmarks overview](/docs/benchmarks) for methodology details and environment information. ::: -*Last generated: 2025-11-17T00:29:04.103Z* +*Last generated: 2025-11-18T00:28:14.219Z* diff --git a/docs/docs/benchmarks/SetupTeardownTests.md b/docs/docs/benchmarks/SetupTeardownTests.md index fefdeeedc8..fa7067085b 100644 --- a/docs/docs/benchmarks/SetupTeardownTests.md +++ b/docs/docs/benchmarks/SetupTeardownTests.md @@ -7,7 +7,7 @@ sidebar_position: 7 # SetupTeardownTests Benchmark :::info Last Updated -This benchmark was automatically generated on **2025-11-17** from the latest CI run. +This benchmark was automatically generated on **2025-11-18** from the latest CI run. **Environment:** Ubuntu Latest • .NET SDK 10.0.100 ::: @@ -16,10 +16,10 @@ This benchmark was automatically generated on **2025-11-17** from the latest CI | Framework | Version | Mean | Median | StdDev | |-----------|---------|------|--------|--------| -| **TUnit** | 1.2.3 | 585.9 ms | 585.4 ms | 3.06 ms | -| NUnit | 4.4.0 | 1,159.2 ms | 1,161.2 ms | 4.69 ms | -| MSTest | 4.0.2 | 1,139.4 ms | 1,140.1 ms | 9.12 ms | -| xUnit3 | 3.2.0 | 1,216.8 ms | 1,216.2 ms | 5.74 ms | +| **TUnit** | 1.2.3 | 578.7 ms | 578.1 ms | 6.18 ms | +| NUnit | 4.4.0 | 1,182.7 ms | 1,182.8 ms | 8.39 ms | +| MSTest | 4.0.2 | 1,152.5 ms | 1,151.9 ms | 6.97 ms | +| xUnit3 | 3.2.0 | 1,229.3 ms | 1,225.5 ms | 9.31 ms | | **TUnit (AOT)** | 1.2.3 | NA | NA | NA | ## 📈 Visual Comparison @@ -58,8 +58,8 @@ This benchmark was automatically generated on **2025-11-17** from the latest CI xychart-beta title "SetupTeardownTests Performance Comparison" x-axis ["TUnit", "NUnit", "MSTest", "xUnit3", "TUnit_AOT"] - y-axis "Time (ms)" 0 --> 1461 - bar [585.9, 1159.2, 1139.4, 1216.8, 0] + y-axis "Time (ms)" 0 --> 1476 + bar [578.7, 1182.7, 1152.5, 1229.3, 0] ``` ## 🎯 Key Insights @@ -72,4 +72,4 @@ This benchmark compares TUnit's performance against NUnit, MSTest, xUnit3 using View the [benchmarks overview](/docs/benchmarks) for methodology details and environment information. ::: -*Last generated: 2025-11-17T00:29:04.103Z* +*Last generated: 2025-11-18T00:28:14.220Z* diff --git a/docs/docs/benchmarks/index.md b/docs/docs/benchmarks/index.md index 575817362c..5dcade5491 100644 --- a/docs/docs/benchmarks/index.md +++ b/docs/docs/benchmarks/index.md @@ -7,7 +7,7 @@ sidebar_position: 1 # Performance Benchmarks :::info Last Updated -These benchmarks were automatically generated on **2025-11-17** from the latest CI run. +These benchmarks were automatically generated on **2025-11-18** from the latest CI run. **Environment:** Ubuntu Latest • .NET SDK 10.0.100 ::: @@ -80,4 +80,4 @@ These benchmarks run automatically daily via [GitHub Actions](https://github.com Each benchmark runs multiple iterations with statistical analysis to ensure accuracy. Results may vary based on hardware and test characteristics. ::: -*Last generated: 2025-11-17T00:29:04.104Z* +*Last generated: 2025-11-18T00:28:14.220Z* diff --git a/docs/static/benchmarks/AsyncTests.json b/docs/static/benchmarks/AsyncTests.json index 5a0c8447db..a7ec56b194 100644 --- a/docs/static/benchmarks/AsyncTests.json +++ b/docs/static/benchmarks/AsyncTests.json @@ -1,5 +1,5 @@ { - "timestamp": "2025-11-17T00:29:04.102Z", + "timestamp": "2025-11-18T00:28:14.218Z", "category": "AsyncTests", "environment": { "benchmarkDotNetVersion": "BenchmarkDotNet v0.15.7, Linux Ubuntu 24.04.3 LTS (Noble Numbat)", @@ -10,42 +10,42 @@ { "Method": "TUnit", "Version": "1.2.3", - "Mean": "572.1 ms", - "Error": "2.41 ms", - "StdDev": "2.01 ms", - "Median": "571.8 ms" + "Mean": "556.5 ms", + "Error": "2.29 ms", + "StdDev": "2.03 ms", + "Median": "556.3 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "696.5 ms", - "Error": "7.35 ms", - "StdDev": "6.14 ms", - "Median": "695.9 ms" + "Mean": "663.3 ms", + "Error": "10.81 ms", + "StdDev": "9.02 ms", + "Median": "662.2 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "666.7 ms", - "Error": "6.92 ms", - "StdDev": "5.78 ms", - "Median": "666.2 ms" + "Mean": "635.0 ms", + "Error": "12.02 ms", + "StdDev": "11.80 ms", + "Median": "632.0 ms" }, { "Method": "xUnit3", "Version": "3.2.0", - "Mean": "742.0 ms", - "Error": "4.82 ms", - "StdDev": "4.27 ms", - "Median": "742.6 ms" + "Mean": "715.8 ms", + "Error": "9.34 ms", + "StdDev": "8.28 ms", + "Median": "716.7 ms" }, { "Method": "TUnit_AOT", "Version": "1.2.3", - "Mean": "124.2 ms", - "Error": "0.49 ms", - "StdDev": "0.46 ms", - "Median": "124.1 ms" + "Mean": "124.5 ms", + "Error": "0.31 ms", + "StdDev": "0.26 ms", + "Median": "124.5 ms" } ] } \ No newline at end of file diff --git a/docs/static/benchmarks/BuildTime.json b/docs/static/benchmarks/BuildTime.json index f9edf1e8b3..a53dfb6c92 100644 --- a/docs/static/benchmarks/BuildTime.json +++ b/docs/static/benchmarks/BuildTime.json @@ -1,5 +1,5 @@ { - "timestamp": "2025-11-17T00:29:04.104Z", + "timestamp": "2025-11-18T00:28:14.220Z", "category": "BuildTime", "environment": { "benchmarkDotNetVersion": "BenchmarkDotNet v0.15.7, Linux Ubuntu 24.04.3 LTS (Noble Numbat)", @@ -10,34 +10,34 @@ { "Method": "Build_TUnit", "Version": "1.2.3", - "Mean": "2.046 s", - "Error": "0.0272 s", - "StdDev": "0.0255 s", - "Median": "2.041 s" + "Mean": "1.947 s", + "Error": "0.0371 s", + "StdDev": "0.0381 s", + "Median": "1.953 s" }, { "Method": "Build_NUnit", "Version": "4.4.0", - "Mean": "1.657 s", - "Error": "0.0306 s", - "StdDev": "0.0286 s", - "Median": "1.655 s" + "Mean": "1.556 s", + "Error": "0.0183 s", + "StdDev": "0.0171 s", + "Median": "1.557 s" }, { "Method": "Build_MSTest", "Version": "4.0.2", - "Mean": "1.743 s", - "Error": "0.0239 s", - "StdDev": "0.0212 s", - "Median": "1.744 s" + "Mean": "1.621 s", + "Error": "0.0167 s", + "StdDev": "0.0156 s", + "Median": "1.624 s" }, { "Method": "Build_xUnit3", "Version": "3.2.0", - "Mean": "1.640 s", - "Error": "0.0162 s", - "StdDev": "0.0151 s", - "Median": "1.638 s" + "Mean": "1.523 s", + "Error": "0.0173 s", + "StdDev": "0.0162 s", + "Median": "1.527 s" } ] } \ No newline at end of file diff --git a/docs/static/benchmarks/DataDrivenTests.json b/docs/static/benchmarks/DataDrivenTests.json index 55272c5497..482e01a132 100644 --- a/docs/static/benchmarks/DataDrivenTests.json +++ b/docs/static/benchmarks/DataDrivenTests.json @@ -1,5 +1,5 @@ { - "timestamp": "2025-11-17T00:29:04.102Z", + "timestamp": "2025-11-18T00:28:14.219Z", "category": "DataDrivenTests", "environment": { "benchmarkDotNetVersion": "BenchmarkDotNet v0.15.7, Linux Ubuntu 24.04.3 LTS (Noble Numbat)", @@ -10,42 +10,42 @@ { "Method": "TUnit", "Version": "1.2.3", - "Mean": "507.19 ms", - "Error": "6.037 ms", - "StdDev": "5.647 ms", - "Median": "507.75 ms" + "Mean": "493.26 ms", + "Error": "3.582 ms", + "StdDev": "3.175 ms", + "Median": "492.16 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "537.32 ms", - "Error": "10.114 ms", - "StdDev": "8.966 ms", - "Median": "536.59 ms" + "Mean": "601.85 ms", + "Error": "11.895 ms", + "StdDev": "13.698 ms", + "Median": "601.22 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "504.77 ms", - "Error": "9.670 ms", - "StdDev": "9.497 ms", - "Median": "506.26 ms" + "Mean": "617.89 ms", + "Error": "11.698 ms", + "StdDev": "12.013 ms", + "Median": "622.69 ms" }, { "Method": "xUnit3", "Version": "3.2.0", - "Mean": "586.00 ms", - "Error": "10.336 ms", - "StdDev": "9.163 ms", - "Median": "585.07 ms" + "Mean": "610.52 ms", + "Error": "11.777 ms", + "StdDev": "11.016 ms", + "Median": "609.06 ms" }, { "Method": "TUnit_AOT", "Version": "1.2.3", - "Mean": "24.73 ms", - "Error": "0.152 ms", - "StdDev": "0.142 ms", - "Median": "24.76 ms" + "Mean": "24.33 ms", + "Error": "0.263 ms", + "StdDev": "0.233 ms", + "Median": "24.31 ms" } ] } \ No newline at end of file diff --git a/docs/static/benchmarks/MassiveParallelTests.json b/docs/static/benchmarks/MassiveParallelTests.json index 48a451c377..8519b4b264 100644 --- a/docs/static/benchmarks/MassiveParallelTests.json +++ b/docs/static/benchmarks/MassiveParallelTests.json @@ -1,5 +1,5 @@ { - "timestamp": "2025-11-17T00:29:04.103Z", + "timestamp": "2025-11-18T00:28:14.219Z", "category": "MassiveParallelTests", "environment": { "benchmarkDotNetVersion": "BenchmarkDotNet v0.15.7, Linux Ubuntu 24.04.3 LTS (Noble Numbat)", @@ -10,42 +10,42 @@ { "Method": "TUnit", "Version": "1.2.3", - "Mean": "580.3 ms", - "Error": "3.44 ms", - "StdDev": "3.05 ms", - "Median": "579.7 ms" + "Mean": "595.5 ms", + "Error": "2.81 ms", + "StdDev": "2.63 ms", + "Median": "595.7 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "1,199.9 ms", - "Error": "8.90 ms", - "StdDev": "7.89 ms", - "Median": "1,199.0 ms" + "Mean": "1,164.1 ms", + "Error": "6.78 ms", + "StdDev": "6.01 ms", + "Median": "1,163.4 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "3,002.4 ms", - "Error": "9.36 ms", - "StdDev": "8.30 ms", - "Median": "3,000.7 ms" + "Mean": "2,947.1 ms", + "Error": "6.79 ms", + "StdDev": "6.02 ms", + "Median": "2,948.2 ms" }, { "Method": "xUnit3", "Version": "3.2.0", - "Mean": "3,080.0 ms", - "Error": "8.54 ms", - "StdDev": "7.99 ms", - "Median": "3,078.2 ms" + "Mean": "3,048.5 ms", + "Error": "15.99 ms", + "StdDev": "14.96 ms", + "Median": "3,046.6 ms" }, { "Method": "TUnit_AOT", "Version": "1.2.3", - "Mean": "130.5 ms", - "Error": "0.48 ms", - "StdDev": "0.45 ms", - "Median": "130.4 ms" + "Mean": "130.9 ms", + "Error": "0.44 ms", + "StdDev": "0.41 ms", + "Median": "130.9 ms" } ] } \ No newline at end of file diff --git a/docs/static/benchmarks/MatrixTests.json b/docs/static/benchmarks/MatrixTests.json index 40e8dd9c42..d48c2a3178 100644 --- a/docs/static/benchmarks/MatrixTests.json +++ b/docs/static/benchmarks/MatrixTests.json @@ -1,5 +1,5 @@ { - "timestamp": "2025-11-17T00:29:04.103Z", + "timestamp": "2025-11-18T00:28:14.219Z", "category": "MatrixTests", "environment": { "benchmarkDotNetVersion": "BenchmarkDotNet v0.15.7, Linux Ubuntu 24.04.3 LTS (Noble Numbat)", @@ -10,42 +10,42 @@ { "Method": "TUnit", "Version": "1.2.3", - "Mean": "567.17 ms", - "Error": "5.532 ms", - "StdDev": "5.174 ms", - "Median": "565.30 ms" + "Mean": "561.43 ms", + "Error": "5.322 ms", + "StdDev": "4.718 ms", + "Median": "562.61 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "1,552.19 ms", - "Error": "5.388 ms", - "StdDev": "4.500 ms", - "Median": "1,551.63 ms" + "Mean": "1,556.36 ms", + "Error": "8.816 ms", + "StdDev": "7.815 ms", + "Median": "1,557.95 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "1,523.45 ms", - "Error": "17.681 ms", - "StdDev": "14.764 ms", - "Median": "1,522.39 ms" + "Mean": "1,517.35 ms", + "Error": "13.710 ms", + "StdDev": "12.824 ms", + "Median": "1,517.64 ms" }, { "Method": "xUnit3", "Version": "3.2.0", - "Mean": "1,607.93 ms", - "Error": "17.000 ms", - "StdDev": "15.902 ms", - "Median": "1,606.94 ms" + "Mean": "1,602.60 ms", + "Error": "12.743 ms", + "StdDev": "11.920 ms", + "Median": "1,603.72 ms" }, { "Method": "TUnit_AOT", "Version": "1.2.3", - "Mean": "79.51 ms", - "Error": "0.269 ms", - "StdDev": "0.251 ms", - "Median": "79.48 ms" + "Mean": "79.00 ms", + "Error": "0.238 ms", + "StdDev": "0.223 ms", + "Median": "79.03 ms" } ] } \ No newline at end of file diff --git a/docs/static/benchmarks/ScaleTests.json b/docs/static/benchmarks/ScaleTests.json index 8b93e3c061..a6dfe8b640 100644 --- a/docs/static/benchmarks/ScaleTests.json +++ b/docs/static/benchmarks/ScaleTests.json @@ -1,5 +1,5 @@ { - "timestamp": "2025-11-17T00:29:04.103Z", + "timestamp": "2025-11-18T00:28:14.219Z", "category": "ScaleTests", "environment": { "benchmarkDotNetVersion": "BenchmarkDotNet v0.15.7, Linux Ubuntu 24.04.3 LTS (Noble Numbat)", @@ -10,42 +10,42 @@ { "Method": "TUnit", "Version": "1.2.3", - "Mean": "524.89 ms", - "Error": "5.963 ms", - "StdDev": "5.578 ms", - "Median": "524.09 ms" + "Mean": "541.57 ms", + "Error": "4.341 ms", + "StdDev": "3.848 ms", + "Median": "540.90 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "577.30 ms", - "Error": "7.326 ms", - "StdDev": "6.118 ms", - "Median": "578.87 ms" + "Mean": "591.49 ms", + "Error": "10.469 ms", + "StdDev": "9.280 ms", + "Median": "591.89 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "501.88 ms", - "Error": "9.090 ms", - "StdDev": "13.036 ms", - "Median": "501.83 ms" + "Mean": "512.41 ms", + "Error": "10.194 ms", + "StdDev": "11.331 ms", + "Median": "507.96 ms" }, { "Method": "xUnit3", "Version": "3.2.0", - "Mean": "576.33 ms", - "Error": "10.205 ms", - "StdDev": "9.545 ms", - "Median": "573.62 ms" + "Mean": "596.95 ms", + "Error": "10.457 ms", + "StdDev": "9.270 ms", + "Median": "593.96 ms" }, { "Method": "TUnit_AOT", "Version": "1.2.3", - "Mean": "45.60 ms", - "Error": "1.239 ms", - "StdDev": "3.635 ms", - "Median": "45.71 ms" + "Mean": "43.84 ms", + "Error": "1.157 ms", + "StdDev": "3.376 ms", + "Median": "43.96 ms" } ] } \ No newline at end of file diff --git a/docs/static/benchmarks/SetupTeardownTests.json b/docs/static/benchmarks/SetupTeardownTests.json index ed69b9c0e9..650a5db7b8 100644 --- a/docs/static/benchmarks/SetupTeardownTests.json +++ b/docs/static/benchmarks/SetupTeardownTests.json @@ -1,5 +1,5 @@ { - "timestamp": "2025-11-17T00:29:04.104Z", + "timestamp": "2025-11-18T00:28:14.220Z", "category": "SetupTeardownTests", "environment": { "benchmarkDotNetVersion": "BenchmarkDotNet v0.15.7, Linux Ubuntu 24.04.3 LTS (Noble Numbat)", @@ -10,34 +10,34 @@ { "Method": "TUnit", "Version": "1.2.3", - "Mean": "585.9 ms", - "Error": "3.28 ms", - "StdDev": "3.06 ms", - "Median": "585.4 ms" + "Mean": "578.7 ms", + "Error": "6.61 ms", + "StdDev": "6.18 ms", + "Median": "578.1 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "1,159.2 ms", - "Error": "5.29 ms", - "StdDev": "4.69 ms", - "Median": "1,161.2 ms" + "Mean": "1,182.7 ms", + "Error": "9.46 ms", + "StdDev": "8.39 ms", + "Median": "1,182.8 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "1,139.4 ms", - "Error": "9.75 ms", - "StdDev": "9.12 ms", - "Median": "1,140.1 ms" + "Mean": "1,152.5 ms", + "Error": "7.45 ms", + "StdDev": "6.97 ms", + "Median": "1,151.9 ms" }, { "Method": "xUnit3", "Version": "3.2.0", - "Mean": "1,216.8 ms", - "Error": "6.48 ms", - "StdDev": "5.74 ms", - "Median": "1,216.2 ms" + "Mean": "1,229.3 ms", + "Error": "9.96 ms", + "StdDev": "9.31 ms", + "Median": "1,225.5 ms" }, { "Method": "TUnit_AOT", diff --git a/docs/static/benchmarks/historical.json b/docs/static/benchmarks/historical.json index d7b44bbe49..26988cc598 100644 --- a/docs/static/benchmarks/historical.json +++ b/docs/static/benchmarks/historical.json @@ -50,5 +50,9 @@ { "date": "2025-11-17", "environment": "Ubuntu" + }, + { + "date": "2025-11-18", + "environment": "Ubuntu" } ] \ No newline at end of file diff --git a/docs/static/benchmarks/latest.json b/docs/static/benchmarks/latest.json index 51a9a35af6..55ec1a5025 100644 --- a/docs/static/benchmarks/latest.json +++ b/docs/static/benchmarks/latest.json @@ -1,5 +1,5 @@ { - "timestamp": "2025-11-17T00:29:04.104Z", + "timestamp": "2025-11-18T00:28:14.221Z", "environment": { "benchmarkDotNetVersion": "BenchmarkDotNet v0.15.7, Linux Ubuntu 24.04.3 LTS (Noble Numbat)", "sdk": ".NET SDK 10.0.100", @@ -10,244 +10,244 @@ { "Method": "TUnit", "Version": "1.2.3", - "Mean": "572.1 ms", - "Error": "2.41 ms", - "StdDev": "2.01 ms", - "Median": "571.8 ms" + "Mean": "556.5 ms", + "Error": "2.29 ms", + "StdDev": "2.03 ms", + "Median": "556.3 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "696.5 ms", - "Error": "7.35 ms", - "StdDev": "6.14 ms", - "Median": "695.9 ms" + "Mean": "663.3 ms", + "Error": "10.81 ms", + "StdDev": "9.02 ms", + "Median": "662.2 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "666.7 ms", - "Error": "6.92 ms", - "StdDev": "5.78 ms", - "Median": "666.2 ms" + "Mean": "635.0 ms", + "Error": "12.02 ms", + "StdDev": "11.80 ms", + "Median": "632.0 ms" }, { "Method": "xUnit3", "Version": "3.2.0", - "Mean": "742.0 ms", - "Error": "4.82 ms", - "StdDev": "4.27 ms", - "Median": "742.6 ms" + "Mean": "715.8 ms", + "Error": "9.34 ms", + "StdDev": "8.28 ms", + "Median": "716.7 ms" }, { "Method": "TUnit_AOT", "Version": "1.2.3", - "Mean": "124.2 ms", - "Error": "0.49 ms", - "StdDev": "0.46 ms", - "Median": "124.1 ms" + "Mean": "124.5 ms", + "Error": "0.31 ms", + "StdDev": "0.26 ms", + "Median": "124.5 ms" } ], "DataDrivenTests": [ { "Method": "TUnit", "Version": "1.2.3", - "Mean": "507.19 ms", - "Error": "6.037 ms", - "StdDev": "5.647 ms", - "Median": "507.75 ms" + "Mean": "493.26 ms", + "Error": "3.582 ms", + "StdDev": "3.175 ms", + "Median": "492.16 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "537.32 ms", - "Error": "10.114 ms", - "StdDev": "8.966 ms", - "Median": "536.59 ms" + "Mean": "601.85 ms", + "Error": "11.895 ms", + "StdDev": "13.698 ms", + "Median": "601.22 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "504.77 ms", - "Error": "9.670 ms", - "StdDev": "9.497 ms", - "Median": "506.26 ms" + "Mean": "617.89 ms", + "Error": "11.698 ms", + "StdDev": "12.013 ms", + "Median": "622.69 ms" }, { "Method": "xUnit3", "Version": "3.2.0", - "Mean": "586.00 ms", - "Error": "10.336 ms", - "StdDev": "9.163 ms", - "Median": "585.07 ms" + "Mean": "610.52 ms", + "Error": "11.777 ms", + "StdDev": "11.016 ms", + "Median": "609.06 ms" }, { "Method": "TUnit_AOT", "Version": "1.2.3", - "Mean": "24.73 ms", - "Error": "0.152 ms", - "StdDev": "0.142 ms", - "Median": "24.76 ms" + "Mean": "24.33 ms", + "Error": "0.263 ms", + "StdDev": "0.233 ms", + "Median": "24.31 ms" } ], "MassiveParallelTests": [ { "Method": "TUnit", "Version": "1.2.3", - "Mean": "580.3 ms", - "Error": "3.44 ms", - "StdDev": "3.05 ms", - "Median": "579.7 ms" + "Mean": "595.5 ms", + "Error": "2.81 ms", + "StdDev": "2.63 ms", + "Median": "595.7 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "1,199.9 ms", - "Error": "8.90 ms", - "StdDev": "7.89 ms", - "Median": "1,199.0 ms" + "Mean": "1,164.1 ms", + "Error": "6.78 ms", + "StdDev": "6.01 ms", + "Median": "1,163.4 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "3,002.4 ms", - "Error": "9.36 ms", - "StdDev": "8.30 ms", - "Median": "3,000.7 ms" + "Mean": "2,947.1 ms", + "Error": "6.79 ms", + "StdDev": "6.02 ms", + "Median": "2,948.2 ms" }, { "Method": "xUnit3", "Version": "3.2.0", - "Mean": "3,080.0 ms", - "Error": "8.54 ms", - "StdDev": "7.99 ms", - "Median": "3,078.2 ms" + "Mean": "3,048.5 ms", + "Error": "15.99 ms", + "StdDev": "14.96 ms", + "Median": "3,046.6 ms" }, { "Method": "TUnit_AOT", "Version": "1.2.3", - "Mean": "130.5 ms", - "Error": "0.48 ms", - "StdDev": "0.45 ms", - "Median": "130.4 ms" + "Mean": "130.9 ms", + "Error": "0.44 ms", + "StdDev": "0.41 ms", + "Median": "130.9 ms" } ], "MatrixTests": [ { "Method": "TUnit", "Version": "1.2.3", - "Mean": "567.17 ms", - "Error": "5.532 ms", - "StdDev": "5.174 ms", - "Median": "565.30 ms" + "Mean": "561.43 ms", + "Error": "5.322 ms", + "StdDev": "4.718 ms", + "Median": "562.61 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "1,552.19 ms", - "Error": "5.388 ms", - "StdDev": "4.500 ms", - "Median": "1,551.63 ms" + "Mean": "1,556.36 ms", + "Error": "8.816 ms", + "StdDev": "7.815 ms", + "Median": "1,557.95 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "1,523.45 ms", - "Error": "17.681 ms", - "StdDev": "14.764 ms", - "Median": "1,522.39 ms" + "Mean": "1,517.35 ms", + "Error": "13.710 ms", + "StdDev": "12.824 ms", + "Median": "1,517.64 ms" }, { "Method": "xUnit3", "Version": "3.2.0", - "Mean": "1,607.93 ms", - "Error": "17.000 ms", - "StdDev": "15.902 ms", - "Median": "1,606.94 ms" + "Mean": "1,602.60 ms", + "Error": "12.743 ms", + "StdDev": "11.920 ms", + "Median": "1,603.72 ms" }, { "Method": "TUnit_AOT", "Version": "1.2.3", - "Mean": "79.51 ms", - "Error": "0.269 ms", - "StdDev": "0.251 ms", - "Median": "79.48 ms" + "Mean": "79.00 ms", + "Error": "0.238 ms", + "StdDev": "0.223 ms", + "Median": "79.03 ms" } ], "ScaleTests": [ { "Method": "TUnit", "Version": "1.2.3", - "Mean": "524.89 ms", - "Error": "5.963 ms", - "StdDev": "5.578 ms", - "Median": "524.09 ms" + "Mean": "541.57 ms", + "Error": "4.341 ms", + "StdDev": "3.848 ms", + "Median": "540.90 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "577.30 ms", - "Error": "7.326 ms", - "StdDev": "6.118 ms", - "Median": "578.87 ms" + "Mean": "591.49 ms", + "Error": "10.469 ms", + "StdDev": "9.280 ms", + "Median": "591.89 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "501.88 ms", - "Error": "9.090 ms", - "StdDev": "13.036 ms", - "Median": "501.83 ms" + "Mean": "512.41 ms", + "Error": "10.194 ms", + "StdDev": "11.331 ms", + "Median": "507.96 ms" }, { "Method": "xUnit3", "Version": "3.2.0", - "Mean": "576.33 ms", - "Error": "10.205 ms", - "StdDev": "9.545 ms", - "Median": "573.62 ms" + "Mean": "596.95 ms", + "Error": "10.457 ms", + "StdDev": "9.270 ms", + "Median": "593.96 ms" }, { "Method": "TUnit_AOT", "Version": "1.2.3", - "Mean": "45.60 ms", - "Error": "1.239 ms", - "StdDev": "3.635 ms", - "Median": "45.71 ms" + "Mean": "43.84 ms", + "Error": "1.157 ms", + "StdDev": "3.376 ms", + "Median": "43.96 ms" } ], "SetupTeardownTests": [ { "Method": "TUnit", "Version": "1.2.3", - "Mean": "585.9 ms", - "Error": "3.28 ms", - "StdDev": "3.06 ms", - "Median": "585.4 ms" + "Mean": "578.7 ms", + "Error": "6.61 ms", + "StdDev": "6.18 ms", + "Median": "578.1 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "1,159.2 ms", - "Error": "5.29 ms", - "StdDev": "4.69 ms", - "Median": "1,161.2 ms" + "Mean": "1,182.7 ms", + "Error": "9.46 ms", + "StdDev": "8.39 ms", + "Median": "1,182.8 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "1,139.4 ms", - "Error": "9.75 ms", - "StdDev": "9.12 ms", - "Median": "1,140.1 ms" + "Mean": "1,152.5 ms", + "Error": "7.45 ms", + "StdDev": "6.97 ms", + "Median": "1,151.9 ms" }, { "Method": "xUnit3", "Version": "3.2.0", - "Mean": "1,216.8 ms", - "Error": "6.48 ms", - "StdDev": "5.74 ms", - "Median": "1,216.2 ms" + "Mean": "1,229.3 ms", + "Error": "9.96 ms", + "StdDev": "9.31 ms", + "Median": "1,225.5 ms" }, { "Method": "TUnit_AOT", @@ -264,34 +264,34 @@ { "Method": "Build_TUnit", "Version": "1.2.3", - "Mean": "2.046 s", - "Error": "0.0272 s", - "StdDev": "0.0255 s", - "Median": "2.041 s" + "Mean": "1.947 s", + "Error": "0.0371 s", + "StdDev": "0.0381 s", + "Median": "1.953 s" }, { "Method": "Build_NUnit", "Version": "4.4.0", - "Mean": "1.657 s", - "Error": "0.0306 s", - "StdDev": "0.0286 s", - "Median": "1.655 s" + "Mean": "1.556 s", + "Error": "0.0183 s", + "StdDev": "0.0171 s", + "Median": "1.557 s" }, { "Method": "Build_MSTest", "Version": "4.0.2", - "Mean": "1.743 s", - "Error": "0.0239 s", - "StdDev": "0.0212 s", - "Median": "1.744 s" + "Mean": "1.621 s", + "Error": "0.0167 s", + "StdDev": "0.0156 s", + "Median": "1.624 s" }, { "Method": "Build_xUnit3", "Version": "3.2.0", - "Mean": "1.640 s", - "Error": "0.0162 s", - "StdDev": "0.0151 s", - "Median": "1.638 s" + "Mean": "1.523 s", + "Error": "0.0173 s", + "StdDev": "0.0162 s", + "Median": "1.527 s" } ] }, @@ -299,6 +299,6 @@ "runtimeCategories": 6, "buildCategories": 1, "totalBenchmarks": 7, - "lastUpdated": "2025-11-17T00:29:04.101Z" + "lastUpdated": "2025-11-18T00:28:14.217Z" } } \ No newline at end of file diff --git a/docs/static/benchmarks/summary.json b/docs/static/benchmarks/summary.json index 465cf055e5..efa7dfbc9c 100644 --- a/docs/static/benchmarks/summary.json +++ b/docs/static/benchmarks/summary.json @@ -10,6 +10,6 @@ "build": [ "BuildTime" ], - "timestamp": "2025-11-17", + "timestamp": "2025-11-18", "environment": "Ubuntu Latest • .NET SDK 10.0.100" } \ No newline at end of file From 433bd47cffe232a37f5a15c85adb7668e67811f5 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Tue, 18 Nov 2025 06:01:08 +0000 Subject: [PATCH 4/8] chore(deps): update dependency polyfill to 9.1.0 (#3866) Co-authored-by: Renovate Bot --- TUnit.Core/TUnit.Core.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TUnit.Core/TUnit.Core.targets b/TUnit.Core/TUnit.Core.targets index ae21f778a1..3e87a68017 100644 --- a/TUnit.Core/TUnit.Core.targets +++ b/TUnit.Core/TUnit.Core.targets @@ -8,7 +8,7 @@ - <_TUnitPolyfillVersion>9.0.3 + <_TUnitPolyfillVersion>9.1.0 <_TUnitNeedsPolyfill Condition="'$(TargetFramework)' == 'netstandard2.0' or '$(TargetFramework)' == 'netstandard2.1' or '$(TargetFrameworkIdentifier)' == '.NETFramework'">true From f44499da90a91be2790ff55ed4ba108fd89d6c0e Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Tue, 18 Nov 2025 06:02:23 +0000 Subject: [PATCH 5/8] chore(deps): update dependency polyfill to 9.1.0 (#3865) Co-authored-by: Renovate Bot --- Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 62e3564e91..cc33abe666 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -59,7 +59,7 @@ - + From a29812b25da76920bb25423e3ea614a1ffcd8ba4 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Tue, 18 Nov 2025 21:44:18 +0000 Subject: [PATCH 6/8] chore: update documentation for dependency injection in TestContext --- docs/docs/test-lifecycle/test-context.md | 163 +---------------------- 1 file changed, 3 insertions(+), 160 deletions(-) diff --git a/docs/docs/test-lifecycle/test-context.md b/docs/docs/test-lifecycle/test-context.md index 6884f43284..5292cce789 100644 --- a/docs/docs/test-lifecycle/test-context.md +++ b/docs/docs/test-lifecycle/test-context.md @@ -28,168 +28,11 @@ if (TestContext.Current?.Result?.State == TestState.Failed) } ``` -## Service Provider Integration +## Dependency Injection -`TestContext` provides access to dependency injection services through the `GetService()` and `GetRequiredService()` methods. This allows you to access registered services within your tests, hooks, and custom extensions. +**Note**: `TestContext` does NOT provide direct access to dependency injection services. The internal service provider in `TestContext` is exclusively for TUnit framework services and is not meant for user-provided dependencies. -### Accessing Services - -```csharp -[Test] -public async Task DatabaseTest() -{ - // Get an optional service (returns null if not registered) - var logger = TestContext.Current?.GetService>(); - logger?.LogInformation("Starting database test"); - - // Get a required service (throws if not registered) - var dbContext = TestContext.Current!.GetRequiredService(); - - // Use the service - var users = await dbContext.Users.ToListAsync(); - await Assert.That(users).IsNotEmpty(); -} -``` - -### Common Use Cases - -#### 1. Accessing Loggers - -```csharp -[Before(HookType.Test)] -public void LogTestStart() -{ - var logger = TestContext.Current?.GetService(); - logger?.LogInformation("Test {TestName} starting", - TestContext.Current?.Metadata.TestName); -} -``` - -#### 2. Working with Scoped Services - -```csharp -[Test] -public async Task ScopedServiceTest() -{ - // Each test gets its own scope, so scoped services are isolated - var service1 = TestContext.Current!.GetRequiredService(); - var service2 = TestContext.Current!.GetRequiredService(); - - // These will be the same instance within the test - await Assert.That(ReferenceEquals(service1, service2)).IsTrue(); -} -``` - -#### 3. Configuration Access - -```csharp -[Test] -public async Task ConfigurationTest() -{ - var configuration = TestContext.Current?.GetService(); - var apiKey = configuration?["ApiSettings:Key"]; - - await Assert.That(apiKey).IsNotNull(); -} -``` - -### Service Provider in Custom Extensions - -When implementing custom test executors or hook executors, you can use the service provider: - -```csharp -public class DatabaseTransactionExecutor : ITestExecutor -{ - public async Task ExecuteAsync(TestContext context, Func testBody) - { - // Get database context from DI - var dbContext = context.GetRequiredService(); - - using var transaction = await dbContext.Database.BeginTransactionAsync(); - - try - { - await testBody(); - await transaction.RollbackAsync(); // Keep tests isolated - } - catch - { - await transaction.RollbackAsync(); - throw; - } - } -} -``` - -### Integration with Test Lifecycle - -The service provider is available throughout the test lifecycle: - -```csharp -public class ServiceIntegrationTests -{ - [Before(HookType.Class)] - public static async Task ClassSetup() - { - // Services available in class-level hooks via the hook context - var context = ClassHookContext.Current; - var cache = context?.GetService(); - cache?.Set("test-data", await LoadTestData()); - } - - [Before(HookType.Test)] - public async Task TestSetup() - { - // Services available in test-level hooks - var cache = TestContext.Current?.GetService(); - var testData = cache?.Get("test-data"); - } - - [Test] - public async Task ActualTest() - { - // Services available in test methods - var service = TestContext.Current!.GetRequiredService(); - var result = await service.PerformOperation(); - await Assert.That(result).IsNotNull(); - } -} -``` - -### Best Practices - -1. **Use GetRequiredService for Essential Services** - ```csharp - // Good - Fails fast if service is missing - var critical = TestContext.Current!.GetRequiredService(); - - // Less ideal - Might hide configuration issues - var critical = TestContext.Current?.GetService() - ?? throw new InvalidOperationException("Service not found"); - ``` - -2. **Null Check When Using GetService** - ```csharp - var optional = TestContext.Current?.GetService(); - if (optional != null) - { - await optional.DoSomething(); - } - ``` - -3. **Consider Service Lifetime** - ```csharp - // Singleton services persist across tests - var singleton = TestContext.Current?.GetService(); - - // Scoped services are unique per test - var scoped = TestContext.Current?.GetService(); - - // Transient services are created each time - var transient1 = TestContext.Current?.GetService(); - var transient2 = TestContext.Current?.GetService(); - // transient1 and transient2 are different instances - ``` +If you need dependency injection in your tests, use the `DependencyInjectionDataSourceAttribute` helper class to set up your own DI container. See the [Dependency Injection guide](./dependency-injection.md) for complete details and examples. ## TestBuilderContext From f79dbfffd6f041077dd3fb8ce337b1791caae3db Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Tue, 18 Nov 2025 23:07:30 +0000 Subject: [PATCH 7/8] test: add regression tests for ClassDataSource with SharedType.None (#3868) --- TUnit.Engine/Building/TestBuilder.cs | 3 +- ...lassDataSourceSharedNoneRegressionTests.cs | 156 ++++++++++++++++++ 2 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 TUnit.TestProject/ClassDataSourceSharedNoneRegressionTests.cs diff --git a/TUnit.Engine/Building/TestBuilder.cs b/TUnit.Engine/Building/TestBuilder.cs index ada56f7737..a854f4e2df 100644 --- a/TUnit.Engine/Building/TestBuilder.cs +++ b/TUnit.Engine/Building/TestBuilder.cs @@ -206,7 +206,8 @@ public async Task> BuildTestsFromMetadataAsy hasAnyClassData = true; classDataLoopIndex++; - var classData = DataUnwrapper.Unwrap(await classDataFactory() ?? []); + var classDataResult = await classDataFactory() ?? []; + var classData = DataUnwrapper.Unwrap(classDataResult); var needsInstanceForMethodDataSources = metadata.DataSources.Any(ds => ds is IAccessesInstanceData); diff --git a/TUnit.TestProject/ClassDataSourceSharedNoneRegressionTests.cs b/TUnit.TestProject/ClassDataSourceSharedNoneRegressionTests.cs new file mode 100644 index 0000000000..15df0b5c36 --- /dev/null +++ b/TUnit.TestProject/ClassDataSourceSharedNoneRegressionTests.cs @@ -0,0 +1,156 @@ +using TUnit.Core.Interfaces; +using TUnit.TestProject.Attributes; + +namespace TUnit.TestProject; + +/// +/// Regression tests for GitHub Issue #3855 +/// Verifies that ClassDataSource with SharedType.None constructs and initializes objects only once per test. +/// +[EngineTest(ExpectedResult.Pass)] +public class ClassDataSourceSharedNoneRegressionTests +{ + // Test helper class that tracks construction + public class ConstructionCounterClass + { + private static int _instanceCounter = 0; + public int InstanceNumber { get; } + + public ConstructionCounterClass() + { + InstanceNumber = Interlocked.Increment(ref _instanceCounter); + } + + public int Value { get; set; } = 42; + } + + // Test helper class that tracks initialization + public class AsyncInitializerCounterClass : IAsyncInitializer + { + private static int _initCounter = 0; + public int InitNumber { get; private set; } = -1; + + public Task InitializeAsync() + { + InitNumber = Interlocked.Increment(ref _initCounter); + return Task.CompletedTask; + } + + public int Value { get; set; } = 99; + } + + /// + /// Test that SharedType.None constructs the class data source exactly once per test. + /// Regression test for issue #3855 where it was being constructed twice. + /// Before the fix, InstanceNumber would be 2 (constructed twice). + /// After the fix, InstanceNumber should be 1 (constructed once). + /// + [Test] + [ClassDataSource(Shared = SharedType.None)] + public async Task SharedTypeNone_ConstructsOnlyOnce(ConstructionCounterClass instance) + { + // Instance should be constructed exactly once, so InstanceNumber should be 1 + // (not 2, which would indicate double construction) + await Assert.That(instance).IsNotNull(); + await Assert.That(instance.InstanceNumber).IsEqualTo(1); + await Assert.That(instance.Value).IsEqualTo(42); + } + + /// + /// Test that SharedType.None with IAsyncInitializer initializes exactly once per test. + /// Regression test for issue #3855 where InitializeAsync was being called twice. + /// Before the fix, InitNumber would be 2 (initialized twice). + /// After the fix, InitNumber should be 1 (initialized once). + /// + [Test] + [ClassDataSource(Shared = SharedType.None)] + public async Task SharedTypeNone_WithAsyncInitializer_InitializesOnlyOnce(AsyncInitializerCounterClass instance) + { + // Instance should be initialized exactly once, so InitNumber should be 1 + // (not 2, which would indicate double initialization) + await Assert.That(instance).IsNotNull(); + await Assert.That(instance.InitNumber).IsEqualTo(1); + await Assert.That(instance.Value).IsEqualTo(99); + } +} + +/// +/// Regression tests for GitHub Issue #3855 - Verify no instance leakage across tests. +/// Tests that multiple test methods in the same class each receive their own unique instance +/// with SharedType.None (no sharing or leakage across tests). +/// +public class ClassDataSourceSharedNoneNoLeakageTests +{ + // Test helper class that tracks instance IDs and mutation + public class UniqueInstanceClass + { + private static int _nextId = 0; + public int InstanceId { get; } + public int MutationValue { get; set; } + + public UniqueInstanceClass() + { + InstanceId = Interlocked.Increment(ref _nextId); + MutationValue = 0; // Initial value + } + + public static void ResetIdCounter() + { + _nextId = 0; + } + } + + [Before(TestSession)] + public static void ResetCounters() + { + UniqueInstanceClass.ResetIdCounter(); + } + + /// + /// First test - should get instance ID 1, mutate it, and not affect other tests + /// + [Test] + [ClassDataSource(Shared = SharedType.None)] + public async Task Test1_GetsUniqueInstance(UniqueInstanceClass instance) + { + // This is the first test, should have InstanceId = 1 + await Assert.That(instance.InstanceId).IsEqualTo(1); + await Assert.That(instance.MutationValue).IsEqualTo(0); + + // Mutate the instance + instance.MutationValue = 100; + } + + /// + /// Second test - should get instance ID 2, with fresh MutationValue = 0 + /// (proves no leakage from Test1) + /// + [Test] + [ClassDataSource(Shared = SharedType.None)] + public async Task Test2_GetsUniqueInstance(UniqueInstanceClass instance) + { + // This is the second test, should have InstanceId = 2 + await Assert.That(instance.InstanceId).IsEqualTo(2); + + // MutationValue should be 0 (not affected by Test1's mutation to 100) + await Assert.That(instance.MutationValue).IsEqualTo(0); + + // Mutate the instance + instance.MutationValue = 200; + } + + /// + /// Third test - should get instance ID 3, with fresh MutationValue = 0 + /// (proves no leakage from Test1 or Test2) + /// + [Test] + [ClassDataSource(Shared = SharedType.None)] + public async Task Test3_GetsUniqueInstance(UniqueInstanceClass instance) + { + // This is the third test, should have InstanceId = 3 + await Assert.That(instance.InstanceId).IsEqualTo(3); + + // MutationValue should be 0 (not affected by Test1 or Test2) + await Assert.That(instance.MutationValue).IsEqualTo(0); + } +} From aea950635b2b9a7f78e66393526ca13d13d0d469 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Tue, 18 Nov 2025 23:15:58 +0000 Subject: [PATCH 8/8] Unmark AOT safe IsEquivalentTo overload (#3854) * feat: enhance AOT compatibility by adding RequiresUnreferencedCode attribute to assertion classes and introducing tests for custom comparer scenarios * feat: update AOT compatibility messages and remove virtual modifiers from GetErrorOutput and GetStandardOutput methods * feat: update GetErrorOutput and GetStandardOutput methods to virtual and add TestBuildContext class * feat: refactor collection assertion classes to expose Comparer property and streamline comparer handling --- .../Generators/AssertionExtensionGenerator.cs | 22 ++++++++++++-- .../CollectionComparerBasedAssertion.cs | 15 ++-------- .../Conditions/IsEquivalentToAssertion.cs | 11 +++---- .../Conditions/NotEquivalentToAssertion.cs | 15 ++++++++-- ...Has_No_API_Changes.DotNet10_0.verified.txt | 17 +++++------ ..._Has_No_API_Changes.DotNet8_0.verified.txt | 17 +++++------ ..._Has_No_API_Changes.DotNet9_0.verified.txt | 17 +++++------ ...ary_Has_No_API_Changes.Net4_7.verified.txt | 5 +++- .../AotCompatibilityTests.cs | 30 +++++++++++++++++++ 9 files changed, 96 insertions(+), 53 deletions(-) diff --git a/TUnit.Assertions.SourceGenerator/Generators/AssertionExtensionGenerator.cs b/TUnit.Assertions.SourceGenerator/Generators/AssertionExtensionGenerator.cs index 8cb0b1debb..1621a2f664 100644 --- a/TUnit.Assertions.SourceGenerator/Generators/AssertionExtensionGenerator.cs +++ b/TUnit.Assertions.SourceGenerator/Generators/AssertionExtensionGenerator.cs @@ -226,6 +226,22 @@ private static void GenerateExtensionMethod( // Skip the first parameter (AssertionContext) var additionalParams = constructor.Parameters.Skip(1).ToArray(); + // Check for RequiresUnreferencedCode attribute on the constructor first, then fall back to class-level + var constructorRequiresUnreferencedCodeAttr = constructor.GetAttributes() + .FirstOrDefault(attr => attr.AttributeClass?.Name == "RequiresUnreferencedCodeAttribute"); + + string? requiresUnreferencedCodeMessage = null; + if (constructorRequiresUnreferencedCodeAttr != null && constructorRequiresUnreferencedCodeAttr.ConstructorArguments.Length > 0) + { + // Constructor-level attribute takes precedence + requiresUnreferencedCodeMessage = constructorRequiresUnreferencedCodeAttr.ConstructorArguments[0].Value?.ToString(); + } + else if (!string.IsNullOrEmpty(data.RequiresUnreferencedCodeMessage)) + { + // Fall back to class-level attribute + requiresUnreferencedCodeMessage = data.RequiresUnreferencedCodeMessage; + } + // Build generic type parameters string // Use the assertion class's own type parameters if it has them var genericParams = new List(); @@ -315,10 +331,10 @@ private static void GenerateExtensionMethod( sourceBuilder.AppendLine($" /// Extension method for {assertionType.Name}."); sourceBuilder.AppendLine(" /// "); - // Add RequiresUnreferencedCode attribute if present - if (!string.IsNullOrEmpty(data.RequiresUnreferencedCodeMessage)) + // Add RequiresUnreferencedCode attribute if present (from constructor or class level) + if (!string.IsNullOrEmpty(requiresUnreferencedCodeMessage)) { - var escapedMessage = data.RequiresUnreferencedCodeMessage!.Replace("\"", "\\\""); + var escapedMessage = requiresUnreferencedCodeMessage!.Replace("\"", "\\\""); sourceBuilder.AppendLine($" [global::System.Diagnostics.CodeAnalysis.RequiresUnreferencedCode(\"{escapedMessage}\")]"); } diff --git a/TUnit.Assertions/Conditions/CollectionComparerBasedAssertion.cs b/TUnit.Assertions/Conditions/CollectionComparerBasedAssertion.cs index 5a2f78fec1..cd168e5bba 100644 --- a/TUnit.Assertions/Conditions/CollectionComparerBasedAssertion.cs +++ b/TUnit.Assertions/Conditions/CollectionComparerBasedAssertion.cs @@ -1,4 +1,3 @@ -using System.Collections; using TUnit.Assertions.Core; using TUnit.Assertions.Sources; @@ -13,7 +12,7 @@ namespace TUnit.Assertions.Conditions; public abstract class CollectionComparerBasedAssertion : CollectionAssertionBase where TCollection : IEnumerable { - private IEqualityComparer? _comparer; + protected IEqualityComparer? Comparer; protected CollectionComparerBasedAssertion(AssertionContext context) : base(context) @@ -26,7 +25,7 @@ protected CollectionComparerBasedAssertion(AssertionContext context /// protected void SetComparer(IEqualityComparer comparer) { - _comparer = comparer; + Comparer = comparer; Context.ExpressionBuilder.Append($".Using({comparer.GetType().Name})"); } @@ -36,14 +35,6 @@ protected void SetComparer(IEqualityComparer comparer) /// protected IEqualityComparer GetComparer() { - return _comparer ?? EqualityComparer.Default; - } - - /// - /// Checks if a custom comparer has been specified. - /// - protected bool HasCustomComparer() - { - return _comparer != null; + return Comparer ?? EqualityComparer.Default; } } diff --git a/TUnit.Assertions/Conditions/IsEquivalentToAssertion.cs b/TUnit.Assertions/Conditions/IsEquivalentToAssertion.cs index 6eaf69bf8b..7592c54f0e 100644 --- a/TUnit.Assertions/Conditions/IsEquivalentToAssertion.cs +++ b/TUnit.Assertions/Conditions/IsEquivalentToAssertion.cs @@ -14,21 +14,19 @@ namespace TUnit.Assertions.Conditions; /// Inherits from CollectionComparerBasedAssertion to preserve collection type awareness in And/Or chains. /// [AssertionExtension("IsEquivalentTo")] -[RequiresUnreferencedCode("Collection equivalency uses structural comparison for complex objects, which requires reflection and is not compatible with AOT")] public class IsEquivalentToAssertion : CollectionComparerBasedAssertion where TCollection : IEnumerable { private readonly IEnumerable _expected; private readonly CollectionOrdering _ordering; + [RequiresUnreferencedCode("Collection equivalency uses structural comparison for complex objects, which requires reflection and is not compatible with AOT")] public IsEquivalentToAssertion( AssertionContext context, IEnumerable expected, CollectionOrdering ordering = CollectionOrdering.Any) - : base(context) + : this(context, expected, StructuralEqualityComparer.Instance, ordering) { - _expected = expected ?? throw new ArgumentNullException(nameof(expected)); - _ordering = ordering; } public IsEquivalentToAssertion( @@ -40,7 +38,7 @@ public IsEquivalentToAssertion( { _expected = expected ?? throw new ArgumentNullException(nameof(expected)); _ordering = ordering; - SetComparer(comparer); + Comparer = comparer; } public IsEquivalentToAssertion Using(IEqualityComparer comparer) @@ -49,7 +47,6 @@ public IsEquivalentToAssertion Using(IEqualityComparer CheckAsync(EvaluationMetadata metadata) { var value = metadata.Value; @@ -60,7 +57,7 @@ protected override Task CheckAsync(EvaluationMetadata.Instance; + var comparer = GetComparer(); var result = CollectionEquivalencyChecker.AreEquivalent( value, diff --git a/TUnit.Assertions/Conditions/NotEquivalentToAssertion.cs b/TUnit.Assertions/Conditions/NotEquivalentToAssertion.cs index 5fbb18d0d7..e32548337f 100644 --- a/TUnit.Assertions/Conditions/NotEquivalentToAssertion.cs +++ b/TUnit.Assertions/Conditions/NotEquivalentToAssertion.cs @@ -13,21 +13,31 @@ namespace TUnit.Assertions.Conditions; /// Inherits from CollectionComparerBasedAssertion to preserve collection type awareness in And/Or chains. /// [AssertionExtension("IsNotEquivalentTo")] -[RequiresUnreferencedCode("Collection equivalency uses structural comparison for complex objects, which requires reflection and is not compatible with AOT")] public class NotEquivalentToAssertion : CollectionComparerBasedAssertion where TCollection : IEnumerable { private readonly IEnumerable _notExpected; private readonly CollectionOrdering _ordering; + [RequiresUnreferencedCode("Collection equivalency uses structural comparison for complex objects, which requires reflection and is not compatible with AOT")] public NotEquivalentToAssertion( AssertionContext context, IEnumerable notExpected, CollectionOrdering ordering = CollectionOrdering.Any) + : this(context, notExpected, StructuralEqualityComparer.Instance, ordering) + { + } + + public NotEquivalentToAssertion( + AssertionContext context, + IEnumerable notExpected, + IEqualityComparer comparer, + CollectionOrdering ordering = CollectionOrdering.Any) : base(context) { _notExpected = notExpected ?? throw new ArgumentNullException(nameof(notExpected)); _ordering = ordering; + Comparer = comparer; } public NotEquivalentToAssertion Using(IEqualityComparer comparer) @@ -36,7 +46,6 @@ public NotEquivalentToAssertion Using(IEqualityComparer CheckAsync(EvaluationMetadata metadata) { var value = metadata.Value; @@ -47,7 +56,7 @@ protected override Task CheckAsync(EvaluationMetadata.Instance; + var comparer = GetComparer(); var result = CollectionEquivalencyChecker.AreEquivalent( value, diff --git a/TUnit.PublicAPI/Tests.Assertions_Library_Has_No_API_Changes.DotNet10_0.verified.txt b/TUnit.PublicAPI/Tests.Assertions_Library_Has_No_API_Changes.DotNet10_0.verified.txt index 35bcf00a2e..9714184f79 100644 --- a/TUnit.PublicAPI/Tests.Assertions_Library_Has_No_API_Changes.DotNet10_0.verified.txt +++ b/TUnit.PublicAPI/Tests.Assertions_Library_Has_No_API_Changes.DotNet10_0.verified.txt @@ -388,9 +388,9 @@ namespace .Conditions public abstract class CollectionComparerBasedAssertion : . where TCollection : . { + protected .? Comparer; protected CollectionComparerBasedAssertion(. context) { } protected . GetComparer() { } - protected bool HasCustomComparer() { } protected void SetComparer(. comparer) { } } [.("Contains")] @@ -817,15 +817,14 @@ namespace .Conditions protected override string GetExpectation() { } public . Using(. comparer) { } } - [.("Collection equivalency uses structural comparison for complex objects, which requ" + - "ires reflection and is not compatible with AOT")] [.("IsEquivalentTo")] public class IsEquivalentToAssertion : . where TCollection : . { + [.("Collection equivalency uses structural comparison for complex objects, which requ" + + "ires reflection and is not compatible with AOT")] public IsEquivalentToAssertion(. context, . expected, . ordering = 0) { } public IsEquivalentToAssertion(. context, . expected, . comparer, . ordering = 0) { } - [.("AOT", "IL3050", Justification="Collection equivalency uses structural comparison which requires reflection")] protected override .<.> CheckAsync(. metadata) { } protected override string GetExpectation() { } public . Using(. comparer) { } @@ -919,14 +918,14 @@ namespace .Conditions public . IgnoringType( type) { } public . IgnoringType() { } } - [.("Collection equivalency uses structural comparison for complex objects, which requ" + - "ires reflection and is not compatible with AOT")] [.("IsNotEquivalentTo")] public class NotEquivalentToAssertion : . where TCollection : . { + [.("Collection equivalency uses structural comparison for complex objects, which requ" + + "ires reflection and is not compatible with AOT")] public NotEquivalentToAssertion(. context, . notExpected, . ordering = 0) { } - [.("AOT", "IL3050", Justification="Collection equivalency uses structural comparison which requires reflection")] + public NotEquivalentToAssertion(. context, . notExpected, . comparer, . ordering = 0) { } protected override .<.> CheckAsync(. metadata) { } protected override string GetExpectation() { } public . Using(. comparer) { } @@ -3085,8 +3084,6 @@ namespace .Extensions "ires reflection and is not compatible with AOT")] public static . IsEquivalentTo(this . source, . expected, . ordering = 0, [.("expected")] string? expectedExpression = null, [.("ordering")] string? orderingExpression = null) where TCollection : . { } - [.("Collection equivalency uses structural comparison for complex objects, which requ" + - "ires reflection and is not compatible with AOT")] public static . IsEquivalentTo(this . source, . expected, . comparer, . ordering = 0, [.("expected")] string? expectedExpression = null, [.("comparer")] string? comparerExpression = null, [.("ordering")] string? orderingExpression = null) where TCollection : . { } } @@ -3184,6 +3181,8 @@ namespace .Extensions "ires reflection and is not compatible with AOT")] public static . IsNotEquivalentTo(this . source, . notExpected, . ordering = 0, [.("notExpected")] string? notExpectedExpression = null, [.("ordering")] string? orderingExpression = null) where TCollection : . { } + public static . IsNotEquivalentTo(this . source, . notExpected, . comparer, . ordering = 0, [.("notExpected")] string? notExpectedExpression = null, [.("comparer")] string? comparerExpression = null, [.("ordering")] string? orderingExpression = null) + where TCollection : . { } } public static class NotSameReferenceAssertionExtensions { diff --git a/TUnit.PublicAPI/Tests.Assertions_Library_Has_No_API_Changes.DotNet8_0.verified.txt b/TUnit.PublicAPI/Tests.Assertions_Library_Has_No_API_Changes.DotNet8_0.verified.txt index 060668995b..665faeb994 100644 --- a/TUnit.PublicAPI/Tests.Assertions_Library_Has_No_API_Changes.DotNet8_0.verified.txt +++ b/TUnit.PublicAPI/Tests.Assertions_Library_Has_No_API_Changes.DotNet8_0.verified.txt @@ -385,9 +385,9 @@ namespace .Conditions public abstract class CollectionComparerBasedAssertion : . where TCollection : . { + protected .? Comparer; protected CollectionComparerBasedAssertion(. context) { } protected . GetComparer() { } - protected bool HasCustomComparer() { } protected void SetComparer(. comparer) { } } [.("Contains")] @@ -814,15 +814,14 @@ namespace .Conditions protected override string GetExpectation() { } public . Using(. comparer) { } } - [.("Collection equivalency uses structural comparison for complex objects, which requ" + - "ires reflection and is not compatible with AOT")] [.("IsEquivalentTo")] public class IsEquivalentToAssertion : . where TCollection : . { + [.("Collection equivalency uses structural comparison for complex objects, which requ" + + "ires reflection and is not compatible with AOT")] public IsEquivalentToAssertion(. context, . expected, . ordering = 0) { } public IsEquivalentToAssertion(. context, . expected, . comparer, . ordering = 0) { } - [.("AOT", "IL3050", Justification="Collection equivalency uses structural comparison which requires reflection")] protected override .<.> CheckAsync(. metadata) { } protected override string GetExpectation() { } public . Using(. comparer) { } @@ -916,14 +915,14 @@ namespace .Conditions public . IgnoringType( type) { } public . IgnoringType() { } } - [.("Collection equivalency uses structural comparison for complex objects, which requ" + - "ires reflection and is not compatible with AOT")] [.("IsNotEquivalentTo")] public class NotEquivalentToAssertion : . where TCollection : . { + [.("Collection equivalency uses structural comparison for complex objects, which requ" + + "ires reflection and is not compatible with AOT")] public NotEquivalentToAssertion(. context, . notExpected, . ordering = 0) { } - [.("AOT", "IL3050", Justification="Collection equivalency uses structural comparison which requires reflection")] + public NotEquivalentToAssertion(. context, . notExpected, . comparer, . ordering = 0) { } protected override .<.> CheckAsync(. metadata) { } protected override string GetExpectation() { } public . Using(. comparer) { } @@ -3068,8 +3067,6 @@ namespace .Extensions "ires reflection and is not compatible with AOT")] public static . IsEquivalentTo(this . source, . expected, . ordering = 0, [.("expected")] string? expectedExpression = null, [.("ordering")] string? orderingExpression = null) where TCollection : . { } - [.("Collection equivalency uses structural comparison for complex objects, which requ" + - "ires reflection and is not compatible with AOT")] public static . IsEquivalentTo(this . source, . expected, . comparer, . ordering = 0, [.("expected")] string? expectedExpression = null, [.("comparer")] string? comparerExpression = null, [.("ordering")] string? orderingExpression = null) where TCollection : . { } } @@ -3166,6 +3163,8 @@ namespace .Extensions "ires reflection and is not compatible with AOT")] public static . IsNotEquivalentTo(this . source, . notExpected, . ordering = 0, [.("notExpected")] string? notExpectedExpression = null, [.("ordering")] string? orderingExpression = null) where TCollection : . { } + public static . IsNotEquivalentTo(this . source, . notExpected, . comparer, . ordering = 0, [.("notExpected")] string? notExpectedExpression = null, [.("comparer")] string? comparerExpression = null, [.("ordering")] string? orderingExpression = null) + where TCollection : . { } } public static class NotSameReferenceAssertionExtensions { diff --git a/TUnit.PublicAPI/Tests.Assertions_Library_Has_No_API_Changes.DotNet9_0.verified.txt b/TUnit.PublicAPI/Tests.Assertions_Library_Has_No_API_Changes.DotNet9_0.verified.txt index 634cbf0e26..c8665a0b6a 100644 --- a/TUnit.PublicAPI/Tests.Assertions_Library_Has_No_API_Changes.DotNet9_0.verified.txt +++ b/TUnit.PublicAPI/Tests.Assertions_Library_Has_No_API_Changes.DotNet9_0.verified.txt @@ -388,9 +388,9 @@ namespace .Conditions public abstract class CollectionComparerBasedAssertion : . where TCollection : . { + protected .? Comparer; protected CollectionComparerBasedAssertion(. context) { } protected . GetComparer() { } - protected bool HasCustomComparer() { } protected void SetComparer(. comparer) { } } [.("Contains")] @@ -817,15 +817,14 @@ namespace .Conditions protected override string GetExpectation() { } public . Using(. comparer) { } } - [.("Collection equivalency uses structural comparison for complex objects, which requ" + - "ires reflection and is not compatible with AOT")] [.("IsEquivalentTo")] public class IsEquivalentToAssertion : . where TCollection : . { + [.("Collection equivalency uses structural comparison for complex objects, which requ" + + "ires reflection and is not compatible with AOT")] public IsEquivalentToAssertion(. context, . expected, . ordering = 0) { } public IsEquivalentToAssertion(. context, . expected, . comparer, . ordering = 0) { } - [.("AOT", "IL3050", Justification="Collection equivalency uses structural comparison which requires reflection")] protected override .<.> CheckAsync(. metadata) { } protected override string GetExpectation() { } public . Using(. comparer) { } @@ -919,14 +918,14 @@ namespace .Conditions public . IgnoringType( type) { } public . IgnoringType() { } } - [.("Collection equivalency uses structural comparison for complex objects, which requ" + - "ires reflection and is not compatible with AOT")] [.("IsNotEquivalentTo")] public class NotEquivalentToAssertion : . where TCollection : . { + [.("Collection equivalency uses structural comparison for complex objects, which requ" + + "ires reflection and is not compatible with AOT")] public NotEquivalentToAssertion(. context, . notExpected, . ordering = 0) { } - [.("AOT", "IL3050", Justification="Collection equivalency uses structural comparison which requires reflection")] + public NotEquivalentToAssertion(. context, . notExpected, . comparer, . ordering = 0) { } protected override .<.> CheckAsync(. metadata) { } protected override string GetExpectation() { } public . Using(. comparer) { } @@ -3085,8 +3084,6 @@ namespace .Extensions "ires reflection and is not compatible with AOT")] public static . IsEquivalentTo(this . source, . expected, . ordering = 0, [.("expected")] string? expectedExpression = null, [.("ordering")] string? orderingExpression = null) where TCollection : . { } - [.("Collection equivalency uses structural comparison for complex objects, which requ" + - "ires reflection and is not compatible with AOT")] public static . IsEquivalentTo(this . source, . expected, . comparer, . ordering = 0, [.("expected")] string? expectedExpression = null, [.("comparer")] string? comparerExpression = null, [.("ordering")] string? orderingExpression = null) where TCollection : . { } } @@ -3184,6 +3181,8 @@ namespace .Extensions "ires reflection and is not compatible with AOT")] public static . IsNotEquivalentTo(this . source, . notExpected, . ordering = 0, [.("notExpected")] string? notExpectedExpression = null, [.("ordering")] string? orderingExpression = null) where TCollection : . { } + public static . IsNotEquivalentTo(this . source, . notExpected, . comparer, . ordering = 0, [.("notExpected")] string? notExpectedExpression = null, [.("comparer")] string? comparerExpression = null, [.("ordering")] string? orderingExpression = null) + where TCollection : . { } } public static class NotSameReferenceAssertionExtensions { diff --git a/TUnit.PublicAPI/Tests.Assertions_Library_Has_No_API_Changes.Net4_7.verified.txt b/TUnit.PublicAPI/Tests.Assertions_Library_Has_No_API_Changes.Net4_7.verified.txt index 03c58549c0..bd832a6fca 100644 --- a/TUnit.PublicAPI/Tests.Assertions_Library_Has_No_API_Changes.Net4_7.verified.txt +++ b/TUnit.PublicAPI/Tests.Assertions_Library_Has_No_API_Changes.Net4_7.verified.txt @@ -381,9 +381,9 @@ namespace .Conditions public abstract class CollectionComparerBasedAssertion : . where TCollection : . { + protected .? Comparer; protected CollectionComparerBasedAssertion(. context) { } protected . GetComparer() { } - protected bool HasCustomComparer() { } protected void SetComparer(. comparer) { } } [.("Contains")] @@ -893,6 +893,7 @@ namespace .Conditions where TCollection : . { public NotEquivalentToAssertion(. context, . notExpected, . ordering = 0) { } + public NotEquivalentToAssertion(. context, . notExpected, . comparer, . ordering = 0) { } protected override .<.> CheckAsync(. metadata) { } protected override string GetExpectation() { } public . Using(. comparer) { } @@ -2881,6 +2882,8 @@ namespace .Extensions { public static . IsNotEquivalentTo(this . source, . notExpected, . ordering = 0, [.("notExpected")] string? notExpectedExpression = null, [.("ordering")] string? orderingExpression = null) where TCollection : . { } + public static . IsNotEquivalentTo(this . source, . notExpected, . comparer, . ordering = 0, [.("notExpected")] string? notExpectedExpression = null, [.("comparer")] string? comparerExpression = null, [.("ordering")] string? orderingExpression = null) + where TCollection : . { } } public static class NotSameReferenceAssertionExtensions { diff --git a/tools/tunit-nuget-tester/TUnit.NugetTester/TUnit.NugetTester/AotCompatibilityTests.cs b/tools/tunit-nuget-tester/TUnit.NugetTester/TUnit.NugetTester/AotCompatibilityTests.cs index 9702c05d5e..454de33ccf 100644 --- a/tools/tunit-nuget-tester/TUnit.NugetTester/TUnit.NugetTester/AotCompatibilityTests.cs +++ b/tools/tunit-nuget-tester/TUnit.NugetTester/TUnit.NugetTester/AotCompatibilityTests.cs @@ -1,3 +1,5 @@ +using TUnit.Assertions.Enums; + namespace TUnit.NugetTester; /// @@ -48,4 +50,32 @@ public async Task PropertyInjection_ShouldNotTriggerAotWarnings() await Assert.That(InjectedProperty).IsNotNull(); await Assert.That(InjectedProperty).IsEqualTo("test value"); } + + /// + /// Tests for issue #3851 - IsEquivalentTo with custom comparer should be AOT-compatible + /// When a custom comparer is provided, no reflection is used, so the method should not + /// have RequiresUnreferencedCode attribute and should be safe for AOT. + /// + [Test] + public async Task IsEquivalentTo_WithCustomComparer_ShouldNotTriggerAotWarnings() + { + // This test verifies that using IsEquivalentTo with a custom comparer doesn't trigger IL2026/IL3050 + // The custom comparer path doesn't use StructuralEqualityComparer which requires reflection + var list1 = new List { 1, 2, 3 }; + var list2 = new List { 3, 2, 1 }; + + // Using explicit comparer - should be AOT-safe + await Assert.That(list1).IsEquivalentTo(list2, EqualityComparer.Default); + } + + [Test] + public async Task IsEquivalentTo_WithCustomComparer_OrderMatching_ShouldNotTriggerAotWarnings() + { + // Verify that custom comparer works with both ordering modes + var list1 = new List { "a", "b", "c" }; + var list2 = new List { "a", "b", "c" }; + + // Using custom comparer with order matching - should be AOT-safe + await Assert.That(list1).IsEquivalentTo(list2, StringComparer.OrdinalIgnoreCase, CollectionOrdering.Matching); + } }