8000 [build] Add windows-11-arm and ubuntu-24.04-arm to CI run-tests.yaml by nietras · Pull Request #2760 · dotnet/BenchmarkDotNet · GitHub
[go: up one dir, main page]
Skip to content

[build] Add windows-11-arm and ubuntu-24.04-arm to CI run-tests.yaml #2760

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 10 commits into
base: master
Choose a base branch
from
Open

[build] Add windows-11-arm and ubuntu-24.04-arm to CI run-tests.yaml #2760

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
75a8b6d
[build] Add windows-11-arm to CI run-tests.yaml
nietras 8000 Jun 8, 2025
5bd15fa
[build] Add ubuntu-24.04-arm to CI run-tests.yaml
nietras Jun 8, 2025
19a5ddb
add os to upload name
nietras Jun 8, 2025
6a2f05d
add os to upload name
nietras Jun 8, 2025
c80dd8a
add os to core name
nietras Jun 8, 2025
3ba5a93
Merge branch 'ubuntu-arm-ci' into windows-11-arm-ci
nietras Jun 9, 2025
60e1c65
SingleBenchmarkCanBeExecutedForMultipleRuntimes remove Platform.X64
nietras Jun 9, 2025
67d1e27
BenchmarkCanAllocateMoreThan2Gb try use platform for switching
nietras Jun 9, 2025
1b3a1a7
Split LargeAddressAwareTests.
timcassell Jun 14, 2025
7134a1f
Skip WasmIsSupported test on Linux Arm.
timcassell Jun 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 15 additions & 6 deletions .github/workflows/run-tests.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,10 @@ concurrency:
jobs:

test-windows-core:
runs-on: windows-latest
strategy:
matrix:
os: [windows-latest, windows-11-arm]
runs-on: ${{ matrix.os }}
steps:
- name: Disable Windows Defender
run: Set-MpPreference -DisableRealtimeMonitoring $true
Expand All @@ -30,11 +33,14 @@ jobs:
uses: actions/upload-artifact@v4
if: always()
with:
name: test-windows-core-trx-${{ github.run_id }}
name: test-windows-core-trx-${{ github.run_id }}-${{ matrix.os }}
path: "**/*.trx"

test-windows-full:
runs-on: windows-latest
strategy:
matrix:
os: [windows-latest, windows-11-arm]
runs-on: ${{ matrix.os }}
steps:
- name: Disable Windows Defender
run: Set-MpPreference -DisableRealtimeMonitoring $true
Expand All @@ -52,11 +58,14 @@ jobs:
uses: actions/upload-artifact@v4
if: always()
with:
name: test-windows-full-trx-${{ github.run_id }}
name: test-windows-full-trx-${{ github.run_id }}-${{ matrix.os }}
path: "**/*.trx"

test-linux:
runs-on: ubuntu-latest
strategy:
matrix:
os: [ubuntu-latest, ubuntu-24.04-arm]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
# Set up the environment
Expand Down Expand Up @@ -87,7 +96,7 @@ jobs:
uses: actions/upload-artifact@v4
if: always()
with:
name: test-linux-trx-${{ github.run_id }}
name: test-linux-trx-${{ github.run_id }}-${{ matrix.os }}
path: "**/*.trx"

test-macos:
Expand Down
6 changes: 6 additions & 0 deletions tests/BenchmarkDotNet.IntegrationTests/FailingProcessSpawnTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,12 @@ public void NoHangs()
_ => Platform.X64
};

if (wrongPlatform == Platform.X64 && RuntimeInformation.IsFullFramework)
{
// It seems full Framework on Arm ignores the platform and simply runs the native platform, causing this test to fail.
return;
}

var invalidPlatformJob = Job.Dry.WithPlatform(wrongPlatform);
var config = CreateSimpleConfig(job: invalidPlatformJob);

Expand Down
57 changes: 40 additions & 17 deletions tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Environments;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Portability;
using BenchmarkDotNet.Running;
using BenchmarkDotNet.Tests.Loggers;
using BenchmarkDotNet.Tests.XUnit;
Expand All @@ -19,35 +20,57 @@ public class LargeAddressAwareTest

public LargeAddressAwareTest(ITestOutputHelper outputHelper) => output = outputHelper;

[FactEnvSpecific("CLR is a valid job only on Windows", EnvRequirement.WindowsOnly)]
8000 public void BenchmarkCanAllocateMoreThan2Gb()
[Fact]
public void BenchmarkCanAllocateMoreThan2Gb_Core()
{
var summary = BenchmarkRunner
.Run<NeedsMoreThan2GB>(
ManualConfig.CreateEmpty()
.AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithPlatform(Platform.X64).WithId("Core"))
.AddJob(Job.Dry.WithRuntime(ClrRuntime.Net462).WithPlatform(Platform.X86).WithGcServer(false).WithLargeAddressAware().WithId("Framework"))
.AddColumnProvider(DefaultColumnProviders.Instance)
.AddLogger(new OutputLogger(output)));
var platform = RuntimeInformation.GetCurrentPlatform();
var config = ManualConfig.CreateEmpty();
// Running 32-bit benchmarks with .Net Core requires passing the path to 32-bit SDK,
// which makes this test more complex than it's worth in CI, so we only test 64-bit.
config.AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithPlatform(platform).WithId(platform.ToString()));
config.AddColumnProvider(DefaultColumnProviders.Instance)
.AddLogger(new OutputLogger(output));

var summary = BenchmarkRunner.Run<NeedsMoreThan2GB>(config);

Assert.True(summary.Reports
.All(report => report.ExecuteResults
.All(executeResult => executeResult.FoundExecutable)));

Assert.True(summary.Reports.All(report => report.AllMeasurements.Any()));
Assert.True(summary.Reports.All(report => report.ExecuteResults.Any()));
Assert.Equal(1, summary.Reports.Count(report => report.BenchmarkCase.Job.Environment.Runtime is CoreRuntime));

Assert.True(summary.Reports
.Single(report => report.BenchmarkCase.Job.Environment.Runtime is ClrRuntime)
.ExecuteResults
.Any());
Assert.Contains(".NET 8.0", summary.AllRuntimes);
}

[FactEnvSpecific("Framework is only on Windows", EnvRequirement.WindowsOnly)]
public void BenchmarkCanAllocateMoreThan2Gb_Framework()
{
var platform = RuntimeInformation.GetCurrentPlatform();
var config = ManualConfig.CreateEmpty();
// Net481 officially only supports x86, x64, and Arm64.
config.AddJob(Job.Dry.WithRuntime(ClrRuntime.Net481).WithPlatform(platform).WithGcServer(false).WithLargeAddressAware().WithId(platform.ToString()));
int jobCount = 1;
if (platform == Platform.X64)
{
++jobCount;
config.AddJob(Job.Dry.WithRuntime(ClrRuntime.Net462).WithPlatform(Platform.X86).WithGcServer(false).WithLargeAddressAware().WithId("X86"));
}
config.AddColumnProvider(DefaultColumnProviders.Instance)
.AddLogger(new OutputLogger(output));

var summary = BenchmarkRunner.Run<NeedsMoreThan2GB>(config);

Assert.True(summary.Reports
.Single(report => report.BenchmarkCase.Job.Environment.Runtime is CoreRuntime)
.ExecuteResults
.Any());
.All(report => report.ExecuteResults
.All(executeResult => executeResult.FoundExecutable)));

Assert.True(summary.Reports.All(report => report.AllMeasurements.Any()));
Assert.True(summary.Reports.All(report => report.ExecuteResults.Any()));
Assert.Equal(jobCount, summary.Reports.Count(report => report.BenchmarkCase.Job.Environment.Runtime is ClrRuntime));

Assert.Contains(".NET Framework", summary.AllRuntimes);
Assert.Contains(".NET 8.0", summary.AllRuntimes);
}
}

Expand Down
2 changes: 1 addition & 1 deletion tests/BenchmarkDotNet.IntegrationTests/MultipleRuntimesTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ public void SingleBenchmarkCanBeExecutedForMultipleRuntimes()
var summary = BenchmarkRunner
.Run<C>(
ManualConfig.CreateEmpty()
.AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithPlatform(Platform.X64).WithId("Core"))
.AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithId("Core"))
.AddJob(Job.Dry.WithRuntime(ClrRuntime.Net462).WithId("Framework"))
.AddColumnProvider(DefaultColumnProviders.Instance)
.AddLogger(new OutputLogger(output)));
Expand Down
7 changes: 7 additions & 0 deletions tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using System.IO;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Detectors;
using BenchmarkDotNet.Environments;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Portability;
Expand All @@ -26,6 +27,12 @@ public class WasmTests(ITestOutputHelper output) : BenchmarkTestExecutor(output)
[FactEnvSpecific("WASM is only supported on Unix", EnvRequirement.NonWindows)]
public void WasmIsSupported()
{
// Test fails on Linux non-x64.
if (OsDetector.IsLinux() && RuntimeInformation.GetCurrentPlatform() != Platform.X64)
{
return;
}

var dotnetVersion = "net8.0";
var logger = new OutputLogger(Output);
var netCoreAppSettings = new NetCoreAppSettings(dotnetVersion, null, "Wasm");
Expand Down
Loading
0