diff --git a/README.md b/README.md index aa6a0760..45b696cd 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,14 @@ Cross-platform performance benchmarking and testing framework for .NET applicati NBench is designed for .NET developers who need to care about performance and want the ability to "unit test" their application's performance just like [XUnit](https://github.com/xunit/xunit) or [NUnit](http://nunit.org/) tests their application code. +## Build Status + +| Stage | Status | +|-------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Windows Tests | [![Build Status](https://dev.azure.com/petabridge/NBench/_apis/build/status/NBench%20PR%20Validation?branchName=dev&jobName=Windows%20Tests)](https://dev.azure.com/petabridge/NBench/_build/latest?definitionId=115&branchName=dev) | +| Integration Tests | [![Build Status](https://dev.azure.com/petabridge/NBench/_apis/build/status/NBench%20PR%20Validation?branchName=dev&jobName=Integration%20Tests)](https://dev.azure.com/petabridge/NBench/_build/latest?definitionId=115&branchName=dev) | +| NuGet Pack | [![Build Status](https://dev.azure.com/petabridge/NBench/_apis/build/status/NBench%20PR%20Validation?branchName=dev&jobName=NuGet%20Pack)](https://dev.azure.com/petabridge/NBench/_build/latest?definitionId=115&branchName=dev) | + ## Documentation To learn more, please read [the official NBench documentation](https://nbench.io). diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index bf3278ba..26f53463 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,4 +1,4 @@ -#### v2.0.0 February 24 2020 +#### v2.0.1 February 25 2020 NBench 2.0.0 is a major departure from NBench 1.2 and preceding versions, and these changes were done in order to support NBench's future as a cutting-edge, cross-platform performance testing and macro benchmarking framework: - `dotnet nbench` and `NBench.Runner` are both now deprecated - [NBench is now run from directly inside a console application created by end-users](https://nbench.io/articles/quickstart.html). This makes it easier to configure, debug, and create benchmarks on new .NET Core platforms without having to wait for additional instrumentation or tooling from NBench itself. @@ -6,4 +6,14 @@ NBench 2.0.0 is a major departure from NBench 1.2 and preceding versions, and th - We've added a new documentation website for NBench: https://nbench.io/ - NBench now supports configuration as code through the [`TestPackage` class](https://nbench.io/api/NBench.Sdk.TestPackage.html). -For a full set of changes, [please see the NBench 2.0.0 milestone on Github](https://github.com/petabridge/NBench/milestone/3). \ No newline at end of file +For a full set of changes, [please see the NBench 2.0.0 milestone on Github](https://github.com/petabridge/NBench/milestone/3). + +--- + +2.0.1 Notes: + +* Fixed error with setting thread priority on Linux; +* Fixed NuGet symbol publication; +* And more. + +For a full set of changes, [please see the NBench 2.0.1 milestone on Github](https://github.com/petabridge/NBench/milestone/8). \ No newline at end of file diff --git a/build-system/pr-validation.yaml b/build-system/pr-validation.yaml index 2132000a..91ec44a9 100644 --- a/build-system/pr-validation.yaml +++ b/build-system/pr-validation.yaml @@ -31,7 +31,16 @@ jobs: scriptFileName: build.cmd scriptArgs: NBench outputDirectory: 'PerfResults' - artifactName: 'integration_tests-$(Build.BuildId)' + artifactName: 'integration_tests_windows-$(Build.BuildId)' + - template: azure-pipeline.template.yaml + parameters: + name: 'linux_integration' + displayName: 'Integration Tests (Linux)' + vmImage: 'ubuntu-16.04' + scriptFileName: build.sh + scriptArgs: NBench + outputDirectory: 'PerfResults' + artifactName: 'integration_tests_linux-$(Build.BuildId)' - template: azure-pipeline.template.yaml parameters: name: 'nuget_pack' diff --git a/src/NBench.TestAssembly/NBench.TestAssembly.csproj b/src/NBench.TestAssembly/NBench.TestAssembly.csproj index 073d2228..14fb2122 100644 --- a/src/NBench.TestAssembly/NBench.TestAssembly.csproj +++ b/src/NBench.TestAssembly/NBench.TestAssembly.csproj @@ -4,6 +4,7 @@ $(NetCoreTestVersion) NBench.TestAssembly NBench.TestAssembly + false diff --git a/src/NBench.Tests.End2End/NBenchIntegrationTest.WithDependencies.cs b/src/NBench.Tests.End2End/NBenchIntegrationTest.WithDependencies.cs index 73c1fba2..7f078c07 100644 --- a/src/NBench.Tests.End2End/NBenchIntegrationTest.WithDependencies.cs +++ b/src/NBench.Tests.End2End/NBenchIntegrationTest.WithDependencies.cs @@ -30,10 +30,23 @@ public void LoadAssemblyCorrect() { var package = LoadPackageWithDependencies().AddOutput(_benchmarkOutput); var result = TestRunner.Run(package); - - result.AllTestsPassed.Should().BeTrue("Expected all tests to pass, but did not."); - result.ExecutedTestsCount.Should().NotBe(0); - result.IgnoredTestsCount.Should().Be(0); + + try + { + result.AllTestsPassed.Should().BeTrue("Expected all tests to pass, but did not."); + result.ExecutedTestsCount.Should().NotBe(0); + result.IgnoredTestsCount.Should().Be(0); + } + catch + { + foreach (var test in result.FullResults) + { + _output.WriteLine($"DEBUG: Checking output for {test.BenchmarkName}"); + foreach(var a in test.AssertionResults) + _output.WriteLine($"ASSERT: {a.MetricName} - Passed? {a.Passed}"); + } + throw; + } } private static TestPackage LoadPackageWithDependencies(IEnumerable include = null, IEnumerable exclude = null) @@ -52,6 +65,9 @@ private static TestPackage LoadPackageWithDependencies(IEnumerable inclu package.AddExclude(e); } + // need to set this to true in order to resolve https://github.com/petabridge/NBench/issues/314 + package.Concurrent = true; + return package; } } diff --git a/src/NBench/Sdk/Benchmark.cs b/src/NBench/Sdk/Benchmark.cs index 7bf8c78b..00dccd65 100644 --- a/src/NBench/Sdk/Benchmark.cs +++ b/src/NBench/Sdk/Benchmark.cs @@ -43,7 +43,7 @@ public class Benchmark /// The invoker used to execute benchmark and setup / cleanup methods. /// The output target this benchmark will write to. /// Uses the to assert benchmark data. - public Benchmark(BenchmarkSettings settings, IBenchmarkInvoker invoker, IBenchmarkOutput writer) + public Benchmark(BenchmarkSettings settings, IBenchmarkInvoker invoker, IBenchmarkOutput writer) : this(settings, invoker, writer, DefaultBenchmarkAssertionRunner.Instance) { } /// @@ -97,7 +97,7 @@ private void WarmUp() var runTime = 0L; /* Pre-Warmup */ - + Trace.Debug("----- BEGIN PRE-WARMUP -----"); /* Estimate */ @@ -128,7 +128,7 @@ private void WarmUp() { runEstimates[i - 1] = runCount; timeEstimates[i - 1] = warmupStopWatch.ElapsedTicks; - + } runCount = 0; @@ -136,8 +136,8 @@ private void WarmUp() } runCount = (long)Math.Ceiling(runEstimates.Average()); - runTime = (long) Math.Ceiling(timeEstimates.Average()); - + runTime = (long)Math.Ceiling(timeEstimates.Average()); + Trace.Debug( $"Throughput mode: executed {runCount} instances of {BenchmarkName} in roughly {targetTime.TotalSeconds}s. Using that figure for benchmark."); } @@ -225,7 +225,7 @@ public void Run() { Output.Warning($"Error during previous run of {BenchmarkName}. Aborting run..."); } - + } public void Shutdown() @@ -262,12 +262,12 @@ protected void PreRun() // the invoker with an inlined loop Invoker.InvokePerfSetup(WarmupData.EstimatedRunsPerSecond, _currentRun.Context); } - else + else { // Invoke user-defined setup method, if any Invoker.InvokePerfSetup(_currentRun.Context); } - + PrepareForRun(); } @@ -344,7 +344,7 @@ private void Complete(bool isEstimate = false) _currentRun.Dispose(); Trace.Info($"Generating report for {PrintWarmupOrRun(_isWarmup)} {1 + Settings.NumberOfIterations - _pendingIterations} of {BenchmarkName}"); var report = _currentRun.ToReport(StopWatch.Elapsed); - if(!isEstimate) + if (!isEstimate) Output.WriteRun(report, _isWarmup); // Change runs, but not on warmup diff --git a/src/NBench/Sdk/TestRunner.cs b/src/NBench/Sdk/TestRunner.cs index ef0c4573..67e2cde2 100644 --- a/src/NBench/Sdk/TestRunner.cs +++ b/src/NBench/Sdk/TestRunner.cs @@ -8,6 +8,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.Contracts; +using System.Runtime.InteropServices; using System.Threading; using IBenchmarkOutput = NBench.Reporting.IBenchmarkOutput; @@ -81,7 +82,7 @@ public static void SetProcessPriority(bool concurrent) /* * Set priority */ - if (!IsMono) + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && !IsMono) Process.GetCurrentProcess().PriorityClass = ProcessPriorityClass.High; if (!concurrent) { diff --git a/src/common.props b/src/common.props index 37dfa612..93afc6c9 100644 --- a/src/common.props +++ b/src/common.props @@ -2,13 +2,19 @@ Copyright © 2015-2020 Petabridge Petabridge - 2.0.0 + 2.0.1 NBench 2.0.0 is a major departure from NBench 1.2 and preceding versions, and these changes were done in order to support NBench's future as a cutting-edge, cross-platform performance testing and macro benchmarking framework: - `dotnet nbench` and `NBench.Runner` are both now deprecated - [NBench is now run from directly inside a console application created by end-users](https://nbench.io/articles/quickstart.html). This makes it easier to configure, debug, and create benchmarks on new .NET Core platforms without having to wait for additional instrumentation or tooling from NBench itself. - NBench no longer supports .NET Framework explicitly; moving forward NBench will only support .NET Standard 2.0 and later (.NET Framework 4.6.1 and greater or .NET Core 2.0 and greater.) - We've added a new documentation website for NBench: https://nbench.io/ - NBench now supports configuration as code through the [`TestPackage` class](https://nbench.io/api/NBench.Sdk.TestPackage.html). -For a full set of changes, [please see the NBench 2.0.0 milestone on Github](https://github.com/petabridge/NBench/milestone/3). +For a full set of changes, [please see the NBench 2.0.0 milestone on Github](https://github.com/petabridge/NBench/milestone/3). +--- +2.0.1 Notes: +Fixed error with setting thread priority on Linux; +Fixed NuGet symbol publication; +And more. +For a full set of changes, [please see the NBench 2.0.1 milestone on Github](https://github.com/petabridge/NBench/milestone/8). https://nbench.io/ @@ -22,7 +28,7 @@ For a full set of changes, [please see the NBench 2.0.0 milestone on Github](htt https://github.com/petabridge/NBench - 4.5.0 + 4.5.1 2.4.1 16.5.0 5.10.2 @@ -31,4 +37,16 @@ For a full set of changes, [please see the NBench 2.0.0 milestone on Github](htt netcoreapp3.1 netstandard2.0 + + + true + + true + + true + snupkg + + + + \ No newline at end of file