Skip to content

Commit

Permalink
Overhead Benchmark for Request Throttling (dotnet#10907)
Browse files Browse the repository at this point in the history
* initial implementation; no tests

* benchmark project

* Better benchmarks

* overhead test for congested queue

* Addressed feedback
  • Loading branch information
DylanDmitri authored Jun 13, 2019
1 parent af812f2 commit 844530d
Show file tree
Hide file tree
Showing 15 changed files with 265 additions and 34 deletions.
15 changes: 15 additions & 0 deletions src/Middleware/Middleware.sln
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.AspNetCore.Metada
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.AspNetCore.Authorization", "..\Security\Authorization\Core\src\Microsoft.AspNetCore.Authorization.csproj", "{CDDD7C43-5BEB-4E3E-8A59-FCDC83C9FBCF}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.AspNetCore.RequestThrottling.Microbenchmarks", "RequestThrottling\perf\Microbenchmarks\Microsoft.AspNetCore.RequestThrottling.Microbenchmarks.csproj", "{737B26B4-CFC6-4B44-9070-DD36334E85B3}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -1599,6 +1601,18 @@ Global
{CDDD7C43-5BEB-4E3E-8A59-FCDC83C9FBCF}.Release|x64.Build.0 = Release|Any CPU
{CDDD7C43-5BEB-4E3E-8A59-FCDC83C9FBCF}.Release|x86.ActiveCfg = Release|Any CPU
{CDDD7C43-5BEB-4E3E-8A59-FCDC83C9FBCF}.Release|x86.Build.0 = Release|Any CPU
{737B26B4-CFC6-4B44-9070-DD36334E85B3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{737B26B4-CFC6-4B44-9070-DD36334E85B3}.Debug|Any CPU.Build.0 = Debug|Any CPU
{737B26B4-CFC6-4B44-9070-DD36334E85B3}.Debug|x64.ActiveCfg = Debug|Any CPU
{737B26B4-CFC6-4B44-9070-DD36334E85B3}.Debug|x64.Build.0 = Debug|Any CPU
{737B26B4-CFC6-4B44-9070-DD36334E85B3}.Debug|x86.ActiveCfg = Debug|Any CPU
{737B26B4-CFC6-4B44-9070-DD36334E85B3}.Debug|x86.Build.0 = Debug|Any CPU
{737B26B4-CFC6-4B44-9070-DD36334E85B3}.Release|Any CPU.ActiveCfg = Release|Any CPU
{737B26B4-CFC6-4B44-9070-DD36334E85B3}.Release|Any CPU.Build.0 = Release|Any CPU
{737B26B4-CFC6-4B44-9070-DD36334E85B3}.Release|x64.ActiveCfg = Release|Any CPU
{737B26B4-CFC6-4B44-9070-DD36334E85B3}.Release|x64.Build.0 = Release|Any CPU
{737B26B4-CFC6-4B44-9070-DD36334E85B3}.Release|x86.ActiveCfg = Release|Any CPU
{737B26B4-CFC6-4B44-9070-DD36334E85B3}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down Expand Up @@ -1725,6 +1739,7 @@ Global
{353AA2B0-1013-486C-B5BD-9379385CA403} = {8C9AA8A2-9D1F-4450-9F8D-56BAB6F3D343}
{7E2EA6E2-31FE-418A-9AE4-955A4C708AE7} = {ACA6DDB9-7592-47CE-A740-D15BF307E9E0}
{CDDD7C43-5BEB-4E3E-8A59-FCDC83C9FBCF} = {ACA6DDB9-7592-47CE-A740-D15BF307E9E0}
{737B26B4-CFC6-4B44-9070-DD36334E85B3} = {8C9AA8A2-9D1F-4450-9F8D-56BAB6F3D343}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {83786312-A93B-4BB4-AB06-7C6913A59AFA}
Expand Down
3 changes: 2 additions & 1 deletion src/Middleware/RequestThrottling/RequestThrottling.slnf
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@
"HttpsPolicy\\src\\Microsoft.AspNetCore.HttpsPolicy.csproj",
"RequestThrottling\\sample\\RequestThrottlingSample.csproj",
"RequestThrottling\\src\\Microsoft.AspNetCore.RequestThrottling.csproj",
"RequestThrottling\\test\\Microsoft.AspNetCore.RequestThrottling.Tests.csproj"
"RequestThrottling\\test\\Microsoft.AspNetCore.RequestThrottling.Tests.csproj",
"RequestThrottling\\perf\\Microbenchmarks\\Microsoft.AspNetCore.RequestThrottling.Microbenchmarks.csproj"
]
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[assembly: BenchmarkDotNet.Attributes.AspNetCoreBenchmark]
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp3.0</TargetFramework>
<!--<StartupObject>Microsoft.AspNetCore.RequestThrottling.Microbenchmarks.Test</StartupObject>-->
</PropertyGroup>

<ItemGroup>
<Reference Include="BenchmarkDotNet" />
<Reference Include="Microsoft.AspNetCore.BenchmarkRunner.Sources" />
<Reference Include="Microsoft.AspNetCore.RequestThrottling" />
</ItemGroup>
</Project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.

using System;
using System.Threading;
using System.Threading.Tasks;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Running;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;

namespace Microsoft.AspNetCore.RequestThrottling.Microbenchmarks
{
public class QueueEmptyOverhead
{
private const int _numRequests = 20000;

private RequestThrottlingMiddleware _middleware;
private RequestDelegate _restOfServer;

[GlobalSetup]
public void GlobalSetup()
{
_restOfServer = YieldsThreadInternally ? (RequestDelegate)YieldsThread : (RequestDelegate)CompletesImmediately;

var options = new RequestThrottlingOptions
{
MaxConcurrentRequests = 8,
RequestQueueLimit = _numRequests
};

_middleware = new RequestThrottlingMiddleware(
next: _restOfServer,
loggerFactory: NullLoggerFactory.Instance,
options: Options.Create(options)
);
}

[Params(false, true)]
public bool YieldsThreadInternally;

[Benchmark(OperationsPerInvoke = _numRequests)]
public async Task Baseline()
{
for (int i = 0; i < _numRequests; i++)
{
await _restOfServer(null);
}
}

[Benchmark(OperationsPerInvoke = _numRequests)]
public async Task WithEmptyQueueOverhead()
{
for (int i = 0; i < _numRequests; i++)
{
await _middleware.Invoke(null);
}
}

private static async Task YieldsThread(HttpContext context)
{
await Task.Yield();
}

private static Task CompletesImmediately(HttpContext context)
{
return Task.CompletedTask;
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using BenchmarkDotNet.Attributes;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;

namespace Microsoft.AspNetCore.RequestThrottling.Microbenchmarks
{
public class QueueFullOverhead
{
private const int _numRequests = 2000;
private int _requestCount = 0;
private ManualResetEventSlim _mres = new ManualResetEventSlim();

private RequestThrottlingMiddleware _middleware;

[Params(8)]
public int MaxConcurrentRequests;

[GlobalSetup]
public void GlobalSetup()
{
var options = new RequestThrottlingOptions
{
MaxConcurrentRequests = MaxConcurrentRequests,
RequestQueueLimit = _numRequests
};

_middleware = new RequestThrottlingMiddleware(
next: (RequestDelegate)_incrementAndCheck,
loggerFactory: NullLoggerFactory.Instance,
options: Options.Create(options)
);
}

[IterationSetup]
public void Setup()
{
_requestCount = 0;
_mres.Reset();
}

private async Task _incrementAndCheck(HttpContext context)
{
if (Interlocked.Increment(ref _requestCount) == _numRequests)
{
_mres.Set();
}

await Task.Yield();
}

[Benchmark(OperationsPerInvoke = _numRequests)]
public void Baseline()
{
for (int i = 0; i < _numRequests; i++)
{
_ = _incrementAndCheck(null);
}

_mres.Wait();
}

[Benchmark(OperationsPerInvoke = _numRequests)]
public void QueueingAll()
{
for (int i = 0; i < _numRequests; i++)
{
_ = _middleware.Invoke(null);
}

_mres.Wait();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ namespace Microsoft.AspNetCore.RequestThrottling
public partial class RequestThrottlingMiddleware
{
public RequestThrottlingMiddleware(Microsoft.AspNetCore.Http.RequestDelegate next, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory, Microsoft.Extensions.Options.IOptions<Microsoft.AspNetCore.RequestThrottling.RequestThrottlingOptions> options) { }
public int ActiveRequestCount { get { throw null; } }
[System.Diagnostics.DebuggerStepThroughAttribute]
public System.Threading.Tasks.Task Invoke(Microsoft.AspNetCore.Http.HttpContext context) { throw null; }
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,20 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<Project Sdk="Microsoft.NET.Sdk.Web">

<PropertyGroup>
<TargetFramework>netcoreapp3.0</TargetFramework>
</PropertyGroup>

<ItemGroup>
<ItemGroup Condition="'$(BenchmarksTargetFramework)' == ''">
<Reference Include="Microsoft.Extensions.Logging.Console" />
<Reference Include="Microsoft.AspNetCore.RequestThrottling" />
<Reference Include="Microsoft.AspNetCore.Server.Kestrel" />
</ItemGroup>

<ItemGroup Condition="'$(BenchmarksTargetFramework)' != ''">
<PackageReference Include="Microsoft.AspNetCore.RequestThrottling" Version="$(MicrosoftAspNetCoreAppPackageVersion)" />

<FrameworkReference Update="Microsoft.AspNetCore.App" RuntimeFrameworkVersion="$(MicrosoftAspNetCoreAppPackageVersion)" />
<FrameworkReference Update="Microsoft.NETCore.App" RuntimeFrameworkVersion="$(MicrosoftNETCoreAppPackageVersion)" />
</ItemGroup>

</Project>
17 changes: 17 additions & 0 deletions src/Middleware/RequestThrottling/src/Internal/IRequestQueue.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information

using System;
using System.Threading.Tasks;

namespace Microsoft.AspNetCore.RequestThrottling.Internal
{
interface IRequestQueue : IDisposable
{
int TotalRequests { get; }

Task<bool> TryEnterQueueAsync();

void Release();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

namespace Microsoft.AspNetCore.RequestThrottling.Internal
{
internal class RequestQueue : IDisposable
internal class TailDrop : IRequestQueue
{
private readonly int _maxConcurrentRequests;
private readonly int _requestQueueLimit;
Expand All @@ -16,7 +16,7 @@ internal class RequestQueue : IDisposable
private object _totalRequestsLock = new object();
public int TotalRequests { get; private set; }

public RequestQueue(int maxConcurrentRequests, int requestQueueLimit)
public TailDrop(int maxConcurrentRequests, int requestQueueLimit)
{
_maxConcurrentRequests = maxConcurrentRequests;
_requestQueueLimit = requestQueueLimit;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<Description>ASP.NET Core middleware for queuing incoming HTTP requests, to avoid threadpool starvation.</Description>
<TargetFramework>netcoreapp3.0</TargetFramework>
<GenerateDocumentationFile>true</GenerateDocumentationFile>
<PackageTags>aspnetcore;queue;queuing</PackageTags>
<IsShippingPackage>true</IsShippingPackage>
</PropertyGroup>

<ItemGroup>
Expand Down
43 changes: 25 additions & 18 deletions src/Middleware/RequestThrottling/src/RequestThrottlingMiddleware.cs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ namespace Microsoft.AspNetCore.RequestThrottling
/// </summary>
public class RequestThrottlingMiddleware
{
private readonly RequestQueue _requestQueue;
private readonly IRequestQueue _requestQueue;
private readonly RequestDelegate _next;
private readonly RequestThrottlingOptions _requestThrottlingOptions;
private readonly ILogger _logger;
Expand All @@ -35,9 +35,9 @@ public RequestThrottlingMiddleware(RequestDelegate next, ILoggerFactory loggerFa
{
throw new ArgumentException("The value of 'options.MaxConcurrentRequests' must be specified.", nameof(options));
}
if (_requestThrottlingOptions.MaxConcurrentRequests < 0)
if (_requestThrottlingOptions.MaxConcurrentRequests <= 0)
{
throw new ArgumentException("The value of 'options.MaxConcurrentRequests' must be a positive integer.", nameof(options));
throw new ArgumentOutOfRangeException(nameof(options), "The value of `options.MaxConcurrentRequests` must be a positive integer.");
}
if (_requestThrottlingOptions.RequestQueueLimit < 0)
{
Expand All @@ -51,9 +51,16 @@ public RequestThrottlingMiddleware(RequestDelegate next, ILoggerFactory loggerFa

_next = next;
_logger = loggerFactory.CreateLogger<RequestThrottlingMiddleware>();
_requestQueue = new RequestQueue(
_requestThrottlingOptions.MaxConcurrentRequests.Value,
_requestThrottlingOptions.RequestQueueLimit);

if (_requestThrottlingOptions.ServerAlwaysBlocks)
{
// note: this option for testing only. Blocks all requests from entering the server.
_requestQueue = new TailDrop(0, _requestThrottlingOptions.RequestQueueLimit);
}
else
{
_requestQueue = new TailDrop(_requestThrottlingOptions.MaxConcurrentRequests.Value, _requestThrottlingOptions.RequestQueueLimit);
}
}

/// <summary>
Expand All @@ -64,24 +71,24 @@ public RequestThrottlingMiddleware(RequestDelegate next, ILoggerFactory loggerFa
public async Task Invoke(HttpContext context)
{
var waitInQueueTask = _requestQueue.TryEnterQueueAsync();
if (waitInQueueTask.IsCompletedSuccessfully && !waitInQueueTask.Result)

if (waitInQueueTask.IsCompletedSuccessfully && waitInQueueTask.Result)
{
RequestThrottlingLog.RequestRejectedQueueFull(_logger);
context.Response.StatusCode = StatusCodes.Status503ServiceUnavailable;
await _requestThrottlingOptions.OnRejected(context);
return;
RequestThrottlingLog.RequestRunImmediately(_logger, ActiveRequestCount);
}
else if (!waitInQueueTask.IsCompletedSuccessfully)
else
{
RequestThrottlingLog.RequestEnqueued(_logger, ActiveRequestCount);
var result = await waitInQueueTask;
await waitInQueueTask;
RequestThrottlingLog.RequestDequeued(_logger, ActiveRequestCount);

Debug.Assert(result);
}
else

if (!waitInQueueTask.Result)
{
RequestThrottlingLog.RequestRunImmediately(_logger, ActiveRequestCount);
RequestThrottlingLog.RequestRejectedQueueFull(_logger);
context.Response.StatusCode = StatusCodes.Status503ServiceUnavailable;
await _requestThrottlingOptions.OnRejected(context);
return;
}

try
Expand All @@ -98,7 +105,7 @@ public async Task Invoke(HttpContext context)
/// The number of requests currently on the server.
/// Cannot exceeed the sum of <see cref="RequestThrottlingOptions.RequestQueueLimit"> and </see>/><see cref="RequestThrottlingOptions.MaxConcurrentRequests"/>.
/// </summary>
internal int ActiveRequestCount
public int ActiveRequestCount
{
get => _requestQueue.TotalRequests;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,5 +33,10 @@ public class RequestThrottlingOptions
{
return Task.CompletedTask;
};

/// <summary>
/// For internal testing only. If true, no requests will enter the server.
/// </summary>
internal bool ServerAlwaysBlocks { get; set; } = false;
}
}
Loading

0 comments on commit 844530d

Please sign in to comment.