From 14bbb7e508927033cb128e74321320c34707b9fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Wed, 21 Aug 2024 11:26:08 +0200 Subject: [PATCH 01/11] Include the WorkItems project in Docker (#3851) --- .../ProductConstructionService.Api/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Dockerfile b/src/ProductConstructionService/ProductConstructionService.Api/Dockerfile index 96dd998461..d85c4f6517 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Dockerfile +++ b/src/ProductConstructionService/ProductConstructionService.Api/Dockerfile @@ -29,6 +29,7 @@ COPY ["src/ProductConstructionService/ProductConstructionService.Common/ProductC COPY ["src/ProductConstructionService/ProductConstructionService.ServiceDefaults/ProductConstructionService.ServiceDefaults.csproj", "./ProductConstructionService.ServiceDefaults/"] COPY ["src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/ProductConstructionService.SubscriptionTriggerer.csproj", "./ProductConstructionService.SubscriptionTriggerer/"] COPY ["src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/ProductConstructionService.LongestBuildPathUpdater.csproj", "./ProductConstructionService.LongestBuildPathUpdater/"] +COPY ["src/ProductConstructionService/ProductConstructionService.WorkItems/ProductConstructionService.WorkItems.csproj", "./ProductConstructionService.WorkItems/"] RUN dotnet restore "./ProductConstructionService.Api/ProductConstructionService.Api.csproj" RUN dotnet restore "./ProductConstructionService.SubscriptionTriggerer/ProductConstructionService.SubscriptionTriggerer.csproj" From c4421d57c26fd8f512a6ea591e48f9738b404301 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Wed, 21 Aug 2024 11:47:15 +0200 Subject: [PATCH 02/11] Split E2E tests by category into 3 jobs (#3849) Co-authored-by: Oleksandr Didyk <106967057+oleksandr-didyk@users.noreply.github.com> --- eng/templates/jobs/e2e-tests.yml | 131 ++++++++++++++++++ eng/templates/stages/deploy.yaml | 121 +++------------- .../ScenarioTests_AzDoFlow.cs | 3 +- .../ScenarioTests_Clone.cs | 1 + .../ScenarioTests_Dependencies.cs | 1 + .../ScenarioTests_GitHubFlow.cs | 2 +- .../ScenarioTests_Subscriptions.cs | 1 + 7 files changed, 159 insertions(+), 101 deletions(-) create mode 100644 eng/templates/jobs/e2e-tests.yml diff --git a/eng/templates/jobs/e2e-tests.yml b/eng/templates/jobs/e2e-tests.yml new file mode 100644 index 0000000000..a1aec97511 --- /dev/null +++ b/eng/templates/jobs/e2e-tests.yml @@ -0,0 +1,131 @@ +parameters: +- name: isProd + type: boolean +- name: runAuthTests + type: boolean + default: false +- name: name + type: string +- name: displayName + type: string +- name: testFilter + type: string + +jobs: +- job: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + timeoutInMinutes: 60 + variables: + # https://dev.azure.com/dnceng/internal/_library?itemType=VariableGroups&view=VariableGroupView&variableGroupId=20&path=Publish-Build-Assets + # Required for MaestroAppClientId, MaestroStagingAppClientId + - group: Publish-Build-Assets + - ${{ if parameters.isProd }}: + - group: MaestroProd KeyVault + - name: MaestroTestEndpoints + value: https://maestro-prod.westus2.cloudapp.azure.com,https://maestro.dot.net + - name: ScenarioTestSubscription + value: "Darc: Maestro Production" + - name: MaestroAppId + value: $(MaestroAppClientId) + - ${{ else }}: + - group: MaestroInt KeyVault + - name: MaestroTestEndpoints + value: https://maestro-int.westus2.cloudapp.azure.com,https://maestro.int-dot.net + - name: ScenarioTestSubscription + value: "Darc: Maestro Staging" + - name: MaestroAppId + value: $(MaestroStagingAppClientId) + steps: + - download: current + displayName: Download Darc + artifact: PackageArtifacts + + - download: current + displayName: Download ScenarioTets + artifact: Maestro.ScenarioTests + + - task: NuGetToolInstaller@1 + displayName: Use NuGet + inputs: + versionSpec: 5.3.x + + - powershell: | + . .\eng\common\tools.ps1 + InitializeDotNetCli -install:$true + .\.dotnet\dotnet workload install aspire + displayName: Install .NET and Aspire Workload + + - powershell: .\eng\common\build.ps1 -restore + displayName: Install .NET + + - powershell: | + mkdir darc + .\.dotnet\dotnet tool install Microsoft.DotNet.Darc --prerelease --tool-path .\darc --add-source $(Pipeline.Workspace)\PackageArtifacts + displayName: Install Darc + + - task: AzureCLI@2 + name: GetAuthInfo + displayName: Get auth information + inputs: + azureSubscription: ${{ variables.ScenarioTestSubscription }} + addSpnToEnvironment: true + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + # Fetch token used for scenario tests + $token = (az account get-access-token --resource "$(MaestroAppId)" | ConvertFrom-Json).accessToken + echo "##vso[task.setvariable variable=Token;isOutput=true;isSecret=true]$token" + + # Set variables with auth info for tests below + echo "##vso[task.setvariable variable=ServicePrincipalId;isOutput=true]$env:servicePrincipalId" + echo "##vso[task.setvariable variable=FederatedToken;isOutput=true;isSecret=true]$env:idToken" + echo "##vso[task.setvariable variable=TenantId;isOutput=true]$env:tenantId" + + # Either of the URIs will do + $barUri = "${{ split(variables.MaestroTestEndpoints, ',')[0] }}" + echo "##vso[task.setvariable variable=BarUri;isOutput=true]$barUri" + + - ${{ if parameters.runAuthTests }}: + - powershell: + az login --service-principal -u "$(GetAuthInfo.ServicePrincipalId)" --federated-token "$(GetAuthInfo.FederatedToken)" --tenant "$(GetAuthInfo.TenantId)" --allow-no-subscriptions + + .\darc\darc.exe get-default-channels --source-repo arcade-services --ci --bar-uri "$(GetAuthInfo.BarUri)" --debug + displayName: Test Azure CLI auth + + - powershell: + .\darc\darc.exe get-default-channels --source-repo arcade-services --ci --password "$(GetAuthInfo.Token)" --bar-uri "$(GetAuthInfo.BarUri)" --debug + displayName: Test BAR token auth + + - task: DotNetCoreCLI@2 + displayName: Run E2E tests + inputs: + command: custom + projects: | + $(Pipeline.Workspace)/Maestro.ScenarioTests/Maestro.ScenarioTests.dll + custom: test + arguments: > + --filter "TestCategory=PostDeployment&${{ parameters.testFilter }}" + --no-build + --logger "trx;LogFilePrefix=TestResults-" + --parallel + -- + "RunConfiguration.ResultsDirectory=$(Build.ArtifactStagingDirectory)\TestResults" + RunConfiguration.MapCpuCount=4 + env: + MAESTRO_BASEURIS: ${{ variables.MaestroTestEndpoints }} + MAESTRO_TOKEN: $(GetAuthInfo.Token) + GITHUB_TOKEN: $(maestro-scenario-test-github-token) + AZDO_TOKEN: $(dn-bot-dnceng-build-rw-code-rw-release-rw) + DARC_PACKAGE_SOURCE: $(Pipeline.Workspace)\PackageArtifacts + DARC_DIR: $(Build.SourcesDirectory)\darc + DARC_IS_CI: true + + - task: PublishTestResults@2 + displayName: Publish Core Test Results + condition: succeededOrFailed() + inputs: + testRunner: VSTest + testResultsFiles: '**/TestResults-*' + searchFolder: $(Build.ArtifactStagingDirectory)\TestResults + testRunTitle: ${{ parameters.displayName }} + mergeTestResults: true diff --git a/eng/templates/stages/deploy.yaml b/eng/templates/stages/deploy.yaml index add2329bee..1f9fe168d6 100644 --- a/eng/templates/stages/deploy.yaml +++ b/eng/templates/stages/deploy.yaml @@ -136,106 +136,29 @@ stages: continueOnError: true - stage: validateDeployment - displayName: Validate deployment + displayName: E2E tests dependsOn: - deploy - - variables: - - group: Publish-Build-Assets - - ${{ if parameters.isProd }}: - - group: MaestroProd KeyVault - - name: MaestroTestEndpoints - value: https://maestro-prod.westus2.cloudapp.azure.com,https://maestro.dot.net - - name: ScenarioTestSubscription - value: "Darc: Maestro Production" - - name: MaestroAppId - value: $(MaestroAppClientId) - - ${{ else }}: - - group: MaestroInt KeyVault - - name: MaestroTestEndpoints - value: https://maestro-int.westus2.cloudapp.azure.com,https://maestro.int-dot.net - - name: ScenarioTestSubscription - value: "Darc: Maestro Staging" - - name: MaestroAppId - value: $(MaestroStagingAppClientId) jobs: - - job: scenario - displayName: Scenario tests - timeoutInMinutes: 120 - steps: - - download: current - displayName: Download Darc - artifact: PackageArtifacts - - - download: current - displayName: Download ScenarioTets - artifact: Maestro.ScenarioTests - - - task: NuGetToolInstaller@1 - displayName: Use NuGet - inputs: - versionSpec: 5.3.x - - - powershell: | - . .\eng\common\tools.ps1 - InitializeDotNetCli -install:$true - .\.dotnet\dotnet workload install aspire - displayName: Install .NET and Aspire Workload - - - powershell: .\eng\common\build.ps1 -restore - displayName: Install .NET - - - powershell: | - mkdir darc - .\.dotnet\dotnet tool install Microsoft.DotNet.Darc --prerelease --tool-path .\darc --add-source $(Pipeline.Workspace)\PackageArtifacts - displayName: Install Darc - - - task: AzureCLI@2 - name: GetAuthInfo - displayName: Get auth information - inputs: - azureSubscription: ${{ variables.ScenarioTestSubscription }} - addSpnToEnvironment: true - scriptType: ps - scriptLocation: inlineScript - inlineScript: | - # Fetch token used for scenario tests - $token = (az account get-access-token --resource "$(MaestroAppId)" | ConvertFrom-Json).accessToken - echo "##vso[task.setvariable variable=Token;isOutput=true;isSecret=true]$token" - - # Set variables with auth info for tests below - echo "##vso[task.setvariable variable=ServicePrincipalId;isOutput=true]$env:servicePrincipalId" - echo "##vso[task.setvariable variable=FederatedToken;isOutput=true;isSecret=true]$env:idToken" - echo "##vso[task.setvariable variable=TenantId;isOutput=true]$env:tenantId" - - # Either of the URIs will do - $barUri = "${{ split(variables.MaestroTestEndpoints, ',')[0] }}" - echo "##vso[task.setvariable variable=BarUri;isOutput=true]$barUri" - - - powershell: - az login --service-principal -u "$(GetAuthInfo.ServicePrincipalId)" --federated-token "$(GetAuthInfo.FederatedToken)" --tenant "$(GetAuthInfo.TenantId)" --allow-no-subscriptions - - .\darc\darc.exe get-default-channels --source-repo arcade-services --ci --bar-uri "$(GetAuthInfo.BarUri)" --debug - displayName: Test Azure CLI auth - - - powershell: - .\darc\darc.exe get-default-channels --source-repo arcade-services --ci --password "$(GetAuthInfo.Token)" --bar-uri "$(GetAuthInfo.BarUri)" --debug - displayName: Test BAR token auth - - - task: VSTest@2 - displayName: Maestro Scenario Tests - inputs: - testSelector: testAssemblies - testAssemblyVer2: | - Maestro.ScenarioTests.dll - searchFolder: $(Pipeline.Workspace)/Maestro.ScenarioTests - runInParallel: true - env: - MAESTRO_BASEURIS: ${{ variables.MaestroTestEndpoints }} - MAESTRO_TOKEN: $(GetAuthInfo.Token) - GITHUB_TOKEN: $(maestro-scenario-test-github-token) - AZDO_TOKEN: $(dn-bot-dnceng-build-rw-code-rw-release-rw) - DARC_PACKAGE_SOURCE: $(Pipeline.Workspace)\PackageArtifacts - DARC_DIR: $(Build.SourcesDirectory)\darc - DARC_IS_CI: true + - template: ../jobs/e2e-tests.yml + parameters: + name: scenarioTests_GitHub + displayName: GitHub tests + testFilter: 'TestCategory=GitHub' + isProd: ${{ parameters.isProd }} + + - template: ../jobs/e2e-tests.yml + parameters: + name: scenarioTests_AzDO + displayName: AzDO tests + testFilter: 'TestCategory=AzDO' + isProd: ${{ parameters.isProd }} + + - template: ../jobs/e2e-tests.yml + parameters: + name: scenarioTests_Other + displayName: Other tests + testFilter: 'TestCategory!=GitHub&TestCategory!=AzDO' + isProd: ${{ parameters.isProd }} + runAuthTests: true diff --git a/test/Maestro.ScenarioTests/ScenarioTests_AzDoFlow.cs b/test/Maestro.ScenarioTests/ScenarioTests_AzDoFlow.cs index 75af61725b..9b2e29936c 100644 --- a/test/Maestro.ScenarioTests/ScenarioTests_AzDoFlow.cs +++ b/test/Maestro.ScenarioTests/ScenarioTests_AzDoFlow.cs @@ -13,8 +13,9 @@ namespace Maestro.ScenarioTests; [TestFixture] -[NonParallelizable] [Category("PostDeployment")] +[Category("AzDO")] +[NonParallelizable] internal class ScenarioTests_AzDoFlow : MaestroScenarioTestBase { private readonly IImmutableList _source1Assets; diff --git a/test/Maestro.ScenarioTests/ScenarioTests_Clone.cs b/test/Maestro.ScenarioTests/ScenarioTests_Clone.cs index 5940d09a93..d48c377a34 100644 --- a/test/Maestro.ScenarioTests/ScenarioTests_Clone.cs +++ b/test/Maestro.ScenarioTests/ScenarioTests_Clone.cs @@ -18,6 +18,7 @@ namespace Maestro.ScenarioTests; internal class ScenarioTests_Clone : MaestroScenarioTestBase { [Test] + [Ignore("We no longer use this functionality")] public async Task Darc_CloneRepo() { TestContext.WriteLine("Darc-Clone repo end to end test"); diff --git a/test/Maestro.ScenarioTests/ScenarioTests_Dependencies.cs b/test/Maestro.ScenarioTests/ScenarioTests_Dependencies.cs index f31c68f905..6f7542ef8d 100644 --- a/test/Maestro.ScenarioTests/ScenarioTests_Dependencies.cs +++ b/test/Maestro.ScenarioTests/ScenarioTests_Dependencies.cs @@ -15,6 +15,7 @@ namespace Maestro.ScenarioTests; [TestFixture] [Category("PostDeployment")] +[Category("GitHub")] [Parallelizable] internal class ScenarioTests_Dependencies : MaestroScenarioTestBase { diff --git a/test/Maestro.ScenarioTests/ScenarioTests_GitHubFlow.cs b/test/Maestro.ScenarioTests/ScenarioTests_GitHubFlow.cs index 38849241e4..0355091a4e 100644 --- a/test/Maestro.ScenarioTests/ScenarioTests_GitHubFlow.cs +++ b/test/Maestro.ScenarioTests/ScenarioTests_GitHubFlow.cs @@ -14,8 +14,8 @@ namespace Maestro.ScenarioTests; [TestFixture] -[NonParallelizable] [Category("PostDeployment")] +[Category("GitHub")] [Parallelizable] internal class ScenarioTests_GitHubFlow : MaestroScenarioTestBase { diff --git a/test/Maestro.ScenarioTests/ScenarioTests_Subscriptions.cs b/test/Maestro.ScenarioTests/ScenarioTests_Subscriptions.cs index ad7d0aa077..a41d191a7a 100644 --- a/test/Maestro.ScenarioTests/ScenarioTests_Subscriptions.cs +++ b/test/Maestro.ScenarioTests/ScenarioTests_Subscriptions.cs @@ -15,6 +15,7 @@ namespace Maestro.ScenarioTests; [TestFixture] [Category("PostDeployment")] +[Category("AzDO")] [Parallelizable] internal class ScenarioTests_Subscriptions : MaestroScenarioTestBase { From 385b6ed2e73380794dc944507c894cd6b8ca19ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Wed, 21 Aug 2024 12:57:22 +0200 Subject: [PATCH 03/11] Migrate `Maestro.Web.Tests` to `PCS.Api.Tests` (#3855) --- .../ApiTestConfiguration.cs | 27 + .../BuildController20190116Tests.cs | 253 +++++++ .../BuildController20200914Tests.cs | 116 ++++ .../ChannelsController20180716Tests.cs | 186 ++++++ .../ChannelsController20200220Tests.cs | 190 ++++++ .../DefaultChannelsController20200220Tests.cs | 356 ++++++++++ .../DependencyRegistrationTests.cs | 16 +- .../LoggingConfigurationTests.cs | 224 +++++++ ...roductConstructionService.Api.Tests.csproj | 6 +- .../RepositoryUrlAttributeTests.cs | 42 ++ .../SubscriptionsController20200220Tests.cs | 630 ++++++++++++++++++ .../TestDatabase.cs | 122 ++++ 12 files changed, 2152 insertions(+), 16 deletions(-) create mode 100644 test/ProductConstructionService.Api.Tests/ApiTestConfiguration.cs create mode 100644 test/ProductConstructionService.Api.Tests/BuildController20190116Tests.cs create mode 100644 test/ProductConstructionService.Api.Tests/BuildController20200914Tests.cs create mode 100644 test/ProductConstructionService.Api.Tests/ChannelsController20180716Tests.cs create mode 100644 test/ProductConstructionService.Api.Tests/ChannelsController20200220Tests.cs create mode 100644 test/ProductConstructionService.Api.Tests/DefaultChannelsController20200220Tests.cs create mode 100644 test/ProductConstructionService.Api.Tests/LoggingConfigurationTests.cs create mode 100644 test/ProductConstructionService.Api.Tests/RepositoryUrlAttributeTests.cs create mode 100644 test/ProductConstructionService.Api.Tests/SubscriptionsController20200220Tests.cs create mode 100644 test/ProductConstructionService.Api.Tests/TestDatabase.cs diff --git a/test/ProductConstructionService.Api.Tests/ApiTestConfiguration.cs b/test/ProductConstructionService.Api.Tests/ApiTestConfiguration.cs new file mode 100644 index 0000000000..662088d34f --- /dev/null +++ b/test/ProductConstructionService.Api.Tests/ApiTestConfiguration.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.Hosting; + +namespace ProductConstructionService.Api.Tests; + +public static class ApiTestConfiguration +{ + public static WebApplicationBuilder CreateTestHostBuilder() + { + Environment.SetEnvironmentVariable("ASPNETCORE_ENVIRONMENT", Environments.Staging); + Environment.SetEnvironmentVariable("APPLICATIONINSIGHTS_CONNECTION_STRING", "InstrumentationKey=value1"); + + var builder = WebApplication.CreateBuilder(); + builder.Configuration["VmrPath"] = "vmrPath"; + builder.Configuration["TmpPath"] = "tmpPath"; + builder.Configuration["VmrUri"] = "https://vmr.com/uri"; + builder.Configuration["github-oauth-id"] = "clientId"; + builder.Configuration["github-oauth-secret"] = "clientSecret"; + builder.Configuration["BuildAssetRegistrySqlConnectionString"] = "connectionString"; + builder.Configuration["DataProtection:DataProtectionKeyUri"] = "https://keyvault.azure.com/secret/key"; + builder.Configuration["DataProtection:KeyBlobUri"] = "https://blobs.azure.com/secret/key"; + return builder; + } +} diff --git a/test/ProductConstructionService.Api.Tests/BuildController20190116Tests.cs b/test/ProductConstructionService.Api.Tests/BuildController20190116Tests.cs new file mode 100644 index 0000000000..4e1ebc40af --- /dev/null +++ b/test/ProductConstructionService.Api.Tests/BuildController20190116Tests.cs @@ -0,0 +1,253 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Net; +using FluentAssertions; +using Maestro.Api.Model.v2019_01_16; +using Maestro.Data; +using Microsoft.AspNetCore.Mvc; +using Microsoft.DotNet.Internal.Testing.DependencyInjection.Abstractions; +using Microsoft.DotNet.Internal.Testing.Utility; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Hosting.Internal; +using Microsoft.Extensions.Internal; +using Microsoft.Extensions.Logging; +using ProductConstructionService.Api.Api.v2019_01_16.Controllers; + +namespace ProductConstructionService.Api.Tests; + +[TestFixture, NonParallelizable] +public partial class BuildController20190116Tests +{ + [Test] + public async Task MinimalBuildIsCreatedAndCanRetrieved() + { + using TestData data = await TestData.Default.BuildAsync(); + + var commitHash = "FAKE-COMMIT"; + var account = "FAKE-ACCOUNT"; + var project = "FAKE-PROJECT"; + var buildNumber = "20.5.19.20"; + var repository = "FAKE-REPOSITORY"; + var branch = "FAKE-BRANCH"; + + int id; + { + IActionResult result = await data.Controller.Create(new BuildData + { + Commit = commitHash, + AzureDevOpsAccount = account, + AzureDevOpsProject = project, + AzureDevOpsBuildNumber = buildNumber, + AzureDevOpsRepository = repository, + AzureDevOpsBranch = branch, + }); + + result.Should().BeAssignableTo(); + var objResult = (ObjectResult)result; + objResult.StatusCode.Should().Be((int)HttpStatusCode.Created); + objResult.Value.Should().BeAssignableTo(); + var build = (Build)objResult.Value!; + + id = build.Id; + build.Commit.Should().Be(commitHash); + build.AzureDevOpsAccount.Should().Be(account); + build.AzureDevOpsProject.Should().Be(project); + build.AzureDevOpsBuildNumber.Should().Be(buildNumber); + build.AzureDevOpsRepository.Should().Be(repository); + build.AzureDevOpsBranch.Should().Be(branch); + build.DateProduced.Should().Be(data.Clock.UtcNow); + } + + { + var result = await data.Controller.GetBuild(id); + result.Should().BeAssignableTo(); + var objResult = (ObjectResult)result; + objResult.StatusCode.Should().Be((int)HttpStatusCode.OK); + objResult.Value.Should().BeAssignableTo(); + var build = (Build)objResult.Value!; + build.Commit.Should().Be(commitHash); + build.AzureDevOpsAccount.Should().Be(account); + build.AzureDevOpsProject.Should().Be(project); + build.AzureDevOpsBuildNumber.Should().Be(buildNumber); + build.AzureDevOpsRepository.Should().Be(repository); + build.AzureDevOpsBranch.Should().Be(branch); + build.DateProduced.Should().Be(data.Clock.UtcNow); + } + } + + [Test] + public async Task NonsenseBuildIdReturnsNotFound() + { + using TestData data = await TestData.Default.BuildAsync(); + var result = await data.Controller.GetBuild(-99999); + result.Should().BeAssignableTo(); + ((StatusCodeResult)result).StatusCode.Should().Be((int)HttpStatusCode.NotFound); + } + + [Test] + public async Task BuildWithDependenciesIsRegistered() + { + using TestData data = await TestData.Default.BuildAsync(); + + var commitHash = "FAKE-COMMIT"; + var account = "FAKE-ACCOUNT"; + var project = "FAKE-PROJECT"; + var buildNumber = "20.5.19.20"; + var branch = "FAKE-BRANCH"; + + Build aBuild; + Build bBuild; + { + IActionResult result = await data.Controller.Create(new BuildData + { + Commit = commitHash, + AzureDevOpsAccount = account, + AzureDevOpsProject = project, + AzureDevOpsBuildNumber = buildNumber + ".1", + AzureDevOpsRepository = "A-REPO", + AzureDevOpsBranch = branch, + }); + aBuild = (Build)((ObjectResult)result).Value!; + } + data.Clock.UtcNow += TimeSpan.FromHours(1); + { + IActionResult result = await data.Controller.Create(new BuildData + { + Commit = commitHash, + AzureDevOpsAccount = account, + AzureDevOpsProject = project, + AzureDevOpsBuildNumber = buildNumber + ".2", + AzureDevOpsRepository = "B-REPO", + AzureDevOpsBranch = branch, + }); + bBuild = (Build)((ObjectResult)result).Value!; + } + data.Clock.UtcNow += TimeSpan.FromHours(1); + Build cBuild; + { + int cBuildId; + { + IActionResult result = await data.Controller.Create(new BuildData + { + Commit = commitHash, + AzureDevOpsAccount = account, + AzureDevOpsProject = project, + AzureDevOpsBuildNumber = buildNumber + ".3", + AzureDevOpsRepository = "C-REPO", + AzureDevOpsBranch = branch, + Dependencies = + [ + new BuildRef(aBuild.Id, isProduct: true), + new BuildRef(bBuild.Id, isProduct: true), + ], + }); + result.Should().BeAssignableTo(); + var objResult = (ObjectResult)result; + objResult.StatusCode.Should().Be((int)HttpStatusCode.Created); + objResult.Value.Should().BeAssignableTo(); + cBuild = (Build)objResult.Value!; + cBuildId = cBuild.Id; + } + + { + IActionResult result = await data.Controller.GetBuild(cBuildId); + result.Should().BeAssignableTo(); + var objResult = (ObjectResult)result; + objResult.StatusCode.Should().Be((int)HttpStatusCode.OK); + objResult.Value.Should().BeAssignableTo(); + cBuild = (Build)objResult.Value!; + } + + cBuild.Dependencies.Should().HaveCount(2); + cBuild.Dependencies.Should().Contain(b => b.BuildId == aBuild.Id); + cBuild.Dependencies.Should().Contain(b => b.BuildId == bBuild.Id); + } + } + + [Test] + public async Task BuildGraphIncludesOnlyRelatedBuilds() + { + using TestData data = await TestData.Default.BuildAsync(); + + var commitHash = "FAKE-COMMIT"; + var account = "FAKE-ACCOUNT"; + var project = "FAKE-PROJECT"; + var buildNumber = "20.5.19.20"; + var branch = "FAKE-BRANCH"; + + async Task CreateBuildAsync(string repo, string build, params Build[] dependencies) + { + var inputBuild = new BuildData + { + Commit = commitHash, + AzureDevOpsAccount = account, + AzureDevOpsProject = project, + AzureDevOpsBuildNumber = buildNumber + "." + build, + AzureDevOpsRepository = repo, + AzureDevOpsBranch = branch, + Dependencies = dependencies.Select(d => new BuildRef(d.Id, true)).ToList(), + }; + + return (Build)((ObjectResult)await data.Controller.Create(inputBuild)).Value!; + } + + Build aBuild = await CreateBuildAsync("A-REPO", "1"); + Build bBuild = await CreateBuildAsync("B-REPO", "2"); + Build cBuild = await CreateBuildAsync("C-REPO", "3", aBuild, bBuild); + await CreateBuildAsync("UNRELATED-REPO", "4"); + + BuildGraph graph; + { + IActionResult result = await data.Controller.GetBuildGraph(cBuild.Id); + result.Should().BeAssignableTo(); + var objResult = (ObjectResult)result; + objResult.StatusCode.Should().Be((int)HttpStatusCode.OK); + objResult.Value.Should().BeAssignableTo(); + graph = (BuildGraph)objResult.Value!; + } + + graph.Builds.Should().HaveCount(3); + graph.Builds.Should().ContainKey(aBuild.Id); + graph.Builds.Should().ContainKey(bBuild.Id); + graph.Builds.Should().ContainKey(cBuild.Id); + graph.Builds[cBuild.Id].Dependencies.Select(r => r.BuildId).Should().Contain(aBuild.Id); + graph.Builds[cBuild.Id].Dependencies.Select(r => r.BuildId).Should().Contain(bBuild.Id); + graph.Builds[aBuild.Id].Dependencies.Should().BeEmpty(); + graph.Builds[bBuild.Id].Dependencies.Should().BeEmpty(); + } + + [TestDependencyInjectionSetup] + private static class TestDataConfiguration + { + public static async Task Default(IServiceCollection collection) + { + var connectionString = await SharedData.Database.GetConnectionString(); + + collection.AddLogging(l => l.AddProvider(new NUnitLogger())); + collection.AddSingleton(new HostingEnvironment + { + EnvironmentName = Environments.Development + }); + collection.AddBuildAssetRegistry(options => + { + options.UseSqlServer(connectionString); + options.EnableServiceProviderCaching(false); + }); + } + + public static Func Clock(IServiceCollection collection) + { + collection.AddSingleton(); + return s => (TestClock)s.GetRequiredService(); + } + + public static Func Controller(IServiceCollection collection) + { + collection.AddTransient(); + return s => s.GetRequiredService(); + } + } +} diff --git a/test/ProductConstructionService.Api.Tests/BuildController20200914Tests.cs b/test/ProductConstructionService.Api.Tests/BuildController20200914Tests.cs new file mode 100644 index 0000000000..fe6fa48bb6 --- /dev/null +++ b/test/ProductConstructionService.Api.Tests/BuildController20200914Tests.cs @@ -0,0 +1,116 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Net; +using FluentAssertions; +using Maestro.Api.Model.v2020_02_20; +using Maestro.Data; +using Microsoft.AspNetCore.Mvc; +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.Internal.Testing.DependencyInjection.Abstractions; +using Microsoft.DotNet.Internal.Testing.Utility; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Hosting.Internal; +using Microsoft.Extensions.Internal; +using Microsoft.Extensions.Logging; +using Moq; + +using ProductConstructionService.Api.Api.v2020_02_20.Controllers; +using Commit = Maestro.Api.Model.v2020_02_20.Commit; + +namespace ProductConstructionService.Api.Tests; + +[TestFixture] +public partial class BuildController20200914Tests +{ + private const string Repository = "FAKE-REPOSITORY"; + private const string CommitHash = "FAKE-COMMIT"; + private const string CommitMessage = "FAKE-COMMIT-MESSAGE"; + private const string Account = "FAKE-ACCOUNT"; + private const string Project = "FAKE-PROJECT"; + private const string Branch = "FAKE-BRANCH"; + private const string BuildNumber = "20.9.18.20"; + + [Test] + public async Task CommitIsFound() + { + using TestData data = await TestData.Default.BuildAsync(); + + int id; + { + IActionResult result = await data.Controller.Create(new BuildData + { + Commit = CommitHash, + AzureDevOpsAccount = Account, + AzureDevOpsProject = Project, + AzureDevOpsRepository = Repository, + AzureDevOpsBuildNumber = BuildNumber, + AzureDevOpsBranch = Branch, + GitHubBranch = Branch, + }); + + result.Should().BeAssignableTo(); + var objResult = (ObjectResult)result; + objResult.StatusCode.Should().Be((int)HttpStatusCode.Created); + objResult.Value.Should().BeAssignableTo(); + var build = (Build)objResult.Value!; + + id = build.Id; + build.Commit.Should().Be(CommitHash); + build.AzureDevOpsAccount.Should().Be(Account); + build.AzureDevOpsProject.Should().Be(Project); + build.AzureDevOpsBuildNumber.Should().Be(BuildNumber); + build.AzureDevOpsRepository.Should().Be(Repository); + build.AzureDevOpsBranch.Should().Be(Branch); + } + + { + var resultCommit = await data.Controller.GetCommit(id); + var objResultCommit = (ObjectResult)resultCommit; + objResultCommit.StatusCode.Should().Be((int)HttpStatusCode.OK); + objResultCommit.Value.Should().BeAssignableTo(); + var commit = (Commit)objResultCommit.Value!; + + commit.Message.Should().Be(CommitMessage); + commit.Sha.Should().Be(CommitHash); + commit.Author.Should().Be(Account); + } + } + + [TestDependencyInjectionSetup] + private static class TestDataConfiguration + { + public static async Task Dependencies(IServiceCollection collection) + { + var connectionString = await SharedData.Database.GetConnectionString(); + collection.AddLogging(l => l.AddProvider(new NUnitLogger())); + + var mockIRemoteFactory = new Mock(); + var mockIRemote = new Mock(); + mockIRemoteFactory.Setup(f => f.GetRemoteAsync(Repository, It.IsAny())).ReturnsAsync(mockIRemote.Object); + mockIRemote.Setup(f => f.GetCommitAsync(Repository, CommitHash)).ReturnsAsync(new Microsoft.DotNet.DarcLib.Commit(Account, CommitHash, CommitMessage)); + + collection.AddSingleton(mockIRemote.Object); + collection.AddSingleton(mockIRemoteFactory.Object); + collection.AddSingleton(Mock.Of()); + collection.AddSingleton(new HostingEnvironment + { + EnvironmentName = Environments.Development + }); + collection.AddBuildAssetRegistry(options => + { + options.UseSqlServer(connectionString); + options.EnableServiceProviderCaching(false); + }); + collection.AddSingleton(); + } + + public static Func Controller(IServiceCollection collection) + { + collection.AddTransient(); + return s => s.GetRequiredService(); + } + } +} diff --git a/test/ProductConstructionService.Api.Tests/ChannelsController20180716Tests.cs b/test/ProductConstructionService.Api.Tests/ChannelsController20180716Tests.cs new file mode 100644 index 0000000000..84946bc258 --- /dev/null +++ b/test/ProductConstructionService.Api.Tests/ChannelsController20180716Tests.cs @@ -0,0 +1,186 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Net; +using FluentAssertions; +using Maestro.Api.Model.v2018_07_16; +using Maestro.Data; +using Microsoft.AspNetCore.Mvc; +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.Internal.Testing.DependencyInjection.Abstractions; +using Microsoft.DotNet.Internal.Testing.Utility; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Hosting.Internal; +using Microsoft.Extensions.Internal; +using Microsoft.Extensions.Logging; +using Moq; +using ProductConstructionService.Api.Api.v2018_07_16.Controllers; + +namespace ProductConstructionService.Api.Tests; + +[TestFixture] +public partial class ChannelsController20180716Tests +{ + [Test] + public async Task CreateChannel() + { + using TestData data = await TestData.Default.BuildAsync(); + Channel channel; + var channelName = "TEST-CHANNEL-BASIC-20180716"; + var classification = "TEST-CLASSIFICATION"; + { + IActionResult result = await data.Controller.CreateChannel(channelName, classification); + result.Should().BeAssignableTo(); + var objResult = (ObjectResult)result; + objResult.StatusCode.Should().Be((int)HttpStatusCode.Created); + objResult.Value.Should().BeAssignableTo(); + channel = (Channel)objResult.Value!; + channel.Name.Should().Be(channelName); + channel.Classification.Should().Be(classification); + } + + { + IActionResult result = await data.Controller.GetChannel(channel.Id); + result.Should().BeAssignableTo(); + var objResult = (ObjectResult)result; + objResult.StatusCode.Should().Be((int)HttpStatusCode.OK); + objResult.Value.Should().BeAssignableTo(); + channel = (Channel)objResult.Value!; + channel.Name.Should().Be(channelName); + channel.Classification.Should().Be(classification); + } + } + + [Test] + public async Task ListRepositories() + { + using TestData data = await TestData.Default.BuildAsync(); + var channelName = "TEST-CHANNEL-LIST-REPOSITORIES-20180716"; + var classification = "TEST-CLASSIFICATION"; + var commitHash = "FAKE-COMMIT"; + var buildNumber = "20.5.19.20"; + var repository = "FAKE-REPOSITORY"; + var branch = "FAKE-BRANCH"; + + Channel channel; + { + var result = await data.Controller.CreateChannel(channelName, classification); + channel = (Channel)((ObjectResult)result).Value!; + } + + Build build; + { + IActionResult result = await data.BuildsController.Create(new BuildData + { + Commit = commitHash, + BuildNumber = buildNumber, + Repository = repository, + Branch = branch, + Assets = [] + }); + build = (Build)((ObjectResult)result).Value!; + } + + await data.Controller.AddBuildToChannel(channel.Id, build.Id); + + List repositories; + { + IActionResult result = await data.Controller.ListRepositories(channel.Id); + result.Should().BeAssignableTo(); + var objResult = (ObjectResult)result; + objResult.StatusCode.Should().Be((int)HttpStatusCode.OK); + objResult.Value.Should().BeAssignableTo>(); + repositories = ((IEnumerable)objResult.Value!).ToList(); + } + + repositories.Should().ContainSingle(); + } + + [Test] + public async Task AddingBuildToChannelTwiceWorks() + { + using TestData data = await TestData.Default.BuildAsync(); + const string channelName = "TEST-CHANNEL-ADD-TWICE-2018"; + const string classification = "TEST-CLASSIFICATION"; + const string commitHash = "FAKE-COMMIT"; + const string buildNumber = "20.5.19.20"; + const string repository = "FAKE-REPOSITORY"; + const string branch = "FAKE-BRANCH"; + + Channel channel; + { + var result = await data.Controller.CreateChannel(channelName, classification); + channel = (Channel)((ObjectResult)result).Value!; + } + + Build build; + { + IActionResult result = await data.BuildsController.Create(new BuildData + { + Commit = commitHash, + BuildNumber = buildNumber, + Repository = repository, + Branch = branch, + Assets = [], + }); + build = (Build)((ObjectResult)result).Value!; + } + + { + IActionResult result = await data.Controller.AddBuildToChannel(channel.Id, build.Id); + result.Should().BeAssignableTo(); + var objResult = (StatusCodeResult)result; + objResult.StatusCode.Should().Be((int)HttpStatusCode.Created); + } + + { + IActionResult result = await data.Controller.AddBuildToChannel(channel.Id, build.Id); + result.Should().BeAssignableTo(); + var objResult = (StatusCodeResult)result; + objResult.StatusCode.Should().Be((int)HttpStatusCode.Created); + } + } + + [TestDependencyInjectionSetup] + private static class TestDataConfiguration + { + public static async Task Dependencies(IServiceCollection collection) + { + var connectionString = await SharedData.Database.GetConnectionString(); + collection.AddLogging(l => l.AddProvider(new NUnitLogger())); + collection.AddSingleton(new HostingEnvironment + { + EnvironmentName = Environments.Development + }); + collection.AddBuildAssetRegistry(options => + { + options.UseSqlServer(connectionString); + options.EnableServiceProviderCaching(false); + }); + collection.AddSingleton(); + collection.AddSingleton(); + collection.AddSingleton(Mock.Of()); + collection.AddSingleton(Mock.Of()); + } + + public static Func Clock(IServiceCollection collection) + { + collection.AddSingleton(); + return s => (TestClock)s.GetRequiredService(); + } + + public static Func Controller(IServiceCollection collection) + { + collection.AddSingleton(); + return s => s.GetRequiredService(); + } + + public static Func BuildsController(IServiceCollection collection) + { + collection.AddSingleton(); + return s => s.GetRequiredService(); + } + } +} diff --git a/test/ProductConstructionService.Api.Tests/ChannelsController20200220Tests.cs b/test/ProductConstructionService.Api.Tests/ChannelsController20200220Tests.cs new file mode 100644 index 0000000000..e6f0dc080e --- /dev/null +++ b/test/ProductConstructionService.Api.Tests/ChannelsController20200220Tests.cs @@ -0,0 +1,190 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Net; +using FluentAssertions; +using Maestro.Api.Model.v2020_02_20; +using Maestro.Data; +using Microsoft.AspNetCore.Mvc; +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.Internal.Testing.DependencyInjection.Abstractions; +using Microsoft.DotNet.Internal.Testing.Utility; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Hosting.Internal; +using Microsoft.Extensions.Internal; +using Microsoft.Extensions.Logging; +using Moq; +using ProductConstructionService.Api.Api.v2020_02_20.Controllers; + +namespace ProductConstructionService.Api.Tests; + +[TestFixture] +public partial class ChannelsController20200220Tests +{ + [Test] + public async Task CreateChannel() + { + using TestData data = await TestData.Default.BuildAsync(); + Channel channel; + var channelName = "TEST-CHANNEL-BASIC"; + var classification = "TEST-CLASSIFICATION"; + { + IActionResult result = await data.Controller.CreateChannel(channelName, classification); + result.Should().BeAssignableTo(); + var objResult = (ObjectResult)result; + objResult.StatusCode.Should().Be((int)HttpStatusCode.Created); + objResult.Value.Should().BeAssignableTo(); + channel = (Channel)objResult.Value!; + channel.Name.Should().Be(channelName); + channel.Classification.Should().Be(classification); + } + + { + IActionResult result = await data.Controller.GetChannel(channel.Id); + result.Should().BeAssignableTo(); + var objResult = (ObjectResult)result; + objResult.StatusCode.Should().Be((int)HttpStatusCode.OK); + objResult.Value.Should().BeAssignableTo(); + channel = (Channel)objResult.Value!; + channel.Name.Should().Be(channelName); + channel.Classification.Should().Be(classification); + } + } + + [Test] + public async Task ListRepositories() + { + using TestData data = await TestData.Default.BuildAsync(); + var channelName = "TEST-CHANNEL-LIST-REPOSITORIES"; + var classification = "TEST-CLASSIFICATION"; + var commitHash = "FAKE-COMMIT"; + var account = "FAKE-ACCOUNT"; + var project = "FAKE-PROJECT"; + var buildNumber = "20.5.19.20"; + var repository = "FAKE-REPOSITORY"; + var branch = "FAKE-BRANCH"; + + Channel channel; + { + var result = await data.Controller.CreateChannel(channelName, classification); + channel = (Channel)((ObjectResult)result).Value!; + } + + Build build; + { + IActionResult result = await data.BuildsController.Create(new BuildData + { + Commit = commitHash, + AzureDevOpsAccount = account, + AzureDevOpsProject = project, + AzureDevOpsBuildNumber = buildNumber, + AzureDevOpsRepository = repository, + AzureDevOpsBranch = branch, + }); + build = (Build)((ObjectResult)result).Value!; + } + + await data.Controller.AddBuildToChannel(channel.Id, build.Id); + + List repositories; + { + IActionResult result = await data.Controller.ListRepositories(channel.Id); + result.Should().BeAssignableTo(); + var objResult = (ObjectResult)result; + objResult.StatusCode.Should().Be((int)HttpStatusCode.OK); + objResult.Value.Should().BeAssignableTo>(); + repositories = ((IEnumerable)objResult.Value!).ToList(); + } + + repositories.Should().ContainSingle(); + } + + [Test] + public async Task AddingBuildToChannelTwiceWorks() + { + using TestData data = await TestData.Default.BuildAsync(); + const string channelName = "TEST-CHANNEL-ADD-TWICE-2020"; + const string classification = "TEST-CLASSIFICATION"; + const string commitHash = "FAKE-COMMIT"; + const string account = "FAKE-ACCOUNT"; + const string project = "FAKE-PROJECT"; + const string buildNumber = "20.5.19.20"; + const string repository = "FAKE-REPOSITORY"; + const string branch = "FAKE-BRANCH"; + + Channel channel; + { + var result = await data.Controller.CreateChannel(channelName, classification); + channel = (Channel)((ObjectResult)result).Value!; + } + + Build build; + { + IActionResult result = await data.BuildsController.Create(new BuildData + { + Commit = commitHash, + AzureDevOpsAccount = account, + AzureDevOpsProject = project, + AzureDevOpsBuildNumber = buildNumber, + AzureDevOpsRepository = repository, + AzureDevOpsBranch = branch, + }); + build = (Build)((ObjectResult)result).Value!; + } + + { + IActionResult result = await data.Controller.AddBuildToChannel(channel.Id, build.Id); + result.Should().BeAssignableTo(); + var objResult = (StatusCodeResult)result; + objResult.StatusCode.Should().Be((int)HttpStatusCode.Created); + } + + { + IActionResult result = await data.Controller.AddBuildToChannel(channel.Id, build.Id); + result.Should().BeAssignableTo(); + var objResult = (StatusCodeResult)result; + objResult.StatusCode.Should().Be((int)HttpStatusCode.Created); + } + } + + [TestDependencyInjectionSetup] + private static class TestDataConfiguration + { + public static async Task Dependencies(IServiceCollection collection) + { + var connectionString = await SharedData.Database.GetConnectionString(); + collection.AddLogging(l => l.AddProvider(new NUnitLogger())); + collection.AddSingleton(new HostingEnvironment + { + EnvironmentName = Environments.Development + }); + collection.AddBuildAssetRegistry(options => + { + options.UseSqlServer(connectionString); + options.EnableServiceProviderCaching(false); + }); + collection.AddSingleton(Mock.Of()); + collection.AddSingleton(Mock.Of()); + } + + public static Func Clock(IServiceCollection collection) + { + collection.AddSingleton(); + return s => (TestClock)s.GetRequiredService(); + } + + public static Func Controller(IServiceCollection collection) + { + collection.AddSingleton(); + return s => s.GetRequiredService(); + } + + public static Func BuildsController(IServiceCollection collection) + { + collection.AddSingleton(); + return s => s.GetRequiredService(); + } + } +} diff --git a/test/ProductConstructionService.Api.Tests/DefaultChannelsController20200220Tests.cs b/test/ProductConstructionService.Api.Tests/DefaultChannelsController20200220Tests.cs new file mode 100644 index 0000000000..c5eb1ddf43 --- /dev/null +++ b/test/ProductConstructionService.Api.Tests/DefaultChannelsController20200220Tests.cs @@ -0,0 +1,356 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Net; +using FluentAssertions; +using Maestro.Api.Model.v2020_02_20; +using Maestro.Data; +using Microsoft.AspNetCore.Mvc; +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.Internal.Testing.DependencyInjection.Abstractions; +using Microsoft.DotNet.Internal.Testing.Utility; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Hosting.Internal; +using Microsoft.Extensions.Internal; +using Microsoft.Extensions.Logging; +using Moq; + +using ProductConstructionService.Api.Api.v2020_02_20.Controllers; +using static Maestro.Api.Model.v2020_02_20.DefaultChannel; + +namespace ProductConstructionService.Api.Tests; + +[TestFixture] +public partial class DefaultChannelsController20200220Tests +{ + [Test] + public async Task CreateAndGetDefaultChannel() + { + using TestData data = await TestData.Default.BuildAsync(); + var channelName = "TEST-CHANNEL-LIST-REPOSITORIES"; + var classification = "TEST-CLASSIFICATION"; + var repository = "FAKE-REPOSITORY"; + var branch = "FAKE-BRANCH"; + + Channel channel1, channel2; + { + var result = await data.ChannelsController.CreateChannel($"{channelName}-1", classification); + channel1 = (Channel)((ObjectResult)result).Value!; + result = await data.ChannelsController.CreateChannel($"{channelName}-2", classification); + channel2 = (Channel)((ObjectResult)result).Value!; + } + + DefaultChannel defaultChannel; + { + var testDefaultChannelData = new DefaultChannelCreateData() + { + Branch = branch, + ChannelId = channel2.Id, + Enabled = true, + Repository = repository + }; + var result = await data.DefaultChannelsController.Create(testDefaultChannelData); + defaultChannel = (DefaultChannel)((ObjectResult)result).Value!; + } + + DefaultChannel singleChannelGetDefaultChannel; + { + IActionResult result = await data.DefaultChannelsController.Get(defaultChannel.Id); + result.Should().BeAssignableTo(); + var objResult = (ObjectResult)result; + objResult.StatusCode.Should().Be((int)HttpStatusCode.OK); + objResult.Value.Should().BeAssignableTo(); + singleChannelGetDefaultChannel = (DefaultChannel)objResult.Value!; + } + singleChannelGetDefaultChannel.Id.Should().Be(defaultChannel.Id); + + List listOfInsertedDefaultChannels; + { + IActionResult result = data.DefaultChannelsController.List(repository, branch, channel2.Id); + result.Should().BeAssignableTo(); + var objResult = (ObjectResult)result; + objResult.StatusCode.Should().Be((int)HttpStatusCode.OK); + objResult.Value.Should().BeAssignableTo>(); + listOfInsertedDefaultChannels = ((IEnumerable)objResult.Value!).ToList(); + } + + listOfInsertedDefaultChannels.Should().ContainSingle(); + listOfInsertedDefaultChannels.Single().Channel.Id.Should().Be(channel2.Id, "Only fake channel #2's id should show up as a default channel"); + } + + [Test] + public async Task UpdateDefaultChannel() + { + using TestData data = await TestData.Default.BuildAsync(); + var channelName = "TEST-CHANNEL-TO-UPDATE"; + var classification = "TEST-CLASSIFICATION"; + var repository = "FAKE-REPOSITORY"; + var branch = "FAKE-BRANCH"; + + Channel channel1, channel2; + { + var result = await data.ChannelsController.CreateChannel($"{channelName}-1", classification); + channel1 = (Channel)((ObjectResult)result).Value!; + result = await data.ChannelsController.CreateChannel($"{channelName}-2", classification); + channel2 = (Channel)((ObjectResult)result).Value!; + } + + DefaultChannel defaultChannel; + { + var testDefaultChannelData = new DefaultChannelCreateData() + { + Branch = branch, + ChannelId = channel1.Id, + Enabled = true, + Repository = repository + }; + var result = await data.DefaultChannelsController.Create(testDefaultChannelData); + defaultChannel = (DefaultChannel)((ObjectResult)result).Value!; + } + + DefaultChannel updatedDefaultChannel; + { + var defaultChannelUpdateData = new DefaultChannelUpdateData() + { + Branch = $"{branch}-UPDATED", + ChannelId = channel2.Id, + Enabled = false, + Repository = $"NEW-{repository}" + }; + var result = await data.DefaultChannelsController.Update(defaultChannel.Id, defaultChannelUpdateData); + updatedDefaultChannel = (DefaultChannel)((ObjectResult)result).Value!; + } + + List defaultChannels; + { + IActionResult result = data.DefaultChannelsController.List($"NEW-{repository}", $"{branch}-UPDATED", channel2.Id, false); + result.Should().BeAssignableTo(); + var objResult = (ObjectResult)result; + objResult.StatusCode.Should().Be((int)HttpStatusCode.OK); + objResult.Value.Should().BeAssignableTo>(); + defaultChannels = ((IEnumerable)objResult.Value!).ToList(); + } + + defaultChannels.Should().ContainSingle(); + defaultChannels.Single().Channel.Id.Should().Be(channel2.Id, "Only fake channel #2's id should show up as a default channel"); + } + + [Test] + public async Task DefaultChannelRegularExpressionMatching() + { + using TestData data = await TestData.Default.BuildAsync(); + var channelName = "TEST-CHANNEL-REGEX-FOR-DEFAULT"; + var classification = "TEST-CLASSIFICATION"; + var repository = "FAKE-REPOSITORY"; + var branch = "-regex:FAKE-BRANCH-REGEX-.*"; + + Channel channel; + { + var result = await data.ChannelsController.CreateChannel($"{channelName}", classification); + channel = (Channel)((ObjectResult)result).Value!; + } + + DefaultChannel defaultChannel; + { + var testDefaultChannelData = new DefaultChannelCreateData() + { + Branch = branch, + ChannelId = channel.Id, + Enabled = true, + Repository = repository + }; + var result = await data.DefaultChannelsController.Create(testDefaultChannelData); + defaultChannel = (DefaultChannel)((ObjectResult)result).Value!; + } + + string[] branchesThatMatch = ["FAKE-BRANCH-REGEX-", "FAKE-BRANCH-REGEX-RELEASE-BRANCH-1", "FAKE-BRANCH-REGEX-RELEASE-BRANCH-2"]; + string[] branchesThatDontMatch = ["I-DONT-MATCH", "REAL-BRANCH-REGEX"]; + + foreach (var branchName in branchesThatMatch) + { + List defaultChannels; + { + IActionResult result = data.DefaultChannelsController.List(repository, branchName, channel.Id); + result.Should().BeAssignableTo(); + var objResult = (ObjectResult)result; + objResult.StatusCode.Should().Be((int)HttpStatusCode.OK); + objResult.Value.Should().BeAssignableTo>(); + defaultChannels = ((IEnumerable)objResult.Value!).ToList(); + } + defaultChannels.Should().ContainSingle(); + defaultChannels.Single().Channel.Id.Should().Be(channel.Id); + } + + foreach (var branchName in branchesThatDontMatch) + { + List defaultChannels; + { + IActionResult result = data.DefaultChannelsController.List(repository, branchName, channel.Id); + result.Should().BeAssignableTo(); + var objResult = (ObjectResult)result; + objResult.StatusCode.Should().Be((int)HttpStatusCode.OK); + objResult.Value.Should().BeAssignableTo>(); + defaultChannels = ((IEnumerable)objResult.Value!).ToList(); + } + defaultChannels.Should().BeEmpty(); + } + } + + [Test] + public async Task TryToAddNonExistentChannel() + { + using TestData data = await TestData.Default.BuildAsync(); + var repository = "FAKE-REPOSITORY"; + var branch = "FAKE-BRANCH"; + + var testDefaultChannelData = new DefaultChannelCreateData() + { + Branch = branch, + ChannelId = 404, + Enabled = true, + Repository = repository + }; + var result = await data.DefaultChannelsController.Create(testDefaultChannelData); + result.Should().BeOfType("Asking for a non-existent channel should give a not-found-object type result"); + } + + [Test] + public async Task TryToGetOrUpdateNonExistentChannel() + { + var channelName = "TEST-CHANNEL-TO-UPDATE"; + var classification = "TEST-CLASSIFICATION"; + var repository = "FAKE-NON-EXISTENT-REPOSITORY-MISSING-CHANNEL-UPDATE"; + var branch = "FAKE-BRANCH-MISSING-CHANNEL-UPDATE"; + + using TestData data = await TestData.Default.BuildAsync(); + var defaultChannelThatDoesntExistUpdateData = new DefaultChannelUpdateData() + { + Branch = branch, + ChannelId = 404, + Enabled = false, + Repository = repository + }; + // First: non-existent default channel + var expectedFailResult = await data.DefaultChannelsController.Update(404, defaultChannelThatDoesntExistUpdateData); + expectedFailResult.Should().BeOfType("Asking for a non-existent channel should give a not-found type result"); + + // Second: Extant default, non-existent channel. + Channel channel; + { + var result = await data.ChannelsController.CreateChannel(channelName, classification); + channel = (Channel)((ObjectResult)result).Value!; + } + + DefaultChannel defaultChannel; + { + var testDefaultChannelData = new DefaultChannelCreateData() + { + Branch = branch, + ChannelId = channel.Id, + Enabled = true, + Repository = repository + }; + var result = await data.DefaultChannelsController.Create(testDefaultChannelData); + defaultChannel = (DefaultChannel)((ObjectResult)result).Value!; + } + + var defaultChannelUpdateData = new DefaultChannelUpdateData() + { + Branch = $"{branch}-UPDATED", + ChannelId = 404, + Enabled = false, + Repository = $"NEW-{repository}" + }; + var secondExpectedFailResult = await data.DefaultChannelsController.Update(defaultChannel.Id, defaultChannelUpdateData); + secondExpectedFailResult.Should().BeOfType("Updating a default channel for a non-existent channel should give a not-found type result"); + // Try to get a default channel that just doesn't exist at all. + var thirdExpectedFailResult = await data.DefaultChannelsController.Get(404); + thirdExpectedFailResult.Should().BeOfType("Getting a default channel for a non-existent default channel should give a not-found type result"); + } + + [Test] + public async Task AddDuplicateDefaultChannels() + { + using TestData data = await TestData.Default.BuildAsync(); + var channelName = "TEST-CHANNEL-DUPLICATE-ENTRY-SCENARIO"; + var classification = "TEST-CLASSIFICATION"; + var repository = "FAKE-REPOSITORY"; + var branch = "FAKE-BRANCH"; + + Channel channel; + { + var result = await data.ChannelsController.CreateChannel(channelName, classification); + channel = (Channel)((ObjectResult)result).Value!; + } + + var testDefaultChannelData = new DefaultChannelCreateData() + { + Branch = branch, + ChannelId = channel.Id, + Enabled = true, + Repository = repository + }; + + DefaultChannel defaultChannel; + { + var result = await data.DefaultChannelsController.Create(testDefaultChannelData); + defaultChannel = (DefaultChannel)((ObjectResult)result).Value!; + } + + defaultChannel.Should().NotBeNull(); + + DefaultChannel defaultChannelDuplicateAdd; + { + var result = await data.DefaultChannelsController.Create(testDefaultChannelData); + defaultChannelDuplicateAdd = (DefaultChannel)((ObjectResult)result).Value!; + } + + // Adding the same thing twice should succeed, as well as provide the correct object in return. + defaultChannelDuplicateAdd.Should().BeEquivalentTo(defaultChannel); + } + + [TestDependencyInjectionSetup] + private static class TestDataConfiguration + { + public static async Task Dependencies(IServiceCollection collection) + { + var connectionString = await SharedData.Database.GetConnectionString(); + + collection.AddLogging(l => l.AddProvider(new NUnitLogger())); + collection.AddSingleton(new HostingEnvironment + { + EnvironmentName = Environments.Development + }); + collection.AddBuildAssetRegistry(options => + { + options.UseSqlServer(connectionString); + options.EnableServiceProviderCaching(false); + }); + collection.AddSingleton(); + collection.AddSingleton(); + collection.AddSingleton(Mock.Of()); + collection.AddSingleton(Mock.Of()); + } + + public static Func Clock(IServiceCollection collection) + { + collection.AddSingleton(); + return s => (TestClock)s.GetRequiredService(); + } + + public static Func ChannelsController(IServiceCollection collection) + { + collection.AddSingleton(); + return s => s.GetRequiredService(); + } + + public static Func DefaultChannelsController( + IServiceCollection collection) + { + collection.AddSingleton(); + return s => s.GetRequiredService(); + } + } +} diff --git a/test/ProductConstructionService.Api.Tests/DependencyRegistrationTests.cs b/test/ProductConstructionService.Api.Tests/DependencyRegistrationTests.cs index 1b769568de..8e04ffe0ef 100644 --- a/test/ProductConstructionService.Api.Tests/DependencyRegistrationTests.cs +++ b/test/ProductConstructionService.Api.Tests/DependencyRegistrationTests.cs @@ -2,11 +2,9 @@ // The .NET Foundation licenses this file to you under the MIT license. using FluentAssertions; -using Microsoft.AspNetCore.Builder; using Microsoft.DotNet.Internal.DependencyInjection.Testing; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; -using Microsoft.Extensions.Hosting; namespace ProductConstructionService.Api.Tests; @@ -15,19 +13,7 @@ public class DependencyRegistrationTests [Test] public async Task AreDependenciesRegistered() { - Environment.SetEnvironmentVariable("ASPNETCORE_ENVIRONMENT", Environments.Staging); - - var builder = WebApplication.CreateBuilder(); - - builder.Configuration["VmrPath"] = "vmrPath"; - builder.Configuration["TmpPath"] = "tmpPath"; - builder.Configuration["VmrUri"] = "https://vmr.com/uri"; - builder.Configuration["github-oauth-id"] = "clientId"; - builder.Configuration["github-oauth-secret"] = "clientSecret"; - builder.Configuration["BuildAssetRegistrySqlConnectionString"] = "connectionString"; - builder.Configuration["DataProtection:DataProtectionKeyUri"] = "https://keyvault.azure.com/secret/key"; - builder.Configuration["DataProtection:KeyBlobUri"] = "https://blobs.azure.com/secret/key"; - + var builder = ApiTestConfiguration.CreateTestHostBuilder(); await builder.ConfigurePcs( addKeyVault: false, addRedis: false, diff --git a/test/ProductConstructionService.Api.Tests/LoggingConfigurationTests.cs b/test/ProductConstructionService.Api.Tests/LoggingConfigurationTests.cs new file mode 100644 index 0000000000..167931ee67 --- /dev/null +++ b/test/ProductConstructionService.Api.Tests/LoggingConfigurationTests.cs @@ -0,0 +1,224 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using FluentAssertions; +using Microsoft.ApplicationInsights; +using Microsoft.ApplicationInsights.Channel; +using Microsoft.ApplicationInsights.DataContracts; +using Microsoft.AspNetCore.Builder; +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.Internal.Logging; +using Microsoft.DotNet.ServiceFabric.ServiceHost; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.Services.Common; +using Moq; +using ProductConstructionService.Api.Telemetry; + +namespace ProductConstructionService.Api.Tests; + +[TestFixture, NonParallelizable] +public class LoggingConfigurationTests +{ + private class TestData + : IDisposable + { + private readonly ServiceProvider _outerProvider; + public ILogger Logger { get; } + public OperationManager OperationManager { get; } + public TelemetryClient TelemetryClient { get; } + public List TelemetryLogged { get; } + public ServiceProvider Provider { get; } + + public TestData( + ILogger logger, + OperationManager operationManager, + TelemetryClient telemetryClient, + List telemetryLogged, + ServiceProvider provider, + ServiceProvider outerProvider) + { + _outerProvider = outerProvider; + Logger = logger; + OperationManager = operationManager; + TelemetryClient = telemetryClient; + TelemetryLogged = telemetryLogged; + Provider = provider; + } + + public void Dispose() + { + Provider.Dispose(); + _outerProvider.Dispose(); + } + } + + private sealed class TrackedDisposable : IDisposable + { + public bool Disposed { get; private set; } + + public void Dispose() + { + Disposed = true; + } + } + + private static async Task Setup() + { + var channel = new Mock(); + var telemetry = new List(); + channel.Setup(s => s.Send(Capture.In(telemetry))); + + var builder = ApiTestConfiguration.CreateTestHostBuilder(); + + // The only scenario we are worried about is when running in the ServiceHost + ServiceHost.ConfigureDefaultServices(builder.Services); + + await builder.ConfigurePcs( + addKeyVault: false, + addRedis: false, + addSwagger: false); + + builder.Services.AddSingleton(channel.Object); + builder.Services.AddSingleton(); + builder.Services.AddScoped(); + builder.Services.AddSingleton(); + + ServiceProvider outerProvider = builder.Services.BuildServiceProvider(); + ServiceProvider innerProvider = builder.Services.BuildServiceProvider(); + + var logger = innerProvider.GetRequiredService>(); + var operations = innerProvider.GetRequiredService(); + var tc = innerProvider.GetRequiredService(); + + Activity.Current = null; + // AppInsights behaves very oddly if the ActivityId is W3C + // It's not ideal for a test to mess with static state, but we need to ensure this works correctly + Activity.DefaultIdFormat = ActivityIdFormat.Hierarchical; + + return new TestData(logger, operations, tc, telemetry, innerProvider, outerProvider); + } + + [Test] + public async Task FullScopeDisposesScopedDependencies() + { + using TestData data = await Setup(); + TrackedDisposable toDispose; + using (var op = data.OperationManager.BeginOperation("TEST-SCOPE:{TEST_KEY}", "TEST_VALUE")) + { + toDispose = op.ServiceProvider.GetRequiredService(); + } + + toDispose.Disposed.Should().BeTrue(); + } + + [Test] + public async Task LoggingScopeDoesNotDisposeScopedDependencies() + { + using TestData data = await Setup(); + TrackedDisposable toDispose; + using (var op = data.OperationManager.BeginLoggingScope("TEST-SCOPE:{TEST_KEY}", "TEST_VALUE")) + { + toDispose = op.ServiceProvider.GetRequiredService(); + } + + toDispose.Disposed.Should().BeFalse(); + } + + [Test] + public async Task LoggingWithLoggingScopes() + { + using TestData data = await Setup(); + using (data.TelemetryClient.StartOperation("Fake operation")) + { + data.Logger.LogError("Outside"); + using (var op = data.OperationManager.BeginLoggingScope("TEST-SCOPE:{TEST_KEY}", "TEST_VALUE")) + { + data.Logger.LogError("Something: {TEST_SOMETHING_KEY}", "TEST_SOMETHING_VALUE"); + data.Logger.LogError("Else"); + } + + data.Logger.LogError("Outside again"); + } + + data.TelemetryClient.Flush(); + var traces = data.TelemetryLogged.OfType().ToList(); + traces.Should().HaveCount(4); + + { + // The operation id should stay constant, it's the root + var opIds = traces.Select(t => t.Context?.Operation?.Id).ToArray(); + opIds[0].Should().NotBeNull(); + opIds[1].Should().Be(opIds[0]); + opIds[2].Should().Be(opIds[1]); + opIds[3].Should().Be(opIds[2]); + } + + { + // The parent ids should flow with the operation start/stop + var parentIds = traces.Select(t => t.Context?.Operation?.ParentId).ToArray(); + parentIds[0].Should().NotBeNull(); + parentIds[1].Should().NotBe(parentIds[0]); + parentIds[1].Should().StartWith(parentIds[0]); + parentIds[2].Should().Be(parentIds[1]); + parentIds[3].Should().NotBe(parentIds[2]); + parentIds[3].Should().Be(parentIds[0]); + } + + // The things in the operation should flow the properties from the BeginOperation + traces[1].Properties.GetValueOrDefault("TEST_KEY").Should().Be("TEST_VALUE"); + + // The things outside the operation should not have those properties + traces[3].Properties.Should().NotContainKey("TEST_VALUE"); + } + + [Test] + public async Task LoggingWithFullScopes() + { + using TestData data = await Setup(); + using (data.TelemetryClient.StartOperation("Fake operation")) + { + data.Logger.LogError("Outside"); + using (var op = data.OperationManager.BeginOperation("TEST-SCOPE:{TEST_KEY}", "TEST_VALUE")) + { + data.Logger.LogError("Something: {TEST_SOMETHING_KEY}", "TEST_SOMETHING_VALUE"); + data.Logger.LogError("Else"); + } + + data.Logger.LogError("Outside again"); + } + + data.TelemetryClient.Flush(); + var traces = data.TelemetryLogged.OfType().ToList(); + traces.Should().HaveCount(4); + + { + // The operation id should stay constant, it's the root + var opIds = traces.Select(t => t.Context?.Operation?.Id).ToArray(); + opIds[0].Should().NotBeNull(); + opIds[1].Should().Be(opIds[0]); + opIds[2].Should().Be(opIds[1]); + opIds[3].Should().Be(opIds[2]); + } + + { + // The parent ids should flow with the operation start/stop + var parentIds = traces.Select(t => t.Context?.Operation?.ParentId).ToArray(); + parentIds[0].Should().NotBeNull(); + parentIds[1].Should().NotBe(parentIds[0]); + parentIds[1].Should().StartWith(parentIds[0]); + parentIds[2].Should().Be(parentIds[1]); + parentIds[3].Should().NotBe(parentIds[2]); + parentIds[3].Should().Be(parentIds[0]); + } + + // The things in the operation should flow the properties from the BeginOperation + traces[1].Properties.GetValueOrDefault("TEST_KEY").Should().Be("TEST_VALUE"); + + // The things outside the operation should not have those properties + traces[3].Properties.Should().NotContainKey("TEST_VALUE"); + } +} diff --git a/test/ProductConstructionService.Api.Tests/ProductConstructionService.Api.Tests.csproj b/test/ProductConstructionService.Api.Tests/ProductConstructionService.Api.Tests.csproj index 6e73600bc5..79acb78c8d 100644 --- a/test/ProductConstructionService.Api.Tests/ProductConstructionService.Api.Tests.csproj +++ b/test/ProductConstructionService.Api.Tests/ProductConstructionService.Api.Tests.csproj @@ -15,8 +15,12 @@ - + + + + + diff --git a/test/ProductConstructionService.Api.Tests/RepositoryUrlAttributeTests.cs b/test/ProductConstructionService.Api.Tests/RepositoryUrlAttributeTests.cs new file mode 100644 index 0000000000..67a4910367 --- /dev/null +++ b/test/ProductConstructionService.Api.Tests/RepositoryUrlAttributeTests.cs @@ -0,0 +1,42 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.ComponentModel.DataAnnotations; +using FluentAssertions; +using Maestro.Api.Model; +using NUnit.Framework; + +namespace ProductConstructionService.Api.Tests; + +[TestFixture] +public class RepositoryUrlAttributeTests +{ + [TestCase("https://github.com/org/validRepo")] + [TestCase("https://github.com/org/valid.Repo")] + [TestCase("https://github.com/org/valid-Repo")] + [TestCase("https://github.com/org/valid-Rep.o12")] + [TestCase("https://dev.azure.com/org/project/_git/validRepo")] + [TestCase("https://dev.azure.com/org/project/_git/valid.Repo")] + [TestCase("https://dev.azure.com/org/project/_git/valid-Repo")] + [TestCase("https://dev.azure.com/org/project/_git/valid-Rep.o12")] + public void IsValidWithValidUrl(string url) + { + var attrib = new RepositoryUrlAttribute(); + attrib.GetValidationResult(url, new ValidationContext(url)).Should().Be(ValidationResult.Success); + } + + [TestCase("https://github.com/org/validRepo$")] + [TestCase("https://github.com/org/valid#Repo")] + [TestCase("https://github.com/org/valid*Repo")] + [TestCase("https://github.com/org/valid(Rep)o")] + [TestCase("https://github.com/validRepo")] + [TestCase("https://dev.azure.com/org/project/_git")] + [TestCase("https://dev.azure.com/org/_git/validRepo")] + [TestCase("https://dev.azure.com/_git/validRepo")] + [TestCase("https://dev.azure.com/org/project/validRepo")] + public void IsValidWithInvalidValidUrl(string url) + { + var attrib = new RepositoryUrlAttribute(); + attrib.GetValidationResult(url, new ValidationContext(url)).Should().NotBe(ValidationResult.Success); + } +} diff --git a/test/ProductConstructionService.Api.Tests/SubscriptionsController20200220Tests.cs b/test/ProductConstructionService.Api.Tests/SubscriptionsController20200220Tests.cs new file mode 100644 index 0000000000..a5114c5749 --- /dev/null +++ b/test/ProductConstructionService.Api.Tests/SubscriptionsController20200220Tests.cs @@ -0,0 +1,630 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Net; +using FluentAssertions; +using Maestro.Api.Model.v2020_02_20; +using Maestro.Data; +using Microsoft.AspNetCore.Mvc; +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.GitHub.Authentication; +using Microsoft.DotNet.Internal.Testing.DependencyInjection.Abstractions; +using Microsoft.DotNet.Internal.Testing.Utility; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Hosting.Internal; +using Microsoft.Extensions.Internal; +using Microsoft.Extensions.Logging; +using Moq; +using ProductConstructionService.Api.Api.v2020_02_20.Controllers; + +namespace ProductConstructionService.Api.Tests; + +[TestFixture] +public partial class SubscriptionsController20200220Tests : IDisposable +{ + private readonly TestData _data; + + public SubscriptionsController20200220Tests() + { + _data = TestData.Default.Build(); + } + + public void Dispose() + { + _data.Dispose(); + } + + [Test] + public async Task CreateGetAndListSubscriptions() + { + var testChannelName = "test-channel-sub-controller20200220"; + var defaultGitHubSourceRepo = "https://github.com/dotnet/sub-controller-test-source-repo"; + var defaultGitHubTargetRepo = "https://github.com/dotnet/sub-controller-test-target-repo"; + var defaultAzdoSourceRepo = "https://dev.azure.com/dnceng/internal/_git/sub-controller-test-source-repo"; + var defaultAzdoTargetRepo = "https://dev.azure.com/dnceng/internal/_git/sub-controller-test-target-repo"; + var defaultBranchName = "main"; + var aValidDependencyFlowNotificationList = "@someMicrosoftUser;@some-github-team"; + + // Create two subscriptions + var subscription1 = new SubscriptionData() + { + ChannelName = testChannelName, + Enabled = true, + SourceRepository = defaultGitHubSourceRepo, + TargetRepository = defaultGitHubTargetRepo, + Policy = new Maestro.Api.Model.v2018_07_16.SubscriptionPolicy() { Batchable = true, UpdateFrequency = Maestro.Api.Model.v2018_07_16.UpdateFrequency.EveryWeek }, + TargetBranch = defaultBranchName, + PullRequestFailureNotificationTags = aValidDependencyFlowNotificationList + }; + + Subscription createdSubscription1; + { + IActionResult result = await _data.SubscriptionsController.Create(subscription1); + result.Should().BeAssignableTo(); + var objResult = (ObjectResult)result; + objResult.StatusCode.Should().Be((int)HttpStatusCode.Created); + objResult.Value.Should().BeAssignableTo(); + createdSubscription1 = (Subscription)objResult.Value!; + createdSubscription1.Channel.Name.Should().Be(testChannelName); + createdSubscription1.Policy.Batchable.Should().Be(true); + createdSubscription1.Policy.UpdateFrequency.Should().Be(Maestro.Api.Model.v2018_07_16.UpdateFrequency.EveryWeek); + createdSubscription1.TargetBranch.Should().Be(defaultBranchName); + createdSubscription1.SourceRepository.Should().Be(defaultGitHubSourceRepo); + createdSubscription1.TargetRepository.Should().Be(defaultGitHubTargetRepo); + createdSubscription1.PullRequestFailureNotificationTags.Should().Be(aValidDependencyFlowNotificationList); + createdSubscription1.SourceEnabled.Should().BeFalse(); + createdSubscription1.ExcludedAssets.Should().BeEmpty(); + } + + var subscription2 = new SubscriptionData() + { + ChannelName = testChannelName, + Enabled = false, + SourceRepository = defaultAzdoSourceRepo, + TargetRepository = defaultAzdoTargetRepo, + Policy = new Maestro.Api.Model.v2018_07_16.SubscriptionPolicy() { Batchable = false, UpdateFrequency = Maestro.Api.Model.v2018_07_16.UpdateFrequency.None }, + TargetBranch = defaultBranchName, + SourceEnabled = true, + SourceDirectory = "sub-controller-test-source-repo", + ExcludedAssets = [DependencyFileManager.ArcadeSdkPackageName, "Foo.Bar"], + }; + + Subscription createdSubscription2; + { + IActionResult result = await _data.SubscriptionsController.Create(subscription2); + result.Should().BeAssignableTo(); + var objResult = (ObjectResult)result; + objResult.StatusCode.Should().Be((int)HttpStatusCode.Created); + objResult.Value.Should().BeAssignableTo(); + createdSubscription2 = (Subscription)objResult.Value!; + createdSubscription2.Channel.Name.Should().Be(testChannelName); + createdSubscription2.Policy.Batchable.Should().Be(false); + createdSubscription2.Policy.UpdateFrequency.Should().Be(Maestro.Api.Model.v2018_07_16.UpdateFrequency.None); + createdSubscription2.TargetBranch.Should().Be(defaultBranchName); + createdSubscription2.SourceRepository.Should().Be(defaultAzdoSourceRepo); + createdSubscription2.TargetRepository.Should().Be(defaultAzdoTargetRepo); + createdSubscription2.PullRequestFailureNotificationTags.Should().BeNull(); + createdSubscription2.SourceEnabled.Should().BeTrue(); + createdSubscription2.SourceDirectory.Should().Be("sub-controller-test-source-repo"); + createdSubscription2.ExcludedAssets.Should().BeEquivalentTo([DependencyFileManager.ArcadeSdkPackageName, "Foo.Bar"]); + } + + // List all (both) subscriptions, spot check that we got both + { + IActionResult result = _data.SubscriptionsController.ListSubscriptions(); + result.Should().BeAssignableTo(); + var objResult = (ObjectResult)result; + objResult.StatusCode.Should().Be((int)HttpStatusCode.OK); + var listedSubs = ((IEnumerable)objResult.Value!).ToList(); + listedSubs.Count.Should().Be(2); + listedSubs[0].Enabled.Should().Be(true); + listedSubs[0].TargetRepository.Should().Be(defaultGitHubTargetRepo); + listedSubs[0].PullRequestFailureNotificationTags.Should().Be(aValidDependencyFlowNotificationList); + listedSubs[0].ExcludedAssets.Should().BeEmpty(); + listedSubs[1].Enabled.Should().Be(false); + listedSubs[1].TargetRepository.Should().Be(defaultAzdoTargetRepo); + listedSubs[1].PullRequestFailureNotificationTags.Should().BeNull(); + listedSubs[1].ExcludedAssets.Should().BeEquivalentTo([DependencyFileManager.ArcadeSdkPackageName, "Foo.Bar"]); + } + // Use ListSubscriptions() params at least superficially to go down those codepaths + { + IActionResult result = _data.SubscriptionsController.ListSubscriptions(defaultAzdoSourceRepo, defaultAzdoTargetRepo, createdSubscription2.Channel.Id, enabled: false, sourceEnabled: true); + result.Should().BeAssignableTo(); + var objResult = (ObjectResult)result; + objResult.StatusCode.Should().Be((int)HttpStatusCode.OK); + var listedSubs = ((IEnumerable)objResult.Value!).ToList(); + listedSubs.Count.Should().Be(1); + listedSubs[0].Enabled.Should().Be(false); + listedSubs[0].TargetRepository.Should().Be(defaultAzdoTargetRepo); + listedSubs[0].PullRequestFailureNotificationTags.Should().BeNull(); // This is sub2 + listedSubs[0].ExcludedAssets.Should().BeEquivalentTo([DependencyFileManager.ArcadeSdkPackageName, "Foo.Bar"]); + } + // Directly get one of the subscriptions + { + IActionResult result = await _data.SubscriptionsController.GetSubscription(createdSubscription1.Id); + result.Should().BeAssignableTo(); + var objResult = (ObjectResult)result; + objResult.StatusCode.Should().Be((int)HttpStatusCode.OK); + var theSubscription = (Subscription)objResult.Value!; + theSubscription.Enabled.Should().Be(true); + theSubscription.TargetRepository.Should().Be(defaultGitHubTargetRepo); + theSubscription.PullRequestFailureNotificationTags.Should().Be(aValidDependencyFlowNotificationList); + theSubscription.ExcludedAssets.Should().BeEmpty(); + } + } + + [Test] + public async Task GetAndListNonexistentSubscriptions() + { + var shouldntExist = Guid.Parse("00000000-0000-0000-0000-000000000042"); + + // No subs added, get a random Guid + { + IActionResult result = await _data.SubscriptionsController.GetSubscription(shouldntExist); + result.Should().BeAssignableTo(); + var notFoundResult = (NotFoundResult)result; + notFoundResult.StatusCode.Should().Be((int)HttpStatusCode.NotFound); + } + + { + IActionResult result = _data.SubscriptionsController.ListSubscriptions( + "https://github.com/dotnet/does-not-exist", + "https://github.com/dotnet/does-not-exist-2", + 123456, + true); + result.Should().BeAssignableTo(); + var objResult = (ObjectResult)result; + objResult.StatusCode.Should().Be((int)HttpStatusCode.OK); + var listedSubs = ((IEnumerable)objResult.Value!).ToList(); + listedSubs.Should().BeEmpty(); + } + } + + [Test] + public async Task CreateSubscriptionForNonMicrosoftUserFails() + { + var testChannelName = "test-channel-sub-controller20200220"; + var defaultGitHubSourceRepo = "https://github.com/dotnet/sub-controller-test-source-repo"; + var defaultGitHubTargetRepo = "https://github.com/dotnet/sub-controller-test-target-repo"; + var defaultBranchName = "main"; + var anInvalidDependencyFlowNotificationList = "@someexternaluser;@somemicrosoftuser;@some-team"; + + // @someexternaluser will resolve as not in the microsoft org and should fail + var subscription = new SubscriptionData() + { + ChannelName = testChannelName, + Enabled = true, + SourceRepository = defaultGitHubSourceRepo, + TargetRepository = defaultGitHubTargetRepo, + Policy = new Maestro.Api.Model.v2018_07_16.SubscriptionPolicy() { Batchable = true, UpdateFrequency = Maestro.Api.Model.v2018_07_16.UpdateFrequency.EveryWeek }, + TargetBranch = defaultBranchName, + PullRequestFailureNotificationTags = anInvalidDependencyFlowNotificationList + }; + + IActionResult result = await _data.SubscriptionsController.Create(subscription); + result.Should().BeAssignableTo(); + } + + [Test] + public async Task CreateSubscriptionForNonExistentChannelFails() + { + var defaultGitHubSourceRepo = "https://github.com/dotnet/sub-controller-test-source-repo"; + var defaultGitHubTargetRepo = "https://github.com/dotnet/sub-controller-test-target-repo"; + var defaultBranchName = "main"; + + // Create two subscriptions + var subscription = new SubscriptionData() + { + ChannelName = "this-channel-does-not-exist", + Enabled = true, + SourceRepository = defaultGitHubSourceRepo, + TargetRepository = defaultGitHubTargetRepo, + Policy = new Maestro.Api.Model.v2018_07_16.SubscriptionPolicy() { Batchable = true, UpdateFrequency = Maestro.Api.Model.v2018_07_16.UpdateFrequency.EveryWeek }, + TargetBranch = defaultBranchName + }; + + IActionResult result = await _data.SubscriptionsController.Create(subscription); + result.Should().BeAssignableTo(); + } + + [Test] + public async Task DeleteSubscription() + { + var testChannelName = "test-channel-sub-controller20200220"; + var deleteScenarioSourceRepo = "https://github.com/dotnet/sub-controller-delete-sub-source-repo"; + var deleteScenarioTargetRepo = "https://github.com/dotnet/sub-controller-delete-sub-target-repo"; + var defaultBranchName = "main"; + + // Create two subscriptions + var subscriptionToDelete = new SubscriptionData() + { + ChannelName = testChannelName, + Enabled = true, + SourceRepository = deleteScenarioSourceRepo, + TargetRepository = deleteScenarioTargetRepo, + Policy = new Maestro.Api.Model.v2018_07_16.SubscriptionPolicy() { Batchable = true, UpdateFrequency = Maestro.Api.Model.v2018_07_16.UpdateFrequency.EveryWeek }, + TargetBranch = defaultBranchName + }; + + { + IActionResult createResult = await _data.SubscriptionsController.Create(subscriptionToDelete); + createResult.Should().BeAssignableTo(); + var objResult = (ObjectResult)createResult; + objResult.StatusCode.Should().Be((int)HttpStatusCode.Created); + var createdSubscription = (Subscription)objResult.Value!; + + IActionResult deleteResult = await _data.SubscriptionsController.DeleteSubscription(createdSubscription.Id); + deleteResult.Should().BeAssignableTo(); + var deleteObjResult = (OkObjectResult)deleteResult; + // Seems like this should be OK but it gives created... + deleteObjResult.StatusCode.Should().Be((int)HttpStatusCode.OK); + } + } + + [Test] + public async Task TriggerSubscription() + { + var testChannelName = "test-channel-sub-controller20200220"; + var triggerScenarioSourceRepo = "https://github.com/dotnet/sub-controller-trigger-sub-source-repo"; + var triggerScenarioTargetRepo = "https://github.com/dotnet/sub-controller-trigger-sub-target-repo"; + var defaultBranchName = "main"; + + // Create two subscriptions + var subscriptionToTrigger = new SubscriptionData() + { + ChannelName = testChannelName, + Enabled = true, + SourceRepository = triggerScenarioSourceRepo, + TargetRepository = triggerScenarioTargetRepo, + Policy = new Maestro.Api.Model.v2018_07_16.SubscriptionPolicy() { Batchable = true, UpdateFrequency = Maestro.Api.Model.v2018_07_16.UpdateFrequency.EveryWeek }, + TargetBranch = defaultBranchName + }; + + Subscription createdSubscription; + { + IActionResult createResult = await _data.SubscriptionsController.Create(subscriptionToTrigger); + createResult.Should().BeAssignableTo(); + var objResult = (ObjectResult)createResult; + objResult.StatusCode.Should().Be((int)HttpStatusCode.Created); + createdSubscription = (Subscription)objResult.Value!; + } + + var build1Data = new BuildData() + { + GitHubRepository = triggerScenarioSourceRepo, + AzureDevOpsBuildId = 123 + }; + var build2Data = new BuildData() + { + GitHubRepository = triggerScenarioSourceRepo, + AzureDevOpsBuildId = 124 + }; + var build3Data = new BuildData() + { + GitHubRepository = $"{triggerScenarioSourceRepo}-different", + AzureDevOpsBuildId = 125 + }; + Build build1, build3; + // Add some builds + { + IActionResult createResult1 = await _data.BuildsController.Create(build1Data); + createResult1.Should().BeAssignableTo(); + var objResult1 = (ObjectResult)createResult1; + objResult1.StatusCode.Should().Be((int)HttpStatusCode.Created); + build1 = (Build)objResult1.Value!; + + // Ignored build, just obviates the previous one. + IActionResult createResult2 = await _data.BuildsController.Create(build2Data); + createResult2.Should().BeAssignableTo(); + var objResult2 = (ObjectResult)createResult2; + objResult2.StatusCode.Should().Be((int)HttpStatusCode.Created); + + IActionResult createResult3 = await _data.BuildsController.Create(build3Data); + createResult3.Should().BeAssignableTo(); + var objResult3 = (ObjectResult)createResult3; + objResult3.StatusCode.Should().Be((int)HttpStatusCode.Created); + build3 = (Build)objResult3.Value!; + } + + // Default scenario; 'trigger a subscription with latest build' codepath. + { + IActionResult triggerResult = await _data.SubscriptionsController.TriggerSubscription(createdSubscription.Id); + triggerResult.Should().BeAssignableTo(); + var latestTriggerResult = (AcceptedResult)triggerResult; + latestTriggerResult.StatusCode.Should().Be((int)HttpStatusCode.Accepted); + } + + // Scenario2: 'trigger a subscription with specific build' codepath. + { + IActionResult triggerResult = await _data.SubscriptionsController.TriggerSubscription(createdSubscription.Id, build1.Id); + triggerResult.Should().BeAssignableTo(); + var latestTriggerResult = (AcceptedResult)triggerResult; + latestTriggerResult.StatusCode.Should().Be((int)HttpStatusCode.Accepted); + } + + // Failure: Trigger a subscription with non-existent build id. + { + IActionResult triggerResult = await _data.SubscriptionsController.TriggerSubscription(createdSubscription.Id, 123456); + triggerResult.Should().BeAssignableTo(); + var latestTriggerResult = (BadRequestObjectResult)triggerResult; + latestTriggerResult.StatusCode.Should().Be((int)HttpStatusCode.BadRequest); + } + + // Failure: Trigger a subscription with non-existent build codepath. + { + IActionResult triggerResult = await _data.SubscriptionsController.TriggerSubscription(createdSubscription.Id, build3.Id); + triggerResult.Should().BeAssignableTo(); + var latestTriggerResult = (BadRequestObjectResult)triggerResult; + latestTriggerResult.StatusCode.Should().Be((int)HttpStatusCode.BadRequest); + } + } + + [Test] + public async Task UpdateSubscription() + { + var testChannelName = "test-channel-sub-controller20200220"; + var defaultGitHubSourceRepo = "https://github.com/dotnet/sub-controller-test-source-repo"; + var defaultGitHubTargetRepo = "https://github.com/dotnet/sub-controller-test-target-repo"; + var defaultBranchName = "main"; + var aValidDependencyFlowNotificationList = "@someMicrosoftUser;@some-github-team"; + var anInvalidDependencyFlowNotificationList = "@someExternalUser;@someMicrosoftUser;@some-team"; + + // Create two subscriptions + var subscription1 = new SubscriptionData() + { + ChannelName = testChannelName, + Enabled = true, + SourceRepository = $"{defaultGitHubSourceRepo}-needsupdate", + TargetRepository = defaultGitHubTargetRepo, + Policy = new Maestro.Api.Model.v2018_07_16.SubscriptionPolicy() { Batchable = true, UpdateFrequency = Maestro.Api.Model.v2018_07_16.UpdateFrequency.EveryWeek }, + TargetBranch = defaultBranchName + }; + + Subscription createdSubscription1; + { + IActionResult result = await _data.SubscriptionsController.Create(subscription1); + result.Should().BeAssignableTo(); + var objResult = (ObjectResult)result; + objResult.StatusCode.Should().Be((int)HttpStatusCode.Created); + objResult.Value.Should().BeAssignableTo(); + createdSubscription1 = (Subscription)objResult.Value!; + createdSubscription1.Channel.Name.Should().Be(testChannelName); + createdSubscription1.Policy.Batchable.Should().Be(true); + createdSubscription1.Policy.UpdateFrequency.Should().Be(Maestro.Api.Model.v2018_07_16.UpdateFrequency.EveryWeek); + createdSubscription1.TargetBranch.Should().Be(defaultBranchName); + createdSubscription1.SourceRepository.Should().Be($"{defaultGitHubSourceRepo}-needsupdate"); + createdSubscription1.TargetRepository.Should().Be(defaultGitHubTargetRepo); + } + + var update = new SubscriptionUpdate() + { + Enabled = !subscription1.Enabled, + Policy = new Maestro.Api.Model.v2018_07_16.SubscriptionPolicy() { Batchable = false, UpdateFrequency = Maestro.Api.Model.v2018_07_16.UpdateFrequency.EveryDay }, + SourceRepository = $"{subscription1.SourceRepository}-updated", + PullRequestFailureNotificationTags = aValidDependencyFlowNotificationList + }; + + { + IActionResult result = await _data.SubscriptionsController.UpdateSubscription(createdSubscription1.Id, update); + result.Should().BeAssignableTo(); + var objResult = (ObjectResult)result; + objResult.StatusCode.Should().Be((int)HttpStatusCode.OK); + objResult.Value.Should().BeAssignableTo(); + // Could also do a get after this; that more tests the underlying data context though. + var updatedSubscription = (Subscription)objResult.Value!; + updatedSubscription.Id.Should().Be(createdSubscription1.Id); + updatedSubscription.Enabled.Should().Be(!subscription1.Enabled.Value); + updatedSubscription.Policy.UpdateFrequency.Should().Be(Maestro.Api.Model.v2018_07_16.UpdateFrequency.EveryDay); + updatedSubscription.SourceRepository.Should().Be($"{subscription1.SourceRepository}-updated"); + updatedSubscription.PullRequestFailureNotificationTags.Should().Be(aValidDependencyFlowNotificationList); + } + + // Update with an invalid list, make sure it fails + var badUpdate = new SubscriptionUpdate() + { + Enabled = !subscription1.Enabled, + Policy = new Maestro.Api.Model.v2018_07_16.SubscriptionPolicy() { Batchable = false, UpdateFrequency = Maestro.Api.Model.v2018_07_16.UpdateFrequency.EveryDay }, + SourceRepository = $"{subscription1.SourceRepository}-updated", + PullRequestFailureNotificationTags = anInvalidDependencyFlowNotificationList + }; + + { + IActionResult result = await _data.SubscriptionsController.UpdateSubscription(createdSubscription1.Id, badUpdate); + result.Should().BeAssignableTo(); + } + } + + private class MockOrg : Octokit.Organization + { + public MockOrg(int id, string login) + { + Id = id; + Login = login; + } + } + + [TestDependencyInjectionSetup] + private static class TestDataConfiguration + { + public static void Dependencies(IServiceCollection collection) + { + collection.AddLogging(l => l.AddProvider(new NUnitLogger())); + collection.AddSingleton(new HostingEnvironment + { + EnvironmentName = Environments.Development + }); + collection.AddSingleton(Mock.Of()); + collection.AddSingleton(Mock.Of()); + } + + public static void GitHub(IServiceCollection collection) + { + var gitHubClient = new Mock(MockBehavior.Strict); + + gitHubClient.Setup(ghc => ghc.Organization.GetAllForUser(It.IsAny())) + .Returns((string userLogin) => CallFakeGetAllForUser(userLogin)); + + var clientFactoryMock = new Mock(); + clientFactoryMock.Setup(f => f.CreateGitHubClient(It.IsAny())) + .Returns((string token) => gitHubClient.Object); + collection.AddSingleton(clientFactoryMock.Object); + collection.Configure(o => + { + o.ProductHeader = new Octokit.ProductHeaderValue("TEST", "1.0"); + }); + + static async Task> CallFakeGetAllForUser(string userLogin) + { + await Task.Delay(0); // Added just to suppress green squiggles + List returnValue = []; + + switch (userLogin.ToLower()) + { + case "somemicrosoftuser": // valid user, in MS org + returnValue.Add(MockOrganization(123, "microsoft")); + break; + case "someexternaluser": // "real" user, but not in MS org + returnValue.Add(MockOrganization(456, "definitely-not-microsoft")); + break; + default: // Any other user; GitHub "teams" will fall through here. + throw new Octokit.NotFoundException("Unknown user", HttpStatusCode.NotFound); + } + + return returnValue.AsReadOnly(); + } + } + + + public static async Task> DataContext(IServiceCollection collection) + { + var connectionString = await SharedData.Database.GetConnectionString(); + collection.AddBuildAssetRegistry(options => + { + options.UseSqlServer(connectionString); + options.EnableServiceProviderCaching(false); + }); + + return async provider => + { + var testChannelName = "test-channel-sub-controller20200220"; + var defaultGitHubSourceRepo = "https://github.com/dotnet/sub-controller-test-source-repo"; + var defaultGitHubTargetRepo = "https://github.com/dotnet/sub-controller-test-target-repo"; + var defaultAzdoSourceRepo = + "https://dev.azure.com/dnceng/internal/_git/sub-controller-test-source-repo"; + var defaultAzdoTargetRepo = + "https://dev.azure.com/dnceng/internal/_git/sub-controller-test-target-repo"; + var deleteScenarioSourceRepo = "https://github.com/dotnet/sub-controller-delete-sub-source-repo"; + var deleteScenarioTargetRepo = "https://github.com/dotnet/sub-controller-delete-sub-target-repo"; + var triggerScenarioSourceRepo = + "https://github.com/dotnet/sub-controller-trigger-sub-source-repo"; + var triggerScenarioTargetRepo = + "https://github.com/dotnet/sub-controller-trigger-sub-target-repo"; + var defaultClassification = "classy-classification"; + uint defaultInstallationId = 1234; + + // Setup common data context stuff for the background + var dataContext = provider.GetRequiredService(); + + await dataContext.Channels.AddAsync( + new Maestro.Data.Models.Channel() + { + Name = testChannelName, + Classification = defaultClassification + } + ); + + // Add some repos + await dataContext.Repositories.AddAsync( + new Maestro.Data.Models.Repository() + { + RepositoryName = defaultGitHubSourceRepo, + InstallationId = defaultInstallationId + } + ); + await dataContext.Repositories.AddAsync( + new Maestro.Data.Models.Repository() + { + RepositoryName = defaultGitHubTargetRepo, + InstallationId = defaultInstallationId + } + ); + await dataContext.Repositories.AddAsync( + new Maestro.Data.Models.Repository() + { + RepositoryName = defaultAzdoSourceRepo, + InstallationId = defaultInstallationId + } + ); + await dataContext.Repositories.AddAsync( + new Maestro.Data.Models.Repository() + { + RepositoryName = defaultAzdoTargetRepo, + InstallationId = defaultInstallationId + } + ); + await dataContext.Repositories.AddAsync( + new Maestro.Data.Models.Repository() + { + RepositoryName = deleteScenarioSourceRepo, + InstallationId = defaultInstallationId + } + ); + await dataContext.Repositories.AddAsync( + new Maestro.Data.Models.Repository() + { + RepositoryName = deleteScenarioTargetRepo, + InstallationId = defaultInstallationId + } + ); + await dataContext.Repositories.AddAsync( + new Maestro.Data.Models.Repository() + { + RepositoryName = triggerScenarioSourceRepo, + InstallationId = defaultInstallationId + } + ); + await dataContext.Repositories.AddAsync( + new Maestro.Data.Models.Repository() + { + RepositoryName = triggerScenarioTargetRepo, + InstallationId = defaultInstallationId + } + ); + + await dataContext.SaveChangesAsync(); + }; + } + + public static Func ChannelsController(IServiceCollection collection) + { + collection.AddSingleton(); + return s => s.GetRequiredService(); + } + + public static Func SubscriptionsController(IServiceCollection collection) + { + collection.AddSingleton(); + return s => s.GetRequiredService(); + } + + public static Func BuildsController(IServiceCollection collection) + { + collection.AddSingleton(); + return s => s.GetRequiredService(); + } + + public static Func Clock(IServiceCollection collection) + { + collection.AddSingleton(); + return s => (TestClock)s.GetRequiredService(); + } + } + + // Copied from GitHubClaimsResolverTests; could refactor if needed in another place + private static Octokit.Organization MockOrganization(int id, string login) + { + return new MockOrg(id, login); + } +} diff --git a/test/ProductConstructionService.Api.Tests/TestDatabase.cs b/test/ProductConstructionService.Api.Tests/TestDatabase.cs new file mode 100644 index 0000000000..db10dad529 --- /dev/null +++ b/test/ProductConstructionService.Api.Tests/TestDatabase.cs @@ -0,0 +1,122 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Maestro.Data; +using Microsoft.Data.SqlClient; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Hosting.Internal; + +namespace ProductConstructionService.Api.Tests; + +[SetUpFixture] +public static class SharedData +{ + public static TestDatabase Database { get; private set; } = null!; + + [OneTimeSetUp] + public static void SetUp() + { + Database = new SharedTestDatabase(); + } + + [OneTimeTearDown] + public static void TearDown() + { + Database.Dispose(); + Database = null!; + } + + private class SharedTestDatabase : TestDatabase + { + } +} + +public class TestDatabase : IDisposable +{ + private const string TestDatabasePrefix = "TFD_"; + private string _databaseName = null!; + private readonly SemaphoreSlim _createLock = new(1); + + protected TestDatabase() + { + } + + public void Dispose() + { + using var connection = new SqlConnection(BuildAssetRegistryContextFactory.GetConnectionString("master")); + connection.Open(); + DropAllTestDatabases(connection).GetAwaiter().GetResult(); + } + + public async Task GetConnectionString() + { + if (_databaseName != null) + { + return ConnectionString; + } + + await _createLock.WaitAsync(); + try + { + var databaseName = $"{TestDatabasePrefix}_{TestContext.CurrentContext.Test.ClassName!.Split('.').Last()}_{TestContext.CurrentContext.Test.MethodName}_{DateTime.Now:yyyyMMddHHmmss}"; + TestContext.WriteLine($"Creating database '{databaseName}'"); + await using (var connection = new SqlConnection(BuildAssetRegistryContextFactory.GetConnectionString("master"))) + { + await connection.OpenAsync(); + + await DropAllTestDatabases(connection); + + await using (SqlCommand createCommand = connection.CreateCommand()) + { + createCommand.CommandText = $"CREATE DATABASE {databaseName}"; + await createCommand.ExecuteNonQueryAsync(); + } + } + + var collection = new ServiceCollection(); + collection.AddSingleton(new HostingEnvironment + { EnvironmentName = Environments.Development }); + collection.AddBuildAssetRegistry(o => + { + o.UseSqlServer(BuildAssetRegistryContextFactory.GetConnectionString(databaseName)); + o.EnableServiceProviderCaching(false); + }); + + await using ServiceProvider provider = collection.BuildServiceProvider(); + await provider.GetRequiredService().Database.MigrateAsync(); + + _databaseName = databaseName; + return ConnectionString; + } + finally + { + _createLock.Dispose(); + } + } + + private static async Task DropAllTestDatabases(SqlConnection connection) + { + var previousTestDbs = new List(); + await using (SqlCommand command = connection.CreateCommand()) + { + command.CommandText = $"SELECT name FROM sys.databases WHERE name LIKE '{TestDatabasePrefix}%'"; + await using SqlDataReader reader = await command.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + previousTestDbs.Add(reader.GetString(0)); + } + } + + foreach (var db in previousTestDbs) + { + TestContext.WriteLine($"Dropping test database '{db}'"); + await using SqlCommand command = connection.CreateCommand(); + command.CommandText = $"ALTER DATABASE {db} SET single_user with rollback immediate; DROP DATABASE {db}"; + await command.ExecuteNonQueryAsync(); + } + } + + private string ConnectionString => BuildAssetRegistryContextFactory.GetConnectionString(_databaseName); +} From f4c27758a26b32a48a1bc480856fd771b0d8b3f0 Mon Sep 17 00:00:00 2001 From: Djuradj Kurepa <91743470+dkurepa@users.noreply.github.com> Date: Thu, 22 Aug 2024 09:42:15 +0200 Subject: [PATCH 04/11] Remove Queue TODOS, and add actual queue support. Add reminders (#3856) --- .../Controllers/RepositoryController.cs | 116 ++-------- .../Controllers/SubscriptionsController.cs | 203 ++++++++---------- .../Controllers/SubscriptionsController.cs | 8 +- .../Controllers/BuildsController.cs | 15 +- .../Controllers/CodeFlowController.cs | 4 +- .../Controllers/SubscriptionsController.cs | 8 +- .../PcsStartup.cs | 26 ++- .../ProductConstructionService.Api/Program.cs | 4 +- .../appsettings.json | 2 +- .../ProductConstructionServiceExtension.cs | 58 ++--- .../LongestBuildPathUpdaterConfiguration.cs | 7 +- .../Program.cs | 6 +- ...uctionService.SubscriptionTriggerer.csproj | 1 + .../Program.cs | 21 +- .../SubscriptionTriggerer.cs | 36 ++-- .../SubscriptionTriggererConfiguration.cs | 17 +- .../appsettings.Development.json | 4 +- .../appsettings.Staging.json | 4 +- .../appsettings.json | 1 + .../WorkItemConfiguration.cs | 28 ++- .../BuildCoherencyInfoWorkItem.cs | 10 + .../UpdateSubscriptionWorkItem.cs | 19 ++ .../WorkItemDefinitions/WorkItem.cs | 2 + .../WorkItemProducer.cs | 14 +- .../WorkItemProducerFactory.cs | 11 +- .../BuildController20200914Tests.cs | 7 + .../ChannelsController20200220Tests.cs | 8 + .../SubscriptionsController20200220Tests.cs | 8 + .../DependencyRegistrationTests.cs | 2 +- .../DependencyRegistrationTests.cs | 21 +- 30 files changed, 325 insertions(+), 346 deletions(-) create mode 100644 src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemDefinitions/BuildCoherencyInfoWorkItem.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemDefinitions/UpdateSubscriptionWorkItem.cs diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/RepositoryController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/RepositoryController.cs index 85cdd6f717..45035b0887 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/RepositoryController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/RepositoryController.cs @@ -4,16 +4,14 @@ using System.Collections.Immutable; using System.ComponentModel.DataAnnotations; using System.Net; -using Maestro.Contracts; using Maestro.Data; using Microsoft.AspNetCore.ApiPagination; using Microsoft.AspNetCore.ApiVersioning; using Microsoft.AspNetCore.ApiVersioning.Swashbuckle; using Microsoft.AspNetCore.Mvc; -using Microsoft.DotNet.ServiceFabric.ServiceHost; using Microsoft.DotNet.Services.Utility; -using Microsoft.EntityFrameworkCore; using Maestro.Api.Model.v2018_07_16; +using ProductConstructionService.WorkItems; namespace ProductConstructionService.Api.Api.v2018_07_16.Controllers; @@ -26,14 +24,12 @@ public class RepositoryController : ControllerBase { public RepositoryController( BuildAssetRegistryContext context, - IActorProxyFactory pullRequestActorFactory) + IWorkItemProducerFactory workItemProducerFactory) { - Context = context; - PullRequestActorFactory = pullRequestActorFactory; + _context = context; } - public BuildAssetRegistryContext Context { get; } - public IActorProxyFactory PullRequestActorFactory { get; } + private BuildAssetRegistryContext _context { get; } /// /// Gets the list of RepositoryBranch, optionally filtered by @@ -47,7 +43,7 @@ public RepositoryController( [ValidateModelState] public IActionResult ListRepositories(string? repository = null, string? branch = null) { - IQueryable query = Context.RepositoryBranches; + IQueryable query = _context.RepositoryBranches; if (!string.IsNullOrEmpty(repository)) { @@ -87,7 +83,7 @@ public async Task GetMergePolicies([Required] string repository, return BadRequest(ModelState); } - Maestro.Data.Models.RepositoryBranch? repoBranch = await Context.RepositoryBranches.FindAsync(repository, branch); + Maestro.Data.Models.RepositoryBranch? repoBranch = await _context.RepositoryBranches.FindAsync(repository, branch); if (repoBranch == null) { return NotFound(); @@ -130,7 +126,7 @@ public async Task SetMergePolicies( Maestro.Data.Models.RepositoryBranch.Policy policy = repoBranch.PolicyObject ?? new Maestro.Data.Models.RepositoryBranch.Policy(); policy.MergePolicies = policies?.Select(p => p.ToDb()).ToList() ?? []; repoBranch.PolicyObject = policy; - await Context.SaveChangesAsync(); + await _context.SaveChangesAsync(); return Ok(); } @@ -159,114 +155,26 @@ public async Task GetHistory([Required] string repository, [Requi return BadRequest(ModelState); } - Maestro.Data.Models.RepositoryBranch? repoBranch = await Context.RepositoryBranches.FindAsync(repository, branch); + Maestro.Data.Models.RepositoryBranch? repoBranch = await _context.RepositoryBranches.FindAsync(repository, branch); if (repoBranch == null) { return NotFound(); } - IOrderedQueryable query = Context.RepositoryBranchUpdateHistory + IOrderedQueryable query = _context.RepositoryBranchUpdateHistory .Where(u => u.Repository == repository && u.Branch == branch) .OrderByDescending(u => u.Timestamp); return Ok(query); } - /// - /// Requests that Maestro++ retry the referenced history item. - /// Links to this api are returned from the api. - /// - /// The repository - /// The branch - /// The timestamp identifying the history item to retry - [HttpPost("retry/{timestamp}")] - [SwaggerApiResponse(HttpStatusCode.Accepted, Description = "Retry successfully requested")] - [SwaggerApiResponse(HttpStatusCode.NotAcceptable, Description = "The requested history item was successful and cannot be retried")] - public async Task RetryActionAsync([Required] string repository, [Required] string branch, long timestamp) - { - if (string.IsNullOrEmpty(repository)) - { - ModelState.TryAddModelError(nameof(repository), "The repository parameter is required"); - } - - if (string.IsNullOrEmpty(branch)) - { - ModelState.TryAddModelError(nameof(branch), "The branch parameter is required"); - } - - if (!ModelState.IsValid) - { - return BadRequest(ModelState); - } - - DateTime ts = DateTimeOffset.FromUnixTimeSeconds(timestamp).UtcDateTime; - - Maestro.Data.Models.RepositoryBranch? repoBranch = await Context.RepositoryBranches.FindAsync(repository, branch); - - if (repoBranch == null) - { - return NotFound(); - } - - RepositoryBranchUpdateHistoryEntry? update = await Context.RepositoryBranchUpdateHistory - .Where(u => u.Repository == repository && u.Branch == branch) - .FirstOrDefaultAsync(u => Math.Abs(EF.Functions.DateDiffSecond(u.Timestamp, ts)) < 1); - - if (update == null) - { - return NotFound(); - } - - if (update.Success) - { - return StatusCode( - (int)HttpStatusCode.NotAcceptable, - new ApiError("That action was successful, it cannot be retried.")); - } - - // TODO (https://github.com/dotnet/arcade-services/issues/3814): Queue.Post(PullRequestActionWorkItem.GetArguments(update)); - - return Accepted(); - } - - // TODO (https://github.com/dotnet/arcade-services/issues/3814): - /*private class PullRequestActionWorkItem : IBackgroundWorkItem - { - private readonly IActorProxyFactory _factory; - - public PullRequestActionWorkItem(IActorProxyFactory factory) - { - _factory = factory; - } - - public Task ProcessAsync(JToken argumentToken) - { - var update = argumentToken.ToObject(); - IPullRequestActor actor = _factory.Lookup(PullRequestActorId.Create(update.Repository, update.Branch)); - return actor.RunActionAsync(update.Method, update.MethodArguments); - } - - public static JToken GetArguments(RepositoryBranchUpdateHistoryEntry update) - { - return JToken.FromObject(new Arguments { Repository = update.Repository, Branch = update.Branch, Method = update.Method, MethodArguments = update.Arguments }); - } - - private struct Arguments - { - public string Repository; - public string Branch; - public string Method; - public string MethodArguments; - } - }*/ - private async Task GetRepositoryBranch(string repository, string branch) { - Maestro.Data.Models.RepositoryBranch? repoBranch = await Context.RepositoryBranches.FindAsync(repository, branch); + Maestro.Data.Models.RepositoryBranch? repoBranch = await _context.RepositoryBranches.FindAsync(repository, branch); if (repoBranch == null) { - Context.RepositoryBranches.Add( + _context.RepositoryBranches.Add( repoBranch = new Maestro.Data.Models.RepositoryBranch { RepositoryName = repository, @@ -275,7 +183,7 @@ private struct Arguments } else { - Context.RepositoryBranches.Update(repoBranch); + _context.RepositoryBranches.Update(repoBranch); } return repoBranch; diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs index 614706843f..3f907a3ffe 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs @@ -11,6 +11,8 @@ using Microsoft.EntityFrameworkCore; using Channel = Maestro.Data.Models.Channel; using Maestro.Api.Model.v2018_07_16; +using ProductConstructionService.WorkItems; +using ProductConstructionService.WorkItems.WorkItemDefinitions; namespace ProductConstructionService.Api.Api.v2018_07_16.Controllers; @@ -22,10 +24,17 @@ namespace ProductConstructionService.Api.Api.v2018_07_16.Controllers; public class SubscriptionsController : ControllerBase { private readonly BuildAssetRegistryContext _context; + private readonly IWorkItemProducerFactory _workItemProducerFactory; + private readonly ILogger _logger; - public SubscriptionsController(BuildAssetRegistryContext context) + public SubscriptionsController( + BuildAssetRegistryContext context, + IWorkItemProducerFactory workItemProducerFactory, + ILogger logger) { _context = context; + _workItemProducerFactory = workItemProducerFactory; + _logger = logger; } /// @@ -127,35 +136,54 @@ protected async Task TriggerSubscriptionCore(Guid id, int buildId return NotFound(); } - var values = new { SubId = id, BuildId = buildId }; - // TODO (https://github.com/dotnet/arcade-services/issues/3814): _queue.Post(JToken.FromObject(values)); + await EnqueueUpdateSubscriptionWorkItemAsync(id, buildId); + return Accepted(new Subscription(subscription)); } - // TODO (https://github.com/dotnet/arcade-services/issues/3814): - /*private class StartSubscriptionUpdateWorkItem : IBackgroundWorkItem + private async Task EnqueueUpdateSubscriptionWorkItemAsync(Guid subscriptionId, int buildId) { - private readonly IDependencyUpdater _dependencyUpdater; - - public StartSubscriptionUpdateWorkItem(IDependencyUpdater dependencyUpdater) + Maestro.Data.Models.Subscription? subscriptionToUpdate; + if (buildId != 0) { - _dependencyUpdater = dependencyUpdater; + // Update using a specific build + subscriptionToUpdate = + (from sub in _context.Subscriptions + where sub.Id == subscriptionId + where sub.Enabled + let specificBuild = + sub.Channel.BuildChannels.Select(bc => bc.Build) + .Where(b => (sub.SourceRepository == b.GitHubRepository || sub.SourceRepository == b.AzureDevOpsRepository)) + .Where(b => b.Id == buildId) + .FirstOrDefault() + where specificBuild != null + select sub).SingleOrDefault(); } - - public Task ProcessAsync(JToken argumentToken) + else { - var buildId = argumentToken.Value("BuildId"); + // Update using the latest build + subscriptionToUpdate = + (from sub in _context.Subscriptions + where sub.Id == subscriptionId + where sub.Enabled + let latestBuild = + sub.Channel.BuildChannels.Select(bc => bc.Build) + .Where(b => (sub.SourceRepository == b.GitHubRepository || sub.SourceRepository == b.AzureDevOpsRepository)) + .OrderByDescending(b => b.DateProduced) + .FirstOrDefault() + where latestBuild != null + select sub).SingleOrDefault(); + } - if (buildId != 0) - { - return _dependencyUpdater.StartSubscriptionUpdateForSpecificBuildAsync(argumentToken.Value("SubId"), buildId); - } - else + if (subscriptionToUpdate != null) + { + await _workItemProducerFactory.Create().ProduceWorkItemAsync(new() { - return _dependencyUpdater.StartSubscriptionUpdateAsync(argumentToken.Value("SubId")); - } + SubscriptionId = subscriptionToUpdate.Id, + BuildId = buildId + }); } - }*/ + } /// /// Trigger daily update @@ -163,28 +191,52 @@ public Task ProcessAsync(JToken argumentToken) [HttpPost("triggerDaily")] [SwaggerApiResponse(HttpStatusCode.Accepted, Description = "Trigger all subscriptions normally updated daily.")] [ValidateModelState] - public virtual IActionResult TriggerDailyUpdate() + public virtual async Task TriggerDailyUpdateAsync() { - // TODO (https://github.com/dotnet/arcade-services/issues/3814): _queue.Post(); + // TODO put this and the code in SubscriptionTriggerer in the same place to avoid dupplication + var enabledSubscriptionsWithTargetFrequency = (await _context.Subscriptions + .Where(s => s.Enabled) + .ToListAsync()) + .Where(s => (int)s.PolicyObject.UpdateFrequency == (int)UpdateFrequency.EveryDay); - return Accepted(); - } - - // TODO (https://github.com/dotnet/arcade-services/issues/3814): - /*private class CheckDailySubscriptionsWorkItem : IBackgroundWorkItem - { - private readonly IDependencyUpdater _dependencyUpdater; + var workitemProducer = _workItemProducerFactory.Create(); - public CheckDailySubscriptionsWorkItem(IDependencyUpdater dependencyUpdater) + foreach (var subscription in enabledSubscriptionsWithTargetFrequency) { - _dependencyUpdater = dependencyUpdater; - } + Maestro.Data.Models.Subscription? subscriptionWithBuilds = await _context.Subscriptions + .Where(s => s.Id == subscription.Id) + .Include(s => s.Channel) + .ThenInclude(c => c.BuildChannels) + .ThenInclude(bc => bc.Build) + .FirstOrDefaultAsync(); - public Task ProcessAsync(JToken ignored) - { - return _dependencyUpdater.CheckDailySubscriptionsAsync(CancellationToken.None); + if (subscriptionWithBuilds == null) + { + _logger.LogWarning("Subscription {subscriptionId} was not found in the BAR. Not triggering updates", subscription.Id.ToString()); + continue; + } + + Maestro.Data.Models.Build? latestBuildInTargetChannel = subscriptionWithBuilds.Channel.BuildChannels.Select(bc => bc.Build) + .Where(b => (subscription.SourceRepository == b.GitHubRepository || subscription.SourceRepository == b.AzureDevOpsRepository)) + .OrderByDescending(b => b.DateProduced) + .FirstOrDefault(); + + bool isThereAnUnappliedBuildInTargetChannel = latestBuildInTargetChannel != null && + (subscription.LastAppliedBuild == null || subscription.LastAppliedBuildId != latestBuildInTargetChannel.Id); + + if (isThereAnUnappliedBuildInTargetChannel && latestBuildInTargetChannel != null) + { + _logger.LogInformation("Will trigger {subscriptionId} to build {latestBuildInTargetChannelId}", subscription.Id, latestBuildInTargetChannel.Id); + await workitemProducer.ProduceWorkItemAsync(new() + { + SubscriptionId = subscription.Id, + BuildId = latestBuildInTargetChannel.Id + }); + } } - }*/ + + return Accepted(); + } /// /// Edit an existing @@ -316,85 +368,6 @@ public virtual async Task GetSubscriptionHistory(Guid id) return Ok(query); } - /// - /// Requests that Maestro++ retry the reference history item. - /// Links to this api are returned from the api. - /// - /// The id of the containing the history item to retry - /// The timestamp identifying the history item to retry - [HttpPost("{id}/retry/{timestamp}")] - [SwaggerApiResponse(HttpStatusCode.Accepted, Description = "Retry successfully requested")] - [SwaggerApiResponse(HttpStatusCode.NotAcceptable, Description = "The requested history item was successful and cannot be retried")] - public virtual async Task RetrySubscriptionActionAsync(Guid id, long timestamp) - { - DateTime ts = DateTimeOffset.FromUnixTimeSeconds(timestamp).UtcDateTime; - - Maestro.Data.Models.Subscription? subscription = await _context.Subscriptions.Where(sub => sub.Id == id) - .FirstOrDefaultAsync(); - - if (subscription == null) - { - return NotFound(); - } - - SubscriptionUpdateHistoryEntry? update = await _context.SubscriptionUpdateHistory - .Where(u => u.SubscriptionId == id) - .FirstOrDefaultAsync(u => Math.Abs(EF.Functions.DateDiffSecond(u.Timestamp, ts)) < 1); - - if (update == null) - { - return NotFound(); - } - - if (update.Success) - { - return StatusCode( - (int)HttpStatusCode.NotAcceptable, - new ApiError("That action was successful, it cannot be retried.")); - } - - // TODO (https://github.com/dotnet/arcade-services/issues/3814): _queue.Post( - //SubscriptionActorActionWorkItem.GetArguments(subscription.Id, update.Method, update.Arguments) - //); - - return Accepted(); - } - - // TODO (https://github.com/dotnet/arcade-services/issues/3814): - //private class SubscriptionActorActionWorkItem : IBackgroundWorkItem - //{ - // private readonly IActorProxyFactory _factory; - - // public SubscriptionActorActionWorkItem(IActorProxyFactory factory) - // { - // _factory = factory; - // } - - // private struct Arguments - // { - // public Guid Subscriptionid; - // public string Method; - // public string MethodArguments; - // } - - // public Task ProcessAsync(JToken argumentToken) - // { - // var args = argumentToken.ToObject(); - // ISubscriptionActor actor = _factory.Lookup(new ActorId(args.Subscriptionid)); - // return actor.RunActionAsync(args.Method, args.MethodArguments); - // } - - // public static JToken GetArguments(Guid subscriptionId, string method, string arguments) - // { - // return JToken.FromObject(new Arguments - // { - // Subscriptionid = subscriptionId, - // Method = method, - // MethodArguments = arguments - // }); - // } - //} - /// /// Creates a new /// diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2019_01_16/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2019_01_16/Controllers/SubscriptionsController.cs index 349e93d402..06d31fe134 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2019_01_16/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2019_01_16/Controllers/SubscriptionsController.cs @@ -9,6 +9,7 @@ using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; using Maestro.Api.Model.v2019_01_16; +using ProductConstructionService.WorkItems; namespace ProductConstructionService.Api.Api.v2019_01_16.Controllers; @@ -21,8 +22,11 @@ public class SubscriptionsController : v2018_07_16.Controllers.SubscriptionsCont { private readonly BuildAssetRegistryContext _context; - public SubscriptionsController(BuildAssetRegistryContext context) - : base(context) + public SubscriptionsController( + BuildAssetRegistryContext context, + IWorkItemProducerFactory workItemProducerFactory, + ILogger logger) + : base(context, workItemProducerFactory, logger) { _context = context; } diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/BuildsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/BuildsController.cs index 9fcdba9c2e..6227aca799 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/BuildsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/BuildsController.cs @@ -12,6 +12,8 @@ using Microsoft.DotNet.DarcLib; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Internal; +using ProductConstructionService.WorkItems; +using ProductConstructionService.WorkItems.WorkItemDefinitions; namespace ProductConstructionService.Api.Api.v2020_02_20.Controllers; @@ -23,14 +25,17 @@ namespace ProductConstructionService.Api.Api.v2020_02_20.Controllers; public class BuildsController : v2019_01_16.Controllers.BuildsController { private readonly IRemoteFactory _factory; + private readonly IWorkItemProducerFactory _workItemProducerFactory; public BuildsController( BuildAssetRegistryContext context, ISystemClock clock, - IRemoteFactory factory) + IRemoteFactory factory, + IWorkItemProducerFactory workItemProducerFactory) : base(context, clock) { _factory = factory; + _workItemProducerFactory = workItemProducerFactory; } /// @@ -318,7 +323,11 @@ await _context.BuildDependencies.AddRangeAsync( // Compute the dependency incoherencies of the build. // Since this might be an expensive operation we do it asynchronously. - // TODO (https://github.com/dotnet/arcade-services/issues/3814): Queue.Post(JToken.FromObject(buildModel.Id)); + await _workItemProducerFactory.Create() + .ProduceWorkItemAsync(new() + { + BuildId = buildModel.Id + }); return CreatedAtRoute( new @@ -329,7 +338,7 @@ await _context.BuildDependencies.AddRangeAsync( new Build(buildModel)); } - // TODO PORT: + // TODO PORT THIS TO A WORKITEM PROCESSOR: /* private class BuildCoherencyInfoWorkItem : IBackgroundWorkItem { diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/CodeFlowController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/CodeFlowController.cs index ffa3e9c9bd..819be48fa8 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/CodeFlowController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/CodeFlowController.cs @@ -15,11 +15,11 @@ namespace ProductConstructionService.Api.Api.v2020_02_20.Controllers; [ApiVersion("2020-02-20")] public class CodeFlowController( IBasicBarClient barClient, - WorkItemProducerFactory workItemProducerFactory) + IWorkItemProducerFactory workItemProducerFactory) : ControllerBase { private readonly IBasicBarClient _barClient = barClient; - private readonly WorkItemProducerFactory _workItemProducerFactory = workItemProducerFactory; + private readonly IWorkItemProducerFactory _workItemProducerFactory = workItemProducerFactory; [HttpPost(Name = "Flow")] public async Task FlowBuild([Required, FromBody] Maestro.Api.Model.v2020_02_20.CodeFlowRequest request) diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs index 0608d0e09a..fc4efabac8 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs @@ -3,7 +3,6 @@ using System.ComponentModel.DataAnnotations; using System.Net; -using Kusto.Cloud.Platform.Utils; using Maestro.Data; using Microsoft.AspNetCore.ApiVersioning; using Microsoft.AspNetCore.ApiVersioning.Swashbuckle; @@ -11,6 +10,7 @@ using Microsoft.DotNet.GitHub.Authentication; using Microsoft.EntityFrameworkCore; using Maestro.Api.Model.v2020_02_20; +using ProductConstructionService.WorkItems; namespace ProductConstructionService.Api.Api.v2020_02_20.Controllers; @@ -28,8 +28,10 @@ public class SubscriptionsController : v2019_01_16.Controllers.SubscriptionsCont public SubscriptionsController( BuildAssetRegistryContext context, - IGitHubClientFactory gitHubClientFactory) - : base(context) + IGitHubClientFactory gitHubClientFactory, + IWorkItemProducerFactory workItemProducerFactory, + ILogger logger) + : base(context, workItemProducerFactory, logger) { _context = context; _gitHubClientFactory = gitHubClientFactory; diff --git a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs index 9c8ef079a1..a70fac02a2 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs @@ -36,6 +36,7 @@ using ProductConstructionService.Api.VirtualMonoRepo; using ProductConstructionService.Common; using ProductConstructionService.WorkItems; +using ProductConstructionService.WorkItems.WorkItemDefinitions; namespace ProductConstructionService.Api; @@ -73,6 +74,7 @@ static PcsStartup() { var context = (BuildAssetRegistryContext)entry.Context; ILogger logger = context.GetService>(); + var workItemProducer = context.GetService().Create(); BuildChannel entity = entry.Entity; Build? build = context.Builds @@ -91,9 +93,23 @@ static PcsStartup() if (hasAssetsWithPublishedLocations) { - // TODO: Only activate this when we want the service to do things - // TODO (https://github.com/dotnet/arcade-services/issues/3814): var queue = context.GetService(); - // queue.Post(StartDependencyUpdate.CreateArgs(entity)); + List subscriptionsToUpdate = context.Subscriptions + .Where(sub => + sub.Enabled && + sub.ChannelId == entity.ChannelId && + (sub.SourceRepository == entity.Build.GitHubRepository || sub.SourceDirectory == entity.Build.AzureDevOpsRepository) && + JsonExtensions.JsonValue(sub.PolicyString, "lax $.UpdateFrequency") == ((int)UpdateFrequency.EveryBuild).ToString()) + .ToList(); + + // TODO: https://github.com/dotnet/arcade-services/issues/3811 Add a feature switch to trigger specific subscriptions + /*foreach (Subscription subscription in subscriptionsToUpdate) + { + workItemProducer.ProduceWorkItemAsync(new() + { + BuildId = entity.BuildId, + SubscriptionId = subscription.Id + }).GetAwaiter().GetResult(); + }*/ } else { @@ -140,7 +156,7 @@ internal static async Task ConfigurePcs( builder.Configuration.AddAzureKeyVault(keyVaultUri, azureCredential); } - builder.Services.RegisterBuildAssetRegistry(builder.Configuration); + builder.RegisterBuildAssetRegistry(); builder.AddWorkItemQueues(azureCredential, waitForInitialization: initializeService); builder.Services.AddWorkItemProcessors(); builder.AddVmrRegistrations(gitHubToken); @@ -164,7 +180,7 @@ internal static async Task ConfigurePcs( if (addRedis) { - await builder.Services.AddRedis(builder.Configuration, isDevelopment); + await builder.AddRedis(); } if (initializeService) diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Program.cs b/src/ProductConstructionService/ProductConstructionService.Api/Program.cs index df5fb5e5c6..f7b8a702e0 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Program.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Program.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.FileProviders; using ProductConstructionService.Api; using ProductConstructionService.Api.Configuration; +using ProductConstructionService.Common; using ProductConstructionService.WorkItems; var builder = WebApplication.CreateBuilder(args); @@ -53,7 +54,8 @@ await builder.ConfigurePcs( new PhysicalFileProvider(Path.Combine(Environment.CurrentDirectory, "wwwroot"))), }); - await app.UseLocalWorkItemQueues(); + await app.Services.UseLocalWorkItemQueues( + app.Configuration.GetRequiredValue(WorkItemConfiguration.WorkItemQueueNameConfigurationKey)); if (useSwagger) { diff --git a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.json b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.json index 9b3a94c957..8e9d94ffd3 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.json +++ b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.json @@ -12,8 +12,8 @@ } }, "AllowedHosts": "*", + "WorkItemQueueName": "pcs-workitems", "WorkItemConsumerOptions": { - "WorkItemQueueName": "pcs-workitems", "QueuePollTimeout": "00:01:00", "MaxWorkItemRetries": 3, "QueueMessageInvisibilityTime": "00:01:00" diff --git a/src/ProductConstructionService/ProductConstructionService.Common/ProductConstructionServiceExtension.cs b/src/ProductConstructionService/ProductConstructionService.Common/ProductConstructionServiceExtension.cs index 7e56b35e98..c3e1291d7d 100644 --- a/src/ProductConstructionService/ProductConstructionService.Common/ProductConstructionServiceExtension.cs +++ b/src/ProductConstructionService/ProductConstructionService.Common/ProductConstructionServiceExtension.cs @@ -1,18 +1,16 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using Azure.Identity; -using Azure.Storage.Queues; using Maestro.Data; using Maestro.DataProviders; using Microsoft.DotNet.DarcLib; using Microsoft.DotNet.GitHub.Authentication; using Microsoft.DotNet.Kusto; using Microsoft.EntityFrameworkCore.Diagnostics; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Hosting; using StackExchange.Redis; using Microsoft.Azure.StackExchangeRedis; @@ -20,42 +18,19 @@ namespace ProductConstructionService.Common; public static class ProductConstructionServiceExtension { - private const string QueueConnectionString = "QueueConnectionString"; private const string RedisConnectionString = "redis"; - private const string ManagedIdentityClientId = "ManagedIdentityClientId"; + public const string ManagedIdentityClientId = "ManagedIdentityClientId"; private const string SqlConnectionStringUserIdPlaceholder = "USER_ID_PLACEHOLDER"; private const string DatabaseConnectionString = "BuildAssetRegistrySqlConnectionString"; - public static void RegisterAzureQueue(this IServiceCollection services, IConfiguration configuration) + public static void RegisterBuildAssetRegistry(this IHostApplicationBuilder builder) { - var connectionString = configuration.GetRequiredValue(QueueConnectionString); - - if (connectionString.Contains("UseDevelopmentStorage=true")) - { - var lastPart = connectionString.LastIndexOf('/'); - services.AddTransient(_ => new QueueClient( - connectionString.Substring(0, lastPart), - connectionString.Substring(lastPart + 1))); - } - else - { - services.AddTransient(_ => new QueueClient( - new Uri(connectionString), - new DefaultAzureCredential(new DefaultAzureCredentialOptions - { - ManagedIdentityClientId = configuration[ManagedIdentityClientId] - }))); - } - } - - public static void RegisterBuildAssetRegistry(this IServiceCollection services, IConfiguration configuration) - { - var managedIdentityClientId = configuration[ManagedIdentityClientId]; - string databaseConnectionString = configuration.GetRequiredValue(DatabaseConnectionString) + var managedIdentityClientId = builder.Configuration[ManagedIdentityClientId]; + string databaseConnectionString = builder.Configuration.GetRequiredValue(DatabaseConnectionString) .Replace(SqlConnectionStringUserIdPlaceholder, managedIdentityClientId); - services.TryAddTransient(); - services.AddDbContext(options => + builder.Services.TryAddTransient(); + builder.Services.AddDbContext(options => { // Do not log DB context initialization options.ConfigureWarnings(w => w.Ignore(CoreEventId.ContextInitialized)); @@ -70,24 +45,21 @@ public static void RegisterBuildAssetRegistry(this IServiceCollection services, if (!string.IsNullOrEmpty(managedIdentityClientId)) { string kustoManagedIdentityIdKey = $"Kusto:{nameof(KustoOptions.ManagedIdentityId)}"; - configuration[kustoManagedIdentityIdKey] = managedIdentityClientId; + builder.Configuration[kustoManagedIdentityIdKey] = managedIdentityClientId; } - services.AddKustoClientProvider("Kusto"); - services.AddSingleton(); ; + builder.Services.AddKustoClientProvider("Kusto"); + builder.Services.AddSingleton(); ; } - public static async Task AddRedis( - this IServiceCollection services, - IConfiguration configuration, - bool isDevelopment) + public static async Task AddRedis(this IHostApplicationBuilder builder) { var redisConfig = ConfigurationOptions.Parse( - configuration.GetSection("ConnectionStrings").GetRequiredValue(RedisConnectionString)); - var managedIdentityId = configuration[ManagedIdentityClientId]; + builder.Configuration.GetSection("ConnectionStrings").GetRequiredValue(RedisConnectionString)); + var managedIdentityId = builder.Configuration[ManagedIdentityClientId]; // Local redis instance should not need authentication - if (!isDevelopment) + if (!builder.Environment.IsDevelopment()) { AzureCacheOptions azureOptions = new(); if (managedIdentityId != "system") @@ -98,6 +70,6 @@ public static async Task AddRedis( await redisConfig.ConfigureForAzureAsync(azureOptions); } - services.AddSingleton(_ => ConnectionMultiplexer.Connect(redisConfig)); + builder.Services.AddSingleton(_ => ConnectionMultiplexer.Connect(redisConfig)); } } diff --git a/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/LongestBuildPathUpdaterConfiguration.cs b/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/LongestBuildPathUpdaterConfiguration.cs index 7d61d54745..71edae82be 100644 --- a/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/LongestBuildPathUpdaterConfiguration.cs +++ b/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/LongestBuildPathUpdaterConfiguration.cs @@ -11,11 +11,10 @@ public static class LongestBuildPathUpdaterConfiguration { public static void ConfigureLongestBuildPathUpdater( this HostApplicationBuilder builder, - ITelemetryChannel telemetryChannel, - bool isDevelopment) + ITelemetryChannel telemetryChannel) { - builder.Services.RegisterLogging(telemetryChannel, isDevelopment); - builder.Services.RegisterBuildAssetRegistry(builder.Configuration); + builder.Services.RegisterLogging(telemetryChannel, builder.Environment.IsDevelopment()); + builder.RegisterBuildAssetRegistry(); builder.Services.Configure(o => { }); diff --git a/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/Program.cs b/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/Program.cs index 301262b801..faeedea526 100644 --- a/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/Program.cs +++ b/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/Program.cs @@ -12,9 +12,11 @@ { var builder = Host.CreateApplicationBuilder(); - var serviceProvider = builder.Services.BuildServiceProvider(); + builder.ConfigureLongestBuildPathUpdater(telemetryChannel); - await serviceProvider.GetRequiredService().UpdateLongestBuildPathAsync(); + var applicationScope = builder.Build().Services.CreateScope(); + + await applicationScope.ServiceProvider.GetRequiredService().UpdateLongestBuildPathAsync(); } finally { diff --git a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/ProductConstructionService.SubscriptionTriggerer.csproj b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/ProductConstructionService.SubscriptionTriggerer.csproj index e9a8127078..639bb2022c 100644 --- a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/ProductConstructionService.SubscriptionTriggerer.csproj +++ b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/ProductConstructionService.SubscriptionTriggerer.csproj @@ -16,6 +16,7 @@ + diff --git a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/Program.cs b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/Program.cs index 3c442db042..4b877f103b 100644 --- a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/Program.cs +++ b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/Program.cs @@ -3,9 +3,12 @@ using Azure.Storage.Queues; using Maestro.Data.Models; using Microsoft.ApplicationInsights.Channel; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using ProductConstructionService.Common; using ProductConstructionService.SubscriptionTriggerer; +using ProductConstructionService.WorkItems; if (args.Count() < 1) { @@ -26,19 +29,19 @@ { var builder = Host.CreateApplicationBuilder(); - bool isDevelopment = builder.Environment.IsDevelopment(); + builder.ConfigureSubscriptionTriggerer(telemetryChannel); - builder.ConfigureSubscriptionTriggerer(telemetryChannel, isDevelopment); + // We're registering BAR context as a scoped service, so we have to create a scope to resolve it + var applicationScope = builder.Build().Services.CreateScope(); - ServiceProvider serviceProvider = builder.Services.BuildServiceProvider(); - - if (isDevelopment) - { - var client = serviceProvider.GetRequiredService(); - client.CreateIfNotExists(); + if (builder.Environment.IsDevelopment()) + { + var config = applicationScope.ServiceProvider.GetRequiredService(); + await applicationScope.ServiceProvider.UseLocalWorkItemQueues( + config.GetRequiredValue(WorkItemConfiguration.WorkItemQueueNameConfigurationKey)); } - var triggerer = serviceProvider.GetRequiredService(); + var triggerer = applicationScope.ServiceProvider.GetRequiredService(); await triggerer.CheckSubscriptionsAsync(frequency); } diff --git a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggerer.cs b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggerer.cs index 0aaeb773b8..fe427b00ed 100644 --- a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggerer.cs +++ b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggerer.cs @@ -1,32 +1,29 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using Azure.Storage.Queues; using Maestro.Data; using Maestro.Data.Models; -using Microsoft.DotNet.DarcLib; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; +using ProductConstructionService.WorkItems; +using ProductConstructionService.WorkItems.WorkItemDefinitions; namespace ProductConstructionService.SubscriptionTriggerer; public class SubscriptionTriggerer { - private readonly IBasicBarClient _barClient; private readonly ILogger _logger; private readonly BuildAssetRegistryContext _context; - private readonly QueueClient _queueClient; + private readonly IWorkItemProducerFactory _workItemProducerFactory; public SubscriptionTriggerer( ILogger logger, BuildAssetRegistryContext context, - IBasicBarClient barClient, - QueueClient queueClient) + IWorkItemProducerFactory workItemProducerFactory) { _logger = logger; _context = context; - _barClient = barClient; - _queueClient = queueClient; + _workItemProducerFactory = workItemProducerFactory; } public async Task CheckSubscriptionsAsync(UpdateFrequency targetUpdateFrequency) @@ -36,7 +33,8 @@ public async Task CheckSubscriptionsAsync(UpdateFrequency targetUpdateFrequency) .ToListAsync()) .Where(s => s.PolicyObject?.UpdateFrequency == targetUpdateFrequency); - int subscriptionsUpdated = 0; + var workItemProducer = + _workItemProducerFactory.Create(); foreach (var subscription in enabledSubscriptionsWithTargetFrequency) { Subscription? subscriptionWithBuilds = await _context.Subscriptions @@ -62,18 +60,16 @@ public async Task CheckSubscriptionsAsync(UpdateFrequency targetUpdateFrequency) if (isThereAnUnappliedBuildInTargetChannel && latestBuildInTargetChannel != null) { - _logger.LogInformation("Will trigger {subscriptionId} to build {latestBuildInTargetChannelId}", subscription.Id, latestBuildInTargetChannel.Id); - UpdateSubscriptionAsync(subscription.Id, latestBuildInTargetChannel.Id); - subscriptionsUpdated++; + // TODO https://github.com/dotnet/arcade-services/issues/3811 add some kind of feature switch to trigger specific subscriptions + /*await _workItemProducerFactory.Create().ProduceWorkItemAsync(new() + { + BuildId = latestBuildInTargetChannel.Id, + SubscriptionId = subscription.Id + });*/ + _logger.LogInformation("Queued update for subscription '{subscriptionId}' with build '{buildId}'", + subscription.Id, + latestBuildInTargetChannel.Id); } } } - - private void UpdateSubscriptionAsync(Guid subscriptionId, int buildId) - { - // TODO https://github.com/dotnet/arcade-services/issues/3802 add item to queue so the subscription gets triggered - _logger.LogInformation("Queued update for subscription '{subscriptionId}' with build '{buildId}'", - subscriptionId, - buildId); - } } diff --git a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggererConfiguration.cs b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggererConfiguration.cs index d2bbb9a1af..32aff1d463 100644 --- a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggererConfiguration.cs +++ b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggererConfiguration.cs @@ -8,6 +8,8 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using ProductConstructionService.Common; +using Azure.Identity; +using ProductConstructionService.WorkItems; namespace ProductConstructionService.SubscriptionTriggerer; @@ -15,13 +17,18 @@ public static class SubscriptionTriggererConfiguration { public static HostApplicationBuilder ConfigureSubscriptionTriggerer( this HostApplicationBuilder builder, - ITelemetryChannel telemetryChannel, - bool isDevelopment) + ITelemetryChannel telemetryChannel) { - builder.Services.RegisterLogging(telemetryChannel, isDevelopment); + DefaultAzureCredential credential = new( + new DefaultAzureCredentialOptions + { + ManagedIdentityClientId = builder.Configuration[ProductConstructionServiceExtension.ManagedIdentityClientId] + }); - builder.Services.RegisterBuildAssetRegistry(builder.Configuration); - builder.Services.RegisterAzureQueue(builder.Configuration); + builder.Services.RegisterLogging(telemetryChannel, builder.Environment.IsDevelopment()); + + builder.RegisterBuildAssetRegistry(); + builder.AddWorkItemProducerFactory(credential); builder.Services.Configure(o => { }); diff --git a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/appsettings.Development.json b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/appsettings.Development.json index 7278e781ab..dd44e43c3f 100644 --- a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/appsettings.Development.json +++ b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/appsettings.Development.json @@ -1,7 +1,9 @@ { "BuildAssetRegistrySqlConnectionString": "Data Source=localhost\\SQLEXPRESS;Initial Catalog=BuildAssetRegistry;Integrated Security=true", // Start the azurite container using: docker run -p 10001:10001 mcr.microsoft.com/azure-storage/azurite - "QueueConnectionString": "UseDevelopmentStorage=true;DevelopmentStorageProxyUri=http://127.0.0.1:10001/pcs-queue", + "ConnectionStrings": { + "queues": "UseDevelopmentStorage=true;DevelopmentStorageProxyUri=http://127.0.0.1:10001" + }, "Kusto": { "Database": "engineeringdata", "KustoClusterUri": "https://engdata.westus2.kusto.windows.net", diff --git a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/appsettings.Staging.json b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/appsettings.Staging.json index 27d62da21c..6e3c66bfe9 100644 --- a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/appsettings.Staging.json +++ b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/appsettings.Staging.json @@ -1,5 +1,7 @@ { - "QueueConnectionString": "https://productconstructionint.queue.core.windows.net/pcs-workitems", + "ConnectionStrings": { + "queues": "https://productconstructionint.queue.core.windows.net" + }, "ManagedIdentityClientId": "9729e72a-f381-4d59-a958-8aa94a18a8d2", "BuildAssetRegistrySqlConnectionString": "Data Source=tcp:maestro-int-server.database.windows.net,1433; Initial Catalog=BuildAssetRegistry; Authentication=Active Directory Managed Identity; Persist Security Info=False; MultipleActiveResultSets=True; Connect Timeout=30; Encrypt=True; TrustServerCertificate=False; User Id=USER_ID_PLACEHOLDER", "Kusto": { diff --git a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/appsettings.json b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/appsettings.json index 2c63c08510..e6cce8db67 100644 --- a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/appsettings.json +++ b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/appsettings.json @@ -1,2 +1,3 @@ { + "WorkItemQueueName": "pcs-workitems" } diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConfiguration.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConfiguration.cs index 108df31b86..c1bcc846d8 100644 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConfiguration.cs +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConfiguration.cs @@ -14,28 +14,36 @@ namespace ProductConstructionService.WorkItems; public static class WorkItemConfiguration { - private const string WorkItemQueueNameConfigurationKey = $"{WorkItemConsumerOptions.ConfigurationKey}:WorkItemQueueName"; + public const string WorkItemQueueNameConfigurationKey = "WorkItemQueueName"; - public static void AddWorkItemQueues(this WebApplicationBuilder builder, DefaultAzureCredential credential, bool waitForInitialization) + public static void AddWorkItemQueues(this IHostApplicationBuilder builder, DefaultAzureCredential credential, bool waitForInitialization) { - builder.AddAzureQueueClient("queues", settings => settings.Credential = credential); - - var queueName = builder.Configuration.GetRequiredValue(WorkItemQueueNameConfigurationKey); + builder.AddWorkItemProducerFactory(credential); // When running the service locally, the WorkItemProcessor should start in the Working state builder.Services.AddSingleton(sp => ActivatorUtilities.CreateInstance(sp, waitForInitialization)); + builder.Configuration[$"{WorkItemConsumerOptions.ConfigurationKey}:${WorkItemQueueNameConfigurationKey}"] = + builder.Configuration.GetRequiredValue(WorkItemQueueNameConfigurationKey); builder.Services.Configure( builder.Configuration.GetSection(WorkItemConsumerOptions.ConfigurationKey)); - builder.Services.AddTransient(sp => - ActivatorUtilities.CreateInstance(sp, queueName)); builder.Services.AddHostedService(); } + public static void AddWorkItemProducerFactory(this IHostApplicationBuilder builder, DefaultAzureCredential credential) + { + builder.AddAzureQueueClient("queues", settings => settings.Credential = credential); + + var queueName = builder.Configuration.GetRequiredValue(WorkItemQueueNameConfigurationKey); + + builder.Services.AddTransient(sp => + ActivatorUtilities.CreateInstance(sp, queueName)); + } + // When running locally, create the workitem queue, if it doesn't already exist - public static async Task UseLocalWorkItemQueues(this WebApplication app) + public static async Task UseLocalWorkItemQueues(this IServiceProvider serviceProvider, string queueName) { - var queueServiceClient = app.Services.GetRequiredService(); - var queueClient = queueServiceClient.GetQueueClient(app.Configuration.GetRequiredValue(WorkItemQueueNameConfigurationKey)); + var queueServiceClient = serviceProvider.GetRequiredService(); + var queueClient = queueServiceClient.GetQueueClient(queueName); await queueClient.CreateIfNotExistsAsync(); } diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemDefinitions/BuildCoherencyInfoWorkItem.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemDefinitions/BuildCoherencyInfoWorkItem.cs new file mode 100644 index 0000000000..09dae90194 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemDefinitions/BuildCoherencyInfoWorkItem.cs @@ -0,0 +1,10 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace ProductConstructionService.WorkItems.WorkItemDefinitions; +public class BuildCoherencyInfoWorkItem : WorkItem +{ + public required int BuildId { get; init; } + + public override string Type => nameof(BuildCoherencyInfoWorkItem); +} diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemDefinitions/UpdateSubscriptionWorkItem.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemDefinitions/UpdateSubscriptionWorkItem.cs new file mode 100644 index 0000000000..a9edfe16bd --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemDefinitions/UpdateSubscriptionWorkItem.cs @@ -0,0 +1,19 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace ProductConstructionService.WorkItems.WorkItemDefinitions; + +public class UpdateSubscriptionWorkItem : WorkItem +{ + /// + /// Subscription that is being triggered. + /// + public required Guid SubscriptionId { get; init; } + + /// + /// Build that is being flown. + /// + public required int BuildId { get; init; } + + public override string Type => nameof(UpdateSubscriptionWorkItem); +} diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemDefinitions/WorkItem.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemDefinitions/WorkItem.cs index 40ee1b5646..f592af8de5 100644 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemDefinitions/WorkItem.cs +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemDefinitions/WorkItem.cs @@ -6,6 +6,8 @@ namespace ProductConstructionService.WorkItems.WorkItemDefinitions; [JsonDerivedType(typeof(CodeFlowWorkItem), typeDiscriminator: nameof(CodeFlowWorkItem))] +[JsonDerivedType(typeof(UpdateSubscriptionWorkItem), typeDiscriminator: nameof(UpdateSubscriptionWorkItem))] +[JsonDerivedType(typeof(BuildCoherencyInfoWorkItem), typeDiscriminator: nameof(BuildCoherencyInfoWorkItem))] public abstract class WorkItem { public Guid Id { get; } = Guid.NewGuid(); diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProducer.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProducer.cs index c2edd9714b..b3de2530d0 100644 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProducer.cs +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProducer.cs @@ -8,14 +8,22 @@ namespace ProductConstructionService.WorkItems; -public class WorkItemProducer(QueueServiceClient queueServiceClient, string queueName) where T : WorkItem +public interface IWorkItemProducer +{ + Task ProduceWorkItemAsync(T payload, TimeSpan delay = default); +} + +public class WorkItemProducer(QueueServiceClient queueServiceClient, string queueName) : IWorkItemProducer where T : WorkItem { private readonly QueueServiceClient _queueServiceClient = queueServiceClient; private readonly string _queueName = queueName; - public async Task ProduceWorkItemAsync(T payload) + /// + /// Puts a WorkItem into the queue, which becomes visible after the specified delay. + /// + public async Task ProduceWorkItemAsync(T payload, TimeSpan delay = default) { var client = _queueServiceClient.GetQueueClient(_queueName); - return await client.SendMessageAsync(JsonSerializer.Serialize(payload)); + return await client.SendMessageAsync(JsonSerializer.Serialize(payload), delay); } } diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProducerFactory.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProducerFactory.cs index cdc862456d..97e5fd8250 100644 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProducerFactory.cs +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProducerFactory.cs @@ -6,11 +6,16 @@ namespace ProductConstructionService.WorkItems; -public class WorkItemProducerFactory(QueueServiceClient queueServiceClient, string queueName) +public interface IWorkItemProducerFactory +{ + public IWorkItemProducer Create() where T : WorkItem; +} + +public class WorkItemProducerFactory(QueueServiceClient queueServiceClient, string queueName) : IWorkItemProducerFactory { private readonly QueueServiceClient _queueServiceClient = queueServiceClient; private readonly string _queueName = queueName; - public WorkItemProducer Create() where T : WorkItem - => new(_queueServiceClient, _queueName); + public IWorkItemProducer Create() where T : WorkItem + => new WorkItemProducer(_queueServiceClient, _queueName); } diff --git a/test/ProductConstructionService.Api.Tests/BuildController20200914Tests.cs b/test/ProductConstructionService.Api.Tests/BuildController20200914Tests.cs index fe6fa48bb6..6bd64452cc 100644 --- a/test/ProductConstructionService.Api.Tests/BuildController20200914Tests.cs +++ b/test/ProductConstructionService.Api.Tests/BuildController20200914Tests.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Net; +using Azure.Storage.Queues.Models; using FluentAssertions; using Maestro.Api.Model.v2020_02_20; using Maestro.Data; @@ -18,6 +19,8 @@ using Moq; using ProductConstructionService.Api.Api.v2020_02_20.Controllers; +using ProductConstructionService.WorkItems; +using ProductConstructionService.WorkItems.WorkItemDefinitions; using Commit = Maestro.Api.Model.v2020_02_20.Commit; namespace ProductConstructionService.Api.Tests; @@ -89,6 +92,9 @@ public static async Task Dependencies(IServiceCollection collection) var mockIRemoteFactory = new Mock(); var mockIRemote = new Mock(); + var mockWorkItemProducerFactory = new Mock(); + var mockWorkItemProducer = new Mock>(); + mockWorkItemProducerFactory.Setup(f => f.Create()).Returns(mockWorkItemProducer.Object); mockIRemoteFactory.Setup(f => f.GetRemoteAsync(Repository, It.IsAny())).ReturnsAsync(mockIRemote.Object); mockIRemote.Setup(f => f.GetCommitAsync(Repository, CommitHash)).ReturnsAsync(new Microsoft.DotNet.DarcLib.Commit(Account, CommitHash, CommitMessage)); @@ -105,6 +111,7 @@ public static async Task Dependencies(IServiceCollection collection) options.EnableServiceProviderCaching(false); }); collection.AddSingleton(); + collection.AddSingleton(mockWorkItemProducerFactory.Object); } public static Func Controller(IServiceCollection collection) diff --git a/test/ProductConstructionService.Api.Tests/ChannelsController20200220Tests.cs b/test/ProductConstructionService.Api.Tests/ChannelsController20200220Tests.cs index e6f0dc080e..87494e7914 100644 --- a/test/ProductConstructionService.Api.Tests/ChannelsController20200220Tests.cs +++ b/test/ProductConstructionService.Api.Tests/ChannelsController20200220Tests.cs @@ -17,6 +17,8 @@ using Microsoft.Extensions.Logging; using Moq; using ProductConstructionService.Api.Api.v2020_02_20.Controllers; +using ProductConstructionService.WorkItems.WorkItemDefinitions; +using ProductConstructionService.WorkItems; namespace ProductConstructionService.Api.Tests; @@ -167,6 +169,12 @@ public static async Task Dependencies(IServiceCollection collection) }); collection.AddSingleton(Mock.Of()); collection.AddSingleton(Mock.Of()); + + var mockWorkItemProducerFactory = new Mock(); + var mockWorkItemProducer = new Mock>(); + mockWorkItemProducerFactory.Setup(f => f.Create()).Returns(mockWorkItemProducer.Object); + + collection.AddSingleton(mockWorkItemProducerFactory.Object); } public static Func Clock(IServiceCollection collection) diff --git a/test/ProductConstructionService.Api.Tests/SubscriptionsController20200220Tests.cs b/test/ProductConstructionService.Api.Tests/SubscriptionsController20200220Tests.cs index a5114c5749..4440cf5c71 100644 --- a/test/ProductConstructionService.Api.Tests/SubscriptionsController20200220Tests.cs +++ b/test/ProductConstructionService.Api.Tests/SubscriptionsController20200220Tests.cs @@ -18,6 +18,8 @@ using Microsoft.Extensions.Logging; using Moq; using ProductConstructionService.Api.Api.v2020_02_20.Controllers; +using ProductConstructionService.WorkItems.WorkItemDefinitions; +using ProductConstructionService.WorkItems; namespace ProductConstructionService.Api.Tests; @@ -450,6 +452,11 @@ private static class TestDataConfiguration { public static void Dependencies(IServiceCollection collection) { + var mockWorkItemProducerFactory = new Mock(); + var mockUpdateSubscriptionWorkItemProducer = new Mock>(); + var mockBuildCoherencyInfoWorkItem = new Mock>(); + mockWorkItemProducerFactory.Setup(f => f.Create()).Returns(mockUpdateSubscriptionWorkItemProducer.Object); + mockWorkItemProducerFactory.Setup(f => f.Create()).Returns(mockBuildCoherencyInfoWorkItem.Object); collection.AddLogging(l => l.AddProvider(new NUnitLogger())); collection.AddSingleton(new HostingEnvironment { @@ -457,6 +464,7 @@ public static void Dependencies(IServiceCollection collection) }); collection.AddSingleton(Mock.Of()); collection.AddSingleton(Mock.Of()); + collection.AddSingleton(mockWorkItemProducerFactory.Object); } public static void GitHub(IServiceCollection collection) diff --git a/test/ProductConstructionService.LongestBuildPathUpdater.Tests/DependencyRegistrationTests.cs b/test/ProductConstructionService.LongestBuildPathUpdater.Tests/DependencyRegistrationTests.cs index 3fc9eb2ee4..dc31fa1f79 100644 --- a/test/ProductConstructionService.LongestBuildPathUpdater.Tests/DependencyRegistrationTests.cs +++ b/test/ProductConstructionService.LongestBuildPathUpdater.Tests/DependencyRegistrationTests.cs @@ -19,7 +19,7 @@ public void AreDependenciesRegistered() builder.Configuration["BuildAssetRegistrySqlConnectionString"] = "barConnectionString"; - builder.ConfigureLongestBuildPathUpdater(new InMemoryChannel(), true); + builder.ConfigureLongestBuildPathUpdater(new InMemoryChannel()); DependencyInjectionValidation.IsDependencyResolutionCoherent(s => { diff --git a/test/ProductConstructionService.SubscriptionTriggerer.Tests/DependencyRegistrationTests.cs b/test/ProductConstructionService.SubscriptionTriggerer.Tests/DependencyRegistrationTests.cs index 5a79fceaac..8d61fc55c0 100644 --- a/test/ProductConstructionService.SubscriptionTriggerer.Tests/DependencyRegistrationTests.cs +++ b/test/ProductConstructionService.SubscriptionTriggerer.Tests/DependencyRegistrationTests.cs @@ -19,18 +19,23 @@ public void AreDependenciesRegistered() { var builder = Host.CreateApplicationBuilder(); - builder.Configuration["QueueConnectionString"] = "queueConnectionString"; + builder.Configuration["ConnectionStrings:queues"] = "queueConnectionString"; + builder.Configuration["WorkItemQueueName"] = "queue"; builder.Configuration["BuildAssetRegistrySqlConnectionString"] = "barConnectionString"; - builder.ConfigureSubscriptionTriggerer(new InMemoryChannel(), false); + builder.ConfigureSubscriptionTriggerer(new InMemoryChannel()); DependencyInjectionValidation.IsDependencyResolutionCoherent(s => - { - foreach (var descriptor in builder.Services) { - s.Add(descriptor); - } - }, - out var message).Should().BeTrue(message); + foreach (var descriptor in builder.Services) + { + s.Add(descriptor); + } + }, + out var message, + additionalExemptTypes: [ + "Microsoft.Extensions.Azure.AzureClientsGlobalOptions" + ]) + .Should().BeTrue(message); } } From 319b40ef26a035b625932160f5f8e47fc567f1e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Thu, 22 Aug 2024 11:15:31 +0200 Subject: [PATCH 05/11] Enable data protection (#3861) --- .../provision.bicep | 24 +++++++++++++++++++ .../Configuration/DataProtection.cs | 22 ++++++++--------- .../PcsStartup.cs | 2 +- .../appsettings.Staging.json | 9 ++++--- 4 files changed, 40 insertions(+), 17 deletions(-) diff --git a/eng/service-templates/ProductConstructionService/provision.bicep b/eng/service-templates/ProductConstructionService/provision.bicep index 19cd2e90a9..95b2017316 100644 --- a/eng/service-templates/ProductConstructionService/provision.bicep +++ b/eng/service-templates/ProductConstructionService/provision.bicep @@ -422,6 +422,8 @@ var storageQueueContrubutorRole = subscriptionResourceId('Microsoft.Authorizatio var contributorRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c') // azure system role Key Vault Reader var keyVaultReaderRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '21090545-7ca7-4776-b22c-e363652d74d2') +// storage account blob contributor +var blobContributorRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'ba92f5b4-2d11-453d-a403-e96b0029c9fe') // application insights for service logging resource applicationInsights 'Microsoft.Insights/components@2020-02-02' = { @@ -730,6 +732,28 @@ resource storageAccount 'Microsoft.Storage/storageAccounts@2022-09-01' = { } } +// Create the dataprotection container in the storage account +resource storageAccountBlobService 'Microsoft.Storage/storageAccounts/blobServices@2022-09-01' = { + name: 'default' + parent: storageAccount +} + +resource dataProtectionContainer 'Microsoft.Storage/storageAccounts/blobServices/containers@2022-09-01' = { + name: 'dataprotection' + parent: storageAccountBlobService +} + +// allow identity access to the storage account +resource storageAccountContributor 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: dataProtectionContainer // Use when specifying a scope that is different than the deployment scope + name: guid(subscription().id, resourceGroup().id, blobContributorRole) + properties: { + roleDefinitionId: blobContributorRole + principalType: 'ServicePrincipal' + principalId: pcsIdentity.properties.principalId + } +} + resource storageAccountQueueService 'Microsoft.Storage/storageAccounts/queueServices@2022-09-01' = { name: 'default' parent: storageAccount diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Configuration/DataProtection.cs b/src/ProductConstructionService/ProductConstructionService.Api/Configuration/DataProtection.cs index 25bb4d69c3..ba9bc9c6ae 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Configuration/DataProtection.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Configuration/DataProtection.cs @@ -13,24 +13,24 @@ internal static class DataProtection private static readonly TimeSpan DataProtectionKeyLifetime = new(days: 240, hours: 0, minutes: 0, seconds: 0); - public static void ConfigureDataProtection(this WebApplicationBuilder builder) + public static void AddDataProtection(this WebApplicationBuilder builder) { var keyBlobUri = builder.Configuration[DataProtectionKeyBlobUri]; var dataProtectionKeyUri = builder.Configuration[DataProtectionKeyUri]; if (string.IsNullOrEmpty(keyBlobUri) || string.IsNullOrEmpty(dataProtectionKeyUri)) { - builder.Services.AddDataProtection() + builder.Services + .AddDataProtection() .SetDefaultKeyLifetime(DataProtectionKeyLifetime); + return; } - else - { - var credential = new DefaultAzureCredential(); - builder.Services.AddDataProtection() - .PersistKeysToAzureBlobStorage(new Uri(keyBlobUri), credential) - .ProtectKeysWithAzureKeyVault(new Uri(dataProtectionKeyUri), credential) - .SetDefaultKeyLifetime(DataProtectionKeyLifetime) - .SetApplicationName(typeof(PcsStartup).FullName!); - } + + var credential = new DefaultAzureCredential(); + builder.Services.AddDataProtection() + .PersistKeysToAzureBlobStorage(new Uri(keyBlobUri), credential) + .ProtectKeysWithAzureKeyVault(new Uri(dataProtectionKeyUri), credential) + .SetDefaultKeyLifetime(DataProtectionKeyLifetime) + .SetApplicationName(typeof(PcsStartup).FullName!); } } diff --git a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs index a70fac02a2..cb9ec7213a 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs @@ -142,7 +142,7 @@ internal static async Task ConfigurePcs( string? gitHubToken = builder.Configuration[ConfigurationKeys.GitHubToken]; builder.Services.Configure(ConfigurationKeys.AzureDevOpsConfiguration, (o, s) => s.Bind(o)); - builder.ConfigureDataProtection(); + builder.AddDataProtection(); builder.AddTelemetry(); DefaultAzureCredential azureCredential = new(new DefaultAzureCredentialOptions diff --git a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Staging.json b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Staging.json index 1d5d3f5102..33f5f2c9dc 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Staging.json +++ b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Staging.json @@ -15,11 +15,10 @@ "Database": "engineeringdata", "KustoClusterUri": "https://engdata.westus2.kusto.windows.net" }, - // TODO (https://github.com/dotnet/arcade-services/issues/3815): Enable dataprotection - //"DataProtection": { - // "KeyBlobUri": "https://productconstructionint.blob.core.windows.net/dataprotection/keys.xml", - // "DataProtectionKeyUri": "https://productconstructionint.vault.azure.net/keys/data-protection-encryption-key/" - //}, + "DataProtection": { + "KeyBlobUri": "https://productconstructionint.blob.core.windows.net/dataprotection/keys.xml", + "DataProtectionKeyUri": "https://productconstructionint.vault.azure.net/keys/data-protection-encryption-key/" + }, "EntraAuthentication": { // https://ms.portal.azure.com/#view/Microsoft_AAD_RegisteredApps/ApplicationMenuBlade/~/Overview/appId/baf98f1b-374e-487d-af42-aa33807f11e4/objectId/a0e22263-aa27-4dc8-81d6-f12e63fb0d96/isMSAApp~/false/defaultBlade/Overview/appSignInAudience/AzureADMyOrg/servicePrincipalCreated~/true "ClientId": "baf98f1b-374e-487d-af42-aa33807f11e4", From 0b6a88ae0ba48b42af0598e5d9443aff97554ff5 Mon Sep 17 00:00:00 2001 From: Oleksandr Didyk <106967057+oleksandr-didyk@users.noreply.github.com> Date: Thu, 22 Aug 2024 13:45:10 +0200 Subject: [PATCH 06/11] Add state manager (#3863) --- .../PcsStartup.cs | 9 ++-- .../ProductConstructionService.Api/Program.cs | 2 +- .../ProductConstructionServiceExtension.cs | 10 ++-- .../StateManager.cs | 51 +++++++++++++++++++ .../StateManagerFactory.cs | 45 ++++++++++++++++ .../DependencyRegistrationTests.cs | 2 +- .../LoggingConfigurationTests.cs | 2 +- 7 files changed, 108 insertions(+), 13 deletions(-) create mode 100644 src/ProductConstructionService/ProductConstructionService.Common/StateManager.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Common/StateManagerFactory.cs diff --git a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs index cb9ec7213a..3ad915681d 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs @@ -124,12 +124,12 @@ static PcsStartup() /// /// /// Use KeyVault for secrets? - /// Use Redis for caching? + /// Use authenticated connection for Redis? /// Add Swagger UI? internal static async Task ConfigurePcs( this WebApplicationBuilder builder, bool addKeyVault, - bool addRedis, + bool authRedis, bool addSwagger) { bool isDevelopment = builder.Environment.IsDevelopment(); @@ -178,10 +178,7 @@ internal static async Task ConfigurePcs( builder.Services.AddMergePolicies(); builder.Services.Configure(builder.Configuration.GetSection(ConfigurationKeys.DependencyFlowSLAs)); - if (addRedis) - { - await builder.AddRedis(); - } + await builder.AddStateManager(authRedis); if (initializeService) { diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Program.cs b/src/ProductConstructionService/ProductConstructionService.Api/Program.cs index f7b8a702e0..70f377e1ca 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Program.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Program.cs @@ -15,7 +15,7 @@ await builder.ConfigurePcs( addKeyVault: true, - addRedis: true, + authRedis: !isDevelopment, addSwagger: useSwagger); var app = builder.Build(); diff --git a/src/ProductConstructionService/ProductConstructionService.Common/ProductConstructionServiceExtension.cs b/src/ProductConstructionService/ProductConstructionService.Common/ProductConstructionServiceExtension.cs index c3e1291d7d..fdba1e6edf 100644 --- a/src/ProductConstructionService/ProductConstructionService.Common/ProductConstructionServiceExtension.cs +++ b/src/ProductConstructionService/ProductConstructionService.Common/ProductConstructionServiceExtension.cs @@ -52,14 +52,15 @@ public static void RegisterBuildAssetRegistry(this IHostApplicationBuilder build builder.Services.AddSingleton(); ; } - public static async Task AddRedis(this IHostApplicationBuilder builder) + public static async Task AddStateManager( + this IHostApplicationBuilder builder, + bool useAuth) { var redisConfig = ConfigurationOptions.Parse( builder.Configuration.GetSection("ConnectionStrings").GetRequiredValue(RedisConnectionString)); var managedIdentityId = builder.Configuration[ManagedIdentityClientId]; - // Local redis instance should not need authentication - if (!builder.Environment.IsDevelopment()) + if (useAuth) { AzureCacheOptions azureOptions = new(); if (managedIdentityId != "system") @@ -70,6 +71,7 @@ public static async Task AddRedis(this IHostApplicationBuilder builder) await redisConfig.ConfigureForAzureAsync(azureOptions); } - builder.Services.AddSingleton(_ => ConnectionMultiplexer.Connect(redisConfig)); + builder.Services.AddSingleton(redisConfig); + builder.Services.AddSingleton(); } } diff --git a/src/ProductConstructionService/ProductConstructionService.Common/StateManager.cs b/src/ProductConstructionService/ProductConstructionService.Common/StateManager.cs new file mode 100644 index 0000000000..bf2c22eb64 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Common/StateManager.cs @@ -0,0 +1,51 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using StackExchange.Redis; + +namespace ProductConstructionService.Common; + +public interface IStateManager +{ + public void Set(string value); + public Task SetAsync(string value); + public bool TryGet(out string? value); + public Task GetAsync(); +} + +public class StateManager : IStateManager +{ + private readonly string _stateKey; + private readonly IConnectionMultiplexer _connection; + + private IDatabase Cache => _connection.GetDatabase(); + + internal StateManager(IConnectionMultiplexer connection, string stateKey) + { + _connection = connection; + _stateKey = stateKey; + } + + public void Set(string value) + { + Cache.StringSet(_stateKey, value); + } + + public async Task SetAsync(string value) + { + await Cache.StringSetAsync(_stateKey, value); + } + + public bool TryGet(out string? value) + { + value = Cache.StringGet(_stateKey); + + return value != null; + } + + public async Task GetAsync() + { + return await Cache.StringGetAsync(_stateKey); + } + +} diff --git a/src/ProductConstructionService/ProductConstructionService.Common/StateManagerFactory.cs b/src/ProductConstructionService/ProductConstructionService.Common/StateManagerFactory.cs new file mode 100644 index 0000000000..faad2863bf --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Common/StateManagerFactory.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using StackExchange.Redis; + +namespace ProductConstructionService.Common; + +public interface IStateManagerFactory +{ + public IStateManager CreateCodeFlowStateManager(string actorId); + public IStateManager CreateInProgressPrStateManager(string actorId); + public IStateManager CreateUpdateAssetsStateManager(string actorId); +} + +public class StateManagerFactory: IStateManagerFactory +{ + private static readonly string CODEFLOW_PREFIX = "CodeFlow"; + private static readonly string IN_PROGRESS_PR_PREFIX = "InProgressPullRequest"; + private static readonly string UPDATE_ASSETS_PARAMETERS_PREFIX = "UpdateAssetsParameters"; + + // State managers should store and re-use `ConnectionMultiplexer`, so we + // only use one instance for every state manager we create + // https://stackexchange.github.io/StackExchange.Redis/Basics + private readonly IConnectionMultiplexer _connection; + + public StateManagerFactory(ConfigurationOptions options) + { + _connection = ConnectionMultiplexer.Connect(options); + } + + public IStateManager CreateCodeFlowStateManager(string actorId) + { + return new StateManager(_connection, $"{CODEFLOW_PREFIX}_{actorId}"); + } + + public IStateManager CreateInProgressPrStateManager(string actorId) + { + return new StateManager(_connection, $"{IN_PROGRESS_PR_PREFIX}_{actorId}"); + } + + public IStateManager CreateUpdateAssetsStateManager(string actorId) + { + return new StateManager(_connection, $"{UPDATE_ASSETS_PARAMETERS_PREFIX}_{actorId}"); + } +} diff --git a/test/ProductConstructionService.Api.Tests/DependencyRegistrationTests.cs b/test/ProductConstructionService.Api.Tests/DependencyRegistrationTests.cs index 8e04ffe0ef..8e149392d1 100644 --- a/test/ProductConstructionService.Api.Tests/DependencyRegistrationTests.cs +++ b/test/ProductConstructionService.Api.Tests/DependencyRegistrationTests.cs @@ -16,7 +16,7 @@ public async Task AreDependenciesRegistered() var builder = ApiTestConfiguration.CreateTestHostBuilder(); await builder.ConfigurePcs( addKeyVault: false, - addRedis: false, + authRedis: false, addSwagger: true); DependencyInjectionValidation.IsDependencyResolutionCoherent( diff --git a/test/ProductConstructionService.Api.Tests/LoggingConfigurationTests.cs b/test/ProductConstructionService.Api.Tests/LoggingConfigurationTests.cs index 167931ee67..d23b7cac54 100644 --- a/test/ProductConstructionService.Api.Tests/LoggingConfigurationTests.cs +++ b/test/ProductConstructionService.Api.Tests/LoggingConfigurationTests.cs @@ -79,7 +79,7 @@ private static async Task Setup() await builder.ConfigurePcs( addKeyVault: false, - addRedis: false, + authRedis: false, addSwagger: false); builder.Services.AddSingleton(channel.Object); From 192cc5fad6fbb2c081f69a78675fb87fef594c1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Thu, 22 Aug 2024 13:54:40 +0200 Subject: [PATCH 07/11] Allow dynamic registration of work item processors (#3857) --- arcade-services.sln | 30 +++ .../Controllers/SubscriptionsController.cs | 11 +- .../Controllers/BuildsController.cs | 5 +- .../Controllers/CodeFlowController.cs | 19 +- .../PcsStartup.cs | 7 +- .../ProductConstructionService.Api.csproj | 1 + ...tConstructionService.DependencyFlow.csproj | 25 ++ .../CodeFlowWorkItemProcessor.cs | 35 ++- .../WorkItems}/BuildCoherencyInfoWorkItem.cs | 6 +- .../WorkItems}/CodeFlowWorkItem.cs | 6 +- .../WorkItems}/UpdateSubscriptionWorkItem.cs | 6 +- ...uctionService.SubscriptionTriggerer.csproj | 1 + .../SubscriptionTriggerer.cs | 4 +- .../Properties/AssemblyInfo.cs | 7 + .../WorkItem.cs | 10 + .../WorkItemConfiguration.cs | 45 +++- .../WorkItemConsumer.cs | 35 ++- .../WorkItemDefinitions/WorkItem.cs | 15 -- .../WorkItemProcessor.cs | 23 ++ .../WorkItemProcessorRegistrations.cs | 18 ++ .../WorkItemProcessors/IWorkItemProcessor.cs | 11 - .../WorkItemProducer.cs | 4 +- .../WorkItemProducerFactory.cs | 5 +- .../WorkItemScope.cs | 56 +++-- .../WorkItemScopeManager.cs | 15 +- .../BuildController20200914Tests.cs | 6 +- .../ChannelsController20200220Tests.cs | 5 +- ...roductConstructionService.Api.Tests.csproj | 1 + .../SubscriptionsController20200220Tests.cs | 9 +- .../WorkItemScopeTests.cs | 87 ------- .../GlobalUsings.cs | 4 + ...tConstructionService.WorkItem.Tests.csproj | 21 ++ .../WorkItemScopeTests.cs | 218 ++++++++++++++++++ .../WorkItemsProcessorScopeManagerTests.cs | 10 +- 34 files changed, 546 insertions(+), 215 deletions(-) create mode 100644 src/ProductConstructionService/ProductConstructionService.DependencyFlow/ProductConstructionService.DependencyFlow.csproj rename src/ProductConstructionService/{ProductConstructionService.WorkItems => ProductConstructionService.DependencyFlow}/WorkItemProcessors/CodeFlowWorkItemProcessor.cs (78%) rename src/ProductConstructionService/{ProductConstructionService.WorkItems/WorkItemDefinitions => ProductConstructionService.DependencyFlow/WorkItems}/BuildCoherencyInfoWorkItem.cs (63%) rename src/ProductConstructionService/{ProductConstructionService.WorkItems/WorkItemDefinitions => ProductConstructionService.DependencyFlow/WorkItems}/CodeFlowWorkItem.cs (86%) rename src/ProductConstructionService/{ProductConstructionService.WorkItems/WorkItemDefinitions => ProductConstructionService.DependencyFlow/WorkItems}/UpdateSubscriptionWorkItem.cs (76%) create mode 100644 src/ProductConstructionService/ProductConstructionService.WorkItems/Properties/AssemblyInfo.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItem.cs delete mode 100644 src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemDefinitions/WorkItem.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProcessor.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProcessorRegistrations.cs delete mode 100644 src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProcessors/IWorkItemProcessor.cs delete mode 100644 test/ProductConstructionService.Api.Tests/WorkItemScopeTests.cs create mode 100644 test/ProductConstructionService.WorkItem.Tests/GlobalUsings.cs create mode 100644 test/ProductConstructionService.WorkItem.Tests/ProductConstructionService.WorkItem.Tests.csproj create mode 100644 test/ProductConstructionService.WorkItem.Tests/WorkItemScopeTests.cs rename test/{ProductConstructionService.Api.Tests => ProductConstructionService.WorkItem.Tests}/WorkItemsProcessorScopeManagerTests.cs (95%) diff --git a/arcade-services.sln b/arcade-services.sln index 182ade03b9..7c6548ad3a 100644 --- a/arcade-services.sln +++ b/arcade-services.sln @@ -132,6 +132,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ProductConstructionService. EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ProductConstructionService.WorkItems", "src\ProductConstructionService\ProductConstructionService.WorkItems\ProductConstructionService.WorkItems.csproj", "{90C7747B-EBEF-4CF5-92A7-7856A3A13CAA}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ProductConstructionService.WorkItem.Tests", "test\ProductConstructionService.WorkItem.Tests\ProductConstructionService.WorkItem.Tests.csproj", "{29A75658-2DC4-4E85-8A53-97198F00F28D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ProductConstructionService.DependencyFlow", "src\ProductConstructionService\ProductConstructionService.DependencyFlow\ProductConstructionService.DependencyFlow.csproj", "{E312686C-A134-486F-9F62-89CE6CA34702}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -623,6 +627,30 @@ Global {90C7747B-EBEF-4CF5-92A7-7856A3A13CAA}.Release|x64.Build.0 = Release|Any CPU {90C7747B-EBEF-4CF5-92A7-7856A3A13CAA}.Release|x86.ActiveCfg = Release|Any CPU {90C7747B-EBEF-4CF5-92A7-7856A3A13CAA}.Release|x86.Build.0 = Release|Any CPU + {29A75658-2DC4-4E85-8A53-97198F00F28D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {29A75658-2DC4-4E85-8A53-97198F00F28D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {29A75658-2DC4-4E85-8A53-97198F00F28D}.Debug|x64.ActiveCfg = Debug|Any CPU + {29A75658-2DC4-4E85-8A53-97198F00F28D}.Debug|x64.Build.0 = Debug|Any CPU + {29A75658-2DC4-4E85-8A53-97198F00F28D}.Debug|x86.ActiveCfg = Debug|Any CPU + {29A75658-2DC4-4E85-8A53-97198F00F28D}.Debug|x86.Build.0 = Debug|Any CPU + {29A75658-2DC4-4E85-8A53-97198F00F28D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {29A75658-2DC4-4E85-8A53-97198F00F28D}.Release|Any CPU.Build.0 = Release|Any CPU + {29A75658-2DC4-4E85-8A53-97198F00F28D}.Release|x64.ActiveCfg = Release|Any CPU + {29A75658-2DC4-4E85-8A53-97198F00F28D}.Release|x64.Build.0 = Release|Any CPU + {29A75658-2DC4-4E85-8A53-97198F00F28D}.Release|x86.ActiveCfg = Release|Any CPU + {29A75658-2DC4-4E85-8A53-97198F00F28D}.Release|x86.Build.0 = Release|Any CPU + {E312686C-A134-486F-9F62-89CE6CA34702}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E312686C-A134-486F-9F62-89CE6CA34702}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E312686C-A134-486F-9F62-89CE6CA34702}.Debug|x64.ActiveCfg = Debug|Any CPU + {E312686C-A134-486F-9F62-89CE6CA34702}.Debug|x64.Build.0 = Debug|Any CPU + {E312686C-A134-486F-9F62-89CE6CA34702}.Debug|x86.ActiveCfg = Debug|Any CPU + {E312686C-A134-486F-9F62-89CE6CA34702}.Debug|x86.Build.0 = Debug|Any CPU + {E312686C-A134-486F-9F62-89CE6CA34702}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E312686C-A134-486F-9F62-89CE6CA34702}.Release|Any CPU.Build.0 = Release|Any CPU + {E312686C-A134-486F-9F62-89CE6CA34702}.Release|x64.ActiveCfg = Release|Any CPU + {E312686C-A134-486F-9F62-89CE6CA34702}.Release|x64.Build.0 = Release|Any CPU + {E312686C-A134-486F-9F62-89CE6CA34702}.Release|x86.ActiveCfg = Release|Any CPU + {E312686C-A134-486F-9F62-89CE6CA34702}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -672,6 +700,8 @@ Global {BE0088E3-A8FF-4F05-9456-E8BAD2E50A19} = {243A4561-BF35-405A-AF12-AC57BB27796D} {D40EADB7-5D48-421B-806D-6E2F79C077F8} = {1A456CF0-C09A-4DE6-89CE-1110EED31180} {90C7747B-EBEF-4CF5-92A7-7856A3A13CAA} = {243A4561-BF35-405A-AF12-AC57BB27796D} + {29A75658-2DC4-4E85-8A53-97198F00F28D} = {1A456CF0-C09A-4DE6-89CE-1110EED31180} + {E312686C-A134-486F-9F62-89CE6CA34702} = {243A4561-BF35-405A-AF12-AC57BB27796D} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {32B9C883-432E-4FC8-A1BF-090EB033DD5B} diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs index 3f907a3ffe..a03f079da2 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs @@ -3,16 +3,17 @@ using System.ComponentModel.DataAnnotations; using System.Net; +using Maestro.Api.Model.v2018_07_16; using Maestro.Data; using Microsoft.AspNetCore.ApiPagination; using Microsoft.AspNetCore.ApiVersioning; using Microsoft.AspNetCore.ApiVersioning.Swashbuckle; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; -using Channel = Maestro.Data.Models.Channel; -using Maestro.Api.Model.v2018_07_16; +using ProductConstructionService.DependencyFlow.WorkItems; using ProductConstructionService.WorkItems; -using ProductConstructionService.WorkItems.WorkItemDefinitions; + +using Channel = Maestro.Data.Models.Channel; namespace ProductConstructionService.Api.Api.v2018_07_16.Controllers; @@ -177,7 +178,7 @@ where sub.Enabled if (subscriptionToUpdate != null) { - await _workItemProducerFactory.Create().ProduceWorkItemAsync(new() + await _workItemProducerFactory.CreateProducer().ProduceWorkItemAsync(new() { SubscriptionId = subscriptionToUpdate.Id, BuildId = buildId @@ -199,7 +200,7 @@ public virtual async Task TriggerDailyUpdateAsync() .ToListAsync()) .Where(s => (int)s.PolicyObject.UpdateFrequency == (int)UpdateFrequency.EveryDay); - var workitemProducer = _workItemProducerFactory.Create(); + var workitemProducer = _workItemProducerFactory.CreateProducer(); foreach (var subscription in enabledSubscriptionsWithTargetFrequency) { diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/BuildsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/BuildsController.cs index 6227aca799..fffec48dd3 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/BuildsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/BuildsController.cs @@ -12,8 +12,9 @@ using Microsoft.DotNet.DarcLib; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Internal; +using ProductConstructionService.Api.VirtualMonoRepo; +using ProductConstructionService.DependencyFlow.WorkItems; using ProductConstructionService.WorkItems; -using ProductConstructionService.WorkItems.WorkItemDefinitions; namespace ProductConstructionService.Api.Api.v2020_02_20.Controllers; @@ -323,7 +324,7 @@ await _context.BuildDependencies.AddRangeAsync( // Compute the dependency incoherencies of the build. // Since this might be an expensive operation we do it asynchronously. - await _workItemProducerFactory.Create() + await _workItemProducerFactory.CreateProducer() .ProduceWorkItemAsync(new() { BuildId = buildModel.Id diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/CodeFlowController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/CodeFlowController.cs index 819be48fa8..418fd78c06 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/CodeFlowController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/CodeFlowController.cs @@ -6,8 +6,9 @@ using Microsoft.AspNetCore.Mvc; using Microsoft.DotNet.DarcLib; using Microsoft.DotNet.Maestro.Client.Models; +using ProductConstructionService.Api.VirtualMonoRepo; +using ProductConstructionService.DependencyFlow.WorkItems; using ProductConstructionService.WorkItems; -using ProductConstructionService.WorkItems.WorkItemDefinitions; namespace ProductConstructionService.Api.Api.v2020_02_20.Controllers; @@ -41,13 +42,15 @@ public async Task FlowBuild([Required, FromBody] Maestro.Api.Mode return NotFound($"Build {request.BuildId} not found"); } - await _workItemProducerFactory.Create().ProduceWorkItemAsync(new() - { - BuildId = request.BuildId, - SubscriptionId = request.SubscriptionId, - PrBranch = request.PrBranch, - PrUrl = request.PrUrl, - }); + await _workItemProducerFactory + .CreateProducer() + .ProduceWorkItemAsync(new() + { + BuildId = request.BuildId, + SubscriptionId = request.SubscriptionId, + PrBranch = request.PrBranch, + PrUrl = request.PrUrl, + }); return Ok(); } diff --git a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs index 3ad915681d..b3ef996bfd 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs @@ -35,8 +35,9 @@ using ProductConstructionService.Api.Telemetry; using ProductConstructionService.Api.VirtualMonoRepo; using ProductConstructionService.Common; +using ProductConstructionService.DependencyFlow.WorkItems; +using ProductConstructionService.DependencyFlow.WorkItemProcessors; using ProductConstructionService.WorkItems; -using ProductConstructionService.WorkItems.WorkItemDefinitions; namespace ProductConstructionService.Api; @@ -74,7 +75,7 @@ static PcsStartup() { var context = (BuildAssetRegistryContext)entry.Context; ILogger logger = context.GetService>(); - var workItemProducer = context.GetService().Create(); + var workItemProducer = context.GetService().CreateProducer(); BuildChannel entity = entry.Entity; Build? build = context.Builds @@ -158,12 +159,12 @@ internal static async Task ConfigurePcs( builder.RegisterBuildAssetRegistry(); builder.AddWorkItemQueues(azureCredential, waitForInitialization: initializeService); - builder.Services.AddWorkItemProcessors(); builder.AddVmrRegistrations(gitHubToken); builder.AddMaestroApiClient(managedIdentityId); builder.AddGitHubClientFactory(); builder.Services.AddGitHubTokenProvider(); builder.Services.AddScoped(); + builder.Services.AddWorkItemProcessor(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.Configure(_ => { }); diff --git a/src/ProductConstructionService/ProductConstructionService.Api/ProductConstructionService.Api.csproj b/src/ProductConstructionService/ProductConstructionService.Api/ProductConstructionService.Api.csproj index 760dcd9c94..22886aac04 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/ProductConstructionService.Api.csproj +++ b/src/ProductConstructionService/ProductConstructionService.Api/ProductConstructionService.Api.csproj @@ -48,6 +48,7 @@ + diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/ProductConstructionService.DependencyFlow.csproj b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/ProductConstructionService.DependencyFlow.csproj new file mode 100644 index 0000000000..5331d100c9 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/ProductConstructionService.DependencyFlow.csproj @@ -0,0 +1,25 @@ + + + + net8.0 + enable + enable + + False + + + + + + + + + + + + + + + + + diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProcessors/CodeFlowWorkItemProcessor.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItemProcessors/CodeFlowWorkItemProcessor.cs similarity index 78% rename from src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProcessors/CodeFlowWorkItemProcessor.cs rename to src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItemProcessors/CodeFlowWorkItemProcessor.cs index 418c2a22d7..3b47fb1d16 100644 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProcessors/CodeFlowWorkItemProcessor.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItemProcessors/CodeFlowWorkItemProcessor.cs @@ -7,11 +7,12 @@ using Microsoft.DotNet.Maestro.Client; using Microsoft.DotNet.Maestro.Client.Models; using Microsoft.Extensions.Logging; -using ProductConstructionService.WorkItems.WorkItemDefinitions; +using ProductConstructionService.DependencyFlow.WorkItems; +using ProductConstructionService.WorkItems; -namespace ProductConstructionService.WorkItems.WorkItemProcessors; +namespace ProductConstructionService.DependencyFlow.WorkItemProcessors; -internal class CodeFlowWorkItemProcessor( +public class CodeFlowWorkItemProcessor( IVmrInfo vmrInfo, IBasicBarClient barClient, IMaestroApi maestroApi, @@ -20,7 +21,7 @@ internal class CodeFlowWorkItemProcessor( ILocalLibGit2Client gitClient, ITelemetryRecorder telemetryRecorder, ILogger logger) - : IWorkItemProcessor + : WorkItemProcessor { private readonly IVmrInfo _vmrInfo = vmrInfo; private readonly IBasicBarClient _barClient = barClient; @@ -31,20 +32,28 @@ internal class CodeFlowWorkItemProcessor( private readonly ITelemetryRecorder _telemetryRecorder = telemetryRecorder; private readonly ILogger _logger = logger; - public async Task ProcessWorkItemAsync(WorkItem workItem, CancellationToken cancellationToken) + public override async Task ProcessWorkItemAsync(CodeFlowWorkItem codeflowWorkItem, CancellationToken cancellationToken) { - var codeflowWorkItem = (CodeFlowWorkItem)workItem; + Subscription? subscription = await _barClient.GetSubscriptionAsync(codeflowWorkItem.SubscriptionId); - Subscription subscription = await _barClient.GetSubscriptionAsync(codeflowWorkItem.SubscriptionId) - ?? throw new Exception($"Subscription {codeflowWorkItem.SubscriptionId} not found"); + if (subscription == null) + { + _logger.LogError("Subscription {subscriptionId} not found", codeflowWorkItem.SubscriptionId); + return false; + } if (!subscription.SourceEnabled || (subscription.SourceDirectory ?? subscription.TargetDirectory) == null) { - throw new Exception($"Subscription {codeflowWorkItem.SubscriptionId} is not source enabled or source directory is not set"); + _logger.LogError("Subscription {subscriptionId} is not source enabled or source directory is not set", codeflowWorkItem.SubscriptionId); + return false; } - Build build = await _barClient.GetBuildAsync(codeflowWorkItem.BuildId) - ?? throw new Exception($"Build {codeflowWorkItem.BuildId} not found"); + Build? build = await _barClient.GetBuildAsync(codeflowWorkItem.BuildId); + if (build == null) + { + _logger.LogError("Build {buildId} not found", codeflowWorkItem.BuildId); + return false; + } var isForwardFlow = subscription.TargetDirectory != null; @@ -94,7 +103,7 @@ public async Task ProcessWorkItemAsync(WorkItem workItem, CancellationToken canc { _logger.LogInformation("There were no code-flow updates for subscription {subscriptionId}", subscription.Id); - return; + return true; } _logger.LogInformation("Code changes for {subscriptionId} ready in local branch {branch}", @@ -118,5 +127,7 @@ public async Task ProcessWorkItemAsync(WorkItem workItem, CancellationToken canc await _maestroApi.Subscriptions.TriggerSubscriptionAsync(codeflowWorkItem.BuildId, subscription.Id, default); } + + return true; } } diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemDefinitions/BuildCoherencyInfoWorkItem.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/BuildCoherencyInfoWorkItem.cs similarity index 63% rename from src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemDefinitions/BuildCoherencyInfoWorkItem.cs rename to src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/BuildCoherencyInfoWorkItem.cs index 09dae90194..79b551311d 100644 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemDefinitions/BuildCoherencyInfoWorkItem.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/BuildCoherencyInfoWorkItem.cs @@ -1,10 +1,10 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -namespace ProductConstructionService.WorkItems.WorkItemDefinitions; +using ProductConstructionService.WorkItems; + +namespace ProductConstructionService.DependencyFlow.WorkItems; public class BuildCoherencyInfoWorkItem : WorkItem { public required int BuildId { get; init; } - - public override string Type => nameof(BuildCoherencyInfoWorkItem); } diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemDefinitions/CodeFlowWorkItem.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/CodeFlowWorkItem.cs similarity index 86% rename from src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemDefinitions/CodeFlowWorkItem.cs rename to src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/CodeFlowWorkItem.cs index a6c8ad9552..f42bb7ff12 100644 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemDefinitions/CodeFlowWorkItem.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/CodeFlowWorkItem.cs @@ -1,7 +1,9 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -namespace ProductConstructionService.WorkItems.WorkItemDefinitions; +using ProductConstructionService.WorkItems; + +namespace ProductConstructionService.DependencyFlow.WorkItems; /// /// Main code flow work item which causes new code changes to be flown to a new branch in the target repo. @@ -27,6 +29,4 @@ public class CodeFlowWorkItem : WorkItem /// URL to the code flow PR. /// public string? PrUrl { get; init; } - - public override string Type => nameof(CodeFlowWorkItem); } diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemDefinitions/UpdateSubscriptionWorkItem.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/UpdateSubscriptionWorkItem.cs similarity index 76% rename from src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemDefinitions/UpdateSubscriptionWorkItem.cs rename to src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/UpdateSubscriptionWorkItem.cs index a9edfe16bd..270dc02cf2 100644 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemDefinitions/UpdateSubscriptionWorkItem.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/UpdateSubscriptionWorkItem.cs @@ -1,7 +1,9 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -namespace ProductConstructionService.WorkItems.WorkItemDefinitions; +using ProductConstructionService.WorkItems; + +namespace ProductConstructionService.DependencyFlow.WorkItems; public class UpdateSubscriptionWorkItem : WorkItem { @@ -14,6 +16,4 @@ public class UpdateSubscriptionWorkItem : WorkItem /// Build that is being flown. /// public required int BuildId { get; init; } - - public override string Type => nameof(UpdateSubscriptionWorkItem); } diff --git a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/ProductConstructionService.SubscriptionTriggerer.csproj b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/ProductConstructionService.SubscriptionTriggerer.csproj index 639bb2022c..6421835cd9 100644 --- a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/ProductConstructionService.SubscriptionTriggerer.csproj +++ b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/ProductConstructionService.SubscriptionTriggerer.csproj @@ -16,6 +16,7 @@ + diff --git a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggerer.cs b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggerer.cs index fe427b00ed..08c22941f0 100644 --- a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggerer.cs +++ b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggerer.cs @@ -5,8 +5,8 @@ using Maestro.Data.Models; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; +using ProductConstructionService.DependencyFlow.WorkItems; using ProductConstructionService.WorkItems; -using ProductConstructionService.WorkItems.WorkItemDefinitions; namespace ProductConstructionService.SubscriptionTriggerer; @@ -34,7 +34,7 @@ public async Task CheckSubscriptionsAsync(UpdateFrequency targetUpdateFrequency) .Where(s => s.PolicyObject?.UpdateFrequency == targetUpdateFrequency); var workItemProducer = - _workItemProducerFactory.Create(); + _workItemProducerFactory.CreateProducer(); foreach (var subscription in enabledSubscriptionsWithTargetFrequency) { Subscription? subscriptionWithBuilds = await _context.Subscriptions diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/Properties/AssemblyInfo.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/Properties/AssemblyInfo.cs new file mode 100644 index 0000000000..a49995dd3e --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/Properties/AssemblyInfo.cs @@ -0,0 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("ProductConstructionService.WorkItem.Tests")] +[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2")] diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItem.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItem.cs new file mode 100644 index 0000000000..2d40877d0b --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItem.cs @@ -0,0 +1,10 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace ProductConstructionService.WorkItems; + +public abstract class WorkItem +{ + public Guid Id { get; set; } = Guid.NewGuid(); + public string Type => GetType().Name; +} diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConfiguration.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConfiguration.cs index c1bcc846d8..780bceece2 100644 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConfiguration.cs +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConfiguration.cs @@ -1,14 +1,14 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Text.Json; using Azure.Identity; using Azure.Storage.Queues; -using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; using ProductConstructionService.Common; -using ProductConstructionService.WorkItems.WorkItemDefinitions; -using ProductConstructionService.WorkItems.WorkItemProcessors; namespace ProductConstructionService.WorkItems; @@ -16,12 +16,20 @@ public static class WorkItemConfiguration { public const string WorkItemQueueNameConfigurationKey = "WorkItemQueueName"; + internal static readonly JsonSerializerOptions JsonSerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = false, + }; + public static void AddWorkItemQueues(this IHostApplicationBuilder builder, DefaultAzureCredential credential, bool waitForInitialization) { builder.AddWorkItemProducerFactory(credential); // When running the service locally, the WorkItemProcessor should start in the Working state - builder.Services.AddSingleton(sp => ActivatorUtilities.CreateInstance(sp, waitForInitialization)); + builder.Services.AddSingleton(sp => + new WorkItemScopeManager(waitForInitialization, sp, sp.GetRequiredService>())); + builder.Configuration[$"{WorkItemConsumerOptions.ConfigurationKey}:${WorkItemQueueNameConfigurationKey}"] = builder.Configuration.GetRequiredValue(WorkItemQueueNameConfigurationKey); builder.Services.Configure( @@ -47,14 +55,29 @@ public static async Task UseLocalWorkItemQueues(this IServiceProvider servicePro await queueClient.CreateIfNotExistsAsync(); } - public static void AddWorkItemProcessors(this IServiceCollection services) - { - services.RegisterWorkItemProcessor(); - } - - private static void RegisterWorkItemProcessor(this IServiceCollection services) + public static void AddWorkItemProcessor( + this IServiceCollection services, + Func? factory = null) + where TWorkItem : WorkItem where TProcessor : class, IWorkItemProcessor { - services.AddKeyedTransient(typeof(TWorkItem).Name); + // We need IOption where we add the registrations + services.AddOptions(); + services.TryAddSingleton(); + + var diKey = typeof(TWorkItem).Name; + if (factory != null) + { + services.TryAddKeyedTransient(diKey, (sp, _) => factory(sp)); + } + else + { + services.TryAddKeyedTransient(diKey); + } + + services.Configure(registrations => + { + registrations.RegisterProcessor(); + }); } } diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConsumer.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConsumer.cs index f94cc15e66..44697d9889 100644 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConsumer.cs +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConsumer.cs @@ -1,12 +1,12 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Text.Json.Nodes; using Azure.Storage.Queues; using Azure.Storage.Queues.Models; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using ProductConstructionService.WorkItems.WorkItemDefinitions; namespace ProductConstructionService.WorkItems; @@ -66,15 +66,26 @@ private async Task ReadAndProcessWorkItemAsync(QueueClient queueClient, WorkItem return; } - var workItem = message.Body.ToObjectFromJson(); - - workItemScope.InitializeScope(workItem); - + string workItemId; + string workItemType; + JsonNode node; try { - _logger.LogInformation("Starting attempt {attemptNumber} for work item {workItemId}, type {workItemType}", message.DequeueCount, workItem.Id, workItem.Type); - await workItemScope.RunWorkItemAsync(cancellationToken); + node = JsonNode.Parse(message.Body)!; + workItemId = node["id"]!.ToString(); + workItemType = node["type"]!.ToString(); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to parse work item message {message}", message.Body.ToString()); + await queueClient.DeleteMessageAsync(message.MessageId, message.PopReceipt, cancellationToken); + return; + } + try + { + _logger.LogInformation("Starting attempt {attemptNumber} for work item {workItemId}, type {workItemType}", message.DequeueCount, workItemId, workItemType); + await workItemScope.RunWorkItemAsync(node, cancellationToken); await queueClient.DeleteMessageAsync(message.MessageId, message.PopReceipt, cancellationToken); } // If the cancellation token gets cancelled, don't retry, just exit without deleting the message, we'll handle it later @@ -85,14 +96,18 @@ private async Task ReadAndProcessWorkItemAsync(QueueClient queueClient, WorkItem catch (Exception ex) { _logger.LogError(ex, "Processing work item {workItemId} attempt {attempt}/{maxAttempts} failed", - workItem.Id, message.DequeueCount, _options.Value.MaxWorkItemRetries); + workItemId, message.DequeueCount, _options.Value.MaxWorkItemRetries); // Let the workItem retry a few times. If it fails a few times, delete it from the queue, it's a bad work item - if (message.DequeueCount == _options.Value.MaxWorkItemRetries) + if (message.DequeueCount == _options.Value.MaxWorkItemRetries || ex is NonRetriableException) { _logger.LogError("Work item {workItemId} has failed {maxAttempts} times. Discarding the message {message} from the queue", - workItem.Id, _options.Value.MaxWorkItemRetries, message.Body.ToString()); + workItemId, _options.Value.MaxWorkItemRetries, message.Body.ToString()); await queueClient.DeleteMessageAsync(message.MessageId, message.PopReceipt, cancellationToken); } } } } + +internal class NonRetriableException(string message) : Exception(message) +{ +} diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemDefinitions/WorkItem.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemDefinitions/WorkItem.cs deleted file mode 100644 index f592af8de5..0000000000 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemDefinitions/WorkItem.cs +++ /dev/null @@ -1,15 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Text.Json.Serialization; - -namespace ProductConstructionService.WorkItems.WorkItemDefinitions; - -[JsonDerivedType(typeof(CodeFlowWorkItem), typeDiscriminator: nameof(CodeFlowWorkItem))] -[JsonDerivedType(typeof(UpdateSubscriptionWorkItem), typeDiscriminator: nameof(UpdateSubscriptionWorkItem))] -[JsonDerivedType(typeof(BuildCoherencyInfoWorkItem), typeDiscriminator: nameof(BuildCoherencyInfoWorkItem))] -public abstract class WorkItem -{ - public Guid Id { get; } = Guid.NewGuid(); - public abstract string Type { get; } -} diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProcessor.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProcessor.cs new file mode 100644 index 0000000000..7f2b89ca09 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProcessor.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace ProductConstructionService.WorkItems; + +public interface IWorkItemProcessor +{ + Task ProcessWorkItemAsync(WorkItem workItem, CancellationToken cancellationToken); +} + +public abstract class WorkItemProcessor : IWorkItemProcessor + where TWorkItem : WorkItem +{ + public abstract Task ProcessWorkItemAsync(TWorkItem workItem, CancellationToken cancellationToken); + public async Task ProcessWorkItemAsync(WorkItem workItem, CancellationToken cancellationToken) + { + if (workItem is not TWorkItem typedWorkItem) + { + throw new NonRetriableException($"Expected work item of type {typeof(TWorkItem)}, but got {workItem.GetType()}"); + } + return await ProcessWorkItemAsync(typedWorkItem, cancellationToken); + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProcessorRegistrations.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProcessorRegistrations.cs new file mode 100644 index 0000000000..b77885b569 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProcessorRegistrations.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace ProductConstructionService.WorkItems; + +internal class WorkItemProcessorRegistrations +{ + private readonly Dictionary _processors = new(); + + public void RegisterProcessor() + where TWorkItem : WorkItem + where TProcessor : IWorkItemProcessor + { + _processors.Add(typeof(TWorkItem).Name, (typeof(TWorkItem), typeof(TProcessor))); + } + + public IReadOnlyDictionary Processors => _processors; +} diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProcessors/IWorkItemProcessor.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProcessors/IWorkItemProcessor.cs deleted file mode 100644 index 5e04125288..0000000000 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProcessors/IWorkItemProcessor.cs +++ /dev/null @@ -1,11 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using ProductConstructionService.WorkItems.WorkItemDefinitions; - -namespace ProductConstructionService.WorkItems.WorkItemProcessors; - -public interface IWorkItemProcessor -{ - Task ProcessWorkItemAsync(WorkItem workItem, CancellationToken cancellationToken); -} diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProducer.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProducer.cs index b3de2530d0..7d5138765a 100644 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProducer.cs +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProducer.cs @@ -4,7 +4,6 @@ using System.Text.Json; using Azure.Storage.Queues; using Azure.Storage.Queues.Models; -using ProductConstructionService.WorkItems.WorkItemDefinitions; namespace ProductConstructionService.WorkItems; @@ -24,6 +23,7 @@ public class WorkItemProducer(QueueServiceClient queueServiceClient, string q public async Task ProduceWorkItemAsync(T payload, TimeSpan delay = default) { var client = _queueServiceClient.GetQueueClient(_queueName); - return await client.SendMessageAsync(JsonSerializer.Serialize(payload), delay); + var json = JsonSerializer.Serialize(payload, WorkItemConfiguration.JsonSerializerOptions); + return await client.SendMessageAsync(json, delay); } } diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProducerFactory.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProducerFactory.cs index 97e5fd8250..3f369b364e 100644 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProducerFactory.cs +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProducerFactory.cs @@ -2,13 +2,12 @@ // The .NET Foundation licenses this file to you under the MIT license. using Azure.Storage.Queues; -using ProductConstructionService.WorkItems.WorkItemDefinitions; namespace ProductConstructionService.WorkItems; public interface IWorkItemProducerFactory { - public IWorkItemProducer Create() where T : WorkItem; + public IWorkItemProducer CreateProducer() where T : WorkItem; } public class WorkItemProducerFactory(QueueServiceClient queueServiceClient, string queueName) : IWorkItemProducerFactory @@ -16,6 +15,6 @@ public class WorkItemProducerFactory(QueueServiceClient queueServiceClient, stri private readonly QueueServiceClient _queueServiceClient = queueServiceClient; private readonly string _queueName = queueName; - public IWorkItemProducer Create() where T : WorkItem + public IWorkItemProducer CreateProducer() where T : WorkItem => new WorkItemProducer(_queueServiceClient, _queueName); } diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemScope.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemScope.cs index eaedb0d313..80633e7bc6 100644 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemScope.cs +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemScope.cs @@ -1,47 +1,63 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Text.Json; +using System.Text.Json.Nodes; using Microsoft.DotNet.DarcLib; using Microsoft.Extensions.DependencyInjection; -using ProductConstructionService.WorkItems.WorkItemDefinitions; -using ProductConstructionService.WorkItems.WorkItemProcessors; +using Microsoft.Extensions.Options; namespace ProductConstructionService.WorkItems; -public class WorkItemScope( +public class WorkItemScope : IDisposable +{ + private readonly WorkItemProcessorRegistrations _processorRegistrations; + private readonly Action _finalizer; + private readonly IServiceScope _serviceScope; + private readonly ITelemetryRecorder _telemetryRecorder; + + internal WorkItemScope( + IOptions processorRegistrations, Action finalizer, IServiceScope serviceScope, ITelemetryRecorder telemetryRecorder) - : IDisposable -{ - private readonly IServiceScope _serviceScope = serviceScope; - private readonly ITelemetryRecorder _telemetryRecorder = telemetryRecorder; - private WorkItem? _workItem = null; + { + _processorRegistrations = processorRegistrations.Value; + _finalizer = finalizer; + _serviceScope = serviceScope; + _telemetryRecorder = telemetryRecorder; + } public void Dispose() { - finalizer.Invoke(); + _finalizer.Invoke(); _serviceScope.Dispose(); } - public void InitializeScope(WorkItem workItem) + public async Task RunWorkItemAsync(JsonNode node, CancellationToken cancellationToken) { - _workItem = workItem; - } + var type = node["type"]!.ToString(); - public async Task RunWorkItemAsync(CancellationToken cancellationToken) - { - if (_workItem is null) + if (!_processorRegistrations.Processors.TryGetValue(type, out (Type WorkItem, Type Processor) processorType)) { - throw new Exception($"{nameof(WorkItemScope)} not initialized! Call InitializeScope before calling {nameof(RunWorkItemAsync)}"); + throw new NonRetriableException($"No processor found for work item type {type}"); } - var workItemProcessor = _serviceScope.ServiceProvider.GetRequiredKeyedService(_workItem.Type); + IWorkItemProcessor processor = _serviceScope.ServiceProvider.GetKeyedService(type) + ?? throw new NonRetriableException($"No processor registration found for work item type {type}"); + + if (JsonSerializer.Deserialize(node, processorType.WorkItem, WorkItemConfiguration.JsonSerializerOptions) is not WorkItem workItem) + { + throw new NonRetriableException($"Failed to deserialize work item of type {type}: {node}"); + } - using (ITelemetryScope telemetryScope = _telemetryRecorder.RecordWorkItemCompletion(_workItem.Type)) + using (ITelemetryScope telemetryScope = _telemetryRecorder.RecordWorkItemCompletion(type)) { - await workItemProcessor.ProcessWorkItemAsync(_workItem, cancellationToken); - telemetryScope.SetSuccess(); + var success = await processor.ProcessWorkItemAsync(workItem, cancellationToken); + if (success) + { + telemetryScope.SetSuccess(); + } } } } diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemScopeManager.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemScopeManager.cs index 77f93a694e..bfff923204 100644 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemScopeManager.cs +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemScopeManager.cs @@ -1,8 +1,10 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using Microsoft.DotNet.DarcLib; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; namespace ProductConstructionService.WorkItems; @@ -21,12 +23,15 @@ private set if (_state != value) { _state = value; - _logger.LogInformation($"WorkItemsProcessor state changing to {value}"); + _logger.LogInformation("WorkItemsProcessor state changing to {newValue}", value); } } } - public WorkItemScopeManager(bool initializingOnStartup, IServiceProvider serviceProvider, ILogger logger) + internal WorkItemScopeManager( + bool initializingOnStartup, + IServiceProvider serviceProvider, + ILogger logger) { _autoResetEvent = new AutoResetEvent(!initializingOnStartup); _logger = logger; @@ -47,7 +52,11 @@ public WorkItemScope BeginWorkItemScopeWhenReady() { _autoResetEvent.WaitOne(); var scope = _serviceProvider.CreateScope(); - return ActivatorUtilities.CreateInstance(scope.ServiceProvider, scope, new Action(WorkItemFinished)); + return new WorkItemScope( + scope.ServiceProvider.GetRequiredService>(), + new Action(WorkItemFinished), + scope, + scope.ServiceProvider.GetRequiredService()); } private void WorkItemFinished() diff --git a/test/ProductConstructionService.Api.Tests/BuildController20200914Tests.cs b/test/ProductConstructionService.Api.Tests/BuildController20200914Tests.cs index 6bd64452cc..34ff5da254 100644 --- a/test/ProductConstructionService.Api.Tests/BuildController20200914Tests.cs +++ b/test/ProductConstructionService.Api.Tests/BuildController20200914Tests.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Net; -using Azure.Storage.Queues.Models; using FluentAssertions; using Maestro.Api.Model.v2020_02_20; using Maestro.Data; @@ -19,8 +18,9 @@ using Moq; using ProductConstructionService.Api.Api.v2020_02_20.Controllers; +using ProductConstructionService.Api.VirtualMonoRepo; +using ProductConstructionService.DependencyFlow.WorkItems; using ProductConstructionService.WorkItems; -using ProductConstructionService.WorkItems.WorkItemDefinitions; using Commit = Maestro.Api.Model.v2020_02_20.Commit; namespace ProductConstructionService.Api.Tests; @@ -94,7 +94,7 @@ public static async Task Dependencies(IServiceCollection collection) var mockIRemote = new Mock(); var mockWorkItemProducerFactory = new Mock(); var mockWorkItemProducer = new Mock>(); - mockWorkItemProducerFactory.Setup(f => f.Create()).Returns(mockWorkItemProducer.Object); + mockWorkItemProducerFactory.Setup(f => f.CreateProducer()).Returns(mockWorkItemProducer.Object); mockIRemoteFactory.Setup(f => f.GetRemoteAsync(Repository, It.IsAny())).ReturnsAsync(mockIRemote.Object); mockIRemote.Setup(f => f.GetCommitAsync(Repository, CommitHash)).ReturnsAsync(new Microsoft.DotNet.DarcLib.Commit(Account, CommitHash, CommitMessage)); diff --git a/test/ProductConstructionService.Api.Tests/ChannelsController20200220Tests.cs b/test/ProductConstructionService.Api.Tests/ChannelsController20200220Tests.cs index 87494e7914..5ced4553ff 100644 --- a/test/ProductConstructionService.Api.Tests/ChannelsController20200220Tests.cs +++ b/test/ProductConstructionService.Api.Tests/ChannelsController20200220Tests.cs @@ -17,8 +17,9 @@ using Microsoft.Extensions.Logging; using Moq; using ProductConstructionService.Api.Api.v2020_02_20.Controllers; -using ProductConstructionService.WorkItems.WorkItemDefinitions; using ProductConstructionService.WorkItems; +using ProductConstructionService.Api.VirtualMonoRepo; +using ProductConstructionService.DependencyFlow.WorkItems; namespace ProductConstructionService.Api.Tests; @@ -172,7 +173,7 @@ public static async Task Dependencies(IServiceCollection collection) var mockWorkItemProducerFactory = new Mock(); var mockWorkItemProducer = new Mock>(); - mockWorkItemProducerFactory.Setup(f => f.Create()).Returns(mockWorkItemProducer.Object); + mockWorkItemProducerFactory.Setup(f => f.CreateProducer()).Returns(mockWorkItemProducer.Object); collection.AddSingleton(mockWorkItemProducerFactory.Object); } diff --git a/test/ProductConstructionService.Api.Tests/ProductConstructionService.Api.Tests.csproj b/test/ProductConstructionService.Api.Tests/ProductConstructionService.Api.Tests.csproj index 79acb78c8d..15f56108dd 100644 --- a/test/ProductConstructionService.Api.Tests/ProductConstructionService.Api.Tests.csproj +++ b/test/ProductConstructionService.Api.Tests/ProductConstructionService.Api.Tests.csproj @@ -11,6 +11,7 @@ + diff --git a/test/ProductConstructionService.Api.Tests/SubscriptionsController20200220Tests.cs b/test/ProductConstructionService.Api.Tests/SubscriptionsController20200220Tests.cs index 4440cf5c71..919775cf48 100644 --- a/test/ProductConstructionService.Api.Tests/SubscriptionsController20200220Tests.cs +++ b/test/ProductConstructionService.Api.Tests/SubscriptionsController20200220Tests.cs @@ -18,8 +18,9 @@ using Microsoft.Extensions.Logging; using Moq; using ProductConstructionService.Api.Api.v2020_02_20.Controllers; -using ProductConstructionService.WorkItems.WorkItemDefinitions; using ProductConstructionService.WorkItems; +using ProductConstructionService.Api.VirtualMonoRepo; +using ProductConstructionService.DependencyFlow.WorkItems; namespace ProductConstructionService.Api.Tests; @@ -455,8 +456,8 @@ public static void Dependencies(IServiceCollection collection) var mockWorkItemProducerFactory = new Mock(); var mockUpdateSubscriptionWorkItemProducer = new Mock>(); var mockBuildCoherencyInfoWorkItem = new Mock>(); - mockWorkItemProducerFactory.Setup(f => f.Create()).Returns(mockUpdateSubscriptionWorkItemProducer.Object); - mockWorkItemProducerFactory.Setup(f => f.Create()).Returns(mockBuildCoherencyInfoWorkItem.Object); + mockWorkItemProducerFactory.Setup(f => f.CreateProducer()).Returns(mockUpdateSubscriptionWorkItemProducer.Object); + mockWorkItemProducerFactory.Setup(f => f.CreateProducer()).Returns(mockBuildCoherencyInfoWorkItem.Object); collection.AddLogging(l => l.AddProvider(new NUnitLogger())); collection.AddSingleton(new HostingEnvironment { @@ -631,7 +632,7 @@ public static Func Clock(IServiceCollection collect } // Copied from GitHubClaimsResolverTests; could refactor if needed in another place - private static Octokit.Organization MockOrganization(int id, string login) + private static MockOrg MockOrganization(int id, string login) { return new MockOrg(id, login); } diff --git a/test/ProductConstructionService.Api.Tests/WorkItemScopeTests.cs b/test/ProductConstructionService.Api.Tests/WorkItemScopeTests.cs deleted file mode 100644 index 38645e8793..0000000000 --- a/test/ProductConstructionService.Api.Tests/WorkItemScopeTests.cs +++ /dev/null @@ -1,87 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using FluentAssertions; -using Microsoft.DotNet.DarcLib; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Moq; -using ProductConstructionService.WorkItems; -using ProductConstructionService.WorkItems.WorkItemDefinitions; -using ProductConstructionService.WorkItems.WorkItemProcessors; - -namespace ProductConstructionService.Api.Tests; - -public class WorkItemScopeTests -{ - [Test] - public async Task WorkItemScopeRecordsMetricsTest() - { - IServiceCollection services = new ServiceCollection(); - - Mock telemetryScope = new(); - Mock metricRecorderMock = new(); - TestWorkItem testWorkItem = new() { Text = string.Empty }; - - metricRecorderMock.Setup(m => m.RecordWorkItemCompletion(testWorkItem.Type)).Returns(telemetryScope.Object); - - services.AddSingleton(metricRecorderMock.Object); - services.AddKeyedSingleton(nameof(TestWorkItem), new Mock().Object); - - IServiceProvider serviceProvider = services.BuildServiceProvider(); - - WorkItemScopeManager scopeManager = new(false, serviceProvider, Mock.Of>()); - - using (WorkItemScope workItemScope = scopeManager.BeginWorkItemScopeWhenReady()) - { - workItemScope.InitializeScope(testWorkItem); - - await workItemScope.RunWorkItemAsync(CancellationToken.None); - } - - metricRecorderMock.Verify(m => m.RecordWorkItemCompletion(testWorkItem.Type), Times.Once); - telemetryScope.Verify(m => m.SetSuccess(), Times.Once); - } - - [Test] - public void WorkItemScopeRecordsMetricsWhenThrowingTest() - { - IServiceCollection services = new ServiceCollection(); - - Mock metricRecorderScopeMock = new(); - Mock metricRecorderMock = new(); - TestWorkItem textWorkItem = new() { Text = string.Empty }; - - metricRecorderMock - .Setup(m => m.RecordWorkItemCompletion(textWorkItem.Type)) - .Returns(metricRecorderScopeMock.Object); - - services.AddSingleton(metricRecorderMock.Object); - - Mock workItemProcessor = new(); - workItemProcessor.Setup(i => i.ProcessWorkItemAsync(textWorkItem, It.IsAny())).Throws(); - services.AddKeyedSingleton(nameof(TestWorkItem), workItemProcessor.Object); - - IServiceProvider serviceProvider = services.BuildServiceProvider(); - - WorkItemScopeManager scopeManager = new(false, serviceProvider, Mock.Of>()); - - using (WorkItemScope workItemScope = scopeManager.BeginWorkItemScopeWhenReady()) - { - workItemScope.InitializeScope(textWorkItem); - - Func func = async () => await workItemScope.RunWorkItemAsync(CancellationToken.None); - func.Should().ThrowAsync(); - } - - metricRecorderMock.Verify(m => m.RecordWorkItemCompletion(textWorkItem.Type), Times.Once); - metricRecorderScopeMock.Verify(m => m.SetSuccess(), Times.Never); - } - - private class TestWorkItem : WorkItem - { - public required string Text { get; set; } - - public override string Type => nameof(TestWorkItem); - } -} diff --git a/test/ProductConstructionService.WorkItem.Tests/GlobalUsings.cs b/test/ProductConstructionService.WorkItem.Tests/GlobalUsings.cs new file mode 100644 index 0000000000..253fe99236 --- /dev/null +++ b/test/ProductConstructionService.WorkItem.Tests/GlobalUsings.cs @@ -0,0 +1,4 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +global using NUnit.Framework; diff --git a/test/ProductConstructionService.WorkItem.Tests/ProductConstructionService.WorkItem.Tests.csproj b/test/ProductConstructionService.WorkItem.Tests/ProductConstructionService.WorkItem.Tests.csproj new file mode 100644 index 0000000000..488296e453 --- /dev/null +++ b/test/ProductConstructionService.WorkItem.Tests/ProductConstructionService.WorkItem.Tests.csproj @@ -0,0 +1,21 @@ + + + + net8.0 + enable + enable + False + false + true + + + + + + + + + + + + diff --git a/test/ProductConstructionService.WorkItem.Tests/WorkItemScopeTests.cs b/test/ProductConstructionService.WorkItem.Tests/WorkItemScopeTests.cs new file mode 100644 index 0000000000..4eddcf9640 --- /dev/null +++ b/test/ProductConstructionService.WorkItem.Tests/WorkItemScopeTests.cs @@ -0,0 +1,218 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json; +using FluentAssertions; +using Microsoft.DotNet.DarcLib; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Moq; +using ProductConstructionService.WorkItems; + +namespace ProductConstructionService.WorkItem.Tests; + +public class WorkItemScopeTests +{ + private ServiceCollection _services = new(); + + [SetUp] + public void TestSetup() + { + _services = new(); + _services.AddOptions(); + _services.AddLogging(); + } + + [Test] + public async Task WorkItemScopeRecordsMetricsTest() + { + Mock telemetryScope = new(); + Mock metricRecorderMock = new(); + TestWorkItem testWorkItem = new() { Text = string.Empty }; + bool processCalled = false; + + metricRecorderMock + .Setup(m => m.RecordWorkItemCompletion(testWorkItem.Type)) + .Returns(telemetryScope.Object); + + _services.AddSingleton(metricRecorderMock.Object); + _services.AddWorkItemProcessor( + _ => new TestWorkItemProcessor(() => { processCalled = true; return true; })); + + IServiceProvider serviceProvider = _services.BuildServiceProvider(); + + WorkItemScopeManager scopeManager = new(false, serviceProvider, Mock.Of>()); + + using (WorkItemScope workItemScope = scopeManager.BeginWorkItemScopeWhenReady()) + { + var workItem = JsonSerializer.SerializeToNode(testWorkItem, WorkItemConfiguration.JsonSerializerOptions)!; + await workItemScope.RunWorkItemAsync(workItem, CancellationToken.None); + } + + metricRecorderMock.Verify(m => m.RecordWorkItemCompletion(testWorkItem.Type), Times.Once); + telemetryScope.Verify(m => m.SetSuccess(), Times.Once); + processCalled.Should().BeTrue(); + } + + [Test] + public void WorkItemScopeRecordsMetricsWhenThrowingTest() + { + Mock metricRecorderScopeMock = new(); + Mock metricRecorderMock = new(); + TestWorkItem testWorkItem = new() { Text = string.Empty }; + + metricRecorderMock + .Setup(m => m.RecordWorkItemCompletion(testWorkItem.Type)) + .Returns(metricRecorderScopeMock.Object); + + _services.AddSingleton(metricRecorderMock.Object); + _services.AddWorkItemProcessor( + _ => new TestWorkItemProcessor(() => throw new Exception())); + + IServiceProvider serviceProvider = _services.BuildServiceProvider(); + + WorkItemScopeManager scopeManager = new(false, serviceProvider, Mock.Of>()); + + using (WorkItemScope workItemScope = scopeManager.BeginWorkItemScopeWhenReady()) + { + var workItem = JsonSerializer.SerializeToNode(testWorkItem, WorkItemConfiguration.JsonSerializerOptions)!; + Func func = async () => await workItemScope.RunWorkItemAsync(workItem, CancellationToken.None); + func.Should().ThrowAsync(); + } + + metricRecorderMock.Verify(m => m.RecordWorkItemCompletion(testWorkItem.Type), Times.Once); + metricRecorderScopeMock.Verify(m => m.SetSuccess(), Times.Never); + } + + private class TestWorkItem : WorkItems.WorkItem + { + public required string Text { get; set; } + } + + private class TestWorkItemProcessor : WorkItemProcessor, IWorkItemProcessor + { + private readonly Func _process; + + public TestWorkItemProcessor(Func process) + { + _process = process; + } + + public override Task ProcessWorkItemAsync(TestWorkItem workItem, CancellationToken cancellationToken) + => Task.FromResult(_process()); + } + + [Test] + public async Task DifferentWorkItemsSameProcessorTest() + { + Mock metricRecorderScopeMock = new(); + Mock metricRecorderMock = new(); + TestWorkItem testWorkItem = new() { Text = string.Empty }; + + metricRecorderMock + .Setup(m => m.RecordWorkItemCompletion(It.IsAny())) + .Returns(metricRecorderScopeMock.Object); + + _services.AddSingleton(metricRecorderMock.Object); + + string? lastText = null; + + _services.AddWorkItemProcessor( + _ => new TestWorkItemProcessor2(s => lastText = s)); + + _services.AddWorkItemProcessor( + _ => new TestWorkItemProcessor2(s => lastText = s)); + + IServiceProvider serviceProvider = _services.BuildServiceProvider(); + + WorkItemScopeManager scopeManager = new(false, serviceProvider, Mock.Of>()); + + using (WorkItemScope workItemScope = scopeManager.BeginWorkItemScopeWhenReady()) + { + var workItem = JsonSerializer.SerializeToNode(new TestWorkItem() { Text = "foo" }, WorkItemConfiguration.JsonSerializerOptions)!; + await workItemScope.RunWorkItemAsync(workItem, CancellationToken.None); + } + + lastText.Should().Be("foo"); + + using (WorkItemScope workItemScope = scopeManager.BeginWorkItemScopeWhenReady()) + { + var workItem = JsonSerializer.SerializeToNode(new TestWorkItem2() { Text2 = "bar" }, WorkItemConfiguration.JsonSerializerOptions)!; + await workItemScope.RunWorkItemAsync(workItem, CancellationToken.None); + } + + lastText.Should().Be("bar"); + } + + + [Test] + public async Task MultipleProcessorsWithoutFactoryMethodTest() + { + Mock metricRecorderScopeMock = new(); + Mock metricRecorderMock = new(); + TestWorkItem testWorkItem = new() { Text = string.Empty }; + + metricRecorderMock + .Setup(m => m.RecordWorkItemCompletion(It.IsAny())) + .Returns(metricRecorderScopeMock.Object); + + _services.AddSingleton(metricRecorderMock.Object); + + string? lastText = null; + + _services.AddSingleton>(() => { lastText = "true"; return true; }); + _services.AddSingleton>(s => lastText = s); + _services.AddWorkItemProcessor(); + _services.AddWorkItemProcessor(); + + IServiceProvider serviceProvider = _services.BuildServiceProvider(); + + WorkItemScopeManager scopeManager = new(false, serviceProvider, Mock.Of>()); + + using (WorkItemScope workItemScope = scopeManager.BeginWorkItemScopeWhenReady()) + { + var workItem = JsonSerializer.SerializeToNode(new TestWorkItem() { Text = "foo" }, WorkItemConfiguration.JsonSerializerOptions)!; + await workItemScope.RunWorkItemAsync(workItem, CancellationToken.None); + } + + lastText.Should().Be("true"); + + using (WorkItemScope workItemScope = scopeManager.BeginWorkItemScopeWhenReady()) + { + var workItem = JsonSerializer.SerializeToNode(new TestWorkItem2() { Text2 = "bar" }, WorkItemConfiguration.JsonSerializerOptions)!; + await workItemScope.RunWorkItemAsync(workItem, CancellationToken.None); + } + + lastText.Should().Be("bar"); + } + + private class TestWorkItem2 : WorkItems.WorkItem + { + public required string Text2 { get; set; } + } + + private class TestWorkItemProcessor2 : IWorkItemProcessor + { + private readonly Action _action; + + public TestWorkItemProcessor2(Action action) + { + _action = action; + } + + public Task ProcessWorkItemAsync(WorkItems.WorkItem workItem, CancellationToken cancellationToken) + { + switch (workItem) + { + case TestWorkItem t1: + _action(t1.Text); + break; + case TestWorkItem2 t2: + _action(t2.Text2); + break; + } + + return Task.FromResult(true); + } + } +} diff --git a/test/ProductConstructionService.Api.Tests/WorkItemsProcessorScopeManagerTests.cs b/test/ProductConstructionService.WorkItem.Tests/WorkItemsProcessorScopeManagerTests.cs similarity index 95% rename from test/ProductConstructionService.Api.Tests/WorkItemsProcessorScopeManagerTests.cs rename to test/ProductConstructionService.WorkItem.Tests/WorkItemsProcessorScopeManagerTests.cs index 69306c82a7..701f9df35f 100644 --- a/test/ProductConstructionService.Api.Tests/WorkItemsProcessorScopeManagerTests.cs +++ b/test/ProductConstructionService.WorkItem.Tests/WorkItemsProcessorScopeManagerTests.cs @@ -8,17 +8,21 @@ using Moq; using ProductConstructionService.WorkItems; -namespace ProductConstructionService.Api.Tests; +namespace ProductConstructionService.WorkItem.Tests; public class WorkItemsProcessorScopeManagerTests { - private readonly IServiceProvider _serviceProvider; + private IServiceProvider _serviceProvider = null!; - public WorkItemsProcessorScopeManagerTests() + [SetUp] + public void SetUp() { ServiceCollection services = new(); services.AddSingleton(new Mock().Object); + services.AddOptions(); + services.AddLogging(); + services.AddSingleton(); _serviceProvider = services.BuildServiceProvider(); } From 9d574fec4d383b3ca00ac5e22aac5615d47475c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Thu, 22 Aug 2024 16:05:31 +0200 Subject: [PATCH 08/11] Add removable reminders and `RedisCache` (#3865) --- .../PcsStartup.cs | 4 +- .../IRedisCacheFactory.cs | 35 +++++ .../ProductConstructionServiceExtension.cs | 6 +- .../RedisCache.cs | 124 ++++++++++++++++++ .../StateManager.cs | 51 ------- .../StateManagerFactory.cs | 45 ------- ...tConstructionService.DependencyFlow.csproj | 6 + .../LongestBuildPathUpdaterConfiguration.cs | 2 +- .../SubscriptionTriggererConfiguration.cs | 2 +- .../ReminderManager.cs | 60 +++++++++ .../ReminderManagerFactory.cs | 28 ++++ .../WorkItemProducer.cs | 17 ++- 12 files changed, 274 insertions(+), 106 deletions(-) create mode 100644 src/ProductConstructionService/ProductConstructionService.Common/IRedisCacheFactory.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Common/RedisCache.cs delete mode 100644 src/ProductConstructionService/ProductConstructionService.Common/StateManager.cs delete mode 100644 src/ProductConstructionService/ProductConstructionService.Common/StateManagerFactory.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.WorkItems/ReminderManager.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.WorkItems/ReminderManagerFactory.cs diff --git a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs index b3ef996bfd..a0303c56f4 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs @@ -157,7 +157,7 @@ internal static async Task ConfigurePcs( builder.Configuration.AddAzureKeyVault(keyVaultUri, azureCredential); } - builder.RegisterBuildAssetRegistry(); + builder.AddBuildAssetRegistry(); builder.AddWorkItemQueues(azureCredential, waitForInitialization: initializeService); builder.AddVmrRegistrations(gitHubToken); builder.AddMaestroApiClient(managedIdentityId); @@ -179,7 +179,7 @@ internal static async Task ConfigurePcs( builder.Services.AddMergePolicies(); builder.Services.Configure(builder.Configuration.GetSection(ConfigurationKeys.DependencyFlowSLAs)); - await builder.AddStateManager(authRedis); + await builder.AddRedisCache(authRedis); if (initializeService) { diff --git a/src/ProductConstructionService/ProductConstructionService.Common/IRedisCacheFactory.cs b/src/ProductConstructionService/ProductConstructionService.Common/IRedisCacheFactory.cs new file mode 100644 index 0000000000..8489124642 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Common/IRedisCacheFactory.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.Logging; +using StackExchange.Redis; + +namespace ProductConstructionService.Common; + +public interface IRedisCacheFactory +{ + IRedisCache Create(string stateKey) where T : class; + IRedisCache Create(string stateKey); +} + +public class RedisCacheFactory : IRedisCacheFactory +{ + private readonly IConnectionMultiplexer _connection; + private readonly ILogger _logger; + + public RedisCacheFactory(ConfigurationOptions options, ILogger logger) + { + _connection = ConnectionMultiplexer.Connect(options); + _logger = logger; + } + + public IRedisCache Create(string stateKey) where T : class + { + return new RedisCache(Create(stateKey), _logger); + } + + public IRedisCache Create(string stateKey) + { + return new RedisCache(_connection, stateKey); + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Common/ProductConstructionServiceExtension.cs b/src/ProductConstructionService/ProductConstructionService.Common/ProductConstructionServiceExtension.cs index fdba1e6edf..99545e3fcd 100644 --- a/src/ProductConstructionService/ProductConstructionService.Common/ProductConstructionServiceExtension.cs +++ b/src/ProductConstructionService/ProductConstructionService.Common/ProductConstructionServiceExtension.cs @@ -23,7 +23,7 @@ public static class ProductConstructionServiceExtension private const string SqlConnectionStringUserIdPlaceholder = "USER_ID_PLACEHOLDER"; private const string DatabaseConnectionString = "BuildAssetRegistrySqlConnectionString"; - public static void RegisterBuildAssetRegistry(this IHostApplicationBuilder builder) + public static void AddBuildAssetRegistry(this IHostApplicationBuilder builder) { var managedIdentityClientId = builder.Configuration[ManagedIdentityClientId]; string databaseConnectionString = builder.Configuration.GetRequiredValue(DatabaseConnectionString) @@ -52,7 +52,7 @@ public static void RegisterBuildAssetRegistry(this IHostApplicationBuilder build builder.Services.AddSingleton(); ; } - public static async Task AddStateManager( + public static async Task AddRedisCache( this IHostApplicationBuilder builder, bool useAuth) { @@ -72,6 +72,6 @@ public static async Task AddStateManager( } builder.Services.AddSingleton(redisConfig); - builder.Services.AddSingleton(); + builder.Services.AddScoped(); } } diff --git a/src/ProductConstructionService/ProductConstructionService.Common/RedisCache.cs b/src/ProductConstructionService/ProductConstructionService.Common/RedisCache.cs new file mode 100644 index 0000000000..2eada09dbe --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Common/RedisCache.cs @@ -0,0 +1,124 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.Serialization; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using StackExchange.Redis; + +namespace ProductConstructionService.Common; + +public interface IRedisCache +{ + Task GetAsync(); + Task SetAsync(string value, TimeSpan? expiration = null); + Task TryDeleteAsync(); + Task TryGetAsync(); +} + +public class RedisCache : IRedisCache +{ + private readonly string _stateKey; + private readonly IConnectionMultiplexer _connection; + + private IDatabase Cache => _connection.GetDatabase(); + + public RedisCache(IConnectionMultiplexer connection, string stateKey) + { + _connection = connection; + _stateKey = stateKey; + } + + public async Task SetAsync(string value, TimeSpan? expiration = null) + { + await Cache.StringSetAsync(_stateKey, value, expiration); + } + + public async Task TryGetAsync() + { + var value = await Cache.StringGetAsync(_stateKey); + return value.HasValue ? value.ToString() : null; + } + + public async Task TryDeleteAsync() + { + return await Cache.StringGetDeleteAsync(_stateKey); + } + + public async Task GetAsync() + { + return await Cache.StringGetAsync(_stateKey); + } +} + +public interface IRedisCache where T : class +{ + Task SetAsync(T value, TimeSpan? expiration = null); + Task TryDeleteAsync(); + Task TryGetStateAsync(); +} + +public class RedisCache : IRedisCache where T : class +{ + public static readonly JsonSerializerOptions JsonSerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = false, + }; + + private readonly IRedisCache _stateManager; + private readonly ILogger _logger; + + public RedisCache( + IRedisCache stateManager, + ILogger logger) + { + _stateManager = stateManager; + _logger = logger; + } + + public async Task TryGetStateAsync() => await TryGetStateAsync(false); + + public async Task TryDeleteAsync() => await TryGetStateAsync(true); + + public async Task SetAsync(T value, TimeSpan? expiration = null) + { + string json; + try + { + json = JsonSerializer.Serialize(value, JsonSerializerOptions); + } + catch (SerializationException e) + { + _logger.LogError(e, "Failed to serialize {type} into cache", typeof(T).Name); + return; + } + + await _stateManager.SetAsync(json, expiration); + } + + private async Task TryGetStateAsync(bool delete) + { + var state = delete + ? await _stateManager.TryDeleteAsync() + : await _stateManager.TryGetAsync(); + if (state == null) + { + return null; + } + + try + { + var result = JsonSerializer.Deserialize(state, JsonSerializerOptions); + return result; + } + catch (SerializationException e) + { + // If we can't deserialize (maybe the model changed?), we drop the state + _logger.LogError(e, "Failed to deserialize state {type}. Removing from state memory. Original value: {value}", + typeof(T).Name, + await TryDeleteAsync()); + return null; + } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Common/StateManager.cs b/src/ProductConstructionService/ProductConstructionService.Common/StateManager.cs deleted file mode 100644 index bf2c22eb64..0000000000 --- a/src/ProductConstructionService/ProductConstructionService.Common/StateManager.cs +++ /dev/null @@ -1,51 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using StackExchange.Redis; - -namespace ProductConstructionService.Common; - -public interface IStateManager -{ - public void Set(string value); - public Task SetAsync(string value); - public bool TryGet(out string? value); - public Task GetAsync(); -} - -public class StateManager : IStateManager -{ - private readonly string _stateKey; - private readonly IConnectionMultiplexer _connection; - - private IDatabase Cache => _connection.GetDatabase(); - - internal StateManager(IConnectionMultiplexer connection, string stateKey) - { - _connection = connection; - _stateKey = stateKey; - } - - public void Set(string value) - { - Cache.StringSet(_stateKey, value); - } - - public async Task SetAsync(string value) - { - await Cache.StringSetAsync(_stateKey, value); - } - - public bool TryGet(out string? value) - { - value = Cache.StringGet(_stateKey); - - return value != null; - } - - public async Task GetAsync() - { - return await Cache.StringGetAsync(_stateKey); - } - -} diff --git a/src/ProductConstructionService/ProductConstructionService.Common/StateManagerFactory.cs b/src/ProductConstructionService/ProductConstructionService.Common/StateManagerFactory.cs deleted file mode 100644 index faad2863bf..0000000000 --- a/src/ProductConstructionService/ProductConstructionService.Common/StateManagerFactory.cs +++ /dev/null @@ -1,45 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using StackExchange.Redis; - -namespace ProductConstructionService.Common; - -public interface IStateManagerFactory -{ - public IStateManager CreateCodeFlowStateManager(string actorId); - public IStateManager CreateInProgressPrStateManager(string actorId); - public IStateManager CreateUpdateAssetsStateManager(string actorId); -} - -public class StateManagerFactory: IStateManagerFactory -{ - private static readonly string CODEFLOW_PREFIX = "CodeFlow"; - private static readonly string IN_PROGRESS_PR_PREFIX = "InProgressPullRequest"; - private static readonly string UPDATE_ASSETS_PARAMETERS_PREFIX = "UpdateAssetsParameters"; - - // State managers should store and re-use `ConnectionMultiplexer`, so we - // only use one instance for every state manager we create - // https://stackexchange.github.io/StackExchange.Redis/Basics - private readonly IConnectionMultiplexer _connection; - - public StateManagerFactory(ConfigurationOptions options) - { - _connection = ConnectionMultiplexer.Connect(options); - } - - public IStateManager CreateCodeFlowStateManager(string actorId) - { - return new StateManager(_connection, $"{CODEFLOW_PREFIX}_{actorId}"); - } - - public IStateManager CreateInProgressPrStateManager(string actorId) - { - return new StateManager(_connection, $"{IN_PROGRESS_PR_PREFIX}_{actorId}"); - } - - public IStateManager CreateUpdateAssetsStateManager(string actorId) - { - return new StateManager(_connection, $"{UPDATE_ASSETS_PARAMETERS_PREFIX}_{actorId}"); - } -} diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/ProductConstructionService.DependencyFlow.csproj b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/ProductConstructionService.DependencyFlow.csproj index 5331d100c9..1a98faf9dc 100644 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/ProductConstructionService.DependencyFlow.csproj +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/ProductConstructionService.DependencyFlow.csproj @@ -8,6 +8,12 @@ False + + + + + + diff --git a/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/LongestBuildPathUpdaterConfiguration.cs b/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/LongestBuildPathUpdaterConfiguration.cs index 71edae82be..08a832dae4 100644 --- a/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/LongestBuildPathUpdaterConfiguration.cs +++ b/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/LongestBuildPathUpdaterConfiguration.cs @@ -14,7 +14,7 @@ public static void ConfigureLongestBuildPathUpdater( ITelemetryChannel telemetryChannel) { builder.Services.RegisterLogging(telemetryChannel, builder.Environment.IsDevelopment()); - builder.RegisterBuildAssetRegistry(); + builder.AddBuildAssetRegistry(); builder.Services.Configure(o => { }); diff --git a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggererConfiguration.cs b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggererConfiguration.cs index 32aff1d463..8ec0b48ae5 100644 --- a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggererConfiguration.cs +++ b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggererConfiguration.cs @@ -27,7 +27,7 @@ public static HostApplicationBuilder ConfigureSubscriptionTriggerer( builder.Services.RegisterLogging(telemetryChannel, builder.Environment.IsDevelopment()); - builder.RegisterBuildAssetRegistry(); + builder.AddBuildAssetRegistry(); builder.AddWorkItemProducerFactory(credential); builder.Services.Configure(o => { }); diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/ReminderManager.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/ReminderManager.cs new file mode 100644 index 0000000000..a0400338e1 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/ReminderManager.cs @@ -0,0 +1,60 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using ProductConstructionService.Common; + +namespace ProductConstructionService.WorkItems; + +public interface IReminderManager where T : WorkItem +{ + Task RegisterReminderAsync(T reminder, TimeSpan visibilityTimeout); + + Task UnregisterReminderAsync(); +} + +public class ReminderManager : IReminderManager where T : WorkItem +{ + private readonly IWorkItemProducerFactory _workItemProducerFactory; + private readonly IRedisCache _receiptCache; + + public ReminderManager( + IWorkItemProducerFactory workItemProducerFactory, + IRedisCacheFactory cacheFactory, + string key) + { + _workItemProducerFactory = workItemProducerFactory; + _receiptCache = cacheFactory.Create($"ReminderReceipt_{key}"); + } + + public async Task RegisterReminderAsync(T payload, TimeSpan visibilityTimeout) + { + var client = _workItemProducerFactory.CreateProducer(); + var sendReceipt = await client.ProduceWorkItemAsync(payload, visibilityTimeout); + await _receiptCache.SetAsync(new ReminderArguments(sendReceipt.PopReceipt, sendReceipt.MessageId), visibilityTimeout); + } + + public async Task UnregisterReminderAsync() + { + var receipt = await _receiptCache.TryDeleteAsync(); + if (receipt == null) + { + return; + } + + var client = _workItemProducerFactory.CreateProducer(); + await client.DeleteWorkItemAsync(receipt.MessageId, receipt.PopReceipt); + } + + private class ReminderArguments + { + public string PopReceipt { get; set; } + + public string MessageId { get; set; } + + public ReminderArguments(string popReceipt, string messageId) + { + PopReceipt = popReceipt; + MessageId = messageId; + } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/ReminderManagerFactory.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/ReminderManagerFactory.cs new file mode 100644 index 0000000000..ec3afd0b5a --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/ReminderManagerFactory.cs @@ -0,0 +1,28 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using ProductConstructionService.Common; + +namespace ProductConstructionService.WorkItems; + +public interface IReminderManagerFactory +{ + IReminderManager CreateReminderManager(string key) where T : WorkItem; +} + +public class ReminderManagerFactory : IReminderManagerFactory +{ + private readonly IWorkItemProducerFactory _workItemProducerFactory; + private readonly IRedisCacheFactory _cacheFactory; + + public ReminderManagerFactory(IWorkItemProducerFactory workItemProducerFactory, IRedisCacheFactory cacheFactory) + { + _workItemProducerFactory = workItemProducerFactory; + _cacheFactory = cacheFactory; + } + + public IReminderManager CreateReminderManager(string key) where T : WorkItem + { + return new ReminderManager(_workItemProducerFactory, _cacheFactory, key); + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProducer.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProducer.cs index 7d5138765a..001db4bcf7 100644 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProducer.cs +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemProducer.cs @@ -9,7 +9,15 @@ namespace ProductConstructionService.WorkItems; public interface IWorkItemProducer { + /// + /// Puts a WorkItem into the queue, which becomes visible after the specified delay. + /// Task ProduceWorkItemAsync(T payload, TimeSpan delay = default); + + /// + /// Deletes a WorkItem from the queue. + /// + Task DeleteWorkItemAsync(string messageId, string popReceipt); } public class WorkItemProducer(QueueServiceClient queueServiceClient, string queueName) : IWorkItemProducer where T : WorkItem @@ -17,13 +25,16 @@ public class WorkItemProducer(QueueServiceClient queueServiceClient, string q private readonly QueueServiceClient _queueServiceClient = queueServiceClient; private readonly string _queueName = queueName; - /// - /// Puts a WorkItem into the queue, which becomes visible after the specified delay. - /// public async Task ProduceWorkItemAsync(T payload, TimeSpan delay = default) { var client = _queueServiceClient.GetQueueClient(_queueName); var json = JsonSerializer.Serialize(payload, WorkItemConfiguration.JsonSerializerOptions); return await client.SendMessageAsync(json, delay); } + + public async Task DeleteWorkItemAsync(string messageId, string popReceipt) + { + var client = _queueServiceClient.GetQueueClient(_queueName); + await client.DeleteMessageAsync(messageId, popReceipt); + } } From 7b4a8c8b6f046010699c5a28ee60020656da30df Mon Sep 17 00:00:00 2001 From: Stephen Halter Date: Thu, 22 Aug 2024 23:57:09 -0700 Subject: [PATCH 09/11] Fix NRE in DependencyFlow's Incoming.cshtml.cs (#3869) --- .../Maestro.Web/Pages/DependencyFlow/Incoming.cshtml.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Maestro/Maestro.Web/Pages/DependencyFlow/Incoming.cshtml.cs b/src/Maestro/Maestro.Web/Pages/DependencyFlow/Incoming.cshtml.cs index 71d3e70d3d..a70a7049c3 100644 --- a/src/Maestro/Maestro.Web/Pages/DependencyFlow/Incoming.cshtml.cs +++ b/src/Maestro/Maestro.Web/Pages/DependencyFlow/Incoming.cshtml.cs @@ -114,7 +114,7 @@ public async Task OnGet(int channelId, string owner, string repo) } IncomingRepositories = incoming; - CurrentRateLimit = _github.GetLastApiInfo().RateLimit; + CurrentRateLimit = _github.GetLastApiInfo()?.RateLimit; return Page(); } @@ -122,7 +122,10 @@ public async Task OnGet(int channelId, string owner, string repo) private async Task GetOldestUnconsumedBuild(int lastConsumedBuildOfDependencyId) { // Note: We fetch `build` again here so that it will have channel information, which it doesn't when coming from the graph :( - var build = await _context.Builds.FindAsync(lastConsumedBuildOfDependencyId); + var build = await _context.Builds.Where(b => b.Id == lastConsumedBuildOfDependencyId) + .Include(b => b.BuildChannels) + .ThenInclude(bc => bc.Channel) + .FirstOrDefaultAsync(); if (build == null) { @@ -131,9 +134,11 @@ public async Task OnGet(int channelId, string owner, string repo) var channelId = build.BuildChannels.FirstOrDefault(bc => bc.Channel.Classification == "product" || bc.Channel.Classification == "tools")?.ChannelId; var publishedBuildsOfDependency = await _context.Builds + .Include(b => b.BuildChannels) .Where(b => b.GitHubRepository == build.GitHubRepository && b.DateProduced >= build.DateProduced.AddSeconds(-5) && b.BuildChannels.Any(bc => bc.ChannelId == channelId)) + .OrderByDescending(b => b.DateProduced) .ToListAsync(); var last = publishedBuildsOfDependency.LastOrDefault(); From 3f0709e1160b8dd318aa9e9d4e3d40d20f8d5a45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Fri, 23 Aug 2024 11:04:59 +0200 Subject: [PATCH 10/11] Port changes of Maestro's Incoming page to PCS (#3874) --- .../Pages/DependencyFlow/Incoming.cshtml.cs | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Pages/DependencyFlow/Incoming.cshtml.cs b/src/ProductConstructionService/ProductConstructionService.Api/Pages/DependencyFlow/Incoming.cshtml.cs index 314573423e..78938e12d3 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Pages/DependencyFlow/Incoming.cshtml.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Pages/DependencyFlow/Incoming.cshtml.cs @@ -108,7 +108,7 @@ public async Task OnGet(int channelId, string owner, string repo) } IncomingRepositories = incoming; - CurrentRateLimit = _github.GetLastApiInfo().RateLimit; + CurrentRateLimit = _github.GetLastApiInfo()?.RateLimit; return Page(); } @@ -116,7 +116,10 @@ public async Task OnGet(int channelId, string owner, string repo) private async Task GetOldestUnconsumedBuild(int lastConsumedBuildOfDependencyId) { // Note: We fetch `build` again here so that it will have channel information, which it doesn't when coming from the graph :( - var build = await _context.Builds.FindAsync(lastConsumedBuildOfDependencyId); + var build = await _context.Builds.Where(b => b.Id == lastConsumedBuildOfDependencyId) + .Include(b => b.BuildChannels) + .ThenInclude(bc => bc.Channel) + .FirstOrDefaultAsync(); if (build == null) { @@ -125,9 +128,11 @@ public async Task OnGet(int channelId, string owner, string repo) var channelId = build.BuildChannels.FirstOrDefault(bc => bc.Channel.Classification == "product" || bc.Channel.Classification == "tools")?.ChannelId; var publishedBuildsOfDependency = await _context.Builds + .Include(b => b.BuildChannels) .Where(b => b.GitHubRepository == build.GitHubRepository && b.DateProduced >= build.DateProduced.AddSeconds(-5) && b.BuildChannels.Any(bc => bc.ChannelId == channelId)) + .OrderByDescending(b => b.DateProduced) .ToListAsync(); var last = publishedBuildsOfDependency.LastOrDefault(); @@ -169,7 +174,7 @@ public string GetBuildUrl(Build? build) ? "(unknown)" : $"https://dev.azure.com/{build.AzureDevOpsAccount}/{build.AzureDevOpsProject}/_build/results?buildId={build.AzureDevOpsBuildId}&view=results"; - private bool IncludeRepo(GitHubInfo? gitHubInfo) + private static bool IncludeRepo(GitHubInfo? gitHubInfo) { if (string.Equals(gitHubInfo?.Owner, "dotnet", StringComparison.OrdinalIgnoreCase) && string.Equals(gitHubInfo?.Repo, "blazor", StringComparison.OrdinalIgnoreCase)) @@ -212,8 +217,11 @@ public string GetDateProduced(Build? build) } catch (NotFoundException) { - _logger.LogWarning("Failed to compare commit history for '{0}/{1}' between '{2}' and '{3}'.", gitHubInfo.Owner, gitHubInfo.Repo, - build.Commit, build.GitHubBranch); + _logger.LogWarning("Failed to compare commit history for '{owner}/{repo}' between '{commit}' and '{branch}'.", + gitHubInfo.Owner, + gitHubInfo.Repo, + build.Commit, + build.GitHubBranch); return null; } } From f51bd3ec89e421531a7f51a96859948bbb3fc8f8 Mon Sep 17 00:00:00 2001 From: Djuradj Kurepa <91743470+dkurepa@users.noreply.github.com> Date: Mon, 26 Aug 2024 10:28:54 +0200 Subject: [PATCH 11/11] Build docker during pr (#3867) --- azure-pipelines-pr.yml | 34 +++++++++------ ...pipelines-product-construction-service.yml | 41 +++---------------- eng/templates/steps/docker-build.yml | 36 ++++++++++++++++ eng/templates/steps/test.yml | 11 +++-- .../ProductConstructionService.Api/Dockerfile | 3 +- 5 files changed, 71 insertions(+), 54 deletions(-) create mode 100644 eng/templates/steps/docker-build.yml diff --git a/azure-pipelines-pr.yml b/azure-pipelines-pr.yml index fabcfee8de..fa1097a0c4 100644 --- a/azure-pipelines-pr.yml +++ b/azure-pipelines-pr.yml @@ -6,6 +6,8 @@ variables: value: https://dev.azure.com/dnceng - name: AzdoProject value: internal +- name: configuration + value: Release pr: branches: @@ -64,29 +66,35 @@ stages: enablePublishUsingPipelines: ${{ variables._PublishUsingPipelines }} jobs: - - job: Windows_NT + - job: Build + displayName: Build Repo timeoutInMinutes: 90 pool: name: NetCore-Public demands: ImageOverride -equals 1es-windows-2019-open - strategy: - matrix: - debug_configuration: - _BuildConfig: Debug - _PublishType: none - _SignType: test - release_configuration: - _BuildConfig: Release - # PRs or external builds are not signed. - _PublishType: none - _SignType: test steps: - checkout: self clean: true - template: /eng/templates/steps/build.yml parameters: - configuration: $(_BuildConfig) + configuration: $(configuration) - template: /eng/templates/steps/test.yml + parameters: + configuration: $(configuration) + + - job: Builder_Docker + displayName: Build Docker Image + pool: + name: NetCore-Public + demands: ImageOverride -equals 1es-ubuntu-2004-open + + steps: + - checkout: self + clean: true + + - template: /eng/templates/steps/docker-build.yml + parameters: + dockerImageName: test diff --git a/azure-pipelines-product-construction-service.yml b/azure-pipelines-product-construction-service.yml index ac9a480cf5..a105ccd795 100644 --- a/azure-pipelines-product-construction-service.yml +++ b/azure-pipelines-product-construction-service.yml @@ -58,44 +58,13 @@ stages: steps: - checkout: self - - powershell: | - Write-Host "Dev branch suffix is $(devBranchSuffix)" - $shortSha = "$(Build.SourceVersion)".Substring(0,10) - $newDockerTag = "$(Build.BuildNumber)-$(System.JobAttempt)-$shortSha$(devBranchSuffix)" - Write-Host "##vso[task.setvariable variable=newDockerImageTag]$newDockerTag" - Write-Host "set newDockerImageTag to $newDockerTag" - displayName: Generate docker image tag - - - powershell: > - docker build . - -f $(Build.SourcesDirectory)/src/ProductConstructionService/ProductConstructionService.Api/Dockerfile - -t "$(dockerRegistryUrl)/$(containerName):$(newDockerImageTag)" - displayName: Build docker image + - template: eng/templates/steps/docker-build.yml + parameters: + devBranchSuffix: $(devBranchSuffix) + dockerImageName: $(dockerRegistryUrl)/$(containerName) - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: - - task: AzureCLI@2 - inputs: - azureSubscription: $(serviceConnectionName) - scriptType: pscore - scriptLocation: inlineScript - inlineScript: | - az acr login --name $(containerRegistryName) - docker push "$(dockerRegistryUrl)/$(containerName):$(newDockerImageTag)" - displayName: Push docker image - - ${{ if ne(variables['Build.SourceBranch'], 'refs/heads/production') }}: - - task: AzureCLI@2 - inputs: - # The Service Connection name needs to be known at compile time, so we can't use a variable for the azure subscription - azureSubscription: $(serviceConnectionName) - scriptType: pscore - scriptLocation: inlineScript - inlineScript: | - New-Item -ItemType Directory -Path $(diffFolder) - $before = az containerapp show --name $(containerappName) -g $(resourceGroupName) --output json - Set-Content -Path $(diffFolder)/before.json -Value $before - displayName: Snapshot configuration (before) - - task: AzureCLI@2 name: GetAuthInfo displayName: Get PCS Token @@ -118,7 +87,7 @@ stages: arguments: > -resourceGroupName $(resourceGroupName) -containerappName $(containerappName) - -newImageTag $(newDockerImageTag) + -newImageTag $(DockerTag.newDockerImageTag) -containerRegistryName $(containerRegistryName) -imageName $(containerName) -token $(GetAuthInfo.Token) diff --git a/eng/templates/steps/docker-build.yml b/eng/templates/steps/docker-build.yml new file mode 100644 index 0000000000..1add68f816 --- /dev/null +++ b/eng/templates/steps/docker-build.yml @@ -0,0 +1,36 @@ +parameters: +- name: devBranchSuffix + type: string + default: '' +- name: dockerImageName + type: string + +steps: +- powershell: | + Write-Host "Dev branch suffix is ${{ parameters.devBranchSuffix }}" + $shortSha = "$(Build.SourceVersion)".Substring(0,10) + $newDockerTag = "$(Build.BuildNumber)-$(System.JobAttempt)-$shortSha${{ parameters.devBranchSuffix }}" + Write-Host "##vso[task.setvariable variable=newDockerImageTag;isOutput=true]$newDockerTag" + Write-Host "set newDockerImageTag to $newDockerTag" + displayName: Generate docker image tag + name: DockerTag + +- powershell: | + mkdir $(Build.SourcesDirectory)/artifacts/log + docker build . ` + -f $(Build.SourcesDirectory)/src/ProductConstructionService/ProductConstructionService.Api/Dockerfile ` + -t "${{ parameters.dockerImageName }}:$(DockerTag.newDockerImageTag)" ` + --progress=plain ` + 2>&1 | tee $(Build.SourcesDirectory)/artifacts/log/docker-build.log + displayName: Build docker image + +- ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: + - task: AzureCLI@2 + inputs: + azureSubscription: $(serviceConnectionName) + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + az acr login --name $(containerRegistryName) + docker push "$(dockerRegistryUrl)/$(containerName):$(DockerTag.newDockerImageTag)" + displayName: Push docker image \ No newline at end of file diff --git a/eng/templates/steps/test.yml b/eng/templates/steps/test.yml index 6c6248f18c..cca64f9705 100644 --- a/eng/templates/steps/test.yml +++ b/eng/templates/steps/test.yml @@ -1,5 +1,8 @@ -steps: +parameters: +- name: configuration + type: string +steps: - task: Powershell@2 displayName: Install SQL Express inputs: @@ -14,12 +17,12 @@ steps: $(Build.SourcesDirectory)\arcade-services.sln custom: test arguments: > - --configuration $(_BuildConfig) + --configuration ${{ parameters.configuration }} --filter "TestCategory!=PostDeployment&TestCategory!=Nightly&TestCategory!=PreDeployment" --no-build --logger "trx;LogFilePrefix=TestResults-" -v normal - /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/UnitTest.binlog + /bl:$(Build.SourcesDirectory)/artifacts/log/${{ parameters.configuration }}/UnitTest.binlog -- "RunConfiguration.ResultsDirectory=$(Build.ArtifactStagingDirectory)\TestResults" RunConfiguration.MapCpuCount=4 @@ -36,7 +39,7 @@ steps: searchFolder: $(Build.ArtifactStagingDirectory)\TestResults testRunTitle: Basic Tests mergeTestResults: true - configuration: $(_BuildConfig) + configuration: ${{ parameters.configuration }} - script: echo export const token = ''; > src/environments/token.ts workingDirectory: $(Build.SourcesDirectory)/src/Maestro/maestro-angular diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Dockerfile b/src/ProductConstructionService/ProductConstructionService.Api/Dockerfile index d85c4f6517..0f51b8d432 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Dockerfile +++ b/src/ProductConstructionService/ProductConstructionService.Api/Dockerfile @@ -26,9 +26,10 @@ COPY ["src/Microsoft.DotNet.Darc/DarcLib/Microsoft.DotNet.DarcLib.csproj", "./Mi WORKDIR /src/ProductConstructionService COPY ["src/ProductConstructionService/ProductConstructionService.Api/ProductConstructionService.Api.csproj", "./ProductConstructionService.Api/"] COPY ["src/ProductConstructionService/ProductConstructionService.Common/ProductConstructionService.Common.csproj", "./ProductConstructionService.Common/"] +COPY ["src/ProductConstructionService/ProductConstructionService.DependencyFlow/ProductConstructionService.DependencyFlow.csproj", "./ProductConstructionService.DependencyFlow/"] +COPY ["src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/ProductConstructionService.LongestBuildPathUpdater.csproj", "./ProductConstructionService.LongestBuildPathUpdater/"] COPY ["src/ProductConstructionService/ProductConstructionService.ServiceDefaults/ProductConstructionService.ServiceDefaults.csproj", "./ProductConstructionService.ServiceDefaults/"] COPY ["src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/ProductConstructionService.SubscriptionTriggerer.csproj", "./ProductConstructionService.SubscriptionTriggerer/"] -COPY ["src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/ProductConstructionService.LongestBuildPathUpdater.csproj", "./ProductConstructionService.LongestBuildPathUpdater/"] COPY ["src/ProductConstructionService/ProductConstructionService.WorkItems/ProductConstructionService.WorkItems.csproj", "./ProductConstructionService.WorkItems/"] RUN dotnet restore "./ProductConstructionService.Api/ProductConstructionService.Api.csproj"